diff --git a/Parallel/Extensions.vb b/Parallel/Extensions.vb new file mode 100644 index 0000000..b32a112 --- /dev/null +++ b/Parallel/Extensions.vb @@ -0,0 +1,4 @@ + +Public Module Extensions + +End Module diff --git a/Parallel/IpcParallel/IDelegate.vb b/Parallel/IpcParallel/IDelegate.vb index 853e11e..e4cca78 100644 --- a/Parallel/IpcParallel/IDelegate.vb +++ b/Parallel/IpcParallel/IDelegate.vb @@ -49,8 +49,15 @@ Imports Microsoft.VisualBasic.ApplicationServices.Development.NetCore5 #End If Imports TypeInfo = Microsoft.VisualBasic.Scripting.MetaData.TypeInfo +''' +''' remote method handler +''' Public Class IDelegate + ''' + ''' the function name + ''' + ''' Public Property name As String Public Property type As TypeInfo ''' @@ -62,6 +69,12 @@ Public Class IDelegate Sub New() End Sub + ''' + ''' + ''' + ''' + ''' The name of this target method should be unique! + ''' Sub New(target As MethodInfo) type = New TypeInfo(target.DeclaringType) name = target.Name diff --git a/Parallel/Parallel.vbproj b/Parallel/Parallel.vbproj index 0eed86c..8527947 100644 --- a/Parallel/Parallel.vbproj +++ b/Parallel/Parallel.vbproj @@ -20,7 +20,9 @@ true bin\Debug\ Parallel.xml - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 + + pdbonly @@ -29,7 +31,20 @@ true bin\Release\ Parallel.xml - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 + + + + + pdbonly + false + true + true + bin\Release\ + Parallel.xml + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 + + On @@ -49,47 +64,70 @@ true bin\x64\Debug\ Parallel.xml - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 full x64 + + + + + true + bin\x64\Release\ + Parallel.xml + false + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 + full + x64 + + + true + true true bin\x64\Release\ Parallel.xml true - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 pdbonly x64 + + true bin\Rsharp_app_release\ Parallel.xml true - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 pdbonly AnyCPU + + true bin\x64\Rsharp_app_release\ Parallel.xml false - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 full x64 true true + + true bin\GCModeller%28UNIX-Release_X64%29\ Parallel.xml true - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 pdbonly AnyCPU + + true @@ -97,27 +135,51 @@ bin\x64\GCModeller%28UNIX-Release_X64%29\ true Parallel.xml - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 full x64 + + true bin\GCModeller%28Release_X64%29\ Parallel.xml true - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 pdbonly AnyCPU + + true bin\x64\GCModeller%28Release_X64%29\ Parallel.xml true - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 pdbonly x64 + + + + + x64 + + + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 + Parallel.xml + true + true + full + true + bin\x64\ + + + + + 41999,42016,42017,42018,42019,42020,42021,42022,42032,42036 + Parallel.xml @@ -153,6 +215,7 @@ + True diff --git a/Parallel/ThreadTask/BatchTasks.vb b/Parallel/ThreadTask/BatchTasks.vb index 249ec05..0c07271 100644 --- a/Parallel/ThreadTask/BatchTasks.vb +++ b/Parallel/ThreadTask/BatchTasks.vb @@ -45,62 +45,67 @@ Imports System.Runtime.CompilerServices Imports Microsoft.VisualBasic.CommandLine Imports Microsoft.VisualBasic.Language -Public Module BatchTasks - - ''' - ''' Folk this program itself for the large amount data batch processing. - ''' - ''' Self folk processing commandline collection. - ''' If this parameter value less than 1, then will be a single - ''' thread task. Any positive value that greater than 1 will be parallel task. - ''' (小于等于零表示非并行化，单线程任务) - ''' - ''' - ''' Returns the total executation time for running this task collection. - ''' (返回任务的执行的总时长) - ''' - Public Function SelfFolks(CLI As IEnumerable(Of String), Optional parallel% = 0) As Long - Dim sw As Stopwatch = Stopwatch.StartNew - - If parallel <= 0 Then - For Each args As String In CLI - Call App.SelfFolk(args).Run() - Next - Else - Dim Tasks As Func(Of Integer)() = LinqAPI.Exec(Of Func(Of Integer)) <= +Namespace ThreadTask + + Public Module BatchTasks + + ''' + ''' Folk this program itself for the large amount data batch processing. + ''' + ''' Self folk processing commandline collection. + ''' If this parameter value less than 1, then will be a single + ''' thread task. Any positive value that greater than 1 will be parallel task. + ''' (小于等于零表示非并行化，单线程任务) + ''' + ''' + ''' Returns the total executation time for running this task collection. + ''' (返回任务的执行的总时长) + ''' + Public Function SelfFolks(CLI As IEnumerable(Of String), Optional parallel% = 0) As Long + Dim sw As Stopwatch = Stopwatch.StartNew + + If parallel <= 0 Then + For Each args As String In CLI + Call App.SelfFolk(args).Run() + Next + Else + Dim Tasks As Func(Of Integer)() = LinqAPI.Exec(Of Func(Of Integer)) <= _ - From args As String - In CLI - Let io As IIORedirectAbstract = App.SelfFolk(args) - Let task As Func(Of Integer) = AddressOf io.Run - Select task - - Call New ThreadTask(Of Integer)(Tasks).WithDegreeOfParallelism(parallel).RunParallel.ToArray - End If - - Return sw.ElapsedMilliseconds - End Function - - ''' - ''' - ''' - ''' - ''' 同时执行的句柄的数目 - ''' - - Public Sub Invoke(tasks As Action(), numOfThreads As Integer) - Dim getTask As Func(Of Action, Func(Of Integer)) = - Function(task) - Return AddressOf New TaskInvokeHelper With { - .task = task - }.RunTask - End Function - Dim invokes = From action As Action In tasks Select getTask(action) - - Call New ThreadTask(Of Integer)(invokes) _ - .WithDegreeOfParallelism(numOfThreads) _ - .RunParallel() _ - .ToArray - End Sub -End Module - + From args As String + In CLI + Let io As IIORedirectAbstract = App.SelfFolk(args) + Let task As Func(Of Integer) = AddressOf io.Run + Select task + + Call New ThreadTask(Of Integer)(Tasks) _ + .WithDegreeOfParallelism(parallel) _ + .RunParallel _ + .ToArray + End If + + Return sw.ElapsedMilliseconds + End Function + + ''' + ''' + ''' + ''' + ''' 同时执行的句柄的数目 + ''' + + Public Sub Invoke(tasks As Action(), numOfThreads As Integer) + Dim getTask As Func(Of Action, Func(Of Integer)) = + Function(task) + Return AddressOf New TaskInvokeHelper With { + .task = task + }.RunTask + End Function + Dim invokes = From action As Action In tasks Select getTask(action) + + Call New ThreadTask(Of Integer)(invokes) _ + .WithDegreeOfParallelism(numOfThreads) _ + .RunParallel() _ + .ToArray + End Sub + End Module +End Namespace \ No newline at end of file diff --git a/Parallel/ThreadTask/TaskInvokeHelper.vb b/Parallel/ThreadTask/TaskInvokeHelper.vb index 49a5a12..1c46912 100644 --- a/Parallel/ThreadTask/TaskInvokeHelper.vb +++ b/Parallel/ThreadTask/TaskInvokeHelper.vb @@ -1,50 +1,53 @@ #Region "Microsoft.VisualBasic::ef1677867edb409e7343869d372b4818, Parallel\ThreadTask\TaskInvokeHelper.vb" - ' Author: - ' - ' asuka (amethyst.asuka@gcmodeller.org) - ' xie (genetics@smrucc.org) - ' xieguigang (xie.guigang@live.com) - ' - ' Copyright (c) 2018 GPL3 Licensed - ' - ' - ' GNU GENERAL PUBLIC LICENSE (GPL3) - ' - ' - ' This program is free software: you can redistribute it and/or modify - ' it under the terms of the GNU General Public License as published by - ' the Free Software Foundation, either version 3 of the License, or - ' (at your option) any later version. - ' - ' This program is distributed in the hope that it will be useful, - ' but WITHOUT ANY WARRANTY; without even the implied warranty of - ' MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - ' GNU General Public License for more details. - ' - ' You should have received a copy of the GNU General Public License - ' along with this program. If not, see . - - - - ' /********************************************************************************/ - - ' Summaries: - - ' Structure TaskInvokeHelper - ' - ' Function: RunTask - ' - ' /********************************************************************************/ +' Author: +' +' asuka (amethyst.asuka@gcmodeller.org) +' xie (genetics@smrucc.org) +' xieguigang (xie.guigang@live.com) +' +' Copyright (c) 2018 GPL3 Licensed +' +' +' GNU GENERAL PUBLIC LICENSE (GPL3) +' +' +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY; without even the implied warranty of +' MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +' GNU General Public License for more details. +' +' You should have received a copy of the GNU General Public License +' along with this program. If not, see . + + + +' /********************************************************************************/ + +' Summaries: + +' Structure TaskInvokeHelper +' +' Function: RunTask +' +' /********************************************************************************/ #End Region -Friend Structure TaskInvokeHelper +Namespace ThreadTask - Dim task As Action + Friend Structure TaskInvokeHelper - Public Function RunTask() As Integer - Call task() - Return 0 - End Function -End Structure + Dim task As Action + + Public Function RunTask() As Integer + Call task() + Return 0 + End Function + End Structure +End Namespace \ No newline at end of file diff --git a/Parallel/ThreadTask/ThreadTask.vb b/Parallel/ThreadTask/ThreadTask.vb index 4d9fe03..3764b2e 100644 --- a/Parallel/ThreadTask/ThreadTask.vb +++ b/Parallel/ThreadTask/ThreadTask.vb @@ -1,179 +1,181 @@ #Region "Microsoft.VisualBasic::c970e504d23e8ca06d6e00adf594cff8, Parallel\ThreadTask\ThreadTask.vb" - ' Author: - ' - ' asuka (amethyst.asuka@gcmodeller.org) - ' xie (genetics@smrucc.org) - ' xieguigang (xie.guigang@live.com) - ' - ' Copyright (c) 2018 GPL3 Licensed - ' - ' - ' GNU GENERAL PUBLIC LICENSE (GPL3) - ' - ' - ' This program is free software: you can redistribute it and/or modify - ' it under the terms of the GNU General Public License as published by - ' the Free Software Foundation, either version 3 of the License, or - ' (at your option) any later version. - ' - ' This program is distributed in the hope that it will be useful, - ' but WITHOUT ANY WARRANTY; without even the implied warranty of - ' MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - ' GNU General Public License for more details. - ' - ' You should have received a copy of the GNU General Public License - ' along with this program. If not, see . - - - - ' /********************************************************************************/ - - ' Summaries: - - ' Class ThreadTask - ' - ' Constructor: (+1 Overloads) Sub New - ' Function: (+2 Overloads) CreateThreads, GetCompleteThread, GetEmptyThread, RunParallel, ToString - ' WithDegreeOfParallelism - ' - ' /********************************************************************************/ +' Author: +' +' asuka (amethyst.asuka@gcmodeller.org) +' xie (genetics@smrucc.org) +' xieguigang (xie.guigang@live.com) +' +' Copyright (c) 2018 GPL3 Licensed +' +' +' GNU GENERAL PUBLIC LICENSE (GPL3) +' +' +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY; without even the implied warranty of +' MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +' GNU General Public License for more details. +' +' You should have received a copy of the GNU General Public License +' along with this program. If not, see . + + + +' /********************************************************************************/ + +' Summaries: + +' Class ThreadTask +' +' Constructor: (+1 Overloads) Sub New +' Function: (+2 Overloads) CreateThreads, GetCompleteThread, GetEmptyThread, RunParallel, ToString +' WithDegreeOfParallelism +' +' /********************************************************************************/ #End Region Imports Microsoft.VisualBasic.Parallel.Tasks -''' -''' Using parallel linq that may stuck the program when a linq task partion wait a long time task to complete. -''' By using this parallel function that you can avoid this problem from parallel linq, and also you can -''' controls the task thread number manually by using this parallel task function. -''' (由于LINQ是分片段来执行的，当某个片段有一个线程被卡住之后整个进程都会被卡住，所以执行大型的计算任务的时候效率不太好， -''' 使用这个并行化函数可以避免这个问题，同时也可以自己手动控制线程的并发数) -''' -''' -Public Class ThreadTask(Of TOut) - - Dim taskList As Queue(Of Func(Of TOut)) - Dim threads As AsyncHandle(Of TOut)() - Dim size As Integer - - ''' - ''' create parallel task pool from a given collection of task handler - ''' - ''' - Sub New(task As IEnumerable(Of Func(Of TOut))) - Me.taskList = New Queue(Of Func(Of TOut))(task) - Me.size = Me.taskList.Count - End Sub - - ''' - ''' Create a parallel thread task pool and then get the task result value - ''' - ''' - ''' - ''' - ''' - Public Shared Function CreateThreads(Of T)(items As IEnumerable(Of T), task As Func(Of T, Func(Of TOut))) As ThreadTask(Of TOut) - Return New ThreadTask(Of TOut)(items.Select(task)) - End Function +Namespace ThreadTask ''' - ''' Create a parallel thread task pool and then get the task result value + ''' Using parallel linq that may stuck the program when a linq task partion wait a long time task to complete. + ''' By using this parallel function that you can avoid this problem from parallel linq, and also you can + ''' controls the task thread number manually by using this parallel task function. + ''' (由于LINQ是分片段来执行的，当某个片段有一个线程被卡住之后整个进程都会被卡住，所以执行大型的计算任务的时候效率不太好， + ''' 使用这个并行化函数可以避免这个问题，同时也可以自己手动控制线程的并发数) ''' - ''' - ''' - ''' - ''' - Public Shared Function CreateThreads(Of T)(items As IEnumerable(Of T), task As Func(Of T, TOut)) As ThreadTask(Of TOut) - Return New ThreadTask(Of TOut)(items.Select(Function(i) New Func(Of TOut)(Function() task(i)))) - End Function - - ''' - ''' You can controls the parallel tasks number from this parameter, smaller or equals to ZERO means auto - ''' config the thread number, If want single thread, not parallel, set this value to 1, and positive - ''' value greater than 1 will makes the tasks parallel. - ''' (可以在这里手动的控制任务的并发数，这个数值小于或者等于零则表示自动配置线程的数量, 1为单线程) - ''' - ''' - ''' - Public Function WithDegreeOfParallelism(n_threads As Integer) As ThreadTask(Of TOut) - threads = New AsyncHandle(Of TOut)(n_threads) {} - Return Me - End Function - - ''' - ''' get a index of thread which is idle - ''' - ''' + ''' + Public Class ThreadTask(Of TOut) + + Dim taskList As Queue(Of Func(Of TOut)) + Dim threads As AsyncHandle(Of TOut)() + Dim size As Integer + + ''' + ''' create parallel task pool from a given collection of task handler + ''' + ''' + Sub New(task As IEnumerable(Of Func(Of TOut))) + Me.taskList = New Queue(Of Func(Of TOut))(task) + Me.size = Me.taskList.Count + End Sub + + Public Shared Function CreateThreads(Of T)(items As IEnumerable(Of T), task As Func(Of T, Func(Of TOut))) As ThreadTask(Of TOut) + Return New ThreadTask(Of TOut)(items.Select(task)) + End Function + ''' + ''' + ''' + ''' + Public Shared Function CreateThreads(Of T)(items As IEnumerable(Of T), task As Func(Of T, Func(Of TOut))) As ThreadTask(Of TOut) + Return New ThreadTask(Of TOut)(items.Select(task)) + End Function + + ''' + ''' Create a parallel thread task pool and then get the task result value + End Function + + ''' + ''' + ''' + Public Shared Function CreateThreads(Of T)(items As IEnumerable(Of T), task As Func(Of T, TOut)) As ThreadTask(Of TOut) + Return New ThreadTask(Of TOut)(items.Select(Function(i) New Func(Of TOut)(Function() task(i)))) + End Function + + ''' + ''' You can controls the parallel tasks number from this parameter, smaller or equals to ZERO means auto + ''' config the thread number, If want single thread, not parallel, set this value to 1, and positive + ''' value greater than 1 will makes the tasks parallel. + ''' (可以在这里手动的控制任务的并发数，这个数值小于或者等于零则表示自动配置线程的数量, 1为单线程) + ''' + ''' + ''' + Public Function WithDegreeOfParallelism(n_threads As Integer) As ThreadTask(Of TOut) + threads = New AsyncHandle(Of TOut)(n_threads) {} + Return Me + End Function + + Private Function GetEmptyThread() As Integer + For i As Integer = 0 To threads.Length - 1 + If threads(i) Is Nothing Then + Return i Private Function GetEmptyThread() As Integer - For i As Integer = 0 To threads.Length - 1 - If threads(i) Is Nothing Then - Return i - End If + For i As Integer = 0 To threads.Length - 1 + If threads(i) Is Nothing Then + Return i + End If + Next + + Return -1 + End Function + + Private Function GetCompleteThread() As Integer + For i As Integer = 0 To threads.Length - 1 + If (Not threads(i) Is Nothing) AndAlso threads(i).IsCompleted Then + Return i + End If + Next + + Return -1 + End Function Next Return -1 - End Function - - ''' - ''' get the index of the first thread which is run task job complete - ''' - ''' - Private Function GetCompleteThread() As Integer - For i As Integer = 0 To threads.Length - 1 - If (Not threads(i) Is Nothing) AndAlso threads(i).IsCompleted Then - Return i - End If - Next - - Return -1 - End Function - - ''' - ''' view thread pool status - ''' - ''' - Public Overrides Function ToString() As String - Dim free$ = threads.Where(Function(t) t Is Nothing).Count - Dim running$ = threads.Where(Function(t) t IsNot Nothing AndAlso Not t.IsCompleted).Count - Dim finished$ = threads.Where(Function(t) t IsNot Nothing AndAlso t.IsCompleted).Count - Dim delta As Integer = size - taskList.Count - - Return $"[free: {free}, running: {running}, finished: {finished}, progress: {delta} - {CInt(delta / size * 100)}%]" - End Function - - ''' - ''' Run parallel task list - ''' - ''' - Public Iterator Function RunParallel() As IEnumerable(Of TOut) - Do While taskList.Count > 0 - Dim i As Integer = GetEmptyThread() - - If i > -1 Then - threads(i) = New AsyncHandle(Of TOut)(taskList.Dequeue).Run - Call Console.WriteLine($"{ToString()} submit new task on thread [{i + 1}]!") - End If - - Dim j As Integer = GetCompleteThread() - - If j > -1 Then - Yield threads(j).GetValue - threads(j) = Nothing - Call Console.WriteLine($"{ToString()} [thread_{j + 1}] job done!") - End If - Loop - - Do While Not threads.All(Function(t) t Is Nothing) - Dim j As Integer = GetCompleteThread() - - If j > -1 Then - Yield threads(j).GetValue - threads(j) = Nothing - Call Console.WriteLine($"{ToString()} [thread_{j + 1}] job done!") - End If - Loop - End Function - -End Class - + End Function + + ''' + ''' view thread pool status + ''' + ''' + Public Overrides Function ToString() As String + + Return $"[free: {free}, running: {running}, finished: {finished}, progress: {delta} - {CInt(delta / size * 100)}%]" + End Function + + ''' + ''' + ''' + ''' + ''' + ''' Run parallel task list + ''' + ''' + Public Iterator Function RunParallel() As IEnumerable(Of TOut) + Do While taskList.Count > 0 + Dim i As Integer = GetEmptyThread() + + If i > -1 Then + threads(i) = New AsyncHandle(Of TOut)(taskList.Dequeue).Run + Call Console.WriteLine($"{ToString()} submit new task on thread [{i + 1}]!") + End If + + Dim j As Integer = GetCompleteThread() + + If j > -1 Then + Yield threads(j).GetValue + threads(j) = Nothing + Call Console.WriteLine($"{ToString()} [thread_{j + 1}] job done!") + End If + Loop + + Do While Not threads.All(Function(t) t Is Nothing) + Dim j As Integer = GetCompleteThread() + + If j > -1 Then + Yield threads(j).GetValue + threads(j) = Nothing + Call Console.WriteLine($"{ToString()} [thread_{j + 1}] job done!") + End If + Loop + End Function + + End Class +End Namespace diff --git a/README.md b/README.md new file mode 100644 index 0000000..0853124 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# Darwinism +Darwinism High performance computing toolkit for VisualBasic.NET on unix .net 5