Multiclass-interface 1.1__tar.gz → 1.3__tar.gz

{Multiclass_interface-1.1 → multiclass_interface-1.3}/.gitlab-ci.yml

@@ -13,7 +13,6 @@ test_ubuntu:  # name the job what we like
   script:
   - pip install -e .[test]
   - pytest --cov-report term-missing:skip-covered --cov=multiclass_interface multiclass_interface/tests
-  - pytest --cov-report term-missing:skip-covered --cov=multiclass_interface multiclass_interface/tests
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   tags:  # only runners with this tag can do the job [3]
   - ci-ubuntu

{Multiclass_interface-1.1 → multiclass_interface-1.3/Multiclass_interface.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Multiclass_interface
-Version: 1.1
+Version: 1.3
 Summary: Multiclass interface
 Author-email: "Mads M. Pedersen" <mmpe@dtu.dk>
 Project-URL: Homepage, https://gitlab.windenergy.dtu.dk/DYNAMIKS/multiclass_interface

{Multiclass_interface-1.1 → multiclass_interface-1.3}/Multiclass_interface.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@
 LICENSE
 README.md
 pyproject.toml
+sequence_diagrams.md
 Multiclass_interface.egg-info/PKG-INFO
 Multiclass_interface.egg-info/SOURCES.txt
 Multiclass_interface.egg-info/dependency_links.txt

{Multiclass_interface-1.1/Multiclass_interface.egg-info → multiclass_interface-1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Multiclass_interface
-Version: 1.1
+Version: 1.3
 Summary: Multiclass interface
 Author-email: "Mads M. Pedersen" <mmpe@dtu.dk>
 Project-URL: Homepage, https://gitlab.windenergy.dtu.dk/DYNAMIKS/multiclass_interface

{Multiclass_interface-1.1 → multiclass_interface-1.3}/multiclass_interface/mpi_interface.py

@@ -1,27 +1,24 @@
 import numpy as np
 import sys
-import os
-import subprocess
 import traceback
 import inspect
-from enum import Enum
 
-if 'SLURM_NTASKS' in os.environ:
-    mpi = int(os.environ['SLURM_NTASKS']) > 1
-elif any([k in os.environ for k in ['MPI_LOCALNRANKS', 'OMPI_COMM_WORLD_SIZE']]):
-    mpi = True
-else:
-    mpi = subprocess.run("python -c 'from mpi4py import MPI'", shell=True,
-                         check=False, stderr=subprocess.PIPE).returncode == 0
+comm = None
+size = 1
+rank = 0
+name = ""
+main = True
 
-if mpi:
+
+def activate_mpi():
+    global comm, size, rank, name, main
     from mpi4py import MPI
     comm = MPI.COMM_WORLD
     size = MPI.COMM_WORLD.Get_size()
     rank = MPI.COMM_WORLD.Get_rank()
     name = MPI.Get_processor_name()
-else:
-    size = 1
+    main = rank == 0
+
 
 MPIClassInterfaceAttributes = {
     'close',
@@ -38,6 +35,7 @@ MPIClassInterfaceAttributes = {
 
 LOOP_UNTIL_CLOSE = True
 TERMINATE_ON_CLOSE = True
+COLLECTIVE_MPI = True
 
 
 class MPIClassInterface():
@@ -56,10 +54,9 @@ class MPIClassInterface():
             self.object = Dummy()
         self.closed = False
 
-        if rank == 0:
-            self.use_rank = np.array([True] * size)
-            self.use_rank[len(args_lst):] = False
-        elif LOOP_UNTIL_CLOSE:
+        self.use_rank = np.array([True] * size)
+        self.use_rank[len(args_lst):] = False
+        if rank > 0 and COLLECTIVE_MPI and LOOP_UNTIL_CLOSE:
             self.work_loop()
 
     def __enter__(self):
@@ -136,13 +133,18 @@ class MPIClassInterface():
     def __getattribute__(self, name):
         if name in MPIClassInterfaceAttributes:
             return object.__getattribute__(self, name)
-        elif rank > 0:
+        elif COLLECTIVE_MPI and rank > 0:
             self.work_loop()
             return lambda *args, **kwargs: 1
+        elif not self.use_rank[rank]:
+            raise ChildProcessError('MPI worker not used')
 
         def wrap(*args, **kwargs):
-            inp = [self.get_input(name, i, args, kwargs) for i in range(size)]
-            return self.run_task(inp)
+            if COLLECTIVE_MPI:
+                inp = [self.get_input(name, i, args, kwargs) for i in range(size)]
+                return self.run_task(inp)
+            else:
+                return self.do_task(name, args, kwargs)
 
         if hasattr(getattr(self.object, name), '__call__'):
             return wrap
@@ -165,11 +167,14 @@ class MPIClassInterface():
 
     def close(self):
         if not self.closed:
-            if rank == 0:
-                res = self.run_task([('close', [], {}) for _ in range(size)])
+            if COLLECTIVE_MPI:
+                if rank == 0:
+                    res = self.run_task([('close', [], {}) for _ in range(size)])
+                else:
+                    self.work_loop()
+                    res = None
             else:
-                self.work_loop()
-                res = None
+                res = self.do_task('close', [], {})
             self.closed = True
 
 

{Multiclass_interface-1.1 → multiclass_interface-1.3}/multiclass_interface/tests/test_mpi.py

@@ -42,13 +42,30 @@ def mpitest_mpi_ProcessClass():
     with ProcessClass(MyTest) as cls:
         myTest = cls(1)
         assert myTest.get_id() == 1
-    print("done, test_mpi_ProcessClass")
+    print("done, test_mpi_ProcessClass", flush=True)
+
+
+def mpitest_non_collective_mpi():
+    mpi_interface.COLLECTIVE_MPI = False
+    rank = mpi_interface.rank
+    import time
+    N = 3
+    with MPIClassInterface(MyTest, [(i + 10,) for i in range(N)]) as m:
+
+        try:
+            assert m.get_id() == rank + 10
+            assert rank < N  # rank > N will fail with ChildProcessError
+        except ChildProcessError:
+            pass
+
+    mpi_interface.COLLECTIVE_MPI = True
 
 
 def mpitest_all():
+    mpi_interface.activate_mpi()
+
     mpi_interface.exit_mpi_on_close = False
-    from mpi4py import MPI
-    rank = MPI.COMM_WORLD.Get_rank()
+    rank = mpi_interface.rank
     from multiclass_interface.tests.test_multiprocessinterface import test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method
     mpi_interface.TERMINATE_ON_CLOSE = False
 
@@ -67,13 +84,14 @@ def mpitest_all():
         run(f)
 
     try:
-        with MPIClassInterface(MyTest, [(1,), (2,)]) as mpici:
+        with MPIClassInterface(MyTest, [(1,), (2,), (3,)]) as mpici:
             for f in [test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method]:
-                break
+
                 run(f, mpici)
 
     except ChildProcessError:
         pass
-    mpi_interface.TERMINATE_ON_CLOSE = True
+
+    run(mpitest_non_collective_mpi)
 
     print(f"Rank {rank} Done test_all")

{Multiclass_interface-1.1 → multiclass_interface-1.3}/multiclass_interface/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.1'
-__version_tuple__ = version_tuple = (1, 1)
+__version__ = version = '1.3'
+__version_tuple__ = version_tuple = (1, 3)

multiclass_interface-1.3/sequence_diagrams.md

@@ -0,0 +1,275 @@
+# Sequence diagrams
+- [ThreadClass](#threadclass)
+- [ProcessClass](#processclass)
+- [MultiThreadClassInterface](#multithreadclassinterface)
+- [MultiProcessClassInterface](#multiprocessclassinterface)
+- [MPIClassInterface](#mpiclassinterface)
+- [MPIClassInterface(LOOP_UNTIL_CLOSE=False)](#mpiclassinterfaceloop_until_closefalse)
+- [MPIClassInterface(COLLECTIVE_MPI=False)](#mpiclassinterfacecollective_mpifalse)
+
+## ThreadClass
+```mermaid
+sequenceDiagram
+    participant U as Main
+    participant TC as ThreadClass
+    
+    U->>+TC: cls=A
+    
+    create participant Q as IOQueue
+    TC->>Q: 
+    TC->>-U: ~A
+    U->>+TC: ~A(*args,**kwargs)
+    create participant T as Thread
+    TC->>T: A, IOQueue, args, kwargs
+    create participant A
+    T->>A: __init__(*args, **kwargs)
+    T->>+T: Run()
+    T->>+Q: get()
+    TC->>-U: ~a
+    U->>+TC: ~a.f(*args,**kwargs)
+    TC->>Q: put(f,args,kwargs)
+    
+    Q->>-T: f,args,kwargs
+    T->>+A: f(*args, **kwargs)
+    A->>-T: res
+    TC->>+Q: get()
+    T->>Q: put(res)
+    Q->>-TC: res
+    TC->>-U: res
+    TC->>T: close()
+    T->>-T: 
+
+
+```
+
+## ProcessClass
+
+The `ProcessClass` is similar to the `ThreadClas` except that it uses a seprate Process instead of a Thread.
+This means:
+- More time used to spawn process
+- Changes to `os.environ['LD_LIBRARY_PATH']` are effective in the process
+- Libraries (dll/so) will have their own memory space (important if multiple instances are launched)
+- Multiple processes allows parallel execution
+
+## MultiThreadClassInterface
+
+```mermaid
+sequenceDiagram
+    participant U as Main
+    participant TC as MultiThreadClassInterface
+    
+    U->>+TC: cls=A, [args1,args2])
+    
+    create participant Q1 as IOQueue1
+    TC->>Q1: 
+    create participant T1 as Thread1
+    TC->>T1: A, IOQueue1, args1
+    create participant A1
+    T1->>A1: __init__(*args1)
+    T1->>+T1: Run()
+    T1->>+Q1: get()
+
+    create participant Q2 as IOQueue2
+    TC->>Q2: 
+    create participant T2 as Thread2
+    TC->>T2: A, IOQueue2, args2
+    create participant A2
+    T2->>A2: __init__(*args2)
+    T2->>+T2: Run()
+    T2->>+Q2: get()
+
+    TC->>-U: ~a
+    U->>+TC: ~a.f(*args,**kwargs)
+    TC->>Q1: put(f,args,kwargs)
+    TC->>Q2: put(f,args,kwargs)
+    
+    Q1->>-T1: f,args,kwargs
+    Q2->>-T2: f,args,kwargs
+    T1->>+A1: f(*args, **kwargs)
+    T2->>+A2: f(*args, **kwargs)
+    A1->>-T1: res1
+    A2->>-T2: res2
+    TC->>+Q1: get()
+    T1->>Q1: put(res)
+    T2->>Q2: put(res)
+    Q1->>-TC: res1
+    TC->>+Q2: get()
+    Q2->>-TC: res2
+    TC->>-U: [res1,res2]
+    
+    TC->>T1: close()
+    T1->>-T1: 
+    TC->>T2: close()
+    T2->>-T2: 
+```
+
+
+## MultiProcessClassInterface
+
+The `MultiProcessClassInterface` is similar to the `MultiThreadClassInterfaces` except that it uses a seprate processes instead of a threads.
+This means:
+- More time used to spawn processes
+- Changes to `os.environ['LD_LIBRARY_PATH']` are effective in the processes
+- Libraries (dll/so) executed in the processes will their own individual memory space (changes to a variable in one dll does not affect the others)
+- Parallel execution of class functions
+
+## MPIClassInterface
+```mermaid
+sequenceDiagram
+    actor U as User
+    
+    participant MPI0 as MPI,rank0<br>MPIClassInterface
+    
+    
+    MPI0->>+MPI0: cls=A, [(args0),(args1)]
+    
+    
+    create participant A0
+    MPI0->>+A0: args0
+    MPI0->>-U: ~a
+    participant MPI1 as MPI,rank1<br>MPIClassInterface
+    MPI1->>+MPI1: cls=A, [(args0),(args1)]
+    create participant A1
+    MPI1->>A1: args1
+    participant MPI2 as MPI,rank2<br>MPIClassInterface
+    MPI2->>MPI2: cls=A, [(args0),(args1)]
+    MPI1->>-U: 
+    MPI2->>U:  
+    
+    U->>+MPI0:~a.f([args0,args1])
+    MPI0->>+MPI0: run_task()
+    MPI1->>+MPI1: work_loop
+    MPI2->>+MPI2: work_loop
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Scatter
+    MPI0->>MPI0: f,args0
+    MPI0->>MPI1: f,args1
+    MPI0->>MPI2: "skip",[]
+    end
+    MPI0->>+A0: f(args0)
+        MPI1->>+A1: f(args1)
+    A0->>-MPI0: res0
+    A1->>-MPI1: res1
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Gather
+    MPI0->>MPI0: res0
+    MPI1->>MPI0: res1
+    MPI2->>MPI0: None
+    end
+
+    
+    MPI0->>-U: [res1,res2]
+    
+    U->>+MPI0:~a.close()
+    MPI0->>+MPI0: run_task()
+    rect rgb(191, 223, 255)
+    note right of MPI0: Scatter
+    MPI0->>MPI0: close
+    MPI0->>MPI1: close
+    MPI0->>MPI2: close
+    end
+    MPI0->>-MPI0: 
+    MPI1->>-MPI1: 
+    MPI2->>-MPI2: 
+```
+
+## MPIClassInterface(LOOP_UNTIL_CLOSE=False)
+
+Setting `mpi_interface.LOOP_UNTIL_CLOSE=False` results in the following workflow
+
+```mermaid
+sequenceDiagram
+    actor U as User
+    
+    participant MPI0 as MPI,rank0<br>MPIClassInterface
+    
+    
+    MPI0->>+MPI0: cls=A, [(args0),(args1)]
+    
+    
+    create participant A0
+    MPI0->>+A0: args0
+    MPI0->>-U: ~a
+    participant MPI1 as MPI,rank1<br>MPIClassInterface
+    MPI1->>+MPI1: cls=A, [(args0),(args1)]
+    create participant A1
+    MPI1->>A1: args1
+    participant MPI2 as MPI,rank2<br>MPIClassInterface
+    MPI2->>MPI2: cls=A, [(args0),(args1)]
+    MPI1->>-U: 
+    MPI2->>U:  
+    
+    U->>+MPI0:~a.f([args0,args1])
+    MPI0->>+MPI0: run_task()
+    MPI1->>+MPI1: work_loop
+    MPI2->>+MPI2: work_loop
+    MPI2->>-MPI2: 
+    MPI2->>U: 
+
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Scatter
+    MPI0->>MPI0: f,args0
+    MPI0->>MPI1: f,args1
+    MPI0->>MPI2: "skip",[]
+    end
+    MPI0->>+A0: f(args0)
+        MPI1->>+A1: f(args1)
+    A0->>-MPI0: res0
+    A1->>-MPI1: res1
+    MPI1->>-MPI1: 
+    MPI1->>U: 
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Gather
+    MPI0->>MPI0: res0
+    MPI1->>MPI0: res1
+    MPI2->>MPI0: None
+    end
+
+    
+    MPI0->>-U: [res1,res2]
+    
+```
+
+
+## MPIClassInterface(COLLECTIVE_MPI=False)
+Setting `mpi_interface.COLLECTIVE_MPI=False` results in the following workflow
+
+```mermaid
+sequenceDiagram
+    
+    participant MPI0 as MPI,rank0<br>MPIClassInterface
+    
+    
+    MPI0->>+MPI0: cls=A, [(args0),(args1)]
+    
+    
+    create participant A0
+    MPI0->>+A0: args0
+    
+    participant MPI1 as MPI,rank1<br>MPIClassInterface
+    MPI1->>+MPI1: cls=A, [(args0),(args1)]
+    create participant A1
+    MPI1->>A1: args1
+    participant MPI2 as MPI,rank2<br>MPIClassInterface
+    MPI2->>MPI2: cls=A, [(args0),(args1)]
+    
+    MPI0->>+MPI0:~a.f([args0,args1])
+    MPI1->>+MPI1:~a.f([args0,args1])
+    MPI2->>+MPI2:~a.f([args0,args1])
+    
+    
+    MPI0->>+A0: f(args0)
+    MPI1->>+A1: f(args1)
+    MPI2->>-MPI2: ChildProcessError
+    A0->>-MPI0: res0
+    A1->>-MPI1: res1
+        
+    MPI0->>-MPI0: res0
+    MPI1->>-MPI1: res1
+    
+```
+