Multiclass-interface 1.1__tar.gz → 1.2__tar.gz

{Multiclass_interface-1.1 → multiclass_interface-1.2}/.gitlab-ci.yml

@@ -13,7 +13,6 @@ test_ubuntu:  # name the job what we like
   script:
   - pip install -e .[test]
   - pytest --cov-report term-missing:skip-covered --cov=multiclass_interface multiclass_interface/tests
-  - pytest --cov-report term-missing:skip-covered --cov=multiclass_interface multiclass_interface/tests
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   tags:  # only runners with this tag can do the job [3]
   - ci-ubuntu

{Multiclass_interface-1.1 → multiclass_interface-1.2/Multiclass_interface.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Multiclass_interface
-Version: 1.1
+Version: 1.2
 Summary: Multiclass interface
 Author-email: "Mads M. Pedersen" <mmpe@dtu.dk>
 Project-URL: Homepage, https://gitlab.windenergy.dtu.dk/DYNAMIKS/multiclass_interface

{Multiclass_interface-1.1 → multiclass_interface-1.2}/Multiclass_interface.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@
 LICENSE
 README.md
 pyproject.toml
+sequence_diagrams.md
 Multiclass_interface.egg-info/PKG-INFO
 Multiclass_interface.egg-info/SOURCES.txt
 Multiclass_interface.egg-info/dependency_links.txt

{Multiclass_interface-1.1/Multiclass_interface.egg-info → multiclass_interface-1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Multiclass_interface
-Version: 1.1
+Version: 1.2
 Summary: Multiclass interface
 Author-email: "Mads M. Pedersen" <mmpe@dtu.dk>
 Project-URL: Homepage, https://gitlab.windenergy.dtu.dk/DYNAMIKS/multiclass_interface

{Multiclass_interface-1.1 → multiclass_interface-1.2}/multiclass_interface/mpi_interface.py

@@ -1,27 +1,24 @@
 import numpy as np
 import sys
-import os
-import subprocess
 import traceback
 import inspect
-from enum import Enum
 
-if 'SLURM_NTASKS' in os.environ:
-    mpi = int(os.environ['SLURM_NTASKS']) > 1
-elif any([k in os.environ for k in ['MPI_LOCALNRANKS', 'OMPI_COMM_WORLD_SIZE']]):
-    mpi = True
-else:
-    mpi = subprocess.run("python -c 'from mpi4py import MPI'", shell=True,
-                         check=False, stderr=subprocess.PIPE).returncode == 0
+comm = None
+size = 1
+rank = 0
+name = ""
+main = True
 
-if mpi:
+
+def activate_mpi():
+    global comm, size, rank, name, main
     from mpi4py import MPI
     comm = MPI.COMM_WORLD
     size = MPI.COMM_WORLD.Get_size()
     rank = MPI.COMM_WORLD.Get_rank()
     name = MPI.Get_processor_name()
-else:
-    size = 1
+    main = rank == 0
+
 
 MPIClassInterfaceAttributes = {
     'close',

{Multiclass_interface-1.1 → multiclass_interface-1.2}/multiclass_interface/tests/test_mpi.py

@@ -46,9 +46,9 @@ def mpitest_mpi_ProcessClass():
 
 
 def mpitest_all():
+    mpi_interface.activate_mpi()
     mpi_interface.exit_mpi_on_close = False
-    from mpi4py import MPI
-    rank = MPI.COMM_WORLD.Get_rank()
+    rank = mpi_interface.rank
     from multiclass_interface.tests.test_multiprocessinterface import test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method
     mpi_interface.TERMINATE_ON_CLOSE = False
 

{Multiclass_interface-1.1 → multiclass_interface-1.2}/multiclass_interface/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.1'
-__version_tuple__ = version_tuple = (1, 1)
+__version__ = version = '1.2'
+__version_tuple__ = version_tuple = (1, 2)

multiclass_interface-1.2/sequence_diagrams.md

@@ -0,0 +1,228 @@
+## ThreadClass
+```mermaid
+sequenceDiagram
+    participant U as Main
+    participant TC as ThreadClass
+    
+    U->>+TC: cls=A
+    
+    create participant Q as IOQueue
+    TC->>Q: 
+    TC->>-U: ~A
+    U->>+TC: ~A(*args,**kwargs)
+    create participant T as Thread
+    TC->>T: A, IOQueue, args, kwargs
+    create participant A
+    T->>A: __init__(*args, **kwargs)
+    T->>+T: Run()
+    T->>+Q: get()
+    TC->>-U: ~a
+    U->>+TC: ~a.f(*args,**kwargs)
+    TC->>Q: put(f,args,kwargs)
+    
+    Q->>-T: f,args,kwargs
+    T->>+A: f(*args, **kwargs)
+    A->>-T: res
+    TC->>+Q: get()
+    T->>Q: put(res)
+    Q->>-TC: res
+    TC->>-U: res
+    TC->>T: close()
+    T->>-T: 
+
+
+```
+
+## ProcessClass
+
+The `ProcessClass` is similar to the `ThreadClas` except that it uses a seprate Process instead of a Thread.
+This means:
+- More time used to spawn process
+- Changes to `os.environ['LD_LIBRARY_PATH']` are effective in the process
+- Libraries (dll/so) will have their own memory space (important if multiple instances are launched)
+- Multiple processes allows parallel execution
+
+## MultiProcessClassInterface
+
+```mermaid
+sequenceDiagram
+    participant U as Main
+    participant TC as MultiThreadClassInterface
+    
+    U->>+TC: cls=A, [args1,args2])
+    
+    create participant Q1 as IOQueue1
+    TC->>Q1: 
+    create participant T1 as Thread1
+    TC->>T1: A, IOQueue1, args1
+    create participant A1
+    T1->>A1: __init__(*args1)
+    T1->>+T1: Run()
+    T1->>+Q1: get()
+
+    create participant Q2 as IOQueue2
+    TC->>Q2: 
+    create participant T2 as Thread2
+    TC->>T2: A, IOQueue2, args2
+    create participant A2
+    T2->>A2: __init__(*args2)
+    T2->>+T2: Run()
+    T2->>+Q2: get()
+
+    TC->>-U: ~a
+    U->>+TC: ~a.f(*args,**kwargs)
+    TC->>Q1: put(f,args,kwargs)
+    TC->>Q2: put(f,args,kwargs)
+    
+    Q1->>-T1: f,args,kwargs
+    Q2->>-T2: f,args,kwargs
+    T1->>+A1: f(*args, **kwargs)
+    T2->>+A2: f(*args, **kwargs)
+    A1->>-T1: res1
+    A2->>-T2: res2
+    TC->>+Q1: get()
+    T1->>Q1: put(res)
+    T2->>Q2: put(res)
+    Q1->>-TC: res1
+    TC->>+Q2: get()
+    Q2->>-TC: res2
+    TC->>-U: [res1,res2]
+    
+    TC->>T1: close()
+    T1->>-T1: 
+    TC->>T2: close()
+    T2->>-T2: 
+```
+
+
+## MultiProcessClassInterface
+
+The `MultiProcessClassInterface` is similar to the `MultiThreadClassInterfaces` except that it uses a seprate processes instead of a threads.
+This means:
+- More time used to spawn processes
+- Changes to `os.environ['LD_LIBRARY_PATH']` are effective in the processes
+- Libraries (dll/so) executed in the processes will their own individual memory space (changes to a variable in one dll does not affect the others)
+- Parallel execution of class functions
+
+## MPIClassInterface
+```mermaid
+sequenceDiagram
+    actor U as User
+    
+    participant MPI0 as MPI,rank0<br>MPIClassInterface
+    
+    
+    MPI0->>+MPI0: cls=A, [(args0),(args1)]
+    
+    
+    create participant A0
+    MPI0->>+A0: args0
+    MPI0->>-U: ~a
+    participant MPI1 as MPI,rank1<br>MPIClassInterface
+    MPI1->>+MPI1: cls=A, [(args0),(args1)]
+    create participant A1
+    MPI1->>A1: args1
+    participant MPI2 as MPI,rank2<br>MPIClassInterface
+    MPI2->>MPI2: cls=A, [(args0),(args1)]
+    MPI1->>-U: 
+    MPI2->>U:  
+    
+    U->>+MPI0:~a.f([args0,args1])
+    MPI0->>+MPI0: run_task()
+    MPI1->>+MPI1: work_loop
+    MPI2->>+MPI2: work_loop
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Scatter
+    MPI0->>MPI0: f,args0
+    MPI0->>MPI1: f,args1
+    MPI0->>MPI2: "skip",[]
+    end
+    MPI0->>+A0: f(args0)
+        MPI1->>+A1: f(args1)
+    A0->>-MPI0: res0
+    A1->>-MPI1: res1
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Gather
+    MPI0->>MPI0: res0
+    MPI1->>MPI0: res1
+    MPI2->>MPI0: None
+    end
+
+    
+    MPI0->>-U: [res1,res2]
+    
+    U->>+MPI0:~a.close()
+    MPI0->>+MPI0: run_task()
+    rect rgb(191, 223, 255)
+    note right of MPI0: Scatter
+    MPI0->>MPI0: close
+    MPI0->>MPI1: close
+    MPI0->>MPI2: close
+    end
+    MPI0->>-MPI0: 
+    MPI1->>-MPI1: 
+    MPI2->>-MPI2: 
+```
+
+## MPIClassInterface
+
+Setting `mpi_interface.LOOP_UNTIL_CLOSE=False` results in the following workflow
+
+```mermaid
+sequenceDiagram
+    actor U as User
+    
+    participant MPI0 as MPI,rank0<br>MPIClassInterface
+    
+    
+    MPI0->>+MPI0: cls=A, [(args0),(args1)]
+    
+    
+    create participant A0
+    MPI0->>+A0: args0
+    MPI0->>-U: ~a
+    participant MPI1 as MPI,rank1<br>MPIClassInterface
+    MPI1->>+MPI1: cls=A, [(args0),(args1)]
+    create participant A1
+    MPI1->>A1: args1
+    participant MPI2 as MPI,rank2<br>MPIClassInterface
+    MPI2->>MPI2: cls=A, [(args0),(args1)]
+    MPI1->>-U: 
+    MPI2->>U:  
+    
+    U->>+MPI0:~a.f([args0,args1])
+    MPI0->>+MPI0: run_task()
+    MPI1->>+MPI1: work_loop
+    MPI2->>+MPI2: work_loop
+    MPI2->>-MPI2: 
+    MPI2->>U: 
+
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Scatter
+    MPI0->>MPI0: f,args0
+    MPI0->>MPI1: f,args1
+    MPI0->>MPI2: "skip",[]
+    end
+    MPI0->>+A0: f(args0)
+        MPI1->>+A1: f(args1)
+    A0->>-MPI0: res0
+    A1->>-MPI1: res1
+    MPI1->>-MPI1: 
+    MPI1->>U: 
+    
+    rect rgb(191, 223, 255)
+    note right of MPI0: Gather
+    MPI0->>MPI0: res0
+    MPI1->>MPI0: res1
+    MPI2->>MPI0: None
+    end
+
+    
+    MPI0->>-U: [res1,res2]
+    
+```
+
+