Multiclass-interface 1.14__tar.gz → 1.15__tar.gz

{multiclass_interface-1.14 → multiclass_interface-1.15}/.gitlab-ci.yml

@@ -2,39 +2,22 @@ image: continuumio/anaconda3
 
 stages:
   - test
-  - build
   - deploy
 
-# ===== TEST Dynamiks ubuntu =====
-test_ubuntu:  # name the job what we like
-  image: continuumio/anaconda3
-  stage:  # build, test, deploy defined by default [2]
-    test
-  script:
-  - pip install -e .[test]
-  - pytest --cov-report term-missing:skip-covered --cov=multiclass_interface multiclass_interface/tests
-  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
-  tags:  # only runners with this tag can do the job [3]
-  - ci-ubuntu
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-
-
-mpitest-linux_openmpi:
+test_mpitest_openmpi:
     image: registry.windenergy.dtu.dk/hawc2/hawc2lib:mpi
     variables:
       GIT_SUBMODULE_STRATEGY: recursive # need to checkout hawc2 and utils
     script:
     - pip install .[test]
-    - mpirun -n 4 python -c "from multiclass_interface.tests.test_mpi import mpitest_all; mpitest_all()"
-    - python -c "from multiclass_interface import version; print (f'{version.__version__}')"
+    - pytest --cov-report term-missing:skip-covered --cov=multiclass_interface multiclass_interface/tests
     rules:
     - if: $CI_COMMIT_BRANCH == "main"
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     tags: 
     - ci-ubuntu
 
+
 # ===== Check code style =====
 check_code_style:  # name the job what we like
   stage:  # build, test, deploy defined by default [2]
@@ -49,8 +32,6 @@ check_code_style:  # name the job what we like
   - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 
 
-
-
 pypi_linux:
   stage:
     deploy

{multiclass_interface-1.14 → multiclass_interface-1.15/Multiclass_interface.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: Multiclass_interface
-Version: 1.14
+Version: 1.15
 Summary: Multiclass interface
 Author-email: "Mads M. Pedersen" <mmpe@dtu.dk>
 Project-URL: Homepage, https://gitlab.windenergy.dtu.dk/DYNAMIKS/multiclass_interface

{multiclass_interface-1.14/Multiclass_interface.egg-info → multiclass_interface-1.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: Multiclass_interface
-Version: 1.14
+Version: 1.15
 Summary: Multiclass interface
 Author-email: "Mads M. Pedersen" <mmpe@dtu.dk>
 Project-URL: Homepage, https://gitlab.windenergy.dtu.dk/DYNAMIKS/multiclass_interface

{multiclass_interface-1.14 → multiclass_interface-1.15}/multiclass_interface/mpi_interface.py

@@ -1,7 +1,7 @@
 import numpy as np
 import sys
 import traceback
-import inspect
+from contextlib import contextmanager
 
 comm = None
 size = 1
@@ -11,8 +11,8 @@ mpi = False
 main = True
 
 
-def activate_mpi(collective_mpi=True, loop_until_close=True, terminate_on_close=True):
-    global comm, size, rank, name, main, mpi, LOOP_UNTIL_CLOSE, TERMINATE_ON_CLOSE, COLLECTIVE_MPI
+def activate_mpi(collective_mpi=True, terminate_on_close=True):
+    global comm, size, rank, name, main, mpi, TERMINATE_ON_CLOSE, COLLECTIVE_MPI
     from mpi4py import MPI
     comm = MPI.COMM_WORLD
     size = MPI.COMM_WORLD.Get_size()
@@ -21,12 +21,12 @@ def activate_mpi(collective_mpi=True, loop_until_close=True, terminate_on_close=
     mpi = True
     main = rank == 0
     COLLECTIVE_MPI = collective_mpi
-    LOOP_UNTIL_CLOSE = loop_until_close
     TERMINATE_ON_CLOSE = terminate_on_close
 
 
 MPIClassInterfaceAttributes = {
     'close',
+    'release_mpi_workers',
     'use_rank',
     'cls',
     'work_loop',
@@ -38,7 +38,6 @@ MPIClassInterfaceAttributes = {
     'closed'}
 
 
-LOOP_UNTIL_CLOSE = True  # MPI workers (rank>0) will enter the work loop and wait for commands from rank 0
 TERMINATE_ON_CLOSE = True  # MPI workers (rank>0) will terminate (sys.exit) when the close command is called via rank0
 COLLECTIVE_MPI = True  # rank0 acts as a single point of contact (scatters command+arg and gathers results)
 
@@ -59,6 +58,7 @@ def main_run(f, *args, **kwargs):
 
 class MPIClassInterface():
     def __init__(self, cls, args_lst):
+        self.closed = False
         if len(args_lst) > size:
             if rank == 0:
                 raise Exception(f"Not enough mpi slots. Slots: {size}, Requested: {len(args_lst)}")
@@ -73,13 +73,12 @@ class MPIClassInterface():
 
                 def __getattr__(self, name):
                     raise AttributeError(
-                        f"rank {rank} trying to access {name}, but only {len(args_lst)} instances of {cls.__name__} exists")
+                        f"Rank {rank} trying to access {name}, but only {len(args_lst)} instances of {cls.__name__} exists")
             self.object = Dummy()
-        self.closed = False
 
         self.use_rank = np.array([True] * size)
         self.use_rank[len(args_lst):] = False
-        if rank > 0 and COLLECTIVE_MPI and LOOP_UNTIL_CLOSE:
+        if rank > 0 and COLLECTIVE_MPI:
             self.work_loop()
 
     def __enter__(self):
@@ -92,20 +91,20 @@ class MPIClassInterface():
         while True:
             method, args, kwargs = comm.scatter(None)
             comm.gather(self.do_task(method, args, kwargs))
-            if LOOP_UNTIL_CLOSE:
-                if method == 'close':
-                    if TERMINATE_ON_CLOSE:
-                        # comm.gather(f'Exit, rank {rank}')
-                        print("sys.exit", rank, flush=True)
-                        sys.exit(0)
-                    else:
-                        raise ChildProcessError('MPI worker done')
-            else:
+            if method == 'exit_work_loop':
                 break
+            if method == 'close':
+                if TERMINATE_ON_CLOSE:
+                    # comm.gather(f'Exit, rank {rank}')
+                    print("sys.exit", rank, flush=True)
+                    self.closed = True
+                    sys.exit(0)
+                else:
+                    raise ChildProcessError('MPI worker done')
 
     def do_task(self, method, args, kwargs):
         try:
-            if method == 'skip':
+            if method == 'skip' or method == 'exit_work_loop':
                 res = None
             elif method == 'setattr':
                 name, value = args
@@ -119,29 +118,28 @@ class MPIClassInterface():
         return res
 
     def run_task(self, input_lst):
+        assert rank == 0, "other ranks should be in work_loop when invoking run_task. For non-collective mpi, do_task should be used"
         comm.scatter(input_lst, root=0)
-        if rank == 0:
-            method, args, kwargs = input_lst[0]
-            res = self.do_task(method, args, kwargs)
-        else:
-            res = None
+        method, args, kwargs = input_lst[0]
+        res = self.do_task(method, args, kwargs)
+
         use_rank = self.use_rank
 
         res = [res for i, res in enumerate(comm.gather(res, root=0)) if use_rank[i]]
-        if rank == 0:
-            for r in res:
-                if isinstance(r, tuple) and len(r) > 1 and isinstance(r[0], BaseException):
-                    raise r[0].__class__(r[1])
+        for r in res:
+            if isinstance(r, tuple) and len(r) > 1 and isinstance(r[0], BaseException):
+                raise r[0].__class__(r[1])
         return res
 
     def get_input(self, name, i, args, kwargs):
         use_rank = self.use_rank
-        if COLLECTIVE_MPI:
-            N = np.sum(use_rank)
-            j = sum(use_rank[:i])
-        else:
-            N = 1
-            j = 0
+        N = np.sum(use_rank)
+        j = sum(use_rank[:i])  # needed for subsets
+
+        # if COLLECTIVE_MPI:
+        #    j = sum(use_rank[:i])
+        # else:
+        #    j = 0
 
         if use_rank[i]:
             def get_arg(arg):
@@ -149,6 +147,7 @@ class MPIClassInterface():
                     return arg[j]
                 else:
                     return arg
+
             return (name, [get_arg(arg) for arg in args], {k: get_arg(v) for k, v in kwargs.items()})
         else:
             return ('skip', [], {})
@@ -156,10 +155,7 @@ class MPIClassInterface():
     def __getattribute__(self, name):
         if name in MPIClassInterfaceAttributes:
             return object.__getattribute__(self, name)
-        elif COLLECTIVE_MPI and rank > 0:
-            self.work_loop()
-            return lambda *args, **kwargs: 1
-        elif rank > 0 and not self.use_rank[rank]:
+        elif not self.use_rank[rank]:
             raise ChildProcessError(f'MPI worker {rank} not used')
 
         def wrap(*args, **kwargs):
@@ -167,20 +163,28 @@ class MPIClassInterface():
                 inp = [self.get_input(name, i, args, kwargs) for i in range(size)]
                 return self.run_task(inp)
             else:
-                method, args, kwargs = self.get_input(name, 0, args, kwargs)
-                return [self.do_task(name, args, kwargs)]
-        if hasattr(getattr(self.object, name), '__call__'):
+                method, args, kwargs = self.get_input(name, rank, args, kwargs)
+                return [self.do_task(method, args, kwargs)]
+        try:
+            att = getattr(self.object, name)
+        except AttributeError as e:
+            raise type(e)(f'Rank {rank}: {e}').with_traceback(sys.exc_info()[2])
+
+        if hasattr(att, '__call__'):
             res = wrap
         else:
             res = wrap()
         return res
 
     def __setattr__(self, name, value):
-        if rank > 0 or name in MPIClassInterfaceAttributes:
+        if (COLLECTIVE_MPI and rank > 0) or name in MPIClassInterfaceAttributes:
             return object.__setattr__(self, name, value)
         else:
             inp = [self.get_input('setattr', i, (name, value), {}) for i in range(size)]
-            return self.run_task(inp)
+            if COLLECTIVE_MPI:
+                return self.run_task(inp)
+            else:
+                return self.do_task(*inp[rank])
 
     def __getitem__(self, slice):
         use_rank = np.full_like(self.use_rank, False)
@@ -189,14 +193,28 @@ class MPIClassInterface():
             return self
         return SubsetMPIClassInterface(self.cls, self.object, use_rank)
 
+    @contextmanager
+    def release_mpi_workers(self):
+        global COLLECTIVE_MPI
+        """Use to release mpi_workers. All object statements including 'release_mpi_workers' must be protected by 'if mpi_interface.main:' """
+        if COLLECTIVE_MPI:
+            if rank == 0:
+                self.run_task([('exit_work_loop', [], {}) for _ in range(size)])
+            COLLECTIVE_MPI = False
+            try:
+                yield self
+            finally:
+                COLLECTIVE_MPI = True
+                if rank > 0 and not self.closed:
+                    self.work_loop()
+
     def close(self):
+
         if not self.closed:
+
             if COLLECTIVE_MPI:
                 if rank == 0:
                     res = self.run_task([('close', [], {}) for _ in range(size)])
-                else:
-                    self.work_loop()
-                    res = None
             else:
                 res = self.do_task('close', [], {})
             self.closed = True

multiclass_interface-1.15/multiclass_interface/tests/test_mpi.py

@@ -0,0 +1,211 @@
+import os
+import sys
+import time
+import traceback
+from numpy import testing as npt
+import pytest
+
+from multiclass_interface.mpi_interface import MPIClassInterface
+from multiclass_interface.multiprocess_interface import ProcessClass
+import numpy as np
+from multiclass_interface.tests.my_test_cls import MyTest
+from multiclass_interface import mpi_interface
+
+
+def mpirun(f):
+    if 'main()' in traceback.format_stack()[0]:
+        n = f.__name__
+
+        def wrap():
+            exe = sys.executable.replace("\\", "/")
+            cmd = f'''mpirun -n 4 {exe} -c "from multiclass_interface.tests.test_mpi import {n}; {n}()"'''
+            # print(cmd)
+            assert os.system(cmd) == 0, n
+        wrap.__name__ = n
+        return wrap
+    else:
+        return f
+
+
+@mpirun
+def test_mpi_collective_too_few_slots():
+    mpi_interface.activate_mpi()
+    N = 4
+    rank = mpi_interface.rank
+
+    try:
+        with MPIClassInterface(MyTest, [(i + 1,) for i in range(N + 1)]) as m:
+            pass
+        assert rank in [1, 2, 3]
+    except Exception as e:
+        assert rank == 0
+        assert str(e) == 'Not enough mpi slots. Slots: 4, Requested: 5'
+
+
+@mpirun
+def test_mpi_collective_MyTest():
+    mpi_interface.activate_mpi()
+    N = 4
+    rank = mpi_interface.rank
+
+    with MPIClassInterface(MyTest, [(i + 1,) for i in range(N)]) as m:
+        if mpi_interface.main:
+            i = m.get_id()
+            npt.assert_array_equal(i, np.arange(N) + 1)
+
+        with m.release_mpi_workers():
+            assert m.get_id() == [rank + 1]
+
+        assert m.get_id() == [1, 2, 3, 4]
+        i = mpi_interface.main_run(lambda: m.get_id())
+        npt.assert_array_equal(i, 1)
+
+        try:
+            npt.assert_array_equal(m[1:3].get_id(), np.arange(1, 3) + 1)
+        except ChildProcessError:
+            pass
+
+        m.x = "hello"
+        assert m.x == ['hello'] * 4
+
+        assert m.return_input([101, 102, 103, 104])[0] == f'1 got: ({101},) and ' + '{}'
+        m.x = 'hello'
+        assert m.x == ['hello'] * 4
+        m[1::2].x = 'world'
+        assert m[:].x == ['hello', 'world', 'hello', 'world']
+
+        m.release_mpi_workers()
+        m.x = [1, 2, 3, 4]
+        assert m.x == [1, 2, 3, 4]
+        m.y = list(np.array(m.x) + 10)
+        assert m.y == [11, 12, 13, 14]
+        t = time.time()
+        m.work(1)
+        t = time.time() - t
+        assert t < 1.1
+
+        with pytest.raises(Exception, match='Cannot close SubsetMPIClassInterface. Please close all instances at once'):
+            m[:3].close()
+        with pytest.raises(Exception, match='Cannot make subset of SubsetMPIClassInterface'):
+            m[:3][1]
+
+
+@mpirun
+def test_mpi_collective_close_released_mpi_workers():
+    mpi_interface.activate_mpi()
+    N = 4
+    rank = mpi_interface.rank
+    with MPIClassInterface(MyTest, [(i + 1,) for i in range(N)]) as m:
+        with m.release_mpi_workers():
+            m.close()
+
+
+@mpirun
+def test_mpi_ProcessClass():
+    mpi_interface.activate_mpi()
+    with ProcessClass(MyTest) as cls:
+        myTest = cls(1)
+        assert myTest.get_id() == 1
+
+
+@mpirun
+def test_non_collective_mpi2():
+    mpi_interface.activate_mpi(collective_mpi=False)
+    rank = mpi_interface.rank
+    N = 3
+    with MPIClassInterface(MyTest, [(i + 10,) for i in range(N)]) as m:
+
+        try:
+            assert m.get_id()[0] == rank + 10, m.get_id()
+            main_id = mpi_interface.main_run(lambda: m.get_id())
+            assert main_id == [10], (rank, main_id)
+            assert rank < N  # rank > N will fail with ChildProcessError
+            assert m.return_input([101, 102, 103])[0] == f'{rank + 10} got: ({rank + 101},) and ' + '{}'
+            m.x = 'hello'
+            assert m.x[0] == 'hello', m.x[0]
+            m.x = [1, 2, 3]
+            assert m.x[0] == rank + 1
+            m.y = m.x[0] + 10
+            assert m.y[0] == rank + 11
+        except ChildProcessError:
+            pass
+
+        if rank == 3:
+            with pytest.raises(AttributeError, match='Rank 3 trying to access cls, but only 3 instances of MyTest exists'):
+                m.object.cls
+
+
+@mpirun
+def test_mpi_non_collective_MyTest():
+    mpi_interface.activate_mpi(collective_mpi=False)
+    rank = mpi_interface.rank
+    N = 4
+
+    with MPIClassInterface(MyTest, [(i + 1,) for i in range(N)]) as m:
+        i = m.get_id()[0]
+        assert i == rank + 1
+        i = mpi_interface.main_run(lambda: m.get_id())
+        npt.assert_array_equal(i, 1)  # id of rank0 broadcast to all
+        try:
+            assert m[1:3].get_id()[0] == rank + 1
+        except ChildProcessError:
+            assert rank in [0, 3]
+
+        # rank>0 fails, but only rank=0 is needed, so should not fail
+        mpi_interface.main_run(lambda rank=rank: 1 / [0, 1][rank == 0])
+
+        assert m.return_input([101, 102, 103, 104])[0] == f'{rank + 1} got: ({rank + 101},) and ' + '{}'
+        m.x = "hello"
+        assert m.x[0] == 'hello'
+
+        m[1::2].x = 'world'
+        assert m.x[0] == ['hello', 'world'][rank % 2]
+
+        try:
+            assert m[1::2].x[0] == 'world'
+            assert m[1::2][:].x[0] == 'world'
+        except ChildProcessError:
+            pass
+
+        m.x = [1, 2, 3, 4]
+        assert m.x[0] == rank + 1
+
+        m.y = m.x[0] + 10
+        assert m.y[0] == rank + 11
+        t = time.time()
+        m.work(1)
+        t = time.time() - t
+        assert t < 1.1
+
+        with pytest.raises(Exception, match='Cannot close SubsetMPIClassInterface. Please close all instances at once'):
+            m[:3].close()
+        with pytest.raises(Exception, match='Cannot make subset of SubsetMPIClassInterface'):
+            m[:3][1]
+
+
+@mpirun
+def test_multiprocessinterface_tests():
+    from multiclass_interface.tests.test_multiprocessinterface import test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method
+    mpi_interface.activate_mpi()
+
+    try:
+        with MPIClassInterface(MyTest, [(1,), (2,), (3,)]) as mpici:
+            for f in [test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method]:
+                f(mpici)
+    except ChildProcessError:
+        pass
+
+
+@mpirun
+def test_multiprocessinterface_tests_diff_interface():
+    from multiclass_interface.tests.test_multiprocessinterface import test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method
+    mpi_interface.activate_mpi(terminate_on_close=False)
+
+    for f in [test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method]:
+        # print (mpi_interface.rank, f.__name__,flush=True)
+        try:
+            with MPIClassInterface(MyTest, [(1,), (2,), (3,)]) as mpici:
+                f(mpici)
+        except ChildProcessError:
+            pass
+    sys.exit(0)

{multiclass_interface-1.14 → multiclass_interface-1.15}/multiclass_interface/version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.14'
-__version_tuple__ = version_tuple = (1, 14)
+__version__ = version = '1.15'
+__version_tuple__ = version_tuple = (1, 15)

{multiclass_interface-1.14 → multiclass_interface-1.15}/pyproject.toml

@@ -41,9 +41,6 @@ version_file = "multiclass_interface/version.py"
 [tool.coverage.run]
 omit = [
     "multiclass_interface/version.py",
-    "multiclass_interface/mpi_utils.py",
-    "multiclass_interface/mpi_interface.py",
-    "multiclass_interface/tests/my_test_cls.py",
     "multiclass_interface/tests/*",    
     ]
 

multiclass_interface-1.14/multiclass_interface/tests/test_mpi.py

@@ -1,99 +0,0 @@
-import os
-import time
-
-from numpy import testing as npt
-import pytest
-
-from multiclass_interface.mpi_interface import MPIClassInterface
-from multiclass_interface.multiprocess_interface import ProcessClass, MultiProcessClassInterface
-import numpy as np
-from multiclass_interface.tests.my_test_cls import MyTest
-from multiclass_interface import mpi_interface
-
-
-def mpitest_mpi_MyTest():
-
-    N = 4
-    try:
-        with MPIClassInterface(MyTest, [(i,) for i in range(N)]) as m:
-
-            time.sleep(.1)
-            i = m.get_id()
-            npt.assert_array_equal(i, np.arange(N))
-
-            i = mpi_interface.main_run(lambda: m.get_id())
-            npt.assert_array_equal(i, 0)
-            npt.assert_array_equal(m[1:3].get_id(), np.arange(1, 3))
-            t = time.time()
-            m.work(1)
-            t = time.time() - t
-            assert t < 1.1
-
-            with pytest.raises(Exception, match='Cannot close SubsetMPIClassInterface. Please close all instances at once'):
-                m[:3].close()
-            with pytest.raises(Exception, match='Cannot make subset of SubsetMPIClassInterface'):
-                m[:3][1]
-
-            print("done, test_mpi_MyTest")
-    except ChildProcessError:
-        pass
-
-
-def mpitest_mpi_ProcessClass():
-
-    with ProcessClass(MyTest) as cls:
-        myTest = cls(1)
-        assert myTest.get_id() == 1
-    print("done, test_mpi_ProcessClass", flush=True)
-
-
-def mpitest_non_collective_mpi():
-    mpi_interface.COLLECTIVE_MPI = False
-    rank = mpi_interface.rank
-    import time
-    N = 3
-    with MPIClassInterface(MyTest, [(i + 10,) for i in range(N)]) as m:
-
-        try:
-            assert m.get_id()[0] == rank + 10, m.get_id()
-            main_id = mpi_interface.main_run(lambda: m.get_id())
-            assert main_id == [10], (rank, main_id)
-            assert rank < N  # rank > N will fail with ChildProcessError
-        except ChildProcessError:
-            pass
-
-    mpi_interface.COLLECTIVE_MPI = True
-
-
-def mpitest_all():
-    mpi_interface.activate_mpi()
-
-    mpi_interface.exit_mpi_on_close = False
-    rank = mpi_interface.rank
-    from multiclass_interface.tests.test_multiprocessinterface import test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method
-    mpi_interface.TERMINATE_ON_CLOSE = False
-
-    def run(f, *args):
-        try:
-            if rank == 0:
-                print(f"Rank {rank} start {f.__name__}", flush=True)
-            f(*args)
-        except ChildProcessError:
-            pass
-        finally:
-            if rank == 0:
-                print(f"Rank {rank} done {f.__name__}", flush=True)
-    for f in [mpitest_mpi_MyTest,
-              mpitest_mpi_ProcessClass]:
-        run(f)
-
-    try:
-        with MPIClassInterface(MyTest, [(1,), (2,), (3,)]) as mpici:
-            for f in [test_attribute, test_missing_attribute, test_execption, test_setattr, test_setattr_method]:
-                run(f, mpici)
-    except ChildProcessError:
-        pass
-
-    run(mpitest_non_collective_mpi)
-
-    print(f"Rank {rank} Done test_all")