foxes 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

foxes/engines/default.py

@@ -12,6 +12,50 @@ class DefaultEngine(Engine):
 
     """
 
+    def map(
+        self,
+        func,
+        inputs,
+        *args,
+        **kwargs,
+    ):
+        """
+        Runs a function on a list of files
+
+        Parameters
+        ----------
+        func: Callable
+            Function to be called on each file,
+            func(input, *args, **kwargs) -> data
+        inputs: array-like
+            The input data list
+        args: tuple, optional
+            Arguments for func
+        kwargs: dict, optional
+            Keyword arguments for func
+
+        Returns
+        -------
+        results: list
+            The list of results
+
+        """
+
+        self.finalize()
+
+        with Engine.new(
+            "process",
+            n_procs=self.n_procs,
+            chunk_size_states=self.chunk_size_states,
+            chunk_size_points=self.chunk_size_points,
+            verbosity=self.verbosity,
+        ) as e:
+            results = e.map(func, inputs, *args, **kwargs)
+
+        self.initialize()
+
+        return results
+
     def run_calculation(
         self,
         algo,
@@ -48,8 +92,7 @@ class DefaultEngine(Engine):
 
         if (algo.n_states >= max_n) or (
             point_data is not None
-            and self.chunk_size_points is not None
-            and point_data.sizes[FC.TARGET] > self.chunk_size_points
+            and algo.n_states * point_data.sizes[FC.TARGET] > 10000
         ):
             ename = "process"
         else:

foxes/engines/mpi.py

@@ -1,17 +1,8 @@
 from foxes.utils import import_module
 
-from .pool import PoolEngine
 from .futures import ProcessEngine
 
 
-def load_mpi():
-    """On-demand loading of the mpi4py package"""
-    global MPIPoolExecutor
-    MPIPoolExecutor = import_module(
-        "mpi4py.futures", hint="pip install mpi4py"
-    ).MPIPoolExecutor
-
-
 class MPIEngine(ProcessEngine):
     """
     The MPI engine for foxes calculations.
@@ -26,13 +17,11 @@ class MPIEngine(ProcessEngine):
 
     """
 
-    def initialize(self):
-        """
-        Initializes the engine.
-        """
-        load_mpi()
-        PoolEngine.initialize(self)
-
     def _create_pool(self):
         """Creates the pool"""
+        MPIPoolExecutor = import_module(
+            "mpi4py.futures",
+            pip_hint="pip install mpi4py",
+            conda_hint="conda install mpi4py -c conda-forge",
+        ).MPIPoolExecutor
         self._pool = MPIPoolExecutor(max_workers=self.n_procs)

foxes/engines/multiprocess.py

@@ -2,15 +2,6 @@ from foxes.utils import import_module
 
 from .pool import PoolEngine
 
-Pool = None
-
-
-def load_multiprocess():
-    """On-demand loading of the multiprocess package"""
-    global Pool
-    if Pool is None:
-        Pool = import_module("multiprocess", hint="pip install multiprocess").Pool
-
 
 class MultiprocessEngine(PoolEngine):
     """
@@ -22,7 +13,7 @@ class MultiprocessEngine(PoolEngine):
 
     def _create_pool(self):
         """Creates the pool"""
-        load_multiprocess()
+        Pool = import_module("multiprocess").Pool
         self._pool = Pool(processes=self.n_procs)
 
     def _submit(self, f, *args, **kwargs):

foxes/engines/numpy.py

@@ -16,6 +16,36 @@ class NumpyEngine(Engine):
 
     """
 
+    def map(
+        self,
+        func,
+        inputs,
+        *args,
+        **kwargs,
+    ):
+        """
+        Runs a function on a list of files
+
+        Parameters
+        ----------
+        func: Callable
+            Function to be called on each file,
+            func(input, *args, **kwargs) -> data
+        inputs: array-like
+            The input data list
+        args: tuple, optional
+            Arguments for func
+        kwargs: dict, optional
+            Keyword arguments for func
+
+        Returns
+        -------
+        results: list
+            The list of results
+
+        """
+        return [func(input, *args, **kwargs) for input in inputs]
+
     def run_calculation(
         self,
         algo,

foxes/engines/pool.py

@@ -1,3 +1,4 @@
+import numpy as np
 import xarray as xr
 from abc import abstractmethod
 from tqdm import tqdm
@@ -15,6 +16,11 @@ def _run(algo, model, data, iterative, chunk_store, i0_t0, **cpars):
     return results, cstore
 
 
+def _run_map(func, inputs, *args, **kwargs):
+    """Helper function for running map func on proc"""
+    return [func(x, *args, **kwargs) for x in inputs]
+
+
 class PoolEngine(Engine):
     """
     Abstract engine for pool type parallelizations.
@@ -82,6 +88,48 @@ class PoolEngine(Engine):
         self._shutdown_pool()
         super().__exit__(*exit_args)
 
+    def map(
+        self,
+        func,
+        inputs,
+        *args,
+        **kwargs,
+    ):
+        """
+        Runs a function on a list of files
+
+        Parameters
+        ----------
+        func: Callable
+            Function to be called on each file,
+            func(input, *args, **kwargs) -> data
+        inputs: array-like
+            The input data list
+        args: tuple, optional
+            Arguments for func
+        kwargs: dict, optional
+            Keyword arguments for func
+
+        Returns
+        -------
+        results: list
+            The list of results
+
+        """
+        if len(inputs) == 0:
+            return []
+        elif len(inputs) == 1:
+            return [func(inputs[0], *args, **kwargs)]
+        else:
+            inptl = np.array_split(inputs, min(self.n_procs, len(inputs)))
+            jobs = []
+            for subi in inptl:
+                jobs.append(self._submit(_run_map, func, subi, *args, **kwargs))
+            results = []
+            for j in jobs:
+                results += self._result(j)
+            return results
+
     def run_calculation(
         self,
         algo,

foxes/engines/ray.py

@@ -12,7 +12,7 @@ def load_ray():
     """On-demand loading of the ray package"""
     global ray
     if ray is None:
-        ray = import_module("ray", hint="pip install ray")
+        ray = import_module("ray")
 
 
 class RayEngine(PoolEngine):

foxes/engines/single.py

@@ -39,6 +39,36 @@ class SingleChunkEngine(Engine):
     def __repr__(self):
         return f"{type(self).__name__}()"
 
+    def map(
+        self,
+        func,
+        inputs,
+        *args,
+        **kwargs,
+    ):
+        """
+        Runs a function on a list of files
+
+        Parameters
+        ----------
+        func: Callable
+            Function to be called on each file,
+            func(input, *args, **kwargs) -> data
+        inputs: array-like
+            The input data list
+        args: tuple, optional
+            Arguments for func
+        kwargs: dict, optional
+            Keyword arguments for func
+
+        Returns
+        -------
+        results: list
+            The list of results
+
+        """
+        return [func(input, *args, **kwargs) for input in inputs]
+
     def run_calculation(
         self,
         algo,

foxes/input/farm_layout/from_csv.py

@@ -1,7 +1,7 @@
 import pandas as pd
 
 from foxes.core import Turbine
-from foxes.config import get_path
+from foxes.config import get_input_path
 
 
 def add_from_csv(
@@ -76,7 +76,7 @@ def add_from_csv(
     else:
         if verbosity:
             print("Reading file", data_source)
-        pth = get_path(data_source)
+        pth = get_input_path(data_source)
         data = pd.read_csv(pth, index_col=col_index)
 
     tmodels = turbine_parameters.pop("turbine_models", [])

foxes/input/farm_layout/from_file.py

@@ -1,5 +1,5 @@
 from foxes.data import FARM, StaticData
-from foxes.config import get_path
+from foxes.config import get_input_path
 
 from .from_json import add_from_json
 from .from_csv import add_from_csv
@@ -31,7 +31,7 @@ def add_from_file(farm, file_path, *args, verbosity=1, dbook=None, **kwargs):
 
     """
 
-    fpath = get_path(file_path)
+    fpath = get_input_path(file_path)
     dbook = StaticData() if dbook is None else dbook
 
     if not fpath.is_file():

foxes/input/farm_layout/from_json.py

@@ -3,7 +3,7 @@ import numpy as np
 from copy import deepcopy
 
 from foxes.core import Turbine
-from foxes.config import get_path
+from foxes.config import get_input_path
 
 
 def add_from_json(
@@ -28,7 +28,7 @@ def add_from_json(
     :group: input.farm_layout
 
     """
-    fpath = get_path(file_path)
+    fpath = get_input_path(file_path)
     if verbosity:
         print("Reading file", fpath)
     with open(fpath) as f:

foxes/input/states/__init__.py

@@ -6,7 +6,6 @@ from .single import SingleStateStates
 from .scan import ScanStates
 from .states_table import StatesTable, Timeseries, TabStates
 from .field_data_nc import FieldDataNC
-from .slice_data_nc import SliceDataNC
 from .multi_height import MultiHeightStates, MultiHeightTimeseries
 from .multi_height import MultiHeightNCStates, MultiHeightNCTimeseries
 from .one_point_flow import (

foxes/input/states/create/random_abl_states.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import numpy as np
 
-from foxes.config import config, get_path
+from foxes.config import config, get_output_path
 import foxes.variables as FV
 import foxes.constants as FC
 
@@ -99,7 +99,7 @@ def write_random_abl_states(
 
     """
 
-    fpath = get_path(file_path)
+    fpath = get_output_path(file_path)
     if verbosity:
         print("Writing file", fpath)