opengris-parfun 7.3.0__py3-none-any.whl

parfun/backend/local_multiprocessing.py

@@ -0,0 +1,92 @@
+import multiprocessing
+from concurrent.futures import Executor, Future, ProcessPoolExecutor, ThreadPoolExecutor
+from threading import BoundedSemaphore
+
+import attrs
+import psutil
+from attrs.validators import instance_of
+
+from parfun.backend.mixins import BackendEngine, BackendSession
+from parfun.backend.profiled_future import ProfiledFuture
+from parfun.profiler.functions import profile, timed_function
+
+
+class LocalMultiprocessingSession(BackendSession):
+    # Additional constant scheduling overhead that cannot be accounted for when measuring the task execution duration.
+    CONSTANT_SCHEDULING_OVERHEAD = 1_500_000  # 1.5ms
+
+    def __init__(self, underlying_executor: Executor):
+        self._underlying_executor = underlying_executor
+        self._concurrent_task_guard = BoundedSemaphore(underlying_executor._max_workers)  # type: ignore[attr-defined]
+
+    def __enter__(self) -> "LocalMultiprocessingSession":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        return None
+
+    def submit(self, fn, *args, **kwargs) -> ProfiledFuture:
+        with profile() as submit_duration:
+            future = ProfiledFuture()
+
+            self._concurrent_task_guard.acquire()
+
+            underlying_future = self._underlying_executor.submit(timed_function, fn, *args, **kwargs)
+
+        def on_done_callback(underlying_future: Future):
+            assert submit_duration.value is not None
+
+            if underlying_future.cancelled():
+                future.cancel()
+                return
+
+            with profile() as release_duration:
+                exception = underlying_future.exception()
+
+                if exception is None:
+                    result, function_duration = underlying_future.result()
+                else:
+                    function_duration = 0
+                    result = None
+
+                self._concurrent_task_guard.release()
+
+            task_duration = (
+                self.CONSTANT_SCHEDULING_OVERHEAD + submit_duration.value + function_duration + release_duration.value
+            )
+
+            if exception is None:
+                future.set_result(result, duration=task_duration)
+            else:
+                future.set_exception(exception, duration=task_duration)
+
+        underlying_future.add_done_callback(on_done_callback)
+
+        return future
+
+
+@attrs.define(init=False)
+class LocalMultiprocessingBackend(BackendEngine):
+    """
+    Concurrent engine that uses Python builtin :mod:`multiprocessing` module.
+    """
+
+    _underlying_executor: Executor = attrs.field(validator=instance_of(Executor), init=False)
+    _concurrent_task_guard: BoundedSemaphore = attrs.field(validator=instance_of(BoundedSemaphore), init=False)
+
+    def __init__(self, max_workers: int = psutil.cpu_count(logical=False) - 1, is_process: bool = True, **kwargs):
+        if is_process:
+            self._underlying_executor = ProcessPoolExecutor(
+                max_workers=max_workers, mp_context=multiprocessing.get_context("spawn"), **kwargs
+            )
+        else:
+            self._underlying_executor = ThreadPoolExecutor(max_workers=max_workers, **kwargs)
+
+    def session(self) -> LocalMultiprocessingSession:
+        return LocalMultiprocessingSession(self._underlying_executor)
+
+    def shutdown(self, wait=True):
+        self._underlying_executor.shutdown(wait=wait)
+
+    def allows_nested_tasks(self) -> bool:
+        return False

parfun/backend/local_single_process.py

@@ -0,0 +1,47 @@
+from typing import Callable
+
+from parfun.backend.mixins import BackendEngine, BackendSession
+from parfun.backend.profiled_future import ProfiledFuture
+from parfun.profiler.functions import profile
+
+
+class LocalSingleProcessSession(BackendSession):
+    # Additional constant scheduling overhead that cannot be accounted for when measuring the task execution duration.
+    CONSTANT_SCHEDULING_OVERHEAD = 5_000  # 5 us
+
+    def __enter__(self) -> "LocalSingleProcessSession":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        return None
+
+    def submit(self, fn: Callable, *args, **kwargs) -> ProfiledFuture:
+        with profile() as function_duration:
+            future = ProfiledFuture()
+
+            try:
+                result = fn(*args, **kwargs)
+                exception = None
+            except Exception as e:
+                exception = e
+                result = None
+
+        task_duration = self.CONSTANT_SCHEDULING_OVERHEAD + function_duration.value
+
+        if exception is None:
+            future.set_result(result, duration=task_duration)
+        else:
+            future.set_exception(exception, duration=task_duration)
+
+        return future
+
+
+class LocalSingleProcessBackend(BackendEngine):
+    def session(self) -> BackendSession:
+        return LocalSingleProcessSession()
+
+    def shutdown(self):
+        pass
+
+    def allows_nested_tasks(self) -> bool:
+        return False

parfun/backend/mixins.py

@@ -0,0 +1,68 @@
+import abc
+from contextlib import AbstractContextManager
+from typing import Any, Callable
+
+from parfun.backend.profiled_future import ProfiledFuture
+
+
+class BackendSession(AbstractContextManager, metaclass=abc.ABCMeta):
+    """
+    A task submitting session to a backend engine that manages the lifecycle of the task objects (preloaded values,
+    argument values and future objects).
+    """
+
+    def preload_value(self, value: Any) -> Any:
+        """
+        Preloads a value to the backend engine.
+
+        The returned value will be used when calling ``submit()`` instead of the original value.
+        """
+        # By default, does not do anything
+        return value
+
+    @abc.abstractmethod
+    def submit(self, fn: Callable, *args, **kwargs) -> ProfiledFuture:
+        """
+        Executes an asynchronous computation.
+
+        **Blocking if no computing resource is available**.
+        """
+
+        raise NotImplementedError()
+
+
+class BackendEngine(metaclass=abc.ABCMeta):
+    """
+    Asynchronous task manager interface.
+    """
+
+    @abc.abstractmethod
+    def session(self) -> BackendSession:
+        """
+        Returns a new managed session for submitting tasks.
+
+        .. code:: python
+
+            with backend.session() as session:
+                arg_ref = session.preload_value(arg)
+
+                future = session.submit(fn, arg_ref)
+
+                print(future.result())
+
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def shutdown(self):
+        """
+        Shutdowns all resources required by the backend engine.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def allows_nested_tasks(self) -> bool:
+        """
+        Indicates if Parfun can submit new tasks from other tasks.
+        """
+        raise NotImplementedError()

parfun/backend/profiled_future.py

@@ -0,0 +1,50 @@
+from concurrent.futures import Future
+from typing import Any, Optional, Tuple
+
+from parfun.profiler.object import TraceTime
+
+
+class ProfiledFuture(Future):
+    """Future that provides an additional duration metric used to profile the task's duration."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._duration = None
+
+    def set_result(self, result: Any, duration: Optional[TraceTime] = None):
+        # Sets the task duration before the result, as set_result() triggers all completion callbacks.
+        self._duration = duration
+        return super().set_result(result)
+
+    def set_exception(self, exception: Optional[BaseException], duration: Optional[TraceTime] = None) -> None:
+        # Sets the task duration before the exception, as set_exception() triggers all completion callbacks.
+        self._duration = duration
+        return super().set_exception(exception)
+
+    def duration(self, timeout: Optional[float] = None) -> Optional[TraceTime]:
+        """
+        The total CPU time (i.e. user + system times) required to run the task, or `None` if the task didn't provide
+        task profiling.
+
+        This **should** include the overhead time required to schedule the task.
+        """
+
+        self.exception(timeout)  # Waits until the task finishes.
+
+        return self._duration
+
+    def result_and_duration(self, timeout: Optional[float] = None) -> Tuple[Any, Optional[TraceTime]]:
+        """
+        Combines the calls to `result() and duration()`:
+
+        .. code:: python
+
+            result, duration = future.result(), future.duration()
+            # is equivalent to
+            result, duration = future.result_and_duration()
+
+        """
+
+        result = self.result(timeout)
+        return result, self._duration

parfun/backend/scaler.py

@@ -0,0 +1,226 @@
+import inspect
+import itertools
+import threading
+from collections import deque
+from threading import BoundedSemaphore
+from typing import Any, Deque, Dict, Optional, Set, Tuple
+
+try:
+    from scaler import Client, SchedulerClusterCombo
+    from scaler.client.future import ScalerFuture
+    from scaler.client.object_reference import ObjectReference
+except ImportError:
+    raise ImportError("Scaler dependency missing. Use `pip install 'opengris-parfun[scaler]'` to install Scaler.")
+
+import psutil
+
+from parfun.backend.mixins import BackendEngine, BackendSession
+from parfun.backend.profiled_future import ProfiledFuture
+from parfun.profiler.functions import profile
+
+
+class ScalerSession(BackendSession):
+    # Additional constant scheduling overhead that cannot be accounted for when measuring the task execution duration.
+    CONSTANT_SCHEDULING_OVERHEAD = 8_000_000  # 8ms
+
+    def __init__(self, scheduler_address: str, n_workers: int, client_kwargs: Dict):
+        self._concurrent_task_guard = BoundedSemaphore(n_workers)
+        self._client = Client(address=scheduler_address, profiling=True, **client_kwargs)
+
+    def __enter__(self) -> "ScalerSession":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self._client.disconnect()
+
+    def preload_value(self, value: Any) -> ObjectReference:
+        return self._client.send_object(value)
+
+    def submit(self, fn, *args, **kwargs) -> Optional[ProfiledFuture]:
+        with profile() as submit_duration:
+            future = ProfiledFuture()
+
+            acquired = self._concurrent_task_guard.acquire()
+            if not acquired:
+                return None
+
+            underlying_future = self._client.submit(fn, *args, **kwargs)
+
+        def on_done_callback(underlying_future: ScalerFuture):
+            assert submit_duration.value is not None
+
+            if underlying_future.cancelled():
+                future.cancel()
+                return
+
+            with profile() as release_duration:
+                exception = underlying_future.exception()
+
+                if exception is None:
+                    result = underlying_future.result()
+                    function_duration = int(underlying_future.profiling_info().cpu_time_s * 1_000_000_000)
+                else:
+                    function_duration = 0
+                    result = None
+
+                self._concurrent_task_guard.release()
+
+            task_duration = (
+                self.CONSTANT_SCHEDULING_OVERHEAD + submit_duration.value + function_duration + release_duration.value
+            )
+
+            if exception is None:
+                future.set_result(result, duration=task_duration)
+            else:
+                future.set_exception(exception, duration=task_duration)
+
+        underlying_future.add_done_callback(on_done_callback)
+
+        return future
+
+
+class ScalerClientPool:
+    def __init__(self, scheduler_address: str, client_kwargs: Dict, max_unused_clients: int = 1):
+        self._scheduler_address = scheduler_address
+        self._client_kwargs = client_kwargs
+        self._max_unused_clients = max_unused_clients
+
+        self._lock = threading.Lock()
+        self._assigned_clients: Dict[bytes, Client] = {}
+        self._unassigned_clients: Deque[Client] = deque()  # a FIFO poll of up to `max_unused_clients`.
+
+    def acquire(self) -> Client:
+        with self._lock:
+            if len(self._unassigned_clients) > 0:
+                client = self._unassigned_clients.popleft()
+            else:
+                client = Client(address=self._scheduler_address, profiling=True, **self._client_kwargs)
+
+            self._assigned_clients[client.identity] = client
+
+            return client
+
+    def release(self, client: Client) -> None:
+        with self._lock:
+            self._assigned_clients.pop(client.identity)
+
+            if len(self._unassigned_clients) < self._max_unused_clients:
+                self._unassigned_clients.append(client)
+            else:
+                client.disconnect()
+
+    def disconnect_all(self) -> None:
+        with self._lock:
+            for client in itertools.chain(self._unassigned_clients, self._assigned_clients.values()):
+                client.disconnect()
+
+            self._unassigned_clients.clear()
+            self._assigned_clients.clear()
+
+
+class ScalerRemoteBackend(BackendEngine):
+    """Connects to a previously instantiated Scaler instance as a backend engine."""
+
+    def __init__(
+        self,
+        scheduler_address: str,
+        n_workers: int = psutil.cpu_count(logical=False) - 1,
+        allows_nested_tasks: bool = True,
+        **client_kwargs,
+    ):
+        self._scheduler_address = scheduler_address
+        self._n_workers = n_workers
+        self._allows_nested_tasks = allows_nested_tasks
+        self._client_kwargs = client_kwargs
+
+    def __getstate__(self) -> dict:
+        return {
+            "scheduler_address": self._scheduler_address,
+            "n_workers": self._n_workers,
+            "allows_nested_tasks": self._allows_nested_tasks,
+            "client_kwargs": self._client_kwargs,
+        }
+
+    def __setstate__(self, state: dict) -> None:
+        self._scheduler_address = state["scheduler_address"]
+        self._n_workers = state["n_workers"]
+        self._allows_nested_tasks = state["allows_nested_tasks"]
+        self._client_kwargs = state["client_kwargs"]
+
+    def session(self) -> ScalerSession:
+        return ScalerSession(self._scheduler_address, self._n_workers, self._client_kwargs)
+
+    def get_scheduler_address(self) -> str:
+        return self._scheduler_address
+
+    def disconnect(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    def allows_nested_tasks(self) -> bool:
+        return self._allows_nested_tasks
+
+
+class ScalerLocalBackend(ScalerRemoteBackend):
+    """Creates a Scaler cluster on the local machine and uses it as a backend engine."""
+
+    def __init__(
+        self,
+        scheduler_address: Optional[str] = None,
+        n_workers: int = psutil.cpu_count(logical=False) - 1,
+        per_worker_task_queue_size: int = 1000,
+        allows_nested_tasks: bool = True,
+        logging_paths: Tuple[str, ...] = ("/dev/stdout",),
+        logging_level: str = "INFO",
+        logging_config_file: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        :param scheduler_address the ``tcp://host:port`` tuple to use as a cluster address. If ``None``, listen to the
+        local host on an available TCP port.
+        """
+
+        client_kwargs = self.__get_constructor_arg_names(Client)
+        scheduler_cluster_combo_kwargs = self.__get_constructor_arg_names(SchedulerClusterCombo)
+
+        self._cluster = SchedulerClusterCombo(
+            address=scheduler_address,
+            n_workers=n_workers,
+            logging_paths=logging_paths,
+            logging_level=logging_level,
+            logging_config_file=logging_config_file,
+            per_worker_task_queue_size=per_worker_task_queue_size,
+            **{kwarg: value for kwarg, value in kwargs.items() if kwarg in scheduler_cluster_combo_kwargs},
+        )
+        scheduler_address = self._cluster.get_address()
+
+        super().__init__(
+            scheduler_address=scheduler_address,
+            allows_nested_tasks=allows_nested_tasks,
+            n_workers=n_workers,
+            **{kwarg: value for kwarg, value in kwargs.items() if kwarg in client_kwargs},
+        )
+
+    def __setstate__(self, state: dict) -> None:
+        super().__setstate__(state)
+        self._cluster = None  # Unserialized instances have no cluster reference.
+
+    @property
+    def cluster(self) -> SchedulerClusterCombo:
+        if self._cluster is None:
+            raise AttributeError("cluster is undefined for serialized instances.")
+
+        return self._cluster
+
+    def shutdown(self):
+        super().shutdown()
+
+        if self._cluster is not None:
+            self._cluster.shutdown()
+            self._cluster = None
+
+    @staticmethod
+    def __get_constructor_arg_names(class_: type) -> Set:
+        return set(inspect.signature(class_).parameters.keys())

parfun/backend/utility.py

@@ -0,0 +1,7 @@
+import socket
+
+
+def get_available_tcp_port(hostname: str = "127.0.0.1") -> int:
+    with socket.socket(socket.AddressFamily.AF_INET, socket.SocketKind.SOCK_STREAM) as sock:
+        sock.bind((hostname, 0))
+        return sock.getsockname()[1]

parfun/combine/collection.py

@@ -0,0 +1,13 @@
+import warnings
+
+from parfun.py_list import concat
+
+
+warnings.warn(
+    "parfun.combine.collection is deprecated and will be removed in a future version, use parfun.py_list.",
+    DeprecationWarning
+)
+
+list_concat = concat
+
+__all__ = ["list_concat"]

parfun/combine/dataframe.py

@@ -0,0 +1,13 @@
+import warnings
+
+from parfun.dataframe import concat
+
+
+warnings.warn(
+    "parfun.combine.dataframe is deprecated and will be removed in a future version, use parfun.dataframe.",
+    DeprecationWarning
+)
+
+df_concat = concat
+
+__all__ = ["df_concat"]

parfun/dataframe.py

@@ -0,0 +1,175 @@
+"""
+A collection of pre-define APIs to partition and combine dataframes.
+"""
+
+from typing import Iterable, List, Tuple
+
+try:
+    import pandas as pd
+except ImportError:
+    raise ImportError("Pandas dependency missing. Use `pip install 'opengris-parfun[pandas]'` to install Pandas.")
+
+from parfun.partition.object import PartitionFunction, PartitionGenerator
+
+
+def concat(dfs: Iterable[pd.DataFrame]) -> pd.DataFrame:
+    """
+    Similar to :py:func:`pandas.concat`.
+
+    .. code:: python
+
+        df_1 = pd.DataFrame([1,2,3])
+        df_2 = pd.DataFrame([4,5,6])
+
+        print(concat([df_1, df_2]))
+        #    0
+        # 0  1
+        # 1  2
+        # 2  3
+        # 3  4
+        # 4  5
+        # 5  6
+
+    """
+
+    return pd.concat(dfs, ignore_index=True)
+
+
+def by_row(*dfs: pd.DataFrame) -> PartitionGenerator[Tuple[pd.DataFrame, ...]]:
+    """
+    Partitions one or multiple Pandas dataframes by rows.
+
+    If multiple dataframes are given, these returned partitions will be of identical number of rows.
+
+    .. code:: python
+
+        df_1 = pd.DataFrame(range(0, 5))
+        df_2 = df_1 ** 2
+
+        with_partition_size(by_row(df_1, df_2), partition_size=2)
+
+        #  (   0
+        #   1  0
+        #   2  1,
+        #      0
+        #   1  0
+        #   2  1),
+        #  (   0
+        #   3  2
+        #   4  3,
+        #      0
+        #   3  4
+        #   4  9),
+        #  (   0
+        #   5  4,
+        #       0
+        #   5  16)]
+
+    """
+
+    __validate_dfs_parameter(*dfs)
+
+    chunk_size = yield None
+
+    def dfs_chunk(rng_start: int, rng_end: int) -> Tuple[pd.DataFrame, ...]:
+        return tuple(df.iloc[rng_start:rng_end] for df in dfs)
+
+    total_size = dfs[0].shape[0]
+    range_start = 0
+    range_end = chunk_size
+    while range_end < total_size:
+        chunk_size = yield chunk_size, dfs_chunk(range_start, range_end)
+
+        range_start = range_end
+        range_end += chunk_size
+
+    if range_start < total_size:
+        yield total_size - range_start, dfs_chunk(range_start, total_size)
+
+
+def by_group(*args, **kwargs) -> PartitionFunction[pd.DataFrame]:
+    """
+    Partitions one or multiple Pandas dataframes by groups of identical numbers of rows, similar to
+    :py:func:`pandas.DataFrame.groupby`.
+
+    See :py:func:`pandas.DataFrame.groupby` for function parameters.
+
+    .. code:: python
+
+        df_1 = pd.DataFrame({"country": ["USA", "China", "Belgium"], "capital": ["Washington", "Beijing", "Brussels"]})
+        df_2 = pd.DataFrame({"country": ["USA", "China", "Belgium"], "iso_code": ["US", "CN", "BE"]})
+
+        with_partition_size(df_by_group(by="country")(df_1, df_2), partition_size=1)
+
+        # [(   country   capital
+        #   2  Belgium  Brussels,
+        #      country iso_code
+        #   2  Belgium       BE),
+        #  (  country  capital
+        #   1   China  Beijing,
+        #     country iso_code
+        #   1   China       CN),
+        #  (  country     capital
+        #   0     USA  Washington,
+        #     country iso_code
+        #   0     USA       US)]
+
+    """
+
+    def generator(*dfs: pd.DataFrame) -> PartitionGenerator[Tuple[pd.DataFrame, ...]]:
+        __validate_dfs_parameter(*dfs)
+
+        groups: Iterable[Tuple[pd.DataFrame, ...]] = zip(
+            *((group for _name, group in df.groupby(*args, **kwargs)) for df in dfs)
+        )
+
+        it = iter(groups)
+
+        chunked_group: Tuple[List[pd.DataFrame], ...] = tuple([] for _ in range(0, len(dfs)))
+        chunked_group_size: int = 0
+
+        target_chunk_size = yield None
+
+        def concat_chunked_group_dfs(chunked_group: Tuple[List[pd.DataFrame], ...]):
+            return tuple(pd.concat(chunked_dfs) for chunked_dfs in chunked_group)
+
+        while True:
+            try:
+                group = next(it)
+                assert isinstance(group, tuple)
+                assert isinstance(group[0], pd.DataFrame)
+
+                group_size = group[0].shape[0]
+
+                if any(group_df.shape[0] != group_size for group_df in group[1:]):
+                    raise ValueError("all dataframe group sizes should be identical.")
+
+                chunked_group_size += group_size
+
+                for i, group_df in enumerate(group):
+                    chunked_group[i].append(group_df)
+
+                if chunked_group_size >= target_chunk_size:
+                    target_chunk_size = yield chunked_group_size, concat_chunked_group_dfs(chunked_group)
+
+                    chunked_group = tuple([] for _ in range(0, len(dfs)))
+                    chunked_group_size = 0
+            except StopIteration:
+                if chunked_group_size > 0:
+                    yield chunked_group_size, concat_chunked_group_dfs(chunked_group)
+
+                return
+
+    return generator
+
+
+def __validate_dfs_parameter(*dfs: pd.DataFrame) -> None:
+    if len(dfs) < 1:
+        raise ValueError("missing `dfs` parameter.")
+
+    if any(not isinstance(df, pd.DataFrame) for df in dfs):
+        raise ValueError("all `dfs` values should be DataFrame instances.")
+
+    total_size = dfs[0].shape[0]
+    if any(df.shape[0] != total_size for df in dfs[1:]):
+        raise ValueError("all DataFrames should have the same number of rows.")