oups 2025.9.5__py3-none-any.whl

oups/__init__.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""
+Created on Wed Dec  1 18:35:00 2021.
+
+@author: pierrot
+
+"""
+import sys
+
+# Import version dynamically from Poetry
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
+
+
+# Conditional import for stateful_ops to avoid numba issues during documentation builds
+if "sphinx" in sys.modules:
+    # During documentation builds, skip heavy stateful_ops imports
+    AggStream = None
+    by_x_rows = None
+else:
+    from .stateful_ops import AggStream
+    from .stateful_ops import by_x_rows
+
+from .stateful_loop import StatefulLoop
+from .stateful_ops import AsofMerger
+from .store import OrderedParquetDataset
+from .store import Store
+from .store import check_cmidx
+from .store import conform_cmidx
+from .store import is_toplevel
+from .store import sublevel
+from .store import toplevel
+from .store import write
+
+
+try:
+    __version__ = version("oups")
+except PackageNotFoundError:
+    # Package is not installed, likely in development
+    __version__ = "development"

oups/date_utils.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Created on Sat May 24 18:35:00 2025.
+
+@author: pierrot
+
+"""
+from pandas import Timedelta
+from pandas import Timestamp
+from pandas import date_range
+
+
+def floor_ts(ts: Timestamp, freq: str) -> Timestamp:
+    """
+    Floor a timestamp even if using non-fixed frequency.
+
+    Parameters
+    ----------
+    ts : Timestamp
+        Timestamp to floor.
+    freq : str
+        Frequency string.
+        https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases
+
+
+    Returns
+    -------
+    Timestamp
+        Floored timestamp.
+
+    """
+    try:
+        return ts.floor(freq)
+    except ValueError:
+        return date_range(end=ts.normalize(), periods=1, freq=freq)[0]
+
+
+def ceil_ts(ts: Timestamp, freq: str) -> Timestamp:
+    """
+    Ceil a timestamp even if using non-fixed frequency.
+
+    Parameters
+    ----------
+    ts : Timestamp
+        Timestamp to ceil.
+    freq : str
+        Frequency string.
+        https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timeseries-offset-aliases
+
+    Returns
+    -------
+    Timestamp
+        Ceiled timestamp.
+
+    """
+    try:
+        # Can't use 'ceil' from pandas because it does not work on 'D'aily
+        # frequency.
+        # return ts.ceil(freq)
+        return (ts + Timedelta(1, unit=freq)).floor(freq)
+    except ValueError:
+        return date_range(start=floor_ts(ts, freq), periods=2, freq=freq)[1]

oups/defines.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""
+Created on Wed Dec  1 18:35:00 2021.
+
+@author: pierrot
+
+"""
+# Central key in oups world, used as ID of the column name according which
+# dataframes are ordered.
+KEY_ORDERED_ON = "ordered_on"
+# Other shared keys.
+KEY_FILE_IDS = "file_ids"
+KEY_N_ROWS = "n_rows"
+KEY_ORDERED_ON_MINS = "ordered_on_mins"
+KEY_ORDERED_ON_MAXS = "ordered_on_maxs"
+# A specific key for a function parameter with a three-fold type:
+# - None, meaning no duplicates check,
+# - an empty list, meaning identify duplicate on all columns of the dataframe,
+# - a not empty list of a string, the columns to identify row duplicates.
+KEY_DUPLICATES_ON = "duplicates_on"
+# In a fastparquet `ParquetFile`, oups-specific metadata is stored as value for
+# key `KEY_METADATA_KEY`.
+KEY_OUPS_METADATA = "oups"
+# Parameters for 'write()' function.
+KEY_ROW_GROUP_TARGET_SIZE = "row_group_target_size"
+KEY_MAX_N_OFF_TARGET_RGS = "max_n_off_target_rgs"

oups/numpy_utils.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""
+Created on Mon Jun 02 18:35:00 2025.
+
+@author: pierrot
+
+"""
+
+from numpy import arange
+from numpy import flip
+from numpy import isnan
+from numpy import maximum
+from numpy import ones
+from numpy import searchsorted
+from numpy import where
+from numpy.typing import NDArray
+
+
+def isnotin_ordered(
+    sorted_array: NDArray,
+    query_elements: NDArray,
+    return_insert_positions: bool = False,
+) -> NDArray | tuple[NDArray, NDArray]:
+    """
+    Check if query elements are not present in a sorted array.
+
+    Parameters
+    ----------
+    sorted_array : NDArray
+        Sorted array in which to search for elements.
+        Must be sorted in ascending order.
+    query_elements : NDArray
+        Array of elements to search for.
+        Must be sorted in ascending order if containing elements which are
+        are larger than the largest element in 'sorted_array'.
+    return_insert_positions : bool, optional
+        If True, also return the insert positions where unfound elements
+        could be inserted into 'sorted_array', maintaining sort order.
+        Default is False.
+
+    Returns
+    -------
+    Union[NDArray, tuple[NDArray, NDArray]]
+        If 'return_insert_positions' is False:
+            Array of booleans with same length as 'query_elements', where
+            True indicates the element is not found in 'sorted_array'.
+        If 'return_insert_positions' is True:
+            Tuple containing:
+            - Array of booleans indicating which query elements are not found,
+            - Array of insert positions for unfound elements (where they could
+              be inserted into 'sorted_array', maintaining sort order)
+
+    Examples
+    --------
+    >>> sorted_arr = np.array([1, 3, 5, 7, 9])
+    >>> queries = np.array([2, 3, 6, 7])
+    >>> isnotin_ordered(sorted_arr, queries)
+    array([True, False, True, False])
+
+    >>> not_found, insert_positions = isnotin_ordered(sorted_arr, queries, return_insert_positions=True)
+    >>> not_found
+    array([True, False, True, False])
+    >>> insert_positions
+    array([1, 3])  # positions where 2 and 6 could be inserted
+
+    """
+    insert_idx = searchsorted(sorted_array, query_elements, side="left")
+    # Check if elements are found: they exist if insert position is valid
+    # and the element at that position matches the query
+    found_max_idx = searchsorted(insert_idx, len(sorted_array))
+    is_not_found = ones(len(query_elements), dtype=bool)
+    is_not_found[:found_max_idx] = sorted_array[insert_idx[:found_max_idx]] != query_elements[:found_max_idx]
+    return (is_not_found, insert_idx[is_not_found]) if return_insert_positions else is_not_found
+
+
+def ffill1d(arr: NDArray) -> NDArray:
+    """
+    Forward fill 1D array in-place.
+
+    Parameters
+    ----------
+    arr : NDArray
+        Array to fill.
+
+    Returns
+    -------
+    None
+
+    Notes
+    -----
+    If first value is nan, it is kept as nan.
+
+    """
+    mask = isnan(arr)
+    idx = where(~mask, arange(mask.size), 0)
+    maximum.accumulate(idx, out=idx)
+    arr[mask] = arr[idx[mask]]
+
+
+def bfill1d(arr: NDArray) -> NDArray:
+    """
+    Backward fill 1D array in-place.
+
+    Parameters
+    ----------
+    arr : NDArray
+        Array to fill.
+
+    Returns
+    -------
+    None
+
+    """
+    ffill1d(flip(arr, 0))

oups/stateful_loop/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+OUPS Stateful Loop Module.
+
+This module provides the core stateful loop functionality.
+
+"""
+
+from .stateful_loop import StatefulLoop
+
+
+__all__ = [
+    "StatefulLoop",
+]

oups/stateful_loop/loop_persistence_io.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""
+Created on Wed Jun  1 18:35:00 2025.
+
+Loop persistence I/O helpers for the stateful loop.
+
+"""
+from pathlib import Path
+from typing import Any
+
+from cloudpickle import dump
+from cloudpickle import load
+
+
+KEY_ID = "id"
+KEY_VERSION = "version"
+KEY_STATES = "states"
+PERSISTENCE_ID = "stateful_loop"
+PERSISTENCE_VERSION = 1
+
+
+class LoopPersistenceIO:
+    """
+    Helper for reading and writing the loop persistence file with schema validation.
+
+    Schema: {"id": "stateful_loop", "version": 1, "states": {...}}.
+
+    """
+
+    @staticmethod
+    def load(path: Path) -> dict[str, dict[str, Any]]:
+        """
+        Load and validate states from ``path``.
+        """
+        with open(path, "rb") as f:
+            payload = load(f)
+        if not isinstance(payload, dict):
+            raise ValueError("invalid persistence file format.")
+        if payload[KEY_ID] != PERSISTENCE_ID:
+            raise ValueError("invalid persistence file id.")
+        if payload[KEY_VERSION] != PERSISTENCE_VERSION:
+            raise ValueError("unsupported persistence file version.")
+        states = payload[KEY_STATES]
+        if not isinstance(states, dict):
+            raise ValueError("invalid 'states' content in persistence file.")
+        return states
+
+    @staticmethod
+    def save(path: Path, states: dict[str, dict[str, Any]]) -> None:
+        """
+        Save ``states`` to ``path`` with schema and version.
+        """
+        payload = {KEY_ID: PERSISTENCE_ID, KEY_VERSION: PERSISTENCE_VERSION, KEY_STATES: states}
+        with open(path, "wb") as f:
+            dump(payload, f)