FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/utils/misc.py

@@ -7,262 +7,7 @@ import time
 from typing import Any
 
 import msgspec
-
-if importlib.util.find_spec("pyarrow"):
-    import pyarrow as pa
-
-    def convert_large_types_to_standard(schema: pa.Schema) -> pa.Schema:
-        # Define mapping of large types to standard types
-        type_mapping = {
-            pa.large_string(): pa.string(),
-            pa.large_binary(): pa.binary(),
-            pa.large_list(pa.null()): pa.list_(pa.null()),
-        }
-
-        # Convert fields
-        new_fields = []
-        for field in schema:
-            field_type = field.type
-            # Check if type exists in mapping
-            if field_type in type_mapping:
-                new_field = pa.field(
-                    name=field.name,
-                    type=type_mapping[field_type],
-                    nullable=field.nullable,
-                    metadata=field.metadata,
-                )
-                new_fields.append(new_field)
-            # Handle large lists with nested types
-            elif isinstance(field_type, pa.LargeListType):
-                new_field = pa.field(
-                    name=field.name,
-                    type=pa.list_(field_type.value_type),
-                    nullable=field.nullable,
-                    metadata=field.metadata,
-                )
-                new_fields.append(new_field)
-            else:
-                new_fields.append(field)
-
-        return pa.schema(new_fields)
-
-
-else:
-
-    def convert_large_types_to_standard(*args, **kwargs):
-        raise ImportError("pyarrow not installed")
-
-
-if importlib.util.find_spec("polars"):
-    import polars as pl
-
-    def _dict_to_dataframe(
-        data: dict | list[dict], unique: bool | list[str] | str = False
-    ) -> pl.DataFrame:
-        """
-        Convert a dictionary or list of dictionaries to a polars DataFrame.
-
-        Args:
-            data: (dict | list[dict]) Data to convert.
-
-        Returns:
-            pl.DataFrame: Converted data.
-
-        Examples:
-            >>> # Single dict with list values
-            >>> data = {'a': [1, 2, 3], 'b': [4, 5, 6]}
-            >>> _dict_to_dataframe(data)
-            shape: (3, 2)
-            ┌─────┬─────┐
-            │ a   ┆ b   │
-            │ --- ┆ --- │
-            │ i64 ┆ i64 │
-            ╞═════╪═════╡
-            │ 1   ┆ 4   │
-            │ 2   ┆ 5   │
-            │ 3   ┆ 6   │
-            └─────┴─────┘
-
-            >>> # Single dict with scalar values
-            >>> data = {'a': 1, 'b': 2}
-            >>> _dict_to_dataframe(data)
-            shape: (1, 2)
-            ┌─────┬─────┐
-            │ a   ┆ b   │
-            │ --- ┆ --- │
-            │ i64 ┆ i64 │
-            ╞═════╪═════╡
-            │ 1   ┆ 2   │
-            └─────┴─────┘
-
-            >>> # List of dicts with scalar values
-            >>> data = [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}]
-            >>> _dict_to_dataframe(data)
-            shape: (2, 2)
-            ┌─────┬─────┐
-            │ a   ┆ b   │
-            │ --- ┆ --- │
-            │ i64 ┆ i64 │
-            ╞═════╪═════╡
-            │ 1   ┆ 2   │
-            │ 3   ┆ 4   │
-            └─────┴─────┘
-
-            >>> # List of dicts with list values
-            >>> data = [{'a': [1, 2], 'b': [3, 4]}, {'a': [5, 6], 'b': [7, 8]}]
-            >>> _dict_to_dataframe(data)
-            shape: (2, 2)
-            ┌───────┬───────┐
-            │ a     ┆ b     │
-            │ ---   ┆ ---   │
-            │ list  ┆ list  │
-            ╞═══════╪═══════╡
-            │ [1,2] ┆ [3,4] │
-            │ [5,6] ┆ [7,8] │
-            └───────┴───────┘
-        """
-        if isinstance(data, list):
-            # If it's a single-element list, just use the first element
-            if len(data) == 1:
-                data = data[0]
-            # If it's a list of dicts
-            else:
-                first_item = data[0]
-                # Check if the dict values are lists/tuples
-                if any(isinstance(v, (list, tuple)) for v in first_item.values()):
-                    # Each dict becomes a row with list/tuple values
-                    data = pl.DataFrame(data)
-                else:
-                    # If values are scalars, convert list of dicts to DataFrame
-                    data = pl.DataFrame(data)
-
-                if unique:
-                    data = data.unique(
-                        subset=None if not isinstance(unique, str | list) else unique,
-                        maintain_order=True,
-                    )
-                return data
-
-        # If it's a single dict
-        if isinstance(data, dict):
-            # Check if values are lists/tuples
-            if any(isinstance(v, (list, tuple)) for v in data.values()):
-                # Get the length of any list value (assuming all lists have same length)
-                length = len(
-                    next(v for v in data.values() if isinstance(v, (list, tuple)))
-                )
-                # Convert to DataFrame where each list element becomes a row
-                data = pl.DataFrame({
-                    k: v if isinstance(v, (list, tuple)) else [v] * length
-                    for k, v in data.items()
-                })
-            else:
-                # If values are scalars, wrap them in a list to create a single row
-                data = pl.DataFrame({k: [v] for k, v in data.items()})
-
-            if unique:
-                data = data.unique(
-                    subset=None if not isinstance(unique, str | list) else unique,
-                    maintain_order=True,
-                )
-            return data
-
-        raise ValueError("Input must be a dictionary or list of dictionaries")
-
-else:
-
-    def _dict_to_dataframe(*args, **kwargs):
-        raise ImportError("polars not installed")
-
-
-if (
-    importlib.util.find_spec("pandas")
-    and importlib.util.find_spec("polars")
-    and importlib.util.find_spec("pyarrow")
-):
-    from typing import Generator
-
-    import pandas as pd
-
-    def to_pyarrow_table(
-        data: pl.DataFrame
-        | pl.LazyFrame
-        | pd.DataFrame
-        | dict
-        | list[pl.DataFrame | pl.LazyFrame | pd.DataFrame | dict],
-        concat: bool = False,
-        unique: bool | list[str] | str = False,
-    ) -> pa.Table:
-        if isinstance(data, dict):
-            data = _dict_to_dataframe(data)
-        if isinstance(data, list):
-            if isinstance(data[0], dict):
-                data = _dict_to_dataframe(data, unique=unique)
-
-        if not isinstance(data, list):
-            data = [data]
-
-        if isinstance(data[0], pl.LazyFrame):
-            data = [dd.collect() for dd in data]
-
-        if isinstance(data[0], pl.DataFrame):
-            if concat:
-                data = pl.concat(data, how="diagonal_relaxed")
-                if unique:
-                    data = data.unique(
-                        subset=None if not isinstance(unique, str | list) else unique,
-                        maintain_order=True,
-                    )
-                data = data.to_arrow()
-                data = data.cast(convert_large_types_to_standard(data.schema))
-            else:
-                data = [dd.to_arrow() for dd in data]
-                data = [
-                    dd.cast(convert_large_types_to_standard(dd.schema)) for dd in data
-                ]
-
-        elif isinstance(data[0], pd.DataFrame):
-            data = [pa.Table.from_pandas(dd, preserve_index=False) for dd in data]
-            if concat:
-                data = pa.concat_tables(data, promote_options="permissive")
-                if unique:
-                    data = (
-                        pl.from_arrow(data)
-                        .unique(
-                            subset=None
-                            if not isinstance(unique, str | list)
-                            else unique,
-                            maintain_order=True,
-                        )
-                        .to_arrow()
-                    )
-                    data = data.cast(convert_large_types_to_standard(data.schema))
-
-        elif isinstance(data[0], pa.RecordBatch | pa.RecordBatchReader | Generator):
-            if concat:
-                data = pa.Table.from_batches(data)
-                if unique:
-                    data = (
-                        pl.from_arrow(data)
-                        .unique(
-                            subset=None
-                            if not isinstance(unique, str | list)
-                            else unique,
-                            maintain_order=True,
-                        )
-                        .to_arrow()
-                    )
-                    data = data.cast(convert_large_types_to_standard(data.schema))
-            else:
-                data = [pa.Table.from_batches([dd]) for dd in data]
-
-        return data
-
-else:
-
-    def to_pyarrow_table(*args, **kwargs):
-        raise ImportError("pandas, polars, or pyarrow not installed")
-
+from fsspec_utils import AbstractFileSystem, filesystem
 
 if importlib.util.find_spec("joblib"):
     from joblib import Parallel, delayed
@@ -484,3 +229,19 @@ def update_nested_dict(
             # Direct update
             result[key] = value
     return result
+
+
+def get_filesystem(fs: AbstractFileSystem | None = None, fs_type: str = "file") -> AbstractFileSystem:
+    """
+    Helper function to get a filesystem instance.
+    
+    Args:
+        fs: An optional filesystem instance to use. If provided, this will be returned directly.
+        fs_type: The type of filesystem to create if fs is None. Defaults to "file".
+        
+    Returns:
+        An AbstractFileSystem instance.
+    """
+    if fs is None:
+        fs = filesystem(fs_type)
+    return fs

flowerpower/utils/monkey.py

@@ -1,85 +1,3 @@
-import importlib
-import sys
+# Placeholder file - APScheduler monkey patches removed
 
-from dill import dumps, loads
 
-
-def patch_pickle():
-    """
-    Patch the pickle serializer in the apscheduler module.
-
-    This function replaces the `dumps` and `loads` functions in the `apscheduler.serializers.pickle` module
-    with custom implementations.
-
-    This is useful when you want to modify the behavior of the pickle serializer used by the apscheduler module.
-
-    Example usage:
-    patch_pickle()
-
-    """
-    sys.modules["apscheduler.serializers.pickle"].dumps = dumps
-    sys.modules["apscheduler.serializers.pickle"].loads = loads
-
-
-if importlib.util.find_spec("apscheduler"):
-    from apscheduler._structures import Job, Schedule, Task
-
-    def job_to_dict(job):
-        return {
-            "id": str(job.id),
-            "task_id": job.task_id,
-            "args": [str(arg) for arg in job.args],
-            "kwargs": job.kwargs,
-            "schedule_id": job.schedule_id,
-            "scheduled_fire_time": (
-                job.scheduled_fire_time.isoformat() if job.scheduled_fire_time else None
-            ),
-            "jitter": job.jitter.total_seconds(),
-            "start_deadline": (
-                job.start_deadline.isoformat() if job.start_deadline else None
-            ),
-            "result_expiration_time": job.result_expiration_time.total_seconds(),
-            "created_at": job.created_at.isoformat(),
-            "acquired_by": job.acquired_by,
-            "acquired_until": (
-                job.acquired_until.isoformat() if job.acquired_until else None
-            ),
-        }
-
-    Job.to_dict = job_to_dict
-
-    def task_to_dict(task):
-        return {
-            "id": task.id,
-            "func": task.func,
-            "job_executor": task.job_executor,
-            "max_running_jobs": task.max_running_jobs,
-            "misfire_grace_time": task.misfire_grace_time,
-        }
-
-    Task.to_dict = task_to_dict
-
-    def schedule_to_dict(schedule):
-        return {
-            "id": schedule.id,
-            "task_id": schedule.task_id,
-            "trigger": str(schedule.trigger),
-            "args": [str(arg) for arg in schedule.args],
-            "kwargs": schedule.kwargs,
-            "paused": schedule.paused,
-            "coalesce": schedule.coalesce.name if schedule.coalesce else None,
-            "misfire_grace_time": schedule.misfire_grace_time,
-            "max_jitter": schedule.max_jitter,
-            "next_fire_time": (
-                schedule.next_fire_time.isoformat() if schedule.next_fire_time else None
-            ),
-            "last_fire_time": (
-                schedule.last_fire_time.isoformat() if schedule.last_fire_time else None
-            ),
-            "acquired_by": schedule.acquired_by,
-            "acquired_until": (
-                schedule.acquired_until.isoformat() if schedule.acquired_until else None
-            ),
-        }
-
-    Schedule.to_dict = schedule_to_dict