FlowerPower 0.11.6.20__py3-none-any.whl → 0.20.0__py3-none-any.whl

flowerpower/plugins/io/loader/mysql.py

@@ -1,26 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseReader
-
-
-# @attrs.define
-class MySQLReader(BaseDatabaseReader, gc=False):
-    """MySQL loader.
-
-    This class is responsible for loading dataframes from MySQL database.
-
-    Examples:
-        ```python
-        loader = MySQLReader(table_name="table", host="localhost",
-                                port=5432, username="user", password="password",
-                                database="database")
-        df = loader.to_polars()
-
-        # or
-        loader = MySQLReader(table_name="table",
-                                connection_string="mssql+pyodbc://user:password@localhost:5432/database")
-        df = loader.to_pyarrow_table("SELECT * FROM table WHERE column = 'value'")
-        ```
-    """
-
-    type_: str = field(default="mysql")

flowerpower/plugins/io/loader/oracle.py

@@ -1,26 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseReader
-
-
-# @attrs.define
-class OracleDBReader(BaseDatabaseReader, gc=False):
-    """OracleDB loader.
-
-    This class is responsible for loading dataframes from OracleDB database.
-
-    Examples:
-        ```python
-        loader = OracleDBReader(table_name="table", host="localhost",
-                                port=5432, username="user", password="password",
-                                database="database")
-        df = loader.to_polars()
-
-        # or
-        loader = OracleDBReader(table_name="table",
-                                connection_string="mssql+pyodbc://user:password@localhost:5432/database")
-        df = loader.to_pyarrow_table("SELECT * FROM table WHERE column = 'value'")
-        ```
-    """
-
-    type_: str = field(default="oracle")

flowerpower/plugins/io/loader/parquet.py

@@ -1,35 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatasetReader, BaseFileReader
-
-
-# @attrs.define
-class ParquetFileReader(BaseFileReader, gc=False):
-    """Parquet file loader.
-
-    This class is responsible for loading dataframes from Parquet files.
-
-    Examples:
-        ```python
-        loader = ParquetFileReader("data.parquet")
-        df = loader.load()
-        ```
-    """
-
-    format: str = field(default="parquet")
-
-
-# @attrs.define
-class ParquetDatasetReader(BaseDatasetReader, gc=False):
-    """Parquet dataset loader.
-
-    This class is responsible for loading dataframes from Parquet dataset.
-
-    Examples:
-        ```python
-        loader = ParquetDatasetReader("parquet_data/")
-        df = loader.load()
-        ```
-    """
-
-    format: str = field(default="parquet")

flowerpower/plugins/io/loader/postgres.py

@@ -1,26 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseReader
-
-
-# @attrs.define
-class PostgreSQLReader(BaseDatabaseReader, gc=False):
-    """PostgreSQL loader.
-
-    This class is responsible for loading dataframes from PostgreSQL database.
-
-    Examples:
-        ```python
-        loader = PostgreSQLReader(table_name="table", host="localhost",
-                                port=5432, username="user", password="password",
-                                database="database")
-        df = loader.to_polars()
-
-        # or
-        loader = PostgreSQLReader(table_name="table",
-                                connection_string="mssql+pyodbc://user:password@localhost:5432/database")
-        df = loader.to_pyarrow_table("SELECT * FROM table WHERE column = 'value'")
-        ```
-    """
-
-    type_: str = field(default="postgres")

flowerpower/plugins/io/loader/pydala.py

@@ -1,19 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatasetReader
-
-
-# @attrs.define
-class PydalaDatasetReader(BaseDatasetReader, gc=False):
-    """Pydala dataset loader.
-
-    This class is responsible for loading dataframes from Pydala dataset.
-
-    Examples:
-        ```python
-        loader = PydalaDatasetReader("pydala_data/")
-        df = loader.load()
-        ```
-    """
-
-    format: str = field(default="parquet")

flowerpower/plugins/io/loader/sqlite.py

@@ -1,23 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseReader
-
-
-# @attrs.define
-class SQLiteReader(BaseDatabaseReader, gc=False):
-    """SQLite loader.
-
-    This class is responsible for loading dataframes from SQLite database.
-
-    Examples:
-        ```python
-        loader = SQLiteReader(table_name="table", path="data.db")
-        df = loader.to_polars("SELECT * FROM table WHERE column = 'value'")
-
-        # or
-        loader = SQLiteReader(table_name="table", connection_string="sqlite://data.db")
-        df = loader.to_pyarrow_table()
-        ```
-    """
-
-    type_: str = field(default="sqlite")

flowerpower/plugins/io/metadata.py

@@ -1,244 +0,0 @@
-import datetime as dt
-import importlib
-import os
-
-import duckdb
-import pandas as pd
-import polars as pl
-import pyarrow as pa
-import pyarrow.dataset as pds
-from deltalake import DeltaTable
-from fsspec import AbstractFileSystem
-
-from ...fs.ext import path_to_glob
-
-
-def get_serializable_schema(
-    data: (
-        pd.DataFrame
-        | pl.DataFrame
-        | pl.LazyFrame
-        | duckdb.DuckDBPyRelation
-        | pa.Table
-        | pa.Schema
-        | pa.RecordBatch
-        | pa.RecordBatchReader
-        | pds.Dataset
-    ),
-) -> dict[str, str]:
-    """
-    Convert DataFrame dtypes to a serializable dictionary.
-
-    Args:
-        data:  DataFrame
-
-    Returns:
-        dict mapping column names to dtype strings
-    """
-    if isinstance(data, pd.DataFrame):
-        return {col: str(dtype) for col, dtype in data.dtypes.items()}
-    elif isinstance(data, pl.DataFrame):
-        return data.schema.to_python()
-    elif isinstance(data, pl.LazyFrame):
-        return data.collect_schema().to_python()
-    elif isinstance(data, duckdb.DuckDBPyRelation):
-        return dict(zip(data.columns, [str(dtype) for dtype in data.types]))
-    elif isinstance(
-        data, pa.Table | pa.RecordBatch | pa.RecordBatchReader | pds.Dataset
-    ):
-        return dict(zip(data.schema.names, [str(dtype) for dtype in data.schema.types]))
-    elif isinstance(data, pa.Schema):
-        return dict(zip(data.names, [str(dtype) for dtype in data.types]))
-
-
-def get_dataframe_metadata(
-    df: pd.DataFrame
-    | pl.DataFrame
-    | pl.LazyFrame
-    | pa.Table
-    | pa.RecordBatch
-    | pa.RecordBatchReader
-    | list[
-        pd.DataFrame
-        | pl.DataFrame
-        | pl.LazyFrame
-        | pa.Table
-        | pa.RecordBatch
-        | pa.RecordBatchReader
-    ],
-    path: str | list[str] | None = None,
-    format: str | None = None,
-    topic: str | None = None,
-    num_files: int | None = None,
-    partition_columns: list[str] | None = None,
-    fs: AbstractFileSystem | None = None,
-    **kwargs,
-) -> dict:
-    """
-    Get metadata for a DataFrame.
-
-    Args:
-        df: DataFrame
-        path: Path to the file(s) that the DataFrame was loaded from
-        fs: Optional filesystem
-        kwargs: Additional metadata fields
-
-    Returns:
-        dict: DataFrame metadata
-    """
-    if isinstance(df, list):
-        schema = get_serializable_schema(df[0])
-        num_rows = sum(df_.shape[0] for df_ in df)
-    else:
-        schema = get_serializable_schema(df)
-        num_rows = df.shape[0] if hasattr(df, "shape") else None
-
-    if path is not None and num_files is None:
-        if isinstance(path, list):
-            num_files = len(path)
-        else:
-            path_ = path_to_glob(path=path, format=format)
-            num_files = len(fs.glob(path_)) if fs is not None else None
-
-    if partition_columns is not None:
-        schema = {k: v for k, v in schema.items() if k not in partition_columns}
-
-    metadata = {
-        "path": path,
-        "topic": topic,
-        "format": format,
-        "timestamp": int(dt.datetime.now().timestamp() * 1000),
-        "schema": schema,
-        "partition_columns": partition_columns,
-        "num_columns": len(schema),
-        "num_rows": num_rows,
-        "num_files": num_files,
-    }
-    metadata.update(kwargs)
-    return {k: v for k, v in metadata.items() if v is not None}
-
-
-def get_duckdb_metadata(
-    rel: duckdb.DuckDBPyRelation,
-    path: str,
-    format: str,
-    fs: AbstractFileSystem | None = None,
-    include_shape: bool = False,
-    include_num_files: bool = False,
-    partition_columns: list[str] | None = None,
-    **kwargs,
-) -> dict:
-    """
-    Get metadata for a DuckDBPyRelation.
-
-    Args:
-        rel: DuckDBPyRelation
-        path: Path to the file(s) that the DuckDBPyRelation was loaded from
-        fs: Filesystem
-        include_shape: Include shape in metadata
-        include_num_files: Include number of files in metadata
-        kwargs: Additional metadata fields
-
-    Returns:
-        dict: DuckDBPyRelation metadata
-    """
-
-    schema = get_serializable_schema(rel)
-    if include_shape:
-        shape = rel.shape
-    else:
-        shape = None
-    if partition_columns is not None:
-        schema = {k: v for k, v in schema.items() if k not in partition_columns}
-
-    metadata = {
-        "path": path,
-        "format": format,
-        "timestamp": dt.datetime.now().timestamp(),
-        "schema": schema,
-        "partition_columns": partition_columns,
-        "num_columns": shape[1] if shape else None,
-        "num_rows": shape[0] if shape else None,
-        "num_files": len(fs.glob(path)) if include_num_files else None,
-    }
-    metadata.update(kwargs)
-    return {k: v for k, v in metadata.items() if v is not None}
-
-
-def get_pyarrow_dataset_metadata(
-    ds: pds.Dataset,
-    path: str,
-    format: str,
-    **kwargs,
-) -> dict:
-    schema = get_serializable_schema(ds.schema)
-    files = ds.files
-
-    metadata = {
-        "path": path or os.path.dirname(files[0]),
-        "format": format,
-        "timestamp": dt.datetime.now().timestamp(),
-        "schema": schema,
-        "partition_columns": ds.partitioning.schema.names if ds.partitioning else None,
-        "num_columns": len(ds.schema),
-        "num_rows": None,
-        "num_files": len(files),
-    }
-    metadata.update(kwargs)
-    return metadata
-
-
-def get_delta_metadata(
-    dtable: DeltaTable,
-    path: str,
-    **kwargs,
-) -> dict:
-    dt_meta = dtable.metadata()
-    dt_schema = dtable.schema().to_pyarrow()
-    metadata = {
-        "path": path,
-        "format": "delta",
-        "timestamp": dt.datetime.now().timestamp(),
-        "schema": dict(zip(dt_schema.names, [str(x) for x in dt_schema.types])),
-        "partition_columns": dt_meta.partition_columns
-        if hasattr(dt_meta, "partition_columns")
-        else None,
-        "num_columns": len(dt_schema),
-        "num_files": len(dtable.files()),
-        "name": dt_meta.name or kwargs.get("name", None),
-        "description": dt_meta.description or kwargs.get("description", None),
-        "id": dt_meta.id or kwargs.get("id", None),
-    }
-
-    return {k: v for k, v in metadata.items() if v is not None}
-
-
-if importlib.util.find_spec("orjson"):
-    import orjson
-
-    def get_mqtt_metadata(
-        payload: bytes | dict[str, any],
-        topic: str | None = None,
-        **kwargs,
-    ) -> dict:
-        if isinstance(payload, bytes):
-            payload = orjson.loads(payload)
-
-        schema = get_serializable_schema(payload)
-        metadata = {
-            "topic": topic,
-            "format": "mqtt",
-            "timestamp": dt.datetime.now().timestamp(),
-            "schema": schema,
-            "num_columns": len(schema),
-            "num_rows": len(payload),
-            "name": kwargs.get("name", None),
-            "description": kwargs.get("description", None),
-            "id": kwargs.get("id", None),
-        }
-        return metadata
-
-else:
-
-    def get_mqtt_metadata(*args, **kwargs):
-        raise ImportError("orjson not installed")

flowerpower/plugins/io/saver/__init__.py

@@ -1,28 +0,0 @@
-from .csv import CSVDatasetWriter, CSVFileWriter
-from .deltatable import DeltaTableWriter
-from .duckdb import DuckDBWriter
-from .json import JsonDatasetWriter, JsonFileWriter
-from .mssql import MSSQLWriter
-from .mysql import MySQLWriter
-from .oracle import OracleDBWriter
-from .parquet import ParquetDatasetWriter, ParquetFileWriter
-from .postgres import PostgreSQLWriter
-from .pydala import PydalaDatasetWriter
-from .sqlite import SQLiteWriter
-
-__all__ = [
-    "CSVFileWriter",
-    "CSVDatasetWriter",
-    "DeltaTableWriter",
-    "DuckDBWriter",
-    "JsonFileWriter",
-    "JsonDatasetWriter",
-    "MSSQLWriter",
-    "MySQLWriter",
-    "OracleDBWriter",
-    "ParquetFileWriter",
-    "ParquetDatasetWriter",
-    "PostgreSQLWriter",
-    "PydalaDatasetWriter",
-    "SQLiteWriter",
-]

flowerpower/plugins/io/saver/csv.py

@@ -1,36 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatasetWriter, BaseFileWriter
-
-
-# @attrs.define
-class CSVFileWriter(BaseFileWriter, gc=False):
-    """CSV file writer.
-
-    This class is responsible for writing dataframes to CSV files.
-
-    Examples:
-        ```python
-        writer = CSVFileWriter(df, "data.csv")
-        writer.write()
-        ```
-    """
-
-    format: str = field(default="csv")
-
-
-# @attrs.define
-class CSVDatasetWriter(BaseDatasetWriter, gc=False):
-    """CSV dataset writer.
-
-    This class is responsible for writing dataframes to CSV dataset.
-
-    Examples:
-        ```python
-        writer = CSVDatasetWriter(df, "csv_data/")
-        writer.write()
-        ```
-
-    """
-
-    format: str = field(default="csv")

flowerpower/plugins/io/saver/deltatable.py

@@ -1,186 +0,0 @@
-from typing import Any
-
-import pandas as pd
-import polars as pl
-import pyarrow as pa
-from deltalake.transaction import CommitProperties, PostCommitHookProperties
-from deltalake.writer import (ColumnProperties, WriterProperties,
-                              write_deltalake)
-from msgspec import field
-from redis import Redis, StrictRedis
-from sherlock import RedisLock
-
-from ....utils.misc import _dict_to_dataframe
-from ..base import BaseDatasetWriter
-from ..metadata import get_dataframe_metadata
-
-
-# @attrs.define
-class DeltaTableWriter(BaseDatasetWriter, gc=False):
-    """Delta table writer.
-
-    This class is responsible for writing dataframes to Delta tables.
-
-    Examples:
-        ```python
-        writer = DeltaTableWriter("data/")
-        writer.write(df)
-        ```
-    """
-
-    description: str | None = None
-    with_lock: bool = False
-    redis: StrictRedis | Redis | None = None
-    format: str = field(default="delta")
-
-    def __post_init__(self):
-        super().__post_init__()
-        if self.with_lock and self.redis is None:
-            raise ValueError("Redis connection is required when using locks.")
-
-    def write(
-        self,
-        data: (
-            pl.DataFrame
-            | pl.LazyFrame
-            | pa.Table
-            | pa.RecordBatch
-            | pa.RecordBatchReader
-            | pd.DataFrame
-            | dict[str, Any]
-            | list[
-                pl.DataFrame
-                | pl.LazyFrame
-                | pa.Table
-                | pa.RecordBatch
-                | pa.RecordBatchReader
-                | pd.DataFrame
-                | dict[str, Any]
-            ]
-        )
-        | None = None,
-        mode: str = "append",  # "overwrite" | "append" | "error | "ignore"
-        # schema: pa.Schema | None = None,
-        schema_mode: str | None = None,  # "merge" | "overwrite"
-        partition_by: list[str] | None = None,
-        # partition_filters: list[tuple[str, str, Any]] | None = None,
-        predicate: str | None = None,
-        target_file_size: int | None = None,
-        # large_dtypes: bool = False,
-        # custom_metadata: dict[str, Any] | None = None,
-        post_commithook_properties: PostCommitHookProperties | None = None,
-        commit_properties: CommitProperties | None = None,
-        # writerproperties
-        data_page_size_limit: int | None = None,
-        dictionary_page_size_limit: int | None = None,
-        data_page_row_count_limit: int | None = None,
-        write_batch_size: int | None = None,
-        max_row_group_size: int | None = None,
-        compression: str | None = None,
-        compression_level: int | None = None,
-        statistics_truncate_length: int | None = None,
-        default_column_properties: ColumnProperties | None = None,
-        column_properties: dict[str, ColumnProperties] | None = None,
-    ) -> dict[str, Any]:
-        """
-        Write data to a Delta table.
-
-        Args:
-            data: Data to write
-            mode: Write mode
-            schema: Schema of the data
-            schema_mode: Schema mode
-            partition_by: Columns to partition by
-            partition_filters: Filters to apply to the partitions
-            predicate: Predicate to apply to the data
-            target_file_size: Target file size
-            large_dtypes: Whether to use large dtypes
-            custom_metadata: Custom metadata
-            post_commithook_properties: Post-commit hook properties
-            commit_properties: Commit properties
-            data_page_size_limit: Data page size limit
-            dictionary_page_size_limit: Dictionary page size limit
-            data_page_row_count_limit: Data page row count limit
-            write_batch_size: Write batch size
-            max_row_group_size: Maximum row group size
-            compression: Compression method
-            compression_level: Compression level
-            statistics_truncate_length: Statistics truncate length
-            default_column_properties: Default column properties
-            column_properties: Column properties
-
-        Returns:
-            Metadata
-        """
-        if data is None:
-            data = self.data
-        if isinstance(data, dict):
-            data = _dict_to_dataframe(data)
-        if not isinstance(data, list):
-            data = [data]
-        if isinstance(data[0], dict):
-            data = [_dict_to_dataframe(d) for d in data]
-        if isinstance(data[0], pl.LazyFrame):
-            data = [d.collect() for d in data]
-        if isinstance(data[0], pl.DataFrame):
-            data = pl.concat(data, how="diagonal_relaxed").to_arrow()
-        if isinstance(data[0], pd.DataFrame):
-            data = pa.concat_tables(
-                [pa.Table.from_pandas(d, preserve_index=False) for d in data],
-                promote_options="permissive",
-            )
-        if isinstance(data[0], pa.RecordBatch | pa.RecordBatchReader):
-            data = pa.Table.from_batches(data)
-        if isinstance(data[0], pa.Table):
-            data = pa.concat_tables(data, promote_options="permissive")
-
-        metadata = get_dataframe_metadata(
-            data, path=self._base_path, format=self.format
-        )
-
-        writer_properties = WriterProperties(
-            data_page_size_limit=data_page_size_limit,
-            dictionary_page_size_limit=dictionary_page_size_limit,
-            data_page_row_count_limit=data_page_row_count_limit,
-            write_batch_size=write_batch_size,
-            max_row_group_size=max_row_group_size or self.row_group_size,
-            compression=compression or self.compression.upper(),
-            compression_level=compression_level,
-            statistics_truncate_length=statistics_truncate_length,
-            default_column_properties=default_column_properties,
-            column_properties=column_properties,
-        )
-
-        def _write():
-            write_deltalake(
-                self._base_path,
-                data,
-                mode=mode,
-                # schema=schema or self.schema_,
-                partition_by=partition_by or self.partition_by,
-                storage_options=self.storage_options.to_object_store_kwargs(),
-                description=self.description,
-                schema_mode=schema_mode,
-                # partition_filters=partition_filters,
-                predicate=predicate,
-                target_file_size=target_file_size,
-                # large_dtypes=large_dtypes,
-                # custom_metadata=custom_metadata,
-                post_commithook_properties=post_commithook_properties,
-                commit_properties=commit_properties,
-                writer_properties=writer_properties,
-            )
-
-        if self.with_lock:
-            with RedisLock(
-                lock_name=self._base_path,
-                namespace="flowerpower",
-                client=self.redis,
-                expire=10,
-                timeout=5,
-                retry_interval=0.1,
-            ):
-                _write()
-        else:
-            _write()
-        return metadata

flowerpower/plugins/io/saver/duckdb.py

@@ -1,19 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseWriter
-
-
-# @attrs.define
-class DuckDBWriter(BaseDatabaseWriter, gc=False):
-    """DuckDB writer.
-
-    This class is responsible for writing dataframes to DuckDB database.
-
-    Examples:
-        ```python
-        writer = DuckDBWriter(table_name="table", path="data.db")
-        writer.write(df)
-        ```
-    """
-
-    type_: str = field(default="duckdb")