FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

flowerpower/plugins/io/helpers/sql.py

@@ -1,202 +0,0 @@
-import re
-from typing import Any
-
-import pyarrow as pa
-import pyarrow.compute as pc
-from sqlglot import exp, parse_one
-
-from .datetime import timestamp_from_string
-from .polars import pl
-
-# Compile regex patterns once for efficiency
-SPLIT_PATTERN = re.compile(
-    r"<=|<|>=|>|=|!=|\s+[n,N][o,O][t,T]\s+[i,I][n,N]\s+|\s+[i,I][n,N]\s+|"
-    r"\s+[i,I][s,S]\s+[n,N][o,O][t,T]\s+[n,N][u,U][l,L]{2}\s+|\s+[i,I][s,S]\s+[n,N][u,U][l,L]{2}\s+"
-)
-LOGICAL_OPERATORS_PATTERN = re.compile(
-    r"\s+[a,A][n,N][d,D] [n,N][o,O][t,T]\s+|\s+[a,A][n,N][d,D]\s+|"
-    r"\s+[o,O][r,R] [n,N][o,O][t,T]\s+|\s+[o,O][r,R]\s+"
-)
-
-
-def sql2pyarrow_filter(string: str, schema: pa.Schema) -> pc.Expression:
-    """
-    Generates a filter expression for PyArrow based on a given string and schema.
-
-    Parameters:
-        string (str): The string containing the filter expression.
-        schema (pa.Schema): The PyArrow schema used to validate the filter expression.
-
-    Returns:
-        pc.Expression: The generated filter expression.
-
-    Raises:
-        ValueError: If the input string is invalid or contains unsupported operations.
-    """
-
-    def parse_value(val: str, type_: pa.DataType) -> Any:
-        """Parse and convert value based on the field type."""
-        if isinstance(val, (tuple, list)):
-            return type(val)(parse_value(v, type_) for v in val)
-
-        if pa.types.is_timestamp(type_):
-            return timestamp_from_string(val, exact=False, tz=type_.tz)
-        elif pa.types.is_date(type_):
-            return timestamp_from_string(val, exact=True).date()
-        elif pa.types.is_time(type_):
-            return timestamp_from_string(val, exact=True).time()
-
-        elif pa.types.is_integer(type_):
-            return int(float(val.strip("'").replace(",", ".")))
-        elif pa.types.is_floating(type_):
-            return float(val.strip("'").replace(",", "."))
-        elif pa.types.is_boolean(type_):
-            return val.lower().strip("'") in ("true", "1", "yes")
-        else:
-            return val.strip("'")
-
-    def _parse_part(part: str) -> pc.Expression:
-        match = SPLIT_PATTERN.search(part)
-        if not match:
-            raise ValueError(f"Invalid condition: {part}")
-
-        sign = match.group().lower().strip()
-        field, val = [p.strip() for p in SPLIT_PATTERN.split(part)]
-
-        if field not in schema.names:
-            raise ValueError(f"Unknown field: {field}")
-
-        type_ = schema.field(field).type
-        val = parse_value(val, type_)
-
-        operations = {
-            ">=": lambda f, v: pc.field(f) >= v,
-            ">": lambda f, v: pc.field(f) > v,
-            "<=": lambda f, v: pc.field(f) <= v,
-            "<": lambda f, v: pc.field(f) < v,
-            "=": lambda f, v: pc.field(f) == v,
-            "!=": lambda f, v: pc.field(f) != v,
-            "in": lambda f, v: pc.field(f).isin(v),
-            "not in": lambda f, v: ~pc.field(f).isin(v),
-            "is null": lambda f, v: pc.field(f).is_null(nan_is_null=True),
-            "is not null": lambda f, v: ~pc.field(f).is_null(nan_is_null=True),
-        }
-
-        if sign not in operations:
-            raise ValueError(f"Unsupported operation: {sign}")
-
-        return operations[sign](field, val)
-
-    parts = LOGICAL_OPERATORS_PATTERN.split(string)
-    operators = [op.lower().strip() for op in LOGICAL_OPERATORS_PATTERN.findall(string)]
-
-    if len(parts) == 1:
-        return _parse_part(parts[0])
-
-    expr = _parse_part(parts[0])
-    for part, operator in zip(parts[1:], operators):
-        if operator == "and":
-            expr = expr & _parse_part(part)
-        elif operator == "and not":
-            expr = expr & ~_parse_part(part)
-        elif operator == "or":
-            expr = expr | _parse_part(part)
-        elif operator == "or not":
-            expr = expr | ~_parse_part(part)
-        else:
-            raise ValueError(f"Unsupported logical operator: {operator}")
-
-    return expr
-
-
-def sql2polars_filter(string: str, schema: pl.Schema) -> pl.Expr:
-    """
-    Generates a filter expression for Polars based on a given string and schema.
-
-    Parameters:
-        string (str): The string containing the filter expression.
-        schema (pl.Schema): The Polars schema used to validate the filter expression.
-
-    Returns:
-        pl.Expr: The generated filter expression.
-
-    Raises:
-        ValueError: If the input string is invalid or contains unsupported operations.
-    """
-
-    def parse_value(val: str, dtype: pl.DataType) -> Any:
-        """Parse and convert value based on the field type."""
-        if isinstance(val, (tuple, list)):
-            return type(val)(parse_value(v, dtype) for v in val)
-
-        if dtype == pl.Datetime:
-            return timestamp_from_string(val, exact=False, tz=dtype.time_zone)
-        elif dtype == pl.Date:
-            return timestamp_from_string(val, exact=True).date()
-        elif dtype == pl.Time:
-            return timestamp_from_string(val, exact=True).time()
-        elif dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64):
-            return int(float(val.strip("'").replace(",", ".")))
-        elif dtype in (pl.Float32, pl.Float64):
-            return float(val.strip("'").replace(",", "."))
-        elif dtype == pl.Boolean:
-            return val.lower().strip("'") in ("true", "1", "yes")
-        else:
-            return val.strip("'")
-
-    def _parse_part(part: str) -> pl.Expr:
-        match = SPLIT_PATTERN.search(part)
-        if not match:
-            raise ValueError(f"Invalid condition: {part}")
-
-        sign = match.group().lower().strip()
-        field, val = [p.strip() for p in SPLIT_PATTERN.split(part)]
-
-        if field not in schema.names():
-            raise ValueError(f"Unknown field: {field}")
-
-        dtype = schema[field]
-        val = parse_value(val, dtype)
-
-        operations = {
-            ">=": lambda f, v: pl.col(f) >= v,
-            ">": lambda f, v: pl.col(f) > v,
-            "<=": lambda f, v: pl.col(f) <= v,
-            "<": lambda f, v: pl.col(f) < v,
-            "=": lambda f, v: pl.col(f) == v,
-            "!=": lambda f, v: pl.col(f) != v,
-            "in": lambda f, v: pl.col(f).is_in(v),
-            "not in": lambda f, v: ~pl.col(f).is_in(v),
-            "is null": lambda f, v: pl.col(f).is_null(),
-            "is not null": lambda f, v: pl.col(f).is_not_null(),
-        }
-
-        if sign not in operations:
-            raise ValueError(f"Unsupported operation: {sign}")
-
-        return operations[sign](field, val)
-
-    parts = LOGICAL_OPERATORS_PATTERN.split(string)
-    operators = [op.lower().strip() for op in LOGICAL_OPERATORS_PATTERN.findall(string)]
-
-    if len(parts) == 1:
-        return _parse_part(parts[0])
-
-    expr = _parse_part(parts[0])
-    for part, operator in zip(parts[1:], operators):
-        if operator == "and":
-            expr = expr & _parse_part(part)
-        elif operator == "and not":
-            expr = expr & ~_parse_part(part)
-        elif operator == "or":
-            expr = expr | _parse_part(part)
-        elif operator == "or not":
-            expr = expr | ~_parse_part(part)
-        else:
-            raise ValueError(f"Unsupported logical operator: {operator}")
-
-    return expr
-
-
-def get_table_names(sql_query):
-    return [table.name for table in parse_one(sql_query).find_all(exp.Table)]

flowerpower/plugins/io/loader/__init__.py

@@ -1,28 +0,0 @@
-from .csv import CSVDatasetReader, CSVFileReader
-from .deltatable import DeltaTableReader
-from .duckdb import DuckDBReader
-from .json import JsonDatasetReader, JsonFileReader
-from .mssql import MSSQLReader
-from .mysql import MySQLReader
-from .oracle import OracleDBReader
-from .parquet import ParquetDatasetReader, ParquetFileReader
-from .postgres import PostgreSQLReader
-from .pydala import PydalaDatasetReader
-from .sqlite import SQLiteReader
-
-__all__ = [
-    "CSVFileReader",
-    "CSVDatasetReader",
-    "DeltaTableReader",
-    "DuckDBReader",
-    "JsonFileReader",
-    "JsonDatasetReader",
-    "MSSQLReader",
-    "MySQLReader",
-    "OracleDBReader",
-    "ParquetFileReader",
-    "ParquetDatasetReader",
-    "PostgreSQLReader",
-    "PydalaDatasetReader",
-    "SQLiteReader",
-]

flowerpower/plugins/io/loader/csv.py

@@ -1,37 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatasetReader, BaseFileReader
-
-
-# @attrs.define
-class CSVFileReader(BaseFileReader, gc=False):
-    """CSV file loader.
-
-    This class is responsible for loading CSV files into several dataframe formats,
-    duckdb and datafusion.
-
-    Examples:
-        ```python
-        loader = CSVFileReader("data.csv")
-        df = loader.to_pandas()
-    ```
-    """
-
-    format: str = field(default="csv")
-
-
-# @attrs.define
-class CSVDatasetReader(BaseDatasetReader, gc=False):
-    """CSV dataset loader.
-
-    This class is responsible for loading CSV files into several dataframe formats,
-    duckdb and datafusion.
-
-    Examples:
-        ```python
-        loader = CSVDatasetReader("csv_data/")
-        df = loader.to_pandas()
-        ```
-    """
-
-    format: str = field(default="csv")

flowerpower/plugins/io/loader/deltatable.py

@@ -1,190 +0,0 @@
-# import datetime as dt
-
-
-import datetime
-
-import pyarrow as pa
-import pyarrow.dataset as pds
-from deltalake import DeltaTable, table
-from deltalake.exceptions import TableNotFoundError
-from deltalake.transaction import CommitProperties, PostCommitHookProperties
-from deltalake.writer import WriterProperties
-from loguru import logger
-from msgspec import field
-from sherlock import RedisLock
-
-from ..base import BaseDatasetReader
-from ..metadata import (get_dataframe_metadata, get_delta_metadata,
-                        get_pyarrow_dataset_metadata)
-
-
-# @attrs.define
-class DeltaTableReader(BaseDatasetReader, gc=False):
-    """Delta table loader.
-
-    This class is responsible for loading Delta tables into several dataframe formats,
-    duckdb and datafusion.
-
-    """
-
-    delta_table: DeltaTable | None = None
-    with_lock: bool = False
-    redis: str | None = None
-    format: str = field(default="delta")
-
-    def __post_init__(self):
-        super().__post_init__()
-
-        self._init_dt()
-        if self.with_lock and self.redis is None:
-            raise ValueError("Redis connection is required when using locks.")
-
-    def _init_dt(self):
-        try:
-            self.delta_table = DeltaTable(
-                self._base_path,
-                storage_options=self.storage_options.to_object_store_kwargs(),
-            )
-        except TableNotFoundError:
-            logger.warning(f"Table {self._base_path} not found.")
-            self.delta_table = None
-
-    @property
-    def dt(self) -> DeltaTable:
-        return self.delta_table
-
-    def _load(self, reload: bool = False):
-        self.to_pyarrow_table(reload=reload)
-
-    def to_pyarrow_dataset(
-        self, metadata: bool = False, reload: bool = False
-    ) -> pds.Dataset | tuple[pds.Dataset, dict[str, any]]:
-        """Converts the DeltaTable to a PyArrow Dataset.
-
-        Args:
-            metadata (bool, optional): Whether to include metadata. Defaults to False.
-            reload (bool, optional): Whether to reload the dataset. Defaults to False.
-
-        Returns:
-            pds.Dataset | tuple[pds.Dataset, dict[str, any]]: PyArrow Dataset or tuple of PyArrow Dataset and metadata.
-        """
-        if self.delta_table is None:
-            self._init_dt()
-            if self.delta_table is None:
-                return None
-
-        if reload or not hasattr(self, "_dataset"):
-            self._dataset = self.delta_table.to_pyarrow_dataset()
-        if metadata:
-            metadata = get_pyarrow_dataset_metadata(
-                self._dataset, self._base_path, "parquet"
-            )
-            return self._dataset, metadata
-        return self._dataset
-
-    def to_pyarrow_table(
-        self, metadata: bool = False, reload: bool = False
-    ) -> pa.Table | tuple[pa.Table, dict[str, any]]:
-        """Converts the DeltaTable to a PyArrow Table.
-
-        Args:
-            metadata (bool, optional): Whether to include metadata. Defaults to False.
-            reload (bool, optional): Whether to reload the table. Defaults to False.
-
-        Returns:
-            pa.Table | tuple[pa.Table, dict[str, any]]: PyArrow Table or tuple of PyArrow Table and metadata.
-        """
-        if self.delta_table is None:
-            self._init_dt()
-            if self.delta_table is None:
-                return None
-
-        if reload or not hasattr(self, "_data"):
-            self._data = self.delta_table.to_pyarrow_table()
-        if metadata:
-            metadata = get_dataframe_metadata(table, self._base_path, "parquet")
-            return self._data, metadata
-        return self._data
-
-    def compact(
-        self,
-        partition_filters: list[tuple[str, str, any]] | None = None,
-        target_size: int = None,
-        max_concurrent_tasks: int = None,
-        min_commit_interval: int | datetime.timedelta | None = None,
-        writer_properties: WriterProperties = None,
-        custom_metadata: dict[str, str] | None = None,
-        post_commithook_properties: PostCommitHookProperties | None = None,
-        commit_properties: CommitProperties | None = None,
-    ) -> dict[str, any]:
-        def _compact():
-            self.delta_table.compact(
-                partition_filters=partition_filters,
-                target_size=target_size,
-                max_concurrent_tasks=max_concurrent_tasks,
-                min_commit_interval=min_commit_interval,
-                writer_properties=writer_properties,
-                custom_metadata=custom_metadata,
-                post_commithook_properties=post_commithook_properties,
-                commit_properties=commit_properties,
-            )
-
-        if self.with_lock:
-            with RedisLock(
-                lock_name=self._base_path,
-                namespace="flowerpower",
-                client=self.redis,
-                expire=10,
-                timeout=5,
-                retry_interval=0.1,
-            ):
-                _compact()
-        else:
-            _compact()
-
-    def z_order(
-        self,
-        columns: list[str],
-        partition_filters: list[tuple[str, str, any]] | None = None,
-        target_size: int = None,
-        max_concurrent_tasks: int = None,
-        min_commit_interval: int | datetime.timedelta | None = None,
-        writer_properties: WriterProperties = None,
-        custom_metadata: dict[str, str] | None = None,
-        post_commithook_properties: PostCommitHookProperties | None = None,
-        commit_properties: CommitProperties | None = None,
-    ) -> dict[str, any]:
-        def _z_order():
-            self.delta_table.z_order(
-                columns=columns,
-                partition_filters=partition_filters,
-                target_size=target_size,
-                max_concurrent_tasks=max_concurrent_tasks,
-                min_commit_interval=min_commit_interval,
-                writer_properties=writer_properties,
-                custom_metadata=custom_metadata,
-                post_commithook_properties=post_commithook_properties,
-                commit_properties=commit_properties,
-            )
-
-        if self.with_lock:
-            with RedisLock(
-                lock_name=self._base_path,
-                namespace="flowerpower",
-                client=self.redis,
-                expire=10,
-                timeout=5,
-                retry_interval=0.1,
-            ):
-                _z_order()
-        else:
-            _z_order()
-
-    @property
-    def metadata(self) -> dict:
-        if not hasattr(self, "_metadata"):
-            self._metadata = get_delta_metadata(self.delta_table, self._base_path)
-        return self._metadata
-        if not hasattr(self, "_metadata"):
-            self._metadata = get_delta_metadata(self.delta_table, self._base_path)
-        return self._metadata

flowerpower/plugins/io/loader/duckdb.py

@@ -1,19 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseReader
-
-
-# @attrs.define
-class DuckDBReader(BaseDatabaseReader, gc=False):
-    """DuckDB loader.
-
-    This class is responsible for loading dataframes from DuckDB database.
-
-    Examples:
-        ```python
-        loader = DuckDBReader(table_name="table", path="data.db")
-        df = loader.to_polars("SELECT * FROM table WHERE column = 'value'")
-        ```
-    """
-
-    type_: str = field(default="duckdb")

flowerpower/plugins/io/loader/json.py

@@ -1,37 +0,0 @@
-from msgspec import field
-
-from ..base import BaseFileReader
-
-
-# @attrs.define
-class JsonFileReader(BaseFileReader, gc=False):
-    """
-    JSON file loader.
-
-    This class is responsible for loading dataframes from JSON files.
-
-    Examples:
-        ```python
-        loader = JsonFileReader("data.json")
-        df = loader.load()
-        ```
-    """
-
-    format: str = field(default="json")
-
-
-# @attrs.define
-class JsonDatasetReader(BaseFileReader, gc=False):
-    """
-    JSON dataset loader.
-
-    This class is responsible for loading dataframes from JSON dataset.
-
-    Examples:
-        ```python
-        loader = JsonDatasetReader("json_data/")
-        df = loader.load()
-        ```
-    """
-
-    format: str = field(default="json")

flowerpower/plugins/io/loader/mqtt.py

@@ -1,159 +0,0 @@
-from typing import Any
-
-import datafusion
-import duckdb
-import msgspec
-import orjson
-import pandas as pd
-import polars as pl
-import pyarrow as pa
-import pyarrow.dataset as pds
-
-from ..helpers.sql import sql2polars_filter
-from ..metadata import get_dataframe_metadata, get_duckdb_metadata
-
-
-class PayloadReader(msgspec.Struct):
-    payload: bytes | dict[str, Any]
-    topic: str | None = None
-    conn: duckdb.DuckDBPyConnection | None = None
-    ctx: datafusion.SessionContext | None = None
-    format: str = "mqtt"
-
-    def __post_init__(self):
-        if isinstance(self.payload, bytes):
-            self.payload = orjson.loads(self.payload)
-
-        self._metadata = {
-            "format": self.format,
-            "timestamp": pd.Timestamp.now(),
-            "topic": self.topic,
-        }
-
-    def to_pyarrow_table(
-        self, metadata: bool = False
-    ) -> pa.Table | tuple[pa.Table, dict[str, Any]]:
-        try:
-            df = pa.Table.from_pydict(self.payload)
-        except pa.ArrowInvalid:
-            df = pa.Table.from_pylist([self.payload])
-        if metadata:
-            self._metadata = get_dataframe_metadata(df, **self._metadata)
-            return df, self._metadata
-        return df
-
-    def to_pandas(
-        self, metadata: bool = False
-    ) -> pd.DataFrame | tuple[pd.DataFrame, dict[str, Any]]:
-        try:
-            df = pd.DataFrame(self.payload)
-        except ValueError:
-            df = pd.DataFrame([self.payload])
-        if metadata:
-            self._metadata = get_dataframe_metadata(df, **self._metadata)
-            return df, self._metadata
-        return df
-
-    def _to_polars_dataframe(
-        self, metadata: bool = False
-    ) -> pl.DataFrame | tuple[pl.DataFrame, dict[str, Any]]:
-        try:
-            df = pl.DataFrame(self.payload)
-        except pl.exceptions.ShapeError:
-            df = pl.DataFrame([self.payload])
-        if metadata:
-            self._metadata = get_dataframe_metadata(df, **self._metadata)
-            return df, self._metadata
-        return df
-
-    def _to_polars_lazyframe(
-        self, metadata: bool = False
-    ) -> pl.LazyFrame | tuple[pl.LazyFrame, dict[str, Any]]:
-        try:
-            df = pl.LazyFrame(self.payload)
-        except pl.exceptions.ShapeError:
-            df = pl.LazyFrame([self.payload])
-        if metadata:
-            self._metadata = get_dataframe_metadata(df, **self._metadata)
-            return df, self._metadata
-        return df
-
-    def to_polars(
-        self, lazy: bool = False, metadata: bool = False
-    ) -> (
-        pl.DataFrame | pl.LazyFrame | tuple[pl.DataFrame | pl.LazyFrame, dict[str, Any]]
-    ):
-        if lazy:
-            return self._to_polars_lazyframe(metadata=metadata)
-        else:
-            return self._to_polars_dataframe(metadata=metadata)
-
-    def to_duckdb_relation(
-        self, conn: duckdb.DuckDBPyConnection | None = None, metadata: bool = False
-    ) -> duckdb.DuckDBPyRelation | tuple[duckdb.DuckDBPyRelation, dict[str, Any]]:
-        if self.conn is None:
-            if conn is None:
-                conn = duckdb.connect()
-            self.conn = conn
-        rel = self.conn.from_arrow(self.to_pyarrow_table())
-        if metadata:
-            self._metadata = get_duckdb_metadata(rel, **self._metadata)
-            return rel, self._metadata
-        return rel
-
-    def to_pyarrow_dataset(
-        self, metadata: bool = False, **kwargs
-    ) -> pds.Dataset | tuple[pds.Dataset, dict[str, Any]]:
-        if metadata:
-            t, self._metadata = self.to_pyarrow_table(metadata=True)
-            return pds.dataset(t, **kwargs), self._metadata
-        return pds.dataset(self.to_pyarrow_table(), **kwargs)
-
-    def register_in_duckdb(
-        self,
-        conn: duckdb.DuckDBPyConnection | None = None,
-        name: str | None = None,
-    ) -> duckdb.DuckDBPyConnection:
-        if name is None:
-            name = f"mqtt:{self.topic}"
-
-        if self.conn is None:
-            if conn is None:
-                conn = duckdb.connect()
-            self.conn = conn
-
-        self.conn.register(name, self.to_pyarrow_table())
-        return self.conn
-
-    def register_in_datafusion(
-        self,
-        ctx: datafusion.SessionContext | None = None,
-        name: str | None = None,
-    ) -> None:
-        if name is None:
-            name = f"mqtt:{self.topic}"
-
-        if self.ctx is None:
-            if ctx is None:
-                ctx = datafusion.SessionContext()
-            self.ctx = ctx
-
-        self.ctx.register(name, [self.to_pyarrow_table()])
-
-        return self.ctx
-
-    def filter(self, filter_expr: str | pl.Expr) -> pl.DataFrame | pl.LazyFrame:
-        self._data = self.to_polars()
-
-        pl_schema = (
-            self._data.schema
-            if isinstance(self._data, pl.DataFrame)
-            else self._data.collect_schema()
-        )
-        filter_expr = (
-            sql2polars_filter(filter_expr, pl_schema)
-            if isinstance(filter_expr, str)
-            else filter_expr
-        )
-        return self._data.filter(filter_expr)
-        return self._data.filter(filter_expr)

flowerpower/plugins/io/loader/mssql.py

@@ -1,26 +0,0 @@
-from msgspec import field
-
-from ..base import BaseDatabaseReader
-
-
-# @attrs.define
-class MSSQLReader(BaseDatabaseReader, gc=False):
-    """MSSQL loader.
-
-    This class is responsible for loading dataframes from MSSQL database.
-
-    Examples:
-        ```python
-        loader = MSSQLReader(table_name="table", host="localhost",
-                                port=5432, username="user", password="password",
-                                database="database")
-        df = loader.to_polars()
-
-        # or
-        loader = MSSQLReader(table_name="table",
-                                connection_string="mssql+pyodbc://user:password@localhost:5432/database")
-        df = loader.to_pyarrow_table("SELECT * FROM table WHERE column = 'value'")
-        ```
-    """
-
-    type_: str = field(default="mssql")