FlowerPower 0.11.5.8__py3-none-any.whl → 0.11.6.1__py3-none-any.whl

flowerpower/plugins/io/helpers/pyarrow.py

@@ -0,0 +1,406 @@
+import numpy as np
+import polars as pl
+import pyarrow as pa
+import pyarrow.compute as pc
+
+# Pre-compiled regex patterns (identical to original)
+INTEGER_REGEX = r"^[-+]?\d+$"
+FLOAT_REGEX = r"^[-+]?(?:\d*[.,])?\d+(?:[eE][-+]?\d+)?$"
+BOOLEAN_REGEX = r"^(true|false|1|0|yes|ja|no|nein|t|f|y|j|n)$"
+BOOLEAN_TRUE_REGEX = r"^(true|1|yes|ja|t|y|j)$"
+DATETIME_REGEX = (
+    r"^("
+    r"\d{4}-\d{2}-\d{2}"  # ISO: 2023-12-31
+    r"|"
+    r"\d{2}/\d{2}/\d{4}"  # US: 12/31/2023
+    r"|"
+    r"\d{2}\.\d{2}\.\d{4}"  # German: 31.12.2023
+    r"|"
+    r"\d{8}"  # Compact: 20231231
+    r")"
+    r"([ T]\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?)?"  # Optional time: 23:59[:59[.123456]]
+    r"([+-]\d{2}:?\d{2}|Z)?"  # Optional timezone: +01:00, -0500, Z
+    r"$"
+)
+
+# Float32 range limits
+F32_MIN = float(np.finfo(np.float32).min)
+F32_MAX = float(np.finfo(np.float32).max)
+
+
+def unify_schemas(
+    schemas: list[pa.Schema], use_large_dtypes: bool = False
+) -> pa.Schema:
+    """
+    Unify a list of PyArrow schemas into a single schema.
+
+    Args:
+        schemas (list[pa.Schema]): List of PyArrow schemas to unify.
+
+    Returns:
+        pa.Schema: A unified PyArrow schema.
+    """
+    try:
+        return pa.unify_schemas(schemas, promote_options="permissive")
+    except (pa.lib.ArrowInvalid, pa.lib.ArrowTypeError) as e:
+        _ = e.args[0]
+        # If unify_schemas fails, we can try to create a schema with empty tables
+        schema = (
+            pl.concat(
+                [
+                    # pl.from_arrow(pa.Table.from_pylist([], schema=schema))
+                    pl.from_arrow(schema.empty_table())
+                    for schema in schemas
+                ],
+                how="diagonal_relaxed",
+            )
+            .to_arrow()
+            .schema
+        )
+        if not use_large_dtypes:
+            return convert_large_types_to_normal(schema)
+        return schema
+
+
+def cast_schema(table: pa.Table, schema: pa.Schema) -> pa.Table:
+    """
+    Cast a PyArrow table to a given schema, updating the schema to match the table's columns.
+
+    Args:
+        table (pa.Table): The PyArrow table to cast.
+        schema (pa.Schema): The target schema to cast the table to.
+
+    Returns:
+        pa.Table: A new PyArrow table with the specified schema.
+    """
+    # Filter schema fields to only those present in the table
+    table_columns = set(table.schema.names)
+    filtered_fields = [field for field in schema if field.name in table_columns]
+    updated_schema = pa.schema(filtered_fields)
+    return table.select(updated_schema.names).cast(updated_schema)
+
+
+def convert_large_types_to_normal(schema: pa.Schema) -> pa.Schema:
+    """
+    Convert large types in a PyArrow schema to their standard types.
+
+    Args:
+        schema (pa.Schema): The PyArrow schema to convert.
+
+    Returns:
+        pa.Schema: A new PyArrow schema with large types converted to standard types.
+    """
+    # Define mapping of large types to standard types
+    type_mapping = {
+        pa.large_string(): pa.string(),
+        pa.large_binary(): pa.binary(),
+        pa.large_utf8(): pa.utf8(),
+        pa.large_list(pa.null()): pa.list_(pa.null()),
+        pa.large_list_view(pa.null()): pa.list_view(pa.null()),
+    }
+    # Convert fields
+    new_fields = []
+    for field in schema:
+        field_type = field.type
+        # Check if type exists in mapping
+        if field_type in type_mapping:
+            new_field = pa.field(
+                name=field.name,
+                type=type_mapping[field_type],
+                nullable=field.nullable,
+                metadata=field.metadata,
+            )
+            new_fields.append(new_field)
+        # Handle large lists with nested types
+        elif isinstance(field_type, pa.LargeListType):
+            new_field = pa.field(
+                name=field.name,
+                type=pa.list_(
+                    type_mapping[field_type.value_type]
+                    if field_type.value_type in type_mapping
+                    else field_type.value_type
+                ),
+                nullable=field.nullable,
+                metadata=field.metadata,
+            )
+            new_fields.append(new_field)
+        # Handle dictionary with large_string, large_utf8, or large_binary values
+        elif isinstance(field_type, pa.DictionaryType):
+            new_field = pa.field(
+                name=field.name,
+                type=pa.dictionary(
+                    field_type.index_type,
+                    type_mapping[field_type.value_type]
+                    if field_type.value_type in type_mapping
+                    else field_type.value_type,
+                    field_type.ordered,
+                ),
+                # nullable=field.nullable,
+                metadata=field.metadata,
+            )
+            new_fields.append(new_field)
+        else:
+            new_fields.append(field)
+
+    return pa.schema(new_fields)
+
+
+def _clean_string_array(array: pa.Array) -> pa.Array:
+    """
+    Clean string values in a PyArrow array using vectorized operations.
+    """
+    if len(array) == 0 or array.null_count == len(array):
+        return array
+
+    # Trim whitespace using compute functions
+    trimmed = pc.utf8_trim_whitespace(array)
+
+    # Create mask for values to convert to null
+    empty_mask = pc.equal(trimmed, "")
+    dash_mask = pc.equal(trimmed, "-")
+    none_mask = pc.equal(trimmed, "None")
+
+    null_mask = pc.or_(pc.or_(empty_mask, dash_mask), none_mask)
+
+    # Apply the mask to set matching values to null
+    return pc.if_else(null_mask, None, trimmed)
+
+
+def _can_downcast_to_float32(array: pa.Array) -> bool:
+    """
+    Check if float values are within Float32 range using vectorized operations.
+    """
+    if len(array) == 0 or array.null_count == len(array):
+        return True
+
+    # Use compute functions to filter finite values and calculate min/max
+    is_finite = pc.is_finite(array)
+
+    # Skip if no finite values
+    if not pc.any(is_finite).as_py():
+        return True
+
+    # Filter out non-finite values
+    finite_array = pc.filter(array, is_finite)
+
+    min_val = pc.min(finite_array).as_py()
+    max_val = pc.max(finite_array).as_py()
+
+    return F32_MIN <= min_val <= max_val <= F32_MAX
+
+
+def _get_optimal_int_type(array: pa.Array) -> pa.DataType:
+    """
+    Determine the most efficient integer type based on data range.
+    """
+    # Handle empty or all-null arrays
+    if len(array) == 0 or array.null_count == len(array):
+        return pa.int8()
+
+    # Use compute functions to get min and max values
+    min_max = pc.min_max(array)
+    min_val = min_max["min"].as_py()
+    max_val = min_max["max"].as_py()
+
+    if min_val >= 0:  # Unsigned
+        if max_val <= 255:
+            return pa.uint8()
+        elif max_val <= 65535:
+            return pa.uint16()
+        elif max_val <= 4294967295:
+            return pa.uint32()
+        else:
+            return pa.uint64()
+    else:  # Signed
+        if -128 <= min_val and max_val <= 127:
+            return pa.int8()
+        elif -32768 <= min_val and max_val <= 32767:
+            return pa.int16()
+        elif -2147483648 <= min_val and max_val <= 2147483647:
+            return pa.int32()
+        else:
+            return pa.int64()
+
+
+def _optimize_numeric_array(array: pa.Array, shrink: bool) -> pa.Array:
+    """
+    Optimize numeric PyArrow array by downcasting when possible.
+    Uses vectorized operations for efficiency.
+    """
+    if not shrink or len(array) == 0 or array.null_count == len(array):
+        return array if len(array) > 0 else pa.array([], type=pa.int8())
+
+    # Handle floating point types
+    if pa.types.is_floating(array.type):
+        if array.type == pa.float64() and _can_downcast_to_float32(array):
+            return pc.cast(array, pa.float32())
+        return array
+
+    # Handle integer types
+    if pa.types.is_integer(array.type):
+        # Skip if already optimized to smallest types
+        if array.type in [pa.int8(), pa.uint8()]:
+            return array
+
+        optimal_type = _get_optimal_int_type(array)
+        return pc.cast(array, optimal_type)
+
+    # Default: return unchanged
+    return array
+
+
+def _all_match_regex(array: pa.Array, pattern: str) -> bool:
+    """
+    Check if all non-null values in array match regex pattern.
+    Uses pyarrow.compute.match_substring_regex for vectorized evaluation.
+    """
+    if len(array) == 0 or array.null_count == len(array):
+        return True
+
+    # Check if al values match the pattern
+    return pc.all(pc.match_substring_regex(array, pattern, ignore_case=True)).as_py()
+
+
+def _optimize_string_array(
+    array: pa.Array, col_name: str, shrink_numerics: bool, time_zone: str | None = None
+) -> pa.Array:
+    """
+    Convert string PyArrow array to appropriate type based on content analysis.
+    Uses fully vectorized operations wherever possible.
+    """
+    # Handle empty or all-null arrays
+    if len(array) == 0:
+        return pa.array([], type=pa.int8())
+    if array.null_count == len(array):
+        return pa.array([None] * len(array), type=pa.int8())
+
+    # Clean string values
+    cleaned_array = _clean_string_array(array)
+
+    try:
+        # Check for boolean values
+        if _all_match_regex(cleaned_array, BOOLEAN_REGEX):
+            # Match with TRUE pattern
+            true_matches = pc.match_substring_regex(
+                array, BOOLEAN_TRUE_REGEX, ignore_case=True
+            )
+
+            # Convert to boolean type
+            return pc.cast(true_matches, pa.bool_())
+
+        elif _all_match_regex(cleaned_array, INTEGER_REGEX):
+            # Convert to integer
+            # First replace commas with periods in Polars, then cast
+            int_array = pc.cast(
+                pc.replace_substring(cleaned_array, ",", "."), pa.int64()
+            )
+
+            if shrink_numerics:
+                optimal_type = _get_optimal_int_type(int_array)
+                return pc.cast(int_array, optimal_type)
+
+            return int_array
+
+        # Check for numeric values
+        elif _all_match_regex(cleaned_array, FLOAT_REGEX):
+            # Convert to float
+            # First replace commas with periods in Polars
+            float_array = pc.cast(
+                pc.replace_substring(cleaned_array, ",", "."), pa.float64()
+            )
+            if shrink_numerics and _can_downcast_to_float32(float_array):
+                return pc.cast(float_array, pa.float32())
+
+            return float_array
+
+        # Check for datetime values - use polars for conversion as specified
+        elif _all_match_regex(cleaned_array, DATETIME_REGEX):
+            # Convert via polars
+
+            pl_series = pl.Series(col_name, cleaned_array)
+            converted = pl_series.str.to_datetime(
+                strict=False, time_unit="us", time_zone=time_zone
+            )
+            # Convert polars datetime back to pyarrow
+            return converted.to_arrow()
+
+    except Exception:
+        # Fallback: return cleaned strings on any error
+        return cleaned_array
+
+    # Default: return cleaned strings
+    return cleaned_array
+
+
+def _process_column(
+    table: pa.Table, col_name: str, shrink_numerics: bool, time_zone: str | None = None
+) -> pa.Array:
+    """
+    Process a single column for type optimization.
+    """
+    array = table[col_name]
+
+    # Handle all-null columns
+    if array.null_count == len(array):
+        return pa.array([None] * len(array), type=pa.int8())
+
+    # Process based on current type
+    if pa.types.is_floating(array.type) or pa.types.is_integer(array.type):
+        return _optimize_numeric_array(array, shrink_numerics)
+    elif pa.types.is_string(array.type):
+        return _optimize_string_array(array, col_name, shrink_numerics, time_zone)
+
+    # Keep original for other types
+    return array
+
+
+def opt_dtype(
+    table: pa.Table,
+    include: str | list[str] | None = None,
+    exclude: str | list[str] | None = None,
+    time_zone: str | None = None,
+    shrink_numerics: bool = True,
+) -> pa.Table:
+    """
+    Optimize data types of a PyArrow Table for performance and memory efficiency.
+
+    This function analyzes each column and converts it to the most appropriate
+    data type based on content, handling string-to-type conversions and
+    numeric type downcasting. It is the PyArrow equivalent of the Polars
+    `opt_dtype` function.
+
+    Args:
+        table: PyArrow Table to optimize
+        include: Column(s) to include in optimization (default: all columns)
+        exclude: Column(s) to exclude from optimization
+        time_zone: Optional time zone for datetime parsing
+        shrink_numerics: Whether to downcast numeric types when possible
+
+    Returns:
+        PyArrow Table with optimized data types
+    """
+    # Normalize include/exclude parameters
+    if isinstance(include, str):
+        include = [include]
+    if isinstance(exclude, str):
+        exclude = [exclude]
+
+    # Determine columns to process
+    cols_to_process = table.column_names
+    if include:
+        cols_to_process = [col for col in include if col in table.column_names]
+    if exclude:
+        cols_to_process = [col for col in cols_to_process if col not in exclude]
+
+    # Process each column and build a new table
+    new_columns = []
+    for col_name in table.column_names:
+        if col_name in cols_to_process:
+            new_columns.append(
+                _process_column(table, col_name, shrink_numerics, time_zone)
+            )
+        else:
+            new_columns.append(table[col_name])
+
+    # Create a new table with the optimized columns
+    return pa.Table.from_arrays(new_columns, names=table.column_names)

flowerpower/settings/general.py

@@ -3,4 +3,4 @@ import os
 PIPELINES_DIR = os.getenv("FP_PIPELINES_DIR", "pipelines")
 CONFIG_DIR = os.getenv("FP_CONFIG_DIR", "conf")
 HOOKS_DIR = os.getenv("FP_HOOKS_DIR", "hooks")
-CACHE_DIR = os.getenv("FP_CACHE_DIR", "~/.flowerpower/cache")
+CACHE_DIR = os.getenv("FP_CACHE_DIR", "~/.flowerpower/cache")

{flowerpower-0.11.5.8.dist-info → flowerpower-0.11.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowerPower
-Version: 0.11.5.8
+Version: 0.11.6.1
 Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
 Author-email: "Volker L." <ligno.blades@gmail.com>
 Project-URL: Homepage, https://github.com/legout/flowerpower

{flowerpower-0.11.5.8.dist-info → flowerpower-0.11.6.1.dist-info}/RECORD

@@ -18,7 +18,7 @@ flowerpower/cli/pipeline.py,sha256=60P6u_QOSgp0jJXEMxazEEo5Sh7-SWFo-Kkuaz21YuI,3
 flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
 flowerpower/fs/__init__.py,sha256=uZaPXErEfQqQRbKRIjkB9yiygd45X5_psYn9-VVrBTQ,910
 flowerpower/fs/base.py,sha256=TqgqBsaFj13O1NpAr8kHuGJ9CTlaSWViMB8Ai_iuCjs,22761
-flowerpower/fs/ext.py,sha256=gsCJ87VxVdy22oVtNRaN4M-SLO8WORVf5JRyDeQjjEs,63834
+flowerpower/fs/ext.py,sha256=2NmhSbCIL0qnONMRNPHcPUuR39bGjWpxJE4hNHU5Rvw,69044
 flowerpower/fs/storage_options.py,sha256=msq5TpxAU8tcE_Bxjw6SyxaFa75UjdYnR4-O9U2wmbk,48034
 flowerpower/job_queue/__init__.py,sha256=a25hIqv2xoFKb4JZlyUukS0ppZ9-2sJKH3XAvbk3rlk,10788
 flowerpower/job_queue/base.py,sha256=YwLunDQSyqkSU_vJ69C5SSybJeJP1bAiZ3teUtOchxA,13640
@@ -37,17 +37,18 @@ flowerpower/job_queue/rq/utils.py,sha256=QjyNhSM1_gMZkV2cO8eR99XeEji4AMwpxE1TANa
 flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py,sha256=X8rKfSbuITpWiwCyNqN-WVeodBkafFuj-zvbZb7z6lw,7511
 flowerpower/job_queue/rq/concurrent_workers/thread_worker.py,sha256=M_jjci-pypEsbHe-gCQS6jmJBrA-Tb7W19sEWi6htFU,7801
 flowerpower/pipeline/__init__.py,sha256=xbEn_RN0vVNqLZMSFOCdV41ggUkYrghFVJYd_EC0C44,75
-flowerpower/pipeline/base.py,sha256=N3N0iqiVo2vUVli_WSADAQMq283mG9OdFql58LXeF2Q,3275
+flowerpower/pipeline/base.py,sha256=CO2PsOACRHFd8-g-0BUTc1ZMVdaBnhQftJda7HDxPZI,3305
 flowerpower/pipeline/io.py,sha256=8Mlw4G7ehHHZEk4Qui-HcKBM3tBF4FuqUbjfNxK09iU,15963
 flowerpower/pipeline/job_queue.py,sha256=hl38-0QZCH5wujUf0qIqYznIPDLsJAoNDcOD7YGVQ6s,26114
 flowerpower/pipeline/manager.py,sha256=KVpOclUEUAETUNJamJJGuKt3oxCaLitQgxWxkE1q028,74460
-flowerpower/pipeline/registry.py,sha256=WWQoaxtgnlntFEIPQzYM1gk0zUXwrH2PmDLGbTzhrZs,18991
+flowerpower/pipeline/registry.py,sha256=6ngmHyKyQsxvIO4qRYxljedY0BE1wE3lpfksEGOzjNs,18963
 flowerpower/pipeline/runner.py,sha256=dsSVYixFXqlxFk8EJfT4wV_7IwgkXq0ErwH_yf_NGS8,25654
 flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
-flowerpower/plugins/io/base.py,sha256=pWQt7vS_n3jR3J9YaVgMCXQ6utWuo3uKYZdTJ3pLiuY,79071
+flowerpower/plugins/io/base.py,sha256=-bZBTdFGUWm60JuFpBG_1TZO7D0hmjgSA3a8Prg1MnY,96644
 flowerpower/plugins/io/metadata.py,sha256=PCrepLilXRWKDsB5BKFF_-OFs712s1zBeitW-84lDLQ,7005
 flowerpower/plugins/io/helpers/datetime.py,sha256=1WBUg2ywcsodJQwoF6JiIGc9yhVobvE2IErWp4i95m4,10649
-flowerpower/plugins/io/helpers/polars.py,sha256=xWLjmZBUhbJPd0m5pkHreMSMeNLyJThJt0R7V7yzWEs,18068
+flowerpower/plugins/io/helpers/polars.py,sha256=346DBHG-HvoGZWF-DWxgz7H3KlZu8bFylKIqMOnVJSk,27031
+flowerpower/plugins/io/helpers/pyarrow.py,sha256=b1JVi-3V5hkb4y4jMpHq8W6ZeTDk-Eox_hWMNy3dH_Y,13574
 flowerpower/plugins/io/helpers/sql.py,sha256=BPIxjarKF3p93EdtUu-md8KislE9q8IWNSeZ5toFU6U,7298
 flowerpower/plugins/io/loader/__init__.py,sha256=MKH42nvVokaWas0wFgX1yrpU5iLpvHjRqqF-KzwLHCg,780
 flowerpower/plugins/io/loader/csv.py,sha256=Q5bmcbbr530sT1kQ2YiJwvsMUPqi0VcZWsLOygmzRyI,827
@@ -81,7 +82,7 @@ flowerpower/plugins/mqtt/manager.py,sha256=WJIKu_0wfrxph_L0AjV-7RYVFtie2cOmdVGjj
 flowerpower/settings/__init__.py,sha256=g1JKxxQKgBSbghzhCvEPOgD4ofjCGJdWCf8T0husyMQ,178
 flowerpower/settings/backend.py,sha256=jb4H3DXil5uN4BDo3ipqP6-1Hpnrm6qvIZQWiHZcDXg,4994
 flowerpower/settings/executor.py,sha256=vNF383g1gMGkq_CXUzateGMNcJZig-vHkVVb0Hi_b74,249
-flowerpower/settings/general.py,sha256=6kGLT1arCYo7jgFgQoNeqNztmIBkk882MFSnR7EAct0,226
+flowerpower/settings/general.py,sha256=RxY6PGF_L8ApFlLPHulZ2I8_-aHYqOj63fUu9kSQTjI,227
 flowerpower/settings/hamilton.py,sha256=GVzWKz3B-wy07etY1mNUstEa4DFrQ_lM2cjE0qG_6qw,623
 flowerpower/settings/job_queue.py,sha256=wrehB9Suwks9y5Gvfg-cex6SQOoeFfSwZDxdKNmKhcA,2972
 flowerpower/settings/logging.py,sha256=xzoiM-87WGduC-RJu8RfyeweHGTM7SnHXHQPbxKweLE,67
@@ -93,9 +94,9 @@ flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2
 flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
 flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
 flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
-flowerpower-0.11.5.8.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
-flowerpower-0.11.5.8.dist-info/METADATA,sha256=ADH8WX3HkFVTcdB-WmwvC1yN0mRcj_zp5EjyQYV3Zfk,21612
-flowerpower-0.11.5.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flowerpower-0.11.5.8.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
-flowerpower-0.11.5.8.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
-flowerpower-0.11.5.8.dist-info/RECORD,,
+flowerpower-0.11.6.1.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
+flowerpower-0.11.6.1.dist-info/METADATA,sha256=KOkDA61ZYzXs3vvwKIQciSsSl-OoniBSpdilRlYXU8g,21612
+flowerpower-0.11.6.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flowerpower-0.11.6.1.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
+flowerpower-0.11.6.1.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
+flowerpower-0.11.6.1.dist-info/RECORD,,