FlowerPower 0.9.12.4__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

flowerpower/settings.py

@@ -0,0 +1,110 @@
+import os
+
+PIPELINES_DIR = os.getenv("FP_PIPELINES_DIR", "pipelines")
+CONFIG_DIR = os.getenv("FP_CONFIG_DIR", "conf")
+HOOKS_DIR = os.getenv("FP_HOOKS_DIR", "hooks")
+
+# EXECUTOR
+EXECUTOR = os.getenv("FP_EXECUTOR", "threadpool")
+EXECUTOR_MAX_WORKERS = int(
+    os.getenv("FP_EXECUTOR_MAX_WORKERS", os.cpu_count() * 5 or 10)
+)
+EXECUTOR_NUM_CPUS = int(os.getenv("FP_EXECUTOR_NUM_CPUS", os.cpu_count() or 1))
+
+# LOGGING
+LOG_LEVEL = os.getenv("FP_LOG_LEVEL", "INFO")
+
+# WORKER
+DEFAULT_JOB_QUEUE = os.getenv("FP_JOB_QUEUE_TYPE", "rq")
+# RQ WORKER
+RQ_BACKEND = os.getenv("FP_RQ_BACKEND", "redis")
+RQ_QUEUES = (
+    os.getenv("FP_RQ_QUEUES", "default, high, low, scheduler")
+    .replace(" ", "")
+    .split(",")
+)
+RQ_NUM_WORKERS = int(os.getenv("FP_RQ_NUM_WORKERS", EXECUTOR_NUM_CPUS))
+
+# APS WORKER
+APS_BACKEND_DS = os.getenv("FP_APS_DS_BACKEND", "postgresql")
+APS_SCHEMA_DS = os.getenv("FP_APS_SCHEMA", "flowerpower")
+APS_BACKEND_EB = os.getenv("FP_APS_EB_BACKEND", "postgresql")
+APS_CLEANUP_INTERVAL = int(os.getenv("FP_APS_CLEANUP_INTERVAL", 300))
+APS_MAX_CONCURRENT_JOBS = int(os.getenv("FP_APS_MAX_CONCURRENT_JOBS", 10))
+APS_DEFAULT_EXECUTOR = os.getenv("FP_APS_DEFAULT_EXECUTOR", EXECUTOR)
+APS_NUM_WORKERS = int(os.getenv("FP_APS_NUM_WORKERS", EXECUTOR_MAX_WORKERS))
+
+# Define backend properties in a dictionary for easier maintenance
+BACKEND_PROPERTIES = {
+    "postgresql": {
+        "uri_prefix": "postgresql+asyncpg://",
+        "default_port": 5432,
+        "default_host": "localhost",
+        "default_database": "postgres",
+        "default_username": "postgres",
+        "is_sqla_type": True,
+    },
+    "mysql": {
+        "uri_prefix": "mysql+aiomysql://",
+        "default_port": 3306,
+        "default_host": "localhost",
+        "default_database": "mysql",
+        "default_username": "root",
+        "is_sqla_type": True,
+    },
+    "sqlite": {
+        "uri_prefix": "sqlite+aiosqlite://",
+        "default_port": None,
+        "default_host": "",
+        "default_database": "",
+        "is_sqla_type": True,
+        "is_sqlite_type": True,
+    },
+    "mongodb": {
+        "uri_prefix": "mongodb://",
+        "default_port": 27017,
+        "default_host": "localhost",
+        "default_database": "admin",
+        "is_sqla_type": False,
+    },
+    "mqtt": {
+        "uri_prefix": "mqtt://",
+        "default_port": 1883,
+        "default_host": "localhost",
+        "default_database": "mqtt",
+    },
+    "redis": {
+        "uri_prefix": "redis://",
+        "default_port": 6379,
+        "default_host": "localhost",
+        "default_database": 0,
+    },
+    "nats_kv": {
+        "uri_prefix": "nats://",
+        "default_port": 4222,
+        "default_host": "localhost",
+        "default_database": "default",
+    },
+    "memory": {
+        "uri_prefix": "memory://",
+        "default_port": None,
+        "default_host": "",
+        "default_database": "",
+    },
+}
+
+# HAMILTON
+HAMILTON_MAX_LIST_LENGTH_CAPTURE = int(
+    os.getenv("HAMILTON_MAX_LIST_LENGTH_CAPTURE", 50)
+)
+HAMILTON_MAX_DICT_LENGTH_CAPTURE = int(
+    os.getenv("HAMILTON_MAX_DICT_LENGTH_CAPTURE", 10)
+)
+HAMILTON_CAPTURE_DATA_STATISTICS = bool(
+    os.getenv("HAMILTON_CAPTURE_DATA_STATISTICS", True)
+)
+
+HAMILTON_AUTOLOAD_EXTENSIONS = int(os.getenv("HAMILTON_AUTOLOAD_EXTENSIONS", 0))
+HAMILTON_TELEMETRY_ENABLED = bool(os.getenv("HAMILTON_TELEMETRY_ENABLED", False))
+HAMILTON_API_URL = os.getenv("HAMILTON_API_URL", "http://localhost:8241")
+HAMILTON_UI_URL = os.getenv("HAMILTON_UI_URL", "http://localhost:8242")

flowerpower/utils/logging.py

@@ -0,0 +1,21 @@
+import sys
+
+from loguru import logger
+
+from ..settings import LOG_LEVEL  # Import the setting
+
+
+def setup_logging(level: str = LOG_LEVEL) -> None:
+    """
+    Configures the Loguru logger.
+
+    Removes the default handler and adds a new one targeting stderr
+    with the level specified by the FP_LOG_LEVEL setting.
+    """
+    logger.remove()  # Remove the default handler added by Loguru
+    logger.add(
+        sys.stderr,
+        level=level.upper(),  # Use the level from the parameter, ensure it's uppercase
+        format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",  # Example format
+    )
+    # logger.info(f"Log level set to: {FP_LOG_LEVEL.upper()}")

flowerpower/utils/misc.py

@@ -3,8 +3,11 @@ import os
 import subprocess
 import tempfile
 import time
+from typing import Any
 
-import tqdm
+import msgspec
+
+# import tqdm
 
 if importlib.util.find_spec("pyarrow"):
     import pyarrow as pa
@@ -150,12 +153,10 @@ if importlib.util.find_spec("polars"):
                     next(v for v in data.values() if isinstance(v, (list, tuple)))
                 )
                 # Convert to DataFrame where each list element becomes a row
-                data = pl.DataFrame(
-                    {
-                        k: v if isinstance(v, (list, tuple)) else [v] * length
-                        for k, v in data.items()
-                    }
-                )
+                data = pl.DataFrame({
+                    k: v if isinstance(v, (list, tuple)) else [v] * length
+                    for k, v in data.items()
+                })
             else:
                 # If values are scalars, wrap them in a list to create a single row
                 data = pl.DataFrame({k: [v] for k, v in data.items()})
@@ -257,6 +258,7 @@ if (
                 data = [pa.Table.from_batches([dd]) for dd in data]
 
         return data
+
 else:
 
     def to_pyarrow_table(*args, **kwargs):
@@ -345,8 +347,8 @@ if importlib.util.find_spec("joblib"):
         all_iterables = iterables + list(iterable_kwargs.values())
         param_combinations = list(zip(*all_iterables))  # Convert to list for tqdm
 
-        if verbose:
-            param_combinations = tqdm.tqdm(param_combinations)
+        # if verbose:
+        #    param_combinations = tqdm.tqdm(param_combinations)
 
         return Parallel(**parallel_kwargs)(
             delayed(func)(
@@ -410,3 +412,49 @@ def view_img(data: str | bytes, format: str = "svg"):
 
     time.sleep(2)  # Wait for viewer to open
     os.unlink(tmp_path)
+
+
+def update_config_from_dict(
+    struct: msgspec.Struct, data: dict[str, Any]
+) -> msgspec.Struct:
+    """
+    Updates a msgspec.Struct instance with values from a dictionary.
+    Handles nested msgspec.Struct objects and nested dictionaries.
+
+    Args:
+        obj: The msgspec.Struct object to update
+        update_dict: Dictionary containing update values
+
+    Returns:
+        Updated msgspec.Struct instance
+    """
+    # Convert the struct to a dictionary for easier manipulation
+    obj_dict = msgspec.to_builtins(struct)
+
+    # Update the dictionary recursively
+    for key, value in data.items():
+        if key in obj_dict:
+            if isinstance(value, dict) and isinstance(obj_dict[key], dict):
+                # Handle nested dictionaries
+                obj_dict[key] = update_nested_dict(obj_dict[key], value)
+            else:
+                # Direct update for non-nested values
+                obj_dict[key] = value
+
+    # Convert back to the original struct type
+    return msgspec.convert(obj_dict, type(struct))
+
+
+def update_nested_dict(
+    original: dict[str, Any], updates: dict[str, Any]
+) -> dict[str, Any]:
+    """Helper function to update nested dictionaries"""
+    result = original.copy()
+    for key, value in updates.items():
+        if key in result and isinstance(value, dict) and isinstance(result[key], dict):
+            # Recursively update nested dictionaries
+            result[key] = update_nested_dict(result[key], value)
+        else:
+            # Direct update
+            result[key] = value
+    return result

flowerpower/utils/sql.py

@@ -2,8 +2,8 @@ import datetime as dt
 import re
 from functools import lru_cache
 from typing import Any
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
-import pendulum as pdl
 import pyarrow as pa
 import pyarrow.compute as pc
 from sqlglot import exp, parse_one
@@ -13,38 +13,136 @@ from .polars import pl
 
 @lru_cache(maxsize=128)
 def timestamp_from_string(
-    timestamp: str,
+    timestamp_str: str,
     tz: str | None = None,
-    exact: bool = True,
-    strict: bool = False,
     naive: bool = False,
-) -> pdl.DateTime | pdl.Date | pdl.Time | dt.datetime | dt.date | dt.time:
+) -> dt.datetime | dt.date | dt.time:
     """
-    Converts a string like "2023-01-01 10:00:00" into a datetime.datetime object.
+    Converts a timestamp string (ISO 8601 format) into a datetime, date, or time object
+    using only standard Python libraries.
+
+    Handles strings with or without timezone information (e.g., '2023-01-01T10:00:00+02:00',
+    '2023-01-01', '10:00:00'). Supports timezone offsets like '+HH:MM' or '+HHMM'.
+    For named timezones (e.g., 'Europe/Paris'), requires Python 3.9+ and the 'tzdata'
+    package to be installed.
 
     Args:
-        string (str): The string representation of the timestamp, e.g. "2023-01-01 10:00:00".
-        tz (str, optional): The timezone to use for the timestamp. Defaults to None.
-        exact (bool, optional): Whether to use exact parsing. Defaults to True.
-        strict (bool, optional): Whether to use strict parsing. Defaults to False.
-        naive (bool, optional): Whether to return a naive datetime without a timezone. Defaults to False.
+        timestamp_str (str): The string representation of the timestamp (ISO 8601 format).
+        tz (str, optional): Target timezone identifier (e.g., 'UTC', '+02:00', 'Europe/Paris').
+            If provided, the output datetime/time will be localized or converted to this timezone.
+            Defaults to None.
+        naive (bool, optional): If True, return a naive datetime/time (no timezone info),
+            even if the input string or `tz` parameter specifies one. Defaults to False.
 
     Returns:
-        datetime.datetime: The datetime object.
-    """
-    # Extract the timezone from the string if not provided
-    # tz = extract_timezone(timestamp) if tz is None else tz
-    # timestamp = timestamp.replace(tz, "").strip() if tz else timestamp
-
-    pdl_timestamp = pdl.parse(timestamp, exact=exact, strict=strict)
+        Union[dt.datetime, dt.date, dt.time]: The parsed datetime, date, or time object.
 
-    if isinstance(pdl_timestamp, pdl.DateTime):
-        if tz is not None:
-            pdl_timestamp = pdl_timestamp.naive().set(tz=tz)
-        if naive or tz is None:
-            pdl_timestamp = pdl_timestamp.naive()
+    Raises:
+        ValueError: If the timestamp string format is invalid or the timezone is
+                    invalid/unsupported.
+    """
 
-    return pdl_timestamp
+    # Regex to parse timezone offsets like +HH:MM or +HHMM
+    _TZ_OFFSET_REGEX = re.compile(r"([+-])(\d{2}):?(\d{2})")
+
+    def _parse_tz_offset(tz_str: str) -> dt.tzinfo | None:
+        """Parses a timezone offset string into a timezone object."""
+        match = _TZ_OFFSET_REGEX.fullmatch(tz_str)
+        if match:
+            sign, hours, minutes = match.groups()
+            offset_seconds = (int(hours) * 3600 + int(minutes) * 60) * (
+                -1 if sign == "-" else 1
+            )
+            if abs(offset_seconds) >= 24 * 3600:
+                raise ValueError(f"Invalid timezone offset: {tz_str}")
+            return dt.timezone(dt.timedelta(seconds=offset_seconds), name=tz_str)
+        return None
+
+    def _get_tzinfo(tz_identifier: str | None) -> dt.tzinfo | None:
+        """Gets a tzinfo object from a string (offset or IANA name)."""
+        if tz_identifier is None:
+            return None
+        if tz_identifier.upper() == "UTC":
+            return dt.timezone.utc
+
+        # Try parsing as offset first
+        offset_tz = _parse_tz_offset(tz_identifier)
+        if offset_tz:
+            return offset_tz
+
+        # Try parsing as IANA name using zoneinfo (if available)
+        if ZoneInfo:
+            try:
+                return ZoneInfo(tz_identifier)
+            except ZoneInfoNotFoundError:
+                raise ValueError(
+                    f"Timezone '{tz_identifier}' not found. Install 'tzdata' or use offset format."
+                )
+            except Exception as e:  # Catch other potential zoneinfo errors
+                raise ValueError(f"Error loading timezone '{tz_identifier}': {e}")
+        else:
+            # zoneinfo not available
+            raise ValueError(
+                f"Invalid timezone: '{tz_identifier}'. Use offset format (e.g., '+02:00') "
+                "or run Python 3.9+ with 'tzdata' installed for named timezones."
+            )
+
+    target_tz: dt.tzinfo | None = _get_tzinfo(tz)
+    parsed_obj: dt.datetime | dt.date | dt.time | None = None
+
+    # Preprocess: Replace space separator, strip whitespace
+    processed_str = timestamp_str.strip().replace(" ", "T")
+
+    # Attempt parsing (datetime, date, time) using fromisoformat
+    try:
+        # Python < 3.11 fromisoformat has limitations (e.g., no Z, no +HHMM offset)
+        # This implementation assumes Python 3.11+ for full ISO 8601 support via fromisoformat
+        # or that input strings use formats compatible with older versions (e.g., +HH:MM)
+        parsed_obj = dt.datetime.fromisoformat(processed_str)
+    except ValueError:
+        try:
+            parsed_obj = dt.date.fromisoformat(processed_str)
+        except ValueError:
+            try:
+                # Time parsing needs care, especially with offsets in older Python
+                parsed_obj = dt.time.fromisoformat(processed_str)
+            except ValueError:
+                # Add fallback for simple HH:MM:SS if needed, though less robust
+                # try:
+                #     parsed_obj = dt.datetime.strptime(processed_str, "%H:%M:%S").time()
+                # except ValueError:
+                raise ValueError(f"Invalid timestamp format: '{timestamp_str}'")
+
+    # Apply timezone logic if we have a datetime or time object
+    if isinstance(parsed_obj, (dt.datetime, dt.time)):
+        is_aware = (
+            parsed_obj.tzinfo is not None
+            and parsed_obj.tzinfo.utcoffset(
+                parsed_obj if isinstance(parsed_obj, dt.datetime) else None
+            )
+            is not None
+        )
+
+        if target_tz:
+            if is_aware:
+                # Convert existing aware object to target timezone (only for datetime)
+                if isinstance(parsed_obj, dt.datetime):
+                    parsed_obj = parsed_obj.astimezone(target_tz)
+                # else: dt.time cannot be converted without a date context. Keep original tz.
+            else:
+                # Localize naive object to target timezone
+                parsed_obj = parsed_obj.replace(tzinfo=target_tz)
+            is_aware = True  # Object is now considered aware
+
+        # Handle naive flag: remove tzinfo if requested
+        if naive and is_aware:
+            parsed_obj = parsed_obj.replace(tzinfo=None)
+
+    # If it's a date object, tz/naive flags are ignored
+    elif isinstance(parsed_obj, dt.date):
+        pass
+
+    return parsed_obj
 
 
 # Compile regex patterns once for efficiency

flowerpower/utils/templates.py

@@ -1,145 +1,5 @@
-# PIPELINE_TEMPLATE = """# ---------------- Pipelines Configuration ----------------- #
-
-# # ------------------------ Example ------------------------- #
-# #
-# # path: pipelines
-# #
-# # ## pipeline parameter
-# #
-# # params:
-# #   flow1:                      ## pipeline name
-# #       step1:                  ## step name
-# #         param1_1: 123         ## step parameters
-# #         param1_2: abc
-# #       step2:
-# #         param2_1: true
-# #
-# # ## run configuration
-# #
-# # run:
-# #   prod: # environment name
-# #     flow1:
-# #       inputs:                 ## input parameters
-# #       final_vars: [step2]     ## final output vars
-# #       with_tracker: true      ## whether to track the run
-# #
-# #   dev:
-# #     flow1:
-# #       inputs:
-# #       final_vars: [step2]
-# #       with_tracker: false
-# #
-# # ---------------------------------------------------------- #
-
-# """
-
-# SCHEDULER_TEMPLATE = """# ---------------- Scheduler Configuration ----------------- #
-
-# # ------------------------ Example ------------------------- #
-# #
-# # ##  data store configuration
-# #
-# # ### postgres
-# # data_store:
-# #   type: sqlalchemy
-# #   url: postgresql+asyncpg://edge:edge@postgres/flowerpower
-# #
-# # ### sqlite
-# # data_store:
-# #   type: sqlalchemy
-# #   url: sqlite+aiosqlite:///flowerpower.db
-# #
-# # ### memory
-# # data_store:
-# #   type: memory
-# #
-# # ### mongodb
-# # data_store:
-# #   type: mongodb
-# #   url: mongodb://localhost:27017/scheduler
-# #
-# # ## event broker configuration
-# #
-# # ### postgres
-# # event_broker:
-# #   type: asyncpg
-# #   url: postgresql+asyncpg://edge:edge@postgres/flowerpower
-# #
-# # ### mqtt
-# # event_broker:
-# #   type: mqtt
-# #   host: localhost
-# #   port: 1883
-# #   username: edge
-# #   password: edge
-
-# # ### redis
-# # event_broker:
-# #   type: redis
-# #   host: localhost
-# #   port: 6379
-
-# # ### local
-# # event_broker:
-# #   type: local # or memory
-# #
-# # ## cleanup interval configuration
-# #
-# # cleanup_interval:
-# #   unit: minutes
-# #   value: 15
-# #
-# # ## pipeline schedule configuration
-# #
-# # pipeline:
-# #   my_flow:
-# #     type: cron                ## options: interval, calendarinterval, date
-# #     start_time:
-# #     end_time:
-# #     ## optional cron arguments
-# #     crontab: * * * * *
-# #     year:
-# #     month:
-# #     week:
-# #     day:
-# #     days_of_week:
-# #     hour:
-# #     minute:
-# #     second:
-# #     timezone:
-# #     ## optional interval arguments
-# #     weeks:
-# #     days:
-# #     hours:
-# #     minutes:
-# #     seconds:
-# #     microseconds:
-# #
-# # ---------------------------------------------------------- #
-
-# """
-
-# TRACKER_TEMPLATE = """# ----------------- Tracker Configuration ------------------ #
-
-# # ------------------------ Example ------------------------- #
-# #
-# # username: your.email@example.com
-# # api_url: http://localhost:8241
-# # ui_url: http://localhost:8242
-# # api_key:
-
-# # pipeline:
-# #   my_flow:
-# #     project_id: 1
-# #     tags:
-# #       environment: dev
-# #       version: 1.0
-# #       TODO: add_more_tags_to_find_your_run_later
-# #     dag_name: my_flow_123
-# #
-# # ---------------------------------------------------------- #
-
-# """
+
+
 
 PIPELINE_PY_TEMPLATE = """# FlowerPower pipeline {name}.py
 # Created on {date}
@@ -172,3 +32,19 @@ PARAMS = Config.load(
 # Pipeline functions
 
 """
+
+HOOK_TEMPLATE__MQTT_BUILD_CONFIG = '''
+def {function_name}(payload: bytes, topic: str) -> dict:
+    """
+    MQTT hook function to build the configuration for the pipeline.
+    This function is called in the on_message callback of the MQTT client.
+    The result of this function will be passed to the hamilton builder as the config for the pipeline.
+    Args:
+        payload (bytes): The payload of the MQTT message.
+        topic (str): The topic of the MQTT message.
+    Returns:
+        dict: The configuration for the pipeline.
+    """
+
+    pass
+'''