Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_frame/selectors.py

@@ -1,5 +1,3 @@
-from typing import List, Union, TYPE_CHECKING  # Added TYPE_CHECKING
-
 import polars as pl
 
 # --- TYPE CHECKING IMPORTS ---
@@ -7,11 +5,12 @@ import polars as pl
 #     Import Expr only for type hints
 from flowfile_frame.expr import Expr
 
-
 # --- Selector Base Classes (Compound, Complement) ---
 
+
 class Selector:
     """Base class for column selectors, inspired by polars.selectors"""
+
     def __init__(self):
         self._repr_str = self._get_repr_str()  # Use base repr calculation method
         # No agg_func state stored here anymore
@@ -28,25 +27,25 @@ class Selector:
     def __repr__(self) -> str:
         return self._repr_str
 
-    def __or__(self, other: 'Selector') -> 'CompoundSelector':
-        return CompoundSelector(self, other, 'union')
+    def __or__(self, other: "Selector") -> "CompoundSelector":
+        return CompoundSelector(self, other, "union")
 
-    def __and__(self, other: 'Selector') -> 'CompoundSelector':
-        return CompoundSelector(self, other, 'intersection')
+    def __and__(self, other: "Selector") -> "CompoundSelector":
+        return CompoundSelector(self, other, "intersection")
 
-    def __sub__(self, other: 'Selector') -> 'CompoundSelector':
-        return CompoundSelector(self, other, 'difference')
+    def __sub__(self, other: "Selector") -> "CompoundSelector":
+        return CompoundSelector(self, other, "difference")
 
-    def __xor__(self, other: 'Selector') -> 'CompoundSelector':
-        return CompoundSelector(self, other, 'symmetric_difference')
+    def __xor__(self, other: "Selector") -> "CompoundSelector":
+        return CompoundSelector(self, other, "symmetric_difference")
 
-    def __invert__(self) -> 'ComplementSelector':
+    def __invert__(self) -> "ComplementSelector":
         return ComplementSelector(self)
 
     # --- Aggregation Methods ---
     # These methods now return Expr objects, importing Expr locally
 
-    def sum(self) -> 'Expr':
+    def sum(self) -> "Expr":
         """Create an expression to sum columns selected by this selector."""
         # Expr init will handle creating the 'pl.sum(selector)' repr
         return Expr(expr=None, selector=self, agg_func="sum")
@@ -54,43 +53,43 @@ class Selector:
     def expr(self):
         return eval(self.repr_str)
 
-    def mean(self) -> 'Expr':
+    def mean(self) -> "Expr":
         """Create an expression to average columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="mean")
 
-    def median(self) -> 'Expr':
+    def median(self) -> "Expr":
         """Create an expression to find the median of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="median")
 
-    def min(self) -> 'Expr':
+    def min(self) -> "Expr":
         """Create an expression to find the minimum of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="min")
 
-    def max(self) -> 'Expr':
+    def max(self) -> "Expr":
         """Create an expression to find the maximum of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="max")
 
-    def std(self, ddof: int = 1) -> 'Expr':
+    def std(self, ddof: int = 1) -> "Expr":
         """Create an expression to find the standard deviation of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="std", ddof=ddof)
 
-    def var(self, ddof: int = 1) -> 'Expr':
+    def var(self, ddof: int = 1) -> "Expr":
         """Create an expression to find the variance of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="var", ddof=ddof)
 
-    def first(self) -> 'Expr':
+    def first(self) -> "Expr":
         """Create an expression to get the first element of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="first")
 
-    def last(self) -> 'Expr':
+    def last(self) -> "Expr":
         """Create an expression to get the last element of columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="last")
 
-    def count(self) -> 'Expr':
+    def count(self) -> "Expr":
         """Create an expression to count elements in columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="count")
 
-    def n_unique(self) -> 'Expr':
+    def n_unique(self) -> "Expr":
         """Create an expression to count unique elements in columns selected by this selector."""
         return Expr(expr=None, selector=self, agg_func="n_unique")
 
@@ -99,6 +98,7 @@ class Selector:
 
 class CompoundSelector(Selector):
     """Selector representing a compound operation between two selectors"""
+
     def __init__(self, left: Selector, right: Selector, operation: str):
         self.left = left
         self.right = right
@@ -106,8 +106,8 @@ class CompoundSelector(Selector):
         super().__init__()
 
     def _get_repr_str(self) -> str:
-        op_map = {'union': '|', 'intersection': '&', 'difference': '-', 'symmetric_difference': '^'}
-        op_symbol = op_map.get(self.operation, '|')
+        op_map = {"union": "|", "intersection": "&", "difference": "-", "symmetric_difference": "^"}
+        op_symbol = op_map.get(self.operation, "|")
         # Use base repr (_repr_str) of operands
         left_repr = f"({self.left._repr_str})" if isinstance(self.left, CompoundSelector) else self.left._repr_str
         right_repr = f"({self.right._repr_str})" if isinstance(self.right, CompoundSelector) else self.right._repr_str
@@ -116,77 +116,95 @@ class CompoundSelector(Selector):
 
 class ComplementSelector(Selector):
     """Selector representing the complement (NOT) of another selector"""
+
     def __init__(self, selector: Selector):
         self.selector = selector
         super().__init__()
 
     def _get_repr_str(self) -> str:
-        selector_repr = f"({self.selector._repr_str})" if isinstance(self.selector, CompoundSelector) else self.selector._repr_str
+        selector_repr = (
+            f"({self.selector._repr_str})" if isinstance(self.selector, CompoundSelector) else self.selector._repr_str
+        )
         return f"~{selector_repr}"
 
 
 class NumericSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.numeric()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.numeric()"
 
 
 class FloatSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.float()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.float()"
 
 
 class IntegerSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.integer()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.integer()"
 
 
 class StringSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.string()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.string()"
 
 
 class TemporalSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.temporal()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.temporal()"
 
 
 class DatetimeSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.datetime()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.datetime()"
 
 
 class DateSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.date()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.date()"
 
 
 class TimeSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.time()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.time()"
 
 
 class DurationSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.duration()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.duration()"
 
 
 class BooleanSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.boolean()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.boolean()"
 
 
 class CategoricalSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.categorical()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.categorical()"
 
 
 class ObjectSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.object()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.object()"
 
 
 class ListSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.list()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.list()"
 
 
 class StructSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.struct()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.struct()"
 
 
 class AllSelector(Selector):
-    def _get_repr_str(self) -> str: return "pl.selectors.all()"
+    def _get_repr_str(self) -> str:
+        return "pl.selectors.all()"
 
 
 class DtypeSelector(Selector):
-    def __init__(self, dtypes: Union[pl.DataType, List[pl.DataType]]):
+    def __init__(self, dtypes: pl.DataType | list[pl.DataType]):
         self.dtypes = dtypes if isinstance(dtypes, list) else [dtypes]
         super().__init__()
 
@@ -208,38 +226,100 @@ class PatternSelector(Selector):
 
 
 class ContainsSelector(PatternSelector):
-    def _get_repr_str(self) -> str: return f"pl.selectors.contains({self.pattern!r})"
+    def _get_repr_str(self) -> str:
+        return f"pl.selectors.contains({self.pattern!r})"
 
 
 class StartsWithSelector(PatternSelector):
-    def _get_repr_str(self) -> str: return f"pl.selectors.starts_with({self.pattern!r})"
+    def _get_repr_str(self) -> str:
+        return f"pl.selectors.starts_with({self.pattern!r})"
 
 
 class EndsWithSelector(PatternSelector):
-    def _get_repr_str(self) -> str: return f"pl.selectors.ends_with({self.pattern!r})"
+    def _get_repr_str(self) -> str:
+        return f"pl.selectors.ends_with({self.pattern!r})"
 
 
 class MatchesSelector(PatternSelector):
-    def _get_repr_str(self) -> str: return f"pl.selectors.matches({self.pattern!r})"
-
-
-def numeric() -> NumericSelector: return NumericSelector()
-def float_() -> FloatSelector: return FloatSelector()
-def integer() -> IntegerSelector: return IntegerSelector()
-def string() -> StringSelector: return StringSelector()
-def temporal() -> TemporalSelector: return TemporalSelector()
-def datetime() -> DatetimeSelector: return DatetimeSelector()
-def date() -> DateSelector: return DateSelector()
-def time() -> TimeSelector: return TimeSelector()
-def duration() -> DurationSelector: return DurationSelector()
-def boolean() -> BooleanSelector: return BooleanSelector()
-def categorical() -> CategoricalSelector: return CategoricalSelector()
-def object_() -> ObjectSelector: return ObjectSelector()
-def list_() -> ListSelector: return ListSelector()
-def struct() -> StructSelector: return StructSelector()
-def all_() -> AllSelector: return AllSelector()
-def by_dtype(dtypes: Union[pl.DataType, List[pl.DataType]]) -> DtypeSelector: return DtypeSelector(dtypes)
-def contains(pattern: str) -> ContainsSelector: return ContainsSelector(pattern)
-def starts_with(pattern: str) -> StartsWithSelector: return StartsWithSelector(pattern)
-def ends_with(pattern: str) -> EndsWithSelector: return EndsWithSelector(pattern)
-def matches(pattern: str) -> MatchesSelector: return MatchesSelector(pattern)
+    def _get_repr_str(self) -> str:
+        return f"pl.selectors.matches({self.pattern!r})"
+
+
+def numeric() -> NumericSelector:
+    return NumericSelector()
+
+
+def float_() -> FloatSelector:
+    return FloatSelector()
+
+
+def integer() -> IntegerSelector:
+    return IntegerSelector()
+
+
+def string() -> StringSelector:
+    return StringSelector()
+
+
+def temporal() -> TemporalSelector:
+    return TemporalSelector()
+
+
+def datetime() -> DatetimeSelector:
+    return DatetimeSelector()
+
+
+def date() -> DateSelector:
+    return DateSelector()
+
+
+def time() -> TimeSelector:
+    return TimeSelector()
+
+
+def duration() -> DurationSelector:
+    return DurationSelector()
+
+
+def boolean() -> BooleanSelector:
+    return BooleanSelector()
+
+
+def categorical() -> CategoricalSelector:
+    return CategoricalSelector()
+
+
+def object_() -> ObjectSelector:
+    return ObjectSelector()
+
+
+def list_() -> ListSelector:
+    return ListSelector()
+
+
+def struct() -> StructSelector:
+    return StructSelector()
+
+
+def all_() -> AllSelector:
+    return AllSelector()
+
+
+def by_dtype(dtypes: pl.DataType | list[pl.DataType]) -> DtypeSelector:
+    return DtypeSelector(dtypes)
+
+
+def contains(pattern: str) -> ContainsSelector:
+    return ContainsSelector(pattern)
+
+
+def starts_with(pattern: str) -> StartsWithSelector:
+    return StartsWithSelector(pattern)
+
+
+def ends_with(pattern: str) -> EndsWithSelector:
+    return EndsWithSelector(pattern)
+
+
+def matches(pattern: str) -> MatchesSelector:
+    return MatchesSelector(pattern)

flowfile_frame/series.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
+
+from typing import Any
+
 import polars as pl
-from typing import Any, Optional, Union, List
 
 
 class Series:
@@ -9,11 +11,11 @@ class Series:
     """
 
     def __init__(
-            self,
-            name: str | list | pl.Series | None = None,
-            values: list | None = None,
-            dtype: Any = None,
-            **kwargs  # Ignored parameters
+        self,
+        name: str | list | pl.Series | None = None,
+        values: list | None = None,
+        dtype: Any = None,
+        **kwargs,  # Ignored parameters
     ):
         """
         Initialize a FlowSeries with the same API as pl.Series.
@@ -67,4 +69,4 @@ class Series:
 
     def __str__(self) -> str:
         """Same as __repr__."""
-        return self.__repr__()
+        return self.__repr__()

flowfile_frame/utils.py

@@ -1,25 +1,23 @@
+import inspect
+import textwrap
 import uuid
-import time
-import os
-import requests
-import subprocess
-from pathlib import Path
+from collections.abc import Iterable
+from typing import Any
+
 import polars as pl
-from typing import Iterable, Any, List, Optional
 
 from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas import schemas
-from tempfile import TemporaryDirectory
-import inspect
-import textwrap
+
 
 def _is_iterable(obj: Any) -> bool:
     # Avoid treating strings as iterables in this context
     return isinstance(obj, Iterable) and not isinstance(obj, (str, bytes))
 
 
-def _check_if_convertible_to_code(expressions: List[Any]) -> bool:
+def _check_if_convertible_to_code(expressions: list[Any]) -> bool:
     from flowfile_frame.expr import Expr
+
     for expr in expressions:
         if isinstance(expr, Expr):
             if not expr.convertable_to_code:
@@ -27,7 +25,9 @@ def _check_if_convertible_to_code(expressions: List[Any]) -> bool:
     return True
 
 
-def _parse_inputs_as_iterable(inputs: tuple[Any, ...] | tuple[Iterable[Any]],) -> List[Any]:
+def _parse_inputs_as_iterable(
+    inputs: tuple[Any, ...] | tuple[Iterable[Any]],
+) -> list[Any]:
     if not inputs:
         return []
 
@@ -55,7 +55,7 @@ def _get_function_source(func):
         source = inspect.getsource(func)
 
         # Check if it's a lambda
-        if func.__name__ == '<lambda>':
+        if func.__name__ == "<lambda>":
             # Extract just the lambda expression
             # This is tricky as getsource returns the entire line
             return None, False
@@ -72,9 +72,9 @@ def _get_function_source(func):
         return None, False
 
 
-def ensure_inputs_as_iterable(inputs: Any | Iterable[Any]) -> List[Any]:
+def ensure_inputs_as_iterable(inputs: Any | Iterable[Any]) -> list[Any]:
     """Convert inputs to list, treating strings as single items."""
-    if inputs is None or (hasattr(inputs, '__len__') and len(inputs) == 0):
+    if inputs is None or (hasattr(inputs, "__len__") and len(inputs) == 0):
         return []
     # Treat strings/bytes as atomic items, everything else check if iterable
     if isinstance(inputs, (str, bytes)) or not _is_iterable(inputs):
@@ -99,13 +99,11 @@ def create_flow_graph(flow_id: int = None) -> FlowGraph:
     """
     if flow_id is None:
         flow_id = _generate_id()
-    flow_settings = schemas.FlowSettings(
-        flow_id=flow_id,
-        name=f"Flow_{flow_id}",
-        path=f"flow_{flow_id}"
-    )
+    flow_settings = schemas.FlowSettings(flow_id=flow_id, name=f"Flow_{flow_id}", path=f"flow_{flow_id}")
     flow_graph = FlowGraph(flow_settings=flow_settings)
-    flow_graph.flow_settings.execution_location = 'local'  # always create a local frame so that the run time does not attempt to use the flowfile_worker process
+    flow_graph.flow_settings.execution_location = (
+        "local"  # always create a local frame so that the run time does not attempt to use the flowfile_worker process
+    )
     return flow_graph
 
 
@@ -140,4 +138,4 @@ def generate_node_id() -> int:
 
 def set_node_id(node_id):
     """Set the node ID to a specific value."""
-    data["c"] = node_id
+    data["c"] = node_id

flowfile_worker/__init__.py

@@ -1,16 +1,24 @@
-from typing import Dict
-import threading
+# ruff: noqa: E402
+
 import multiprocessing
+import threading
+from importlib.metadata import PackageNotFoundError, version
+
 from shared.storage_config import storage
 
+try:
+    __version__ = version("Flowfile")
+except PackageNotFoundError:
+    __version__ = "0.5.0"
 multiprocessing.set_start_method('spawn', force=True)
 
 from multiprocessing import get_context
+
 from flowfile_worker.models import Status
 
 mp_context = get_context("spawn")
 
-status_dict: Dict[str, Status] = dict()
+status_dict: dict[str, Status] = dict()
 process_dict = dict()
 
 status_dict_lock = threading.Lock()
@@ -23,4 +31,4 @@ CACHE_EXPIRATION_TIME = 24 * 60 * 60
 CACHE_DIR = storage.cache_directory
 
 
-PROCESS_MEMORY_USAGE: Dict[str, float] = dict()
+PROCESS_MEMORY_USAGE: dict[str, float] = dict()

flowfile_worker/configs.py

@@ -1,15 +1,15 @@
 # flowfile_worker.configs
 
-import logging
-import platform
 import argparse
+import logging
 import os
-from connectorx import __version__
+import platform
 
+from connectorx import __version__
 
 # Configure logging
-logging.basicConfig(format='%(asctime)s: %(message)s')
-logger = logging.getLogger('FlowfileWorker')
+logging.basicConfig(format="%(asctime)s: %(message)s")
+logger = logging.getLogger("FlowfileWorker")
 logger.setLevel(logging.INFO)
 
 # Constants for worker and core configuration
@@ -17,18 +17,14 @@ DEFAULT_SERVICE_HOST = "0.0.0.0" if platform.system() != "Windows" else "127.0.0
 DEFAULT_SERVICE_PORT = 63579
 DEFAULT_CORE_HOST = "0.0.0.0" if platform.system() != "Windows" else "127.0.0.1"
 DEFAULT_CORE_PORT = 63578
-TEST_MODE = True if 'TEST_MODE' in os.environ else False
+TEST_MODE = True if "TEST_MODE" in os.environ else False
 
 
 def parse_args():
     """Parse command line arguments"""
     parser = argparse.ArgumentParser(description="Flowfile Worker Server")
-    parser.add_argument(
-        "--host", type=str, default=DEFAULT_SERVICE_HOST, help="Host to bind worker to"
-    )
-    parser.add_argument(
-        "--port", type=int, default=DEFAULT_SERVICE_PORT, help="Port to bind worker to"
-    )
+    parser.add_argument("--host", type=str, default=DEFAULT_SERVICE_HOST, help="Host to bind worker to")
+    parser.add_argument("--port", type=int, default=DEFAULT_SERVICE_PORT, help="Port to bind worker to")
     parser.add_argument(
         "--core-host",
         type=str,
@@ -47,14 +43,10 @@ def parse_args():
 
     # Validate arguments
     if args.port < 1 or args.port > 65535:
-        raise ValueError(
-            f"Invalid port number: {args.port}. Port must be between 1 and 65535."
-        )
+        raise ValueError(f"Invalid port number: {args.port}. Port must be between 1 and 65535.")
 
     if args.core_port < 1 or args.core_port > 65535:
-        raise ValueError(
-            f"Invalid core port number: {args.core_port}. Port must be between 1 and 65535."
-        )
+        raise ValueError(f"Invalid core port number: {args.core_port}. Port must be between 1 and 65535.")
 
     # Check if hosts are valid (basic check)
     if not args.host:
@@ -92,4 +84,4 @@ FLOWFILE_CORE_URI = get_core_url(CORE_HOST, CORE_PORT)
 logger.info(f"ConnectorX version: {__version__}")
 # Log configuration
 logger.info(f"Worker configured at {SERVICE_HOST}:{SERVICE_PORT}")
-logger.info(f"Core service configured at {FLOWFILE_CORE_URI}")
+logger.info(f"Core service configured at {FLOWFILE_CORE_URI}")

flowfile_worker/create/__init__.py

@@ -1,37 +1,24 @@
-from flowfile_worker.create.models import (ReceivedCsvTable, ReceivedParquetTable, ReceivedExcelTable,
-                                           ReceivedJsonTable)
-from flowfile_worker.create.funcs import (create_from_path_csv, create_from_path_parquet, create_from_path_excel,
-                                          create_from_path_json)
-from typing import Dict, Literal
+from typing import Literal
 
-ReceivedTableCollection = ReceivedCsvTable | ReceivedParquetTable | ReceivedJsonTable | ReceivedExcelTable
-FileType = Literal['csv', 'parquet', 'json', 'excel']
+from flowfile_worker.create.funcs import (
+    create_from_path_csv,
+    create_from_path_excel,
+    create_from_path_json,
+    create_from_path_parquet,
+)
 
-
-def received_table_parser(received_table_raw: Dict, file_type: FileType) -> ReceivedTableCollection:
-    match file_type:
-        case 'csv':
-            received_table = ReceivedCsvTable.model_validate(received_table_raw)
-        case 'parquet':
-            received_table = ReceivedParquetTable.model_validate(received_table_raw)
-        case 'excel':
-            received_table = ReceivedExcelTable.model_validate(received_table_raw)
-        case 'json':
-            return ReceivedJsonTable.model_validate(received_table_raw)
-        case _:
-            raise ValueError(f'Unsupported file type: {file_type}')
-    return received_table
+FileType = Literal["csv", "parquet", "json", "excel"]
 
 
-def table_creator_factory_method(file_type: Literal['csv', 'parquet', 'json', 'excel']) -> callable:
+def table_creator_factory_method(file_type: FileType) -> callable:
     match file_type:
-        case 'csv':
+        case "csv":
             return create_from_path_csv
-        case 'parquet':
+        case "parquet":
             return create_from_path_parquet
-        case 'excel':
+        case "excel":
             return create_from_path_excel
-        case 'json':
+        case "json":
             return create_from_path_json
         case _:
-            raise ValueError(f'Unsupported file type: {file_type}')
+            raise ValueError(f"Unsupported file type: {file_type}")