Flowfile 0.2.2__py3-none-any.whl

flowfile_core/utils/arrow_reader.py

@@ -0,0 +1,247 @@
+import pyarrow as pa
+from typing import List, Iterator, Tuple, Callable
+from flowfile_core.configs import logger
+
+
+def open_validated_file(file_path: str, n: int) -> pa.OSFile:
+    """
+    Validate and open an Arrow file with input parameter checking.
+
+    This function performs validation on the input parameters and opens the Arrow file
+    in binary read mode. It includes checks for file existence and parameter types.
+
+    Args:
+        file_path (str): Path to the Arrow file to be opened.
+        n (int): Number of rows to be read. Used for validation purposes.
+            Must be non-negative.
+
+    Returns:
+        pa.OSFile: An open PyArrow file object ready for reading.
+
+    Raises:
+        ValueError: If n is negative.
+        TypeError: If file_path is not a string.
+        FileNotFoundError: If the specified file does not exist.
+
+    Example:
+        >>> file = open_validated_file("data.arrow", 1000)
+        >>> # Use the file object
+        >>> file.close()
+    """
+    logger.debug(f"Attempting to open file: {file_path} with n={n}")
+    if n < 0:
+        logger.error(f"Invalid negative row count requested: {n}")
+        raise ValueError("Number of rows must be non-negative")
+    if not isinstance(file_path, str):
+        logger.error(f"Invalid file_path type: {type(file_path)}")
+        raise TypeError("file_path must be a string")
+    try:
+        file = pa.OSFile(file_path, 'rb')
+        logger.info(f"Successfully opened file: {file_path}")
+        return file
+    except FileNotFoundError:
+        logger.error(f"File not found: {file_path}")
+        raise FileNotFoundError(f"Could not find file: {file_path}")
+
+
+def create_reader(source: pa.OSFile) -> pa.ipc.RecordBatchFileReader:
+    """
+    Create a RecordBatchFileReader from an open Arrow file.
+
+    This function initializes a reader that can process Arrow IPC file formats.
+    It handles the creation of the reader and validates the file format.
+
+    Args:
+        source (pa.OSFile): An open PyArrow file object.
+
+    Returns:
+        pa.ipc.RecordBatchFileReader: A reader object for processing Arrow record batches.
+
+    Raises:
+        ValueError: If the file is not a valid Arrow format.
+
+    Example:
+        >>> with open_validated_file("data.arrow", 1000) as source:
+        ...     reader = create_reader(source)
+        ...     # Use the reader
+    """
+    try:
+        reader = pa.ipc.open_file(source)
+        logger.debug(f"Created reader with {reader.num_record_batches} batches")
+        return reader
+    except pa.ArrowInvalid:
+        logger.error("Failed to create reader: Invalid Arrow file format")
+        raise ValueError("Invalid Arrow file format")
+
+
+def iter_batches(reader: pa.ipc.RecordBatchFileReader, n: int, rows_collected: int) -> Iterator[pa.RecordBatch]:
+    """
+    Iterator over record batches with row limit handling.
+
+    Yields record batches from the reader, handling the case where the last batch
+    needs to be sliced to meet the requested row count. This function provides
+    efficient batch-wise processing of Arrow data.
+
+    Args:
+        reader (pa.ipc.RecordBatchFileReader): The Arrow file reader.
+        n (int): Maximum number of rows to read in total.
+        rows_collected (int): Number of rows already collected before this iteration.
+
+    Yields:
+        pa.RecordBatch: Record batches containing the data, with the last batch
+            potentially sliced to meet the row count requirement.
+
+    Example:
+        >>> reader = create_reader(source)
+        >>> for batch in iter_batches(reader, 1000, 0):
+        ...     # Process each batch
+        ...     process_batch(batch)
+    """
+    logger.debug(f"Starting batch iteration: target={n}, collected={rows_collected}")
+    for i in range(reader.num_record_batches):
+        batch = reader.get_record_batch(i)
+        batch_rows = batch.num_rows
+        logger.debug(f"Processing batch {i}: {batch_rows} rows")
+
+        if rows_collected + batch_rows <= n:
+            yield batch
+        else:
+            remaining_rows = n - rows_collected
+            logger.debug(f"Slicing final batch to {remaining_rows} rows")
+            yield batch.slice(0, remaining_rows)
+            break
+
+
+def collect_batches(reader: pa.ipc.RecordBatchFileReader, n: int) -> Tuple[List[pa.RecordBatch], int]:
+    """
+    Collect record batches from a reader up to a specified number of rows.
+
+    This function aggregates record batches while respecting the total row count
+    limit. It's useful for building a complete dataset from multiple batches.
+
+    Args:
+        reader (pa.ipc.RecordBatchFileReader): The Arrow file reader.
+        n (int): Maximum number of rows to collect.
+
+    Returns:
+        Tuple[List[pa.RecordBatch], int]: A tuple containing:
+            - List of collected record batches
+            - Total number of rows collected
+
+    Example:
+        >>> reader = create_reader(source)
+        >>> batches, row_count = collect_batches(reader, 1000)
+        >>> print(f"Collected {row_count} rows in {len(batches)} batches")
+    """
+    logger.debug(f"Collecting batches up to {n} rows")
+    batches: List[pa.RecordBatch] = []
+    rows_collected = 0
+
+    for batch in iter_batches(reader, n, rows_collected):
+        batches.append(batch)
+        rows_collected += batch.num_rows
+        logger.debug(f"Collected batch: total rows now {rows_collected}")
+        if rows_collected >= n:
+            break
+
+    logger.info(f"Finished collecting {len(batches)} batches with {rows_collected} total rows")
+    return batches, rows_collected
+
+
+def read(file_path: str) -> pa.Table:
+    """
+    Read an entire Arrow file into a Table.
+
+    This function provides a simple interface to read all data from an Arrow file.
+    It handles file opening, reading, and proper resource cleanup.
+
+    Args:
+        file_path (str): Path to the Arrow file to read.
+
+    Returns:
+        pa.Table: PyArrow Table containing all data from the file.
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist.
+        ValueError: If the file is not a valid Arrow format.
+
+    Example:
+        >>> table = read("data.arrow")
+        >>> print(f"Read {len(table)} rows")
+        >>> print(f"Columns: {table.column_names}")
+    """
+    logger.info(f"Reading entire file: {file_path}")
+    with open_validated_file(file_path, 0) as source:
+        reader = create_reader(source)
+        batches, total_rows = collect_batches(reader, float('inf'))
+        table = pa.Table.from_batches(batches)  # type: ignore
+        logger.info(f"Successfully read {total_rows} rows from {file_path}")
+        return table
+
+
+def read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> pa.Table:
+    """
+    Read the first N rows from an Arrow file.
+
+    This function provides efficient reading of a limited number of rows from an
+    Arrow file. It's useful for data sampling and preview operations.
+
+    Args:
+        file_path (str): Path to the Arrow file to read.
+        n (int, optional): Number of rows to read. Defaults to 1000.
+        strict (bool, optional): If True, raises an error when fewer than n rows
+            are available. Defaults to False.
+
+    Returns:
+        pa.Table: PyArrow Table containing up to n rows of data.
+
+    Raises:
+        ValueError: If strict=True and fewer than n rows are available,
+            or if n is negative.
+        FileNotFoundError: If the file doesn't exist.
+
+    Example:
+        >>> # Read first 1000 rows
+        >>> table = read_top_n("data.arrow")
+        >>> # Read exactly 500 rows with strict checking
+        >>> table = read_top_n("data.arrow", n=500, strict=True)
+    """
+    logger.info(f"Reading top {n} rows from {file_path} (strict={strict})")
+    with open_validated_file(file_path, n) as source:
+        reader = create_reader(source)
+        batches, rows_collected = collect_batches(reader, n)
+
+        if strict and rows_collected < n:
+            logger.error(f"Strict mode: requested {n} rows but only {rows_collected} available")
+            raise ValueError(f"Requested {n} rows but only {rows_collected} available")
+
+        table = pa.Table.from_batches(batches)  # type: ignore
+        logger.info(f"Successfully read {rows_collected} rows from {file_path}")
+        return table
+
+
+def get_read_top_n(file_path: str, n: int = 1000, strict: bool = False) -> Callable[[], pa.Table]:
+    """
+    Create a callable that reads the first N rows from an Arrow file.
+
+    This function returns a closure that can be called later to read data.
+    It's useful for creating reusable data reading functions with fixed parameters.
+
+    Args:
+        file_path (str): Path to the Arrow file to read.
+        n (int, optional): Number of rows to read. Defaults to 1000.
+        strict (bool, optional): If True, raises an error when fewer than n rows
+            are available. Defaults to False.
+
+    Returns:
+        Callable[[], pa.Table]: A function that when called, reads and returns
+            the data as a PyArrow Table.
+
+    Example:
+        >>> # Create a reader function for the first 500 rows
+        >>> reader_func = get_read_top_n("data.arrow", n=500)
+        >>> # Later, use the function to read the data
+        >>> table = reader_func()
+    """
+    logger.info(f"Creating reader function for {file_path} with n={n}, strict={strict}")
+    return lambda: read_top_n(file_path, n, strict)

flowfile_core/utils/excel_file_manager.py

@@ -0,0 +1,18 @@
+import os
+import fastexcel
+from typing import List
+from functools import lru_cache
+
+from flowfile_core.configs import logger
+
+
+@lru_cache(maxsize=32)
+def get_sheet_names(file_path: str) -> List[str] | None:
+    if not os.path.exists(file_path):
+        logger.error(f"File does not exist: {file_path}")
+        return
+    try:
+        return fastexcel.read_excel(file_path).sheet_names
+    except Exception as e:
+        logger.error(e)
+        return

flowfile_core/utils/fileManager.py

@@ -0,0 +1,45 @@
+import os
+from flowfile_core.schemas.input_schema import NewDirectory, RemoveItem, RemoveItemsInput
+from typing import Tuple, Optional
+from flowfile_core.configs import logger
+
+local_database_connection = None
+
+
+def create_dir(new_directory: NewDirectory) -> Tuple[bool, Optional[Exception]]:
+    full_path: str = os.path.join(new_directory.source_path,new_directory.dir_name)
+    try:
+        os.mkdir(full_path)
+        logger.info("Successfully created a new folder")
+        return True, None
+    except Exception as e:
+        return False, e
+
+
+def remove_path(path: str) -> Tuple[bool, Optional[Exception]]:
+    try:
+        os.remove(path)
+        logger.info(f"Succesfully removed {path}")
+        return True, None
+    except Exception as e:
+        return False, e
+
+
+def remove_item(item_to_remove: RemoveItem):
+    if item_to_remove.id >= 0:
+        os.remove(item_to_remove.path)
+
+    elif os.path.isfile(item_to_remove.path):
+        os.remove(item_to_remove.path)
+    elif os.path.isdir(item_to_remove.path):
+        os.rmdir(item_to_remove.path)
+
+
+def remove_paths(remove_items: RemoveItemsInput) -> Tuple[bool, Optional[Exception]]:
+    try:
+        for path in remove_items.paths:
+            remove_item(path)
+        logger.info(f'Successfully removed {remove_items.paths}')
+        return True, None
+    except Exception as e:
+        return False, e

flowfile_core/utils/fl_executor.py

@@ -0,0 +1,38 @@
+from flowfile_core.configs import logger
+from inspect import isfunction
+from loky import get_reusable_executor
+from functools import wraps
+from concurrent.futures import ThreadPoolExecutor
+import atexit
+
+
+# process_executor: Uses loky for process-based parallelism
+def process_executor(wait_on_completion: bool = False, max_workers: int = 12):
+    max_workers = max_workers if not wait_on_completion else 1
+
+    def executor(f):
+        @wraps(f)
+        def inner(*args, **kwargs):
+            logger.debug(f'Added task {f.__name__} to a process executor')
+            logger.debug(f'max_workers: {max_workers}')
+
+            # Create a new executor with the required number of workers
+            func_executor = get_reusable_executor(max_workers=max_workers, timeout=2, kill_workers=False, reuse=True)
+            r = func_executor.submit(f, *args, **kwargs)
+            if wait_on_completion:
+                result = r.result()
+                logger.info(f'done executing {f.__name__}')
+                return result
+
+            logger.info(f'done submitting {f.__name__} to a process executor')
+            return r
+
+        return inner
+
+    if isfunction(wait_on_completion):
+        f = wait_on_completion
+        wait_on_completion = False
+        return executor(f)
+    return executor
+
+

flowfile_core/utils/utils.py

@@ -0,0 +1,8 @@
+import re
+
+
+def camel_case_to_snake_case(text: str) -> str:
+    # Use a regular expression to find capital letters and replace them with _ followed by the lowercase letter
+    transformed_text = re.sub(r'(?<!^)(?=[A-Z])', '_', text).lower()
+    return transformed_text
+

flowfile_frame/__init__.py

@@ -0,0 +1,56 @@
+# flowframe/__init__.py
+"""A Polars-like API for building ETL graphs."""
+
+# Core classes
+from flowfile_frame.flow_frame import FlowFrame   # noqa: F401
+
+from flowfile_frame.utils import create_etl_graph  # noqa: F401
+
+# Commonly used functions
+from flowfile_frame.expr import (  # noqa: F401
+    col, lit, column,
+    cum_count, len,
+    sum, min, max, mean, count, when
+)
+
+# Selector utilities
+from flowfile_frame.selectors import (  # noqa: F401
+    numeric, float_, integer, string, temporal,
+    datetime, date, time, duration, boolean,
+    categorical, object_, list_, struct, all_,
+    by_dtype, contains, starts_with, ends_with, matches
+)
+
+# File I/O
+from flowfile_frame.flow_frame import (  # noqa: F401
+    read_csv, read_parquet, from_dict, concat
+)
+
+# Import Polars data types for user convenience
+from polars.datatypes import (  # noqa: F401
+    # Integer types
+    Int8, Int16, Int32, Int64, Int128,
+    UInt8, UInt16, UInt32, UInt64,
+    IntegerType,
+
+    # Float types
+    Float32, Float64,
+
+    # Other primitive types
+    Boolean, String, Utf8, Binary, Null,
+
+    # Complex types
+    List, Array, Struct, Object,
+
+    # Date/time types
+    Date, Time, Datetime, Duration,
+    TemporalType,
+
+    # Special types
+    Categorical, Decimal, Enum, Unknown,
+
+    # Type classes
+    DataType, DataTypeClass, Field
+)
+
+__version__ = "0.1.0"

flowfile_frame/__main__.py

@@ -0,0 +1,12 @@
+"""Main entry point for the FlowFrame CLI."""
+
+def main():
+    """Main entry point for the FlowFrame CLI."""
+    print("FlowFrame - A Polars-like API for building ETL graphs")
+    print("Usage: import flowframe as ff")
+    print("       df = ff.from_dict({'a': [1, 2, 3]})")
+    print("       result = df.filter(ff.col('a') > 1)")
+    print("       print(result.collect())")
+
+if __name__ == "__main__":
+    main()

flowfile_frame/adapters.py

@@ -0,0 +1,17 @@
+# flowframe/adapters.py
+"""Adapters to connect FlowFrame with the flowfile-core library."""
+
+# Import from your existing project
+from flowfile_core.flowfile.FlowfileFlow import FlowGraph, add_connection
+from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.schemas import input_schema, schemas, transform_schema
+
+# Export these for use in FlowFrame
+__all__ = [
+    'FlowGraph',
+    'add_connection',
+    'FlowDataEngine',
+    'input_schema',
+    'schemas',
+    'transform_schema'
+]