Flowfile 0.4.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

flowfile_worker/create/models.py

@@ -1,5 +1,5 @@
-from pydantic import BaseModel, Field, model_validator
-from typing import List, Optional
+from pydantic import BaseModel, Field, model_validator, field_validator
+from typing import List, Optional, Literal, Annotated
 import os
 from pathlib import Path
 
@@ -9,78 +9,146 @@ class MinimalFieldInfo(BaseModel):
     data_type: str
 
 
-class ReceivedTableBase(BaseModel):
-    id: Optional[int] = None
-    name: str
-    path: str
-    directory: Optional[str] = None
-    analysis_file_available: Optional[bool] = False
-    status: Optional[str] = None
-    file_type: Optional[str] = None
-    fields: List[MinimalFieldInfo] = Field(default_factory=list)
-    abs_file_path: Optional[str] = None
+class InputTableBase(BaseModel):
+    """Base settings for input file operations."""
+    file_type: str  # Will be overridden with Literal in subclasses
 
-    @classmethod
-    def create_from_path(cls, path: str):
-        filename = os.path.basename(path)
-        return cls(name=filename, path=path)
 
-    @property
-    def file_path(self) -> str:
-        if self.name not in self.path:
-            return os.path.join(self.path, self.name)
-        return self.path
-
-    @model_validator(mode="after")
-    def set_abs_file_path(cls, values):
-        abs_file_path = getattr(values, "abs_file_path", None)
-        if abs_file_path is None:
-            path = getattr(values, "path", None)
-            if not path:
-                raise ValueError("Field 'path' is required to compute abs_file_path")
-            setattr(values, "abs_file_path", str(Path(path).absolute()))
-        return values
-
-
-class ReceivedCsvTable(ReceivedTableBase):
-    file_type: Optional[str] = 'csv'
-    reference: Optional[str] = ''
-    starting_from_line: Optional[int] = 0
-    delimiter: Optional[str] = ','
-    has_headers: Optional[bool] = True
-    encoding: Optional[str] = 'utf-8'
+class InputCsvTable(InputTableBase):
+    """Defines settings for reading a CSV file."""
+    file_type: Literal['csv'] = 'csv'
+    reference: str = ''
+    starting_from_line: int = 0
+    delimiter: str = ','
+    has_headers: bool = True
+    encoding: str = 'utf-8'
     parquet_ref: Optional[str] = None
-    row_delimiter: Optional[str] = '\n'
-    quote_char: Optional[str] = '"'
-    infer_schema_length: Optional[int] = 10_000
-    truncate_ragged_lines: Optional[bool] = False
-    ignore_errors: Optional[bool] = False
+    row_delimiter: str = '\n'
+    quote_char: str = '"'
+    infer_schema_length: int = 10_000
+    truncate_ragged_lines: bool = False
+    ignore_errors: bool = False
 
 
-class ReceivedJsonTable(ReceivedCsvTable):
-    pass
+class InputJsonTable(InputCsvTable):
+    """Defines settings for reading a JSON file."""
+    file_type: Literal['json'] = 'json'
 
 
-class ReceivedParquetTable(ReceivedTableBase):
-    file_type: Optional[str] = 'parquet'
+class InputParquetTable(InputTableBase):
+    """Defines settings for reading a Parquet file."""
+    file_type: Literal['parquet'] = 'parquet'
 
 
-class ReceivedExcelTable(ReceivedTableBase):
+class InputExcelTable(InputTableBase):
+    """Defines settings for reading an Excel file."""
+    file_type: Literal['excel'] = 'excel'
     sheet_name: Optional[str] = None
-    start_row: Optional[int] = 0  # optional
-    start_column: Optional[int] = 0  # optional
-    end_row: Optional[int] = 0  # optional
-    end_column: Optional[int] = 0  # optional
-    has_headers: Optional[bool] = True  # optional
-    type_inference: Optional[bool] = False  # optional
-
+    start_row: int = 0
+    start_column: int = 0
+    end_row: int = 0
+    end_column: int = 0
+    has_headers: bool = True
+    type_inference: bool = False
+
+    @model_validator(mode='after')
     def validate_range_values(self):
-        # Validate that start and end rows/columns are non-negative integers
+        """Validates that the Excel cell range is logical."""
         for attribute in [self.start_row, self.start_column, self.end_row, self.end_column]:
             if not isinstance(attribute, int) or attribute < 0:
                 raise ValueError("Row and column indices must be non-negative integers")
+        if (self.end_row > 0 and self.start_row > self.end_row) or \
+                (self.end_column > 0 and self.start_column > self.end_column):
+            raise ValueError("Start row/column must not be greater than end row/column")
+        return self
+
+
+# Create the discriminated union (similar to OutputTableSettings)
+InputTableSettings = Annotated[
+    InputCsvTable | InputJsonTable | InputParquetTable | InputExcelTable,
+    Field(discriminator='file_type')
+]
+
+
+# Now create the main ReceivedTable model
+class ReceivedTable(BaseModel):
+    """Model for defining a table received from an external source."""
+    # Metadata fields
+    id: Optional[int] = None
+    name: Optional[str] = None
+    path: str  # This can be an absolute or relative path
+    directory: Optional[str] = None
+    analysis_file_available: bool = False
+    status: Optional[str] = None
+    fields: List[MinimalFieldInfo] = Field(default_factory=list)
+    abs_file_path: Optional[str] = None
+
+    file_type: Literal['csv', 'json', 'parquet', 'excel']
+
+    table_settings: InputTableSettings
+
+    @classmethod
+    def create_from_path(cls, path: str, file_type: Literal['csv', 'json', 'parquet', 'excel'] = 'csv'):
+        """Creates an instance from a file path string."""
+        filename = Path(path).name
+
+        # Create appropriate table_settings based on file_type
+        settings_map = {
+            'csv': InputCsvTable(),
+            'json': InputJsonTable(),
+            'parquet': InputParquetTable(),
+            'excel': InputExcelTable(),
+        }
+
+        return cls(
+            name=filename,
+            path=path,
+            file_type=file_type,
+            table_settings=settings_map.get(file_type, InputCsvTable())
+        )
+
+    @property
+    def file_path(self) -> str:
+        """Constructs the full file path from the directory and name."""
+        if self.name and self.name not in self.path:
+            return os.path.join(self.path, self.name)
+        else:
+            return self.path
+
+    def set_absolute_filepath(self):
+        """Resolves the path to an absolute file path."""
+        base_path = Path(self.path).expanduser()
+        if not base_path.is_absolute():
+            base_path = Path.cwd() / base_path
+        if self.name and self.name not in base_path.name:
+            base_path = base_path / self.name
+        self.abs_file_path = str(base_path.resolve())
+
+    @field_validator('table_settings', mode='before')
+    @classmethod
+    def validate_table_settings(cls, v, info):
+        """Ensures table_settings matches the file_type."""
+        if v is None:
+            file_type = info.data.get('file_type', 'csv')
+            # Create default based on file_type
+            settings_map = {
+                'csv': InputCsvTable(),
+                'json': InputJsonTable(),
+                'parquet': InputParquetTable(),
+                'excel': InputExcelTable(),
+            }
+            return settings_map.get(file_type, InputCsvTable())
+
+        # If it's a dict, add file_type if missing
+        if isinstance(v, dict) and 'file_type' not in v:
+            v['file_type'] = info.data.get('file_type', 'csv')
+
+        return v
+
+    @model_validator(mode='after')
+    def populate_abs_file_path(self):
+        """Ensures the absolute file path is populated after validation."""
+        if not self.abs_file_path:
+            self.set_absolute_filepath()
+        return self
 
-        # Validate that start is before end if end is specified (non-zero)
-        if (0 < self.end_row < self.start_row) or \
-                (0 < self.end_column < self.start_column):
-            raise ValueError("Start row/column must not be greater than end row/column if specified")

flowfile_worker/routes.py

@@ -8,11 +8,11 @@ from base64 import encodebytes
 from flowfile_worker import status_dict, CACHE_DIR, PROCESS_MEMORY_USAGE, status_dict_lock
 from flowfile_worker import models
 from flowfile_worker.spawner import start_process, start_fuzzy_process, start_generic_process, process_manager
-from flowfile_worker.create import table_creator_factory_method, received_table_parser, FileType
+from flowfile_worker.create import table_creator_factory_method, FileType
 from flowfile_worker.configs import logger
 from flowfile_worker.external_sources.sql_source.models import DatabaseReadSettings
-from flowfile_worker.external_sources.sql_source.main import read_sql_source, write_serialized_df_to_database
-
+from flowfile_worker.external_sources.sql_source.main import read_sql_source
+from flowfile_worker.create.models import ReceivedTable
 
 router = APIRouter()
 
@@ -237,7 +237,7 @@ def store_sql_db_result(database_read_settings: DatabaseReadSettings, background
 
 
 @router.post('/create_table/{file_type}')
-def create_table(file_type: FileType, received_table: Dict, background_tasks: BackgroundTasks,
+def create_table(file_type: FileType, received_table: ReceivedTable, background_tasks: BackgroundTasks,
                  flowfile_flow_id: int = 1, flowfile_node_id: int | str = -1) -> models.Status:
     """
     Create a Polars table from received dictionary data based on specified file type.
@@ -253,18 +253,15 @@ def create_table(file_type: FileType, received_table: Dict, background_tasks: Ba
         models.Status: Status object tracking the table creation
     """
     logger.info(f"Creating table of type: {file_type}")
-
     try:
         task_id = str(uuid.uuid4())
         file_ref = os.path.join(create_and_get_default_cache_dir(flowfile_flow_id), f"{task_id}.arrow")
-
         status = models.Status(background_task_id=task_id, status="Starting", file_ref=file_ref,
                                result_type="polars")
         status_dict[task_id] = status
         func_ref = table_creator_factory_method(file_type)
-        received_table_parsed = received_table_parser(received_table, file_type)
         background_tasks.add_task(start_generic_process, func_ref=func_ref, file_ref=file_ref,
-                                  task_id=task_id, kwargs={'received_table': received_table_parsed},
+                                  task_id=task_id, kwargs={'received_table': received_table},
                                   flowfile_flow_id=flowfile_flow_id,
                                   flowfile_node_id=flowfile_node_id)
         logger.info(f"Started table creation task: {task_id}")

tools/migrate/README.md

@@ -0,0 +1,56 @@
+# Flowfile Migration Tool
+
+Migrates `.flowfile` (pickle format, used in v0.4.1 and earlier) to YAML (v0.5+).
+
+## Installation
+
+```bash
+pip install pyyaml  # Required for YAML output
+```
+
+## Usage
+
+```bash
+# Single file
+python -m tools.migrate path/to/flow.flowfile
+
+# Directory (recursive)
+python -m tools.migrate ./flows/
+
+# Specify output path
+python -m tools.migrate flow.flowfile -o /output/path/flow.yaml
+
+# Output as JSON
+python -m tools.migrate flow.flowfile --format json
+
+# Dry run
+python -m tools.migrate ./flows/ --dry-run
+
+# Verbose (show tracebacks)
+python -m tools.migrate flow.flowfile -v
+```
+
+## Output Structure
+
+```yaml
+_version: '2.0'
+_migrated_from: pickle
+flow_id: 1
+flow_name: my_analysis
+flow_settings:
+  name: my_analysis
+  description: null
+  execution_mode: Development
+nodes:
+  - id: 1
+    type: read
+    position: {x: 100, y: 200}
+    settings:
+      received_file:
+        path: data/input.csv
+        file_type: csv
+connections:
+  - [1, 2]
+node_starts:
+  - 1
+```

tools/migrate/__init__.py

@@ -0,0 +1,12 @@
+"""
+Flowfile Migration Tool
+
+Converts old pickle-based .flowfile format to new YAML format.
+
+Usage:
+    python -m tools.migrate <path>
+    python -m tools.migrate old_flow.flowfile
+    python -m tools.migrate ./flows/  # migrate entire directory
+"""
+
+__version__ = "1.0.0"

tools/migrate/__main__.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+"""
+Flowfile Migration Tool - CLI Entry Point
+
+Converts old pickle-based .flowfile format to new YAML format.
+
+Usage:
+    python -m tools.migrate <path> [options]
+
+Examples:
+    # Migrate a single file
+    python -m tools.migrate my_flow.flowfile
+
+    # Migrate to specific output
+    python -m tools.migrate my_flow.flowfile -o my_flow.yaml
+
+    # Migrate entire directory
+    python -m tools.migrate ./flows/
+
+    # Migrate to JSON instead of YAML
+    python -m tools.migrate my_flow.flowfile --format json
+
+    # Migrate directory to different output location
+    python -m tools.migrate ./old_flows/ -o ./new_flows/
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from tools.migrate.migrate import migrate_flowfile, migrate_directory
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='flowfile-migrate',
+        description='Migrate old .flowfile pickles to YAML format',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s my_flow.flowfile              Migrate single file to YAML
+  %(prog)s ./flows/                      Migrate all files in directory
+  %(prog)s flow.flowfile -o flow.yaml    Specify output path
+  %(prog)s ./flows/ --format json        Output as JSON instead of YAML
+        """
+    )
+
+    parser.add_argument(
+        'path',
+        type=Path,
+        help='Path to .flowfile or directory containing .flowfile files'
+    )
+
+    parser.add_argument(
+        '-o', '--output',
+        type=Path,
+        default=None,
+        help='Output path (file or directory). Default: same location with new extension'
+    )
+
+    parser.add_argument(
+        '-f', '--format',
+        choices=['yaml', 'json'],
+        default='yaml',
+        help='Output format (default: yaml)'
+    )
+
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help='Verbose output'
+    )
+
+    parser.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='Show what would be migrated without actually migrating'
+    )
+
+    args = parser.parse_args()
+
+    # Validate input path
+    if not args.path.exists():
+        print(f"Error: Path not found: {args.path}", file=sys.stderr)
+        sys.exit(1)
+
+    # Dry run mode
+    if args.dry_run:
+        if args.path.is_file():
+            print(f"Would migrate: {args.path}")
+            suffix = '.yaml' if args.format == 'yaml' else '.json'
+            output = args.output or args.path.with_suffix(suffix)
+            print(f"  → {output}")
+        else:
+            flowfiles = list(args.path.glob('**/*.flowfile'))
+            print(f"Would migrate {len(flowfiles)} file(s):")
+            for f in flowfiles:
+                print(f"  - {f}")
+        sys.exit(0)
+
+    # Check for yaml dependency
+    if args.format == 'yaml':
+        try:
+            import yaml
+        except ImportError:
+            print("Error: PyYAML is required for YAML output.", file=sys.stderr)
+            print("Install with: pip install pyyaml", file=sys.stderr)
+            sys.exit(1)
+
+    # Run migration
+    try:
+        if args.path.is_file():
+            migrate_flowfile(args.path, args.output, args.format)
+        elif args.path.is_dir():
+            migrate_directory(args.path, args.output, args.format)
+        else:
+            print(f"Error: {args.path} is neither a file nor a directory", file=sys.stderr)
+            sys.exit(1)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+
+    print("\nMigration complete!")
+
+
+if __name__ == '__main__':
+    main()