Flowfile 0.2.2__py3-none-any.whl → 0.3.0.1__py3-none-any.whl

flowfile_core/schemas/input_schema.py

@@ -164,8 +164,7 @@ class OutputSettings(BaseModel):
 
     @model_validator(mode='after')
     def populate_abs_file_path(self):
-        if not self.abs_file_path:
-            self.set_absolute_filepath()
+        self.set_absolute_filepath()
         return self
 
 
@@ -297,12 +296,6 @@ class DatabaseSettings(BaseModel):
     query: Optional[str] = None
     query_mode: Literal['query', 'table', 'reference'] = 'table'
 
-    @model_validator(mode='after')
-    def validate_table_or_query(self):
-        if (not self.table_name and not self.query) and self.query_mode == 'inline':
-            raise ValueError("Either 'table' or 'query' must be provided")
-        return self
-
     @model_validator(mode='after')
     def validate_table_or_query(self):
         # Validate that either table_name or query is provided

flowfile_frame/__init__.py

@@ -4,7 +4,7 @@
 # Core classes
 from flowfile_frame.flow_frame import FlowFrame   # noqa: F401
 
-from flowfile_frame.utils import create_etl_graph  # noqa: F401
+from flowfile_frame.utils import create_flow_graph  # noqa: F401
 
 # Commonly used functions
 from flowfile_frame.expr import (  # noqa: F401
@@ -26,7 +26,6 @@ from flowfile_frame.flow_frame import (  # noqa: F401
     read_csv, read_parquet, from_dict, concat
 )
 
-# Import Polars data types for user convenience
 from polars.datatypes import (  # noqa: F401
     # Integer types
     Int8, Int16, Int32, Int64, Int128,

flowfile_frame/flow_frame.py

@@ -16,7 +16,7 @@ from flowfile_core.schemas import input_schema, transform_schema
 from flowfile_frame.expr import Expr, Column, lit, col
 from flowfile_frame.selectors import Selector
 from flowfile_frame.group_frame import GroupByFrame
-from flowfile_frame.utils import _parse_inputs_as_iterable, create_etl_graph
+from flowfile_frame.utils import _parse_inputs_as_iterable, create_flow_graph
 from flowfile_frame.join import _normalize_columns_to_list, _create_join_mappings
 
 node_id_counter = 0
@@ -92,7 +92,7 @@ class FlowFrame:
 
         # Create a new flow graph if none is provided
         if flow_graph is None:
-            flow_graph = create_etl_graph()
+            flow_graph = create_flow_graph()
 
         flow_id = flow_graph.flow_id
 
@@ -198,7 +198,7 @@ class FlowFrame:
 
         # Initialize graph
         if flow_graph is None:
-            flow_graph = create_etl_graph()
+            flow_graph = create_flow_graph()
         self.flow_graph = flow_graph
         # Set up data
         if isinstance(data, FlowDataEngine):
@@ -1922,7 +1922,7 @@ def read_csv(file_path, *, flow_graph: FlowGraph = None, separator: str = ';',
     # Create new node ID
     node_id = generate_node_id()
     if flow_graph is None:
-        flow_graph = create_etl_graph()
+        flow_graph = create_flow_graph()
 
     flow_id = flow_graph.flow_id
 
@@ -1982,7 +1982,7 @@ def read_parquet(file_path, *, flow_graph: FlowGraph = None, description: str =
     node_id = generate_node_id()
 
     if flow_graph is None:
-        flow_graph = create_etl_graph()
+        flow_graph = create_flow_graph()
 
     flow_id = flow_graph.flow_id
 
@@ -2028,7 +2028,7 @@ def from_dict(data, *, flow_graph: FlowGraph = None, description: str = None) ->
     node_id = generate_node_id()
 
     if not flow_graph:
-        flow_graph = create_etl_graph()
+        flow_graph = create_flow_graph()
     flow_id = flow_graph.flow_id
 
     input_node = input_schema.NodeManualInput(

flowfile_frame/utils.py

@@ -33,7 +33,7 @@ def _generate_id() -> int:
     return int(uuid.uuid4().int % 100000)
 
 
-def create_etl_graph() -> FlowGraph:
+def create_flow_graph() -> FlowGraph:
     flow_id = _generate_id()
     flow_settings = schemas.FlowSettings(
         flow_id=flow_id,
@@ -43,142 +43,3 @@ def create_etl_graph() -> FlowGraph:
     flow_graph = FlowGraph(flow_id=flow_id, flow_settings=flow_settings)
     flow_graph.flow_settings.execution_location = 'local'  # always create a local frame so that the run time does not attempt to use the flowfile_worker process
     return flow_graph
-
-
-def is_flowfile_running() -> bool:
-    """Check if the Flowfile application is running by testing its API endpoint."""
-    try:
-        response = requests.get("http://0.0.0.0:63578/docs", timeout=2)
-        return response.status_code == 200
-    except (requests.ConnectionError, requests.Timeout):
-        return False
-
-
-def start_flowfile_application() -> bool:
-    """Start the Flowfile application on macOS."""
-    try:
-        # Attempt to start the Flowfile application
-        subprocess.Popen(['open', '-a', 'Flowfile'],
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE)
-
-        # Wait for the application to start up (max 10 seconds)
-        start_time = time.time()
-        while time.time() - start_time < 10:
-            if is_flowfile_running():
-                return True
-            time.sleep(0.5)  # Check every half second
-
-        # If we get here, the app didn't start in time
-        return False
-    except Exception as e:
-        print(f"Error starting Flowfile application: {e}")
-        return False
-
-
-def get_auth_token() -> Optional[str]:
-    """Get an authentication token from the Flowfile API."""
-    try:
-        response = requests.post(
-            "http://0.0.0.0:63578/auth/token",
-            json={},  # Empty body as specified
-            timeout=5
-        )
-
-        if response.status_code == 200:
-            token_data = response.json()
-            return token_data.get("access_token")
-        else:
-            print(f"Failed to get auth token: {response.status_code} - {response.text}")
-            return None
-    except Exception as e:
-        print(f"Error getting auth token: {e}")
-        return None
-
-
-def import_flow_to_editor(flow_path: str, auth_token: str) -> Optional[int]:
-    """Import the flow into the Flowfile editor using the API endpoint."""
-    try:
-        flow_path = Path(flow_path).resolve()  # Get absolute path
-        if not flow_path.exists():
-            print(f"Flow file not found: {flow_path}")
-            return None
-
-        # Set authorization header with the token
-        headers = {"Authorization": f"Bearer {auth_token}"}
-
-        # Make a GET request to the import endpoint
-        response = requests.get(
-            "http://0.0.0.0:63578/import_flow/",
-            params={"flow_path": str(flow_path)},
-            headers=headers,
-            timeout=10
-        )
-
-        if response.status_code == 200:
-            flow_id = response.json()
-            print(f"Flow imported successfully with ID: {flow_id}")
-            return flow_id
-        else:
-            print(f"Failed to import flow: {response.status_code} - {response.text}")
-            return None
-    except Exception as e:
-        print(f"Error importing flow: {e}")
-        return None
-
-
-def open_graph_in_editor(etl_graph: FlowGraph, storage_location: str = None) -> bool:
-    """
-    Save the ETL graph and open it in the Flowfile editor.
-
-    Parameters:
-    -----------
-    etl_graph : FlowGraph
-        The graph to save and open
-    storage_location : str, optional
-        Where to save the flowfile. If None, a default name is used.
-
-    Returns:
-    --------
-    bool
-        True if the graph was successfully opened in the editor, False otherwise
-    """
-    # Create a temporary directory if needed
-    temp_dir = None
-    if storage_location is None:
-        temp_dir = TemporaryDirectory()
-        storage_location = os.path.join(temp_dir.name, 'temp_flow.flowfile')
-    else:
-        # Ensure path is absolute
-        storage_location = os.path.abspath(storage_location)
-
-    etl_graph.apply_layout()
-    etl_graph.save_flow(storage_location)
-    print(f"Flow saved to: {storage_location}")
-
-    # Check if Flowfile is running, and start it if not
-    if not is_flowfile_running():
-        print("Flowfile application is not running. Starting it...")
-        if not start_flowfile_application():
-            print("Failed to start Flowfile application")
-            if temp_dir:
-                temp_dir.cleanup()
-            return False
-        print("Flowfile application started successfully")
-
-    # Get authentication token
-    auth_token = get_auth_token()
-    if not auth_token:
-        print("Failed to authenticate with Flowfile API")
-        if temp_dir:
-            temp_dir.cleanup()
-        return False
-
-    # Import the flow into the editor
-    flow_id = import_flow_to_editor(storage_location, auth_token)
-
-    # Clean up temporary directory if we created one
-    if temp_dir:
-        temp_dir.cleanup()
-
-    return flow_id is not None

flowfile-0.2.2.dist-info/METADATA

@@ -1,225 +0,0 @@
-Metadata-Version: 2.3
-Name: Flowfile
-Version: 0.2.2
-Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
-Author: Edward van Eechoud
-Author-email: evaneechoud@gmail.com
-Requires-Python: >=3.10,<3.13
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: XlsxWriter (>=3.2.0,<3.3.0)
-Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
-Requires-Dist: airbyte-cdk (==6.47.2)
-Requires-Dist: bcrypt (>=4.3.0,<5.0.0)
-Requires-Dist: connectorx (>=0.4.2,<0.5.0)
-Requires-Dist: databases (>=0.9.0,<0.10.0)
-Requires-Dist: faker (>=23.1.0,<23.2.0)
-Requires-Dist: fastapi (>=0.115.2,<0.116.0)
-Requires-Dist: fastexcel (>=0.12.0,<0.13.0)
-Requires-Dist: google-api-python-client (>=2.149.0,<2.150.0)
-Requires-Dist: gspread (>=6.1.3,<6.2.0)
-Requires-Dist: loky (>=3.4.1,<3.5.0)
-Requires-Dist: methodtools (>=0.4.7,<0.5.0)
-Requires-Dist: openpyxl (>=3.1.2,<3.2.0)
-Requires-Dist: passlib (>=1.7.4,<1.8.0)
-Requires-Dist: pendulum (==2.1.2) ; python_version < "3.12"
-Requires-Dist: polars (>1.8.2,<=1.25.2)
-Requires-Dist: polars-distance (>=0.4.3,<0.5.0)
-Requires-Dist: polars-ds (>=0.6.0)
-Requires-Dist: polars-expr-transformer (>0.4.7.0)
-Requires-Dist: polars-grouper (>=0.3.0,<0.4.0)
-Requires-Dist: polars_simed (>=0.3.4,<0.4.0)
-Requires-Dist: pyairbyte-flowfile (==0.20.2)
-Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
-Requires-Dist: pydantic (>=2.9.2,<2.10.0)
-Requires-Dist: pyinstaller (>=6.11.0,<7.0.0)
-Requires-Dist: pytest (>=8.3.4,<9.0.0)
-Requires-Dist: python-jose (>=3.4.0,<4.0.0)
-Requires-Dist: python-multipart (>=0.0.12,<0.1.0)
-Requires-Dist: uvicorn (>=0.32.0,<0.33.0)
-Description-Content-Type: text/markdown
-
-<h1 align="center">
-  <img src=".github/images/logo.png" alt="Flowfile Logo" width="100">
-  <br>
-  Flowfile
-</h1>
-<p align="center">
-  <b>Documentation</b>:
-  <a href="https://edwardvaneechoud.github.io/Flowfile/">Website</a>
-  -
-  <a href="flowfile_core/README.md">Core</a>
-  -
-  <a href="flowfile_worker/README.md">Worker</a>
-  -
-  <a href="flowfile_frontend/README.md">Frontend</a>
-  -
-  <a href="https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c">Technical Architecture</a>
-</p>
-<p>
-Flowfile is a visual ETL tool that combines drag-and-drop workflow building with the speed of Polars dataframes. Build data pipelines visually, transform data using powerful nodes, and analyze results - all without writing code.
-</p>
-
-<div align="center">
-  <img src=".github/images/group_by_screenshot.png" alt="Flowfile Interface" width="800"/>
-</div>
-
-## ⚡ Technical Design
-
-Flowfile operates as three interconnected services:
-
-- **Designer** (Electron + Vue): Visual interface for building data flows
-- **Core** (FastAPI): ETL engine using Polars for data transformations (`:63578`)
-- **Worker** (FastAPI): Handles computation and caching of data operations (`:63579`)
-
-Each flow is represented as a directed acyclic graph (DAG), where nodes represent data operations and edges represent data flow between operations.
-
-For a deeper dive into the technical architecture, check out [this article](https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c) on how Flowfile leverages Polars for efficient data processing.
-
-## 🔥 Example Use Cases
-
-- **Data Cleaning & Transformation**
-  - Complex joins (fuzzy matching)
-  - Text to rows transformations
-  - Advanced filtering and grouping
-  - Custom formulas and expressions
-  - Filter data based on conditions
-
-<div align="center">
-  <img src=".github/images/flowfile_demo_1.gif" alt="Flowfile Layout" width="800"/>
-</div>
-
----
-
-- **Performance**
-  - Build to scale out of core
-  - Using polars for data processing
-
-<div align="center">
-  <img src=".github/images/demo_flowfile_write.gif" alt="Flowfile Layout" width="800"/>
-</div>
-
----
-
-### **Data Integration**
-  - Standardize data formats
-  - Handle messy Excel files
-
-
-<div align="center">
-  <img src=".github/images/read_excel_flowfile.gif" alt="Flowfile Layout" width="800"/>
-</div>
-
-
----
-
-- **ETL Operations**
-  - Data quality checks
-
-
-## 🚀 Getting Started
-
-### Prerequisites
-- Python 3.10+
-- Node.js 16+
-- Poetry (Python package manager)
-- Docker & Docker Compose (option, for Docker setup)
-- Make (optional, for build automation)
-
-### Installation Options
-
-#### 1. Desktop Application
-The desktop version offers the best experience with a native interface and integrated services. You can either:
-
-**Option A: Download Pre-built Application** 
-- Download the latest release from [GitHub Releases](https://github.com/Edwardvaneechoud/Flowfile/releases)
-- Run the installer for your platform (Windows, macOS, or Linux)
-  - Note: You may see security warnings since the installer isn't signed. On Windows, click "More info" then "Run anyway". On macOS, right-click the app, select "Open", then confirm. These warnings appear because the app isn't signed with a developer certificate.
-
-**Option B: Build from Source:**
-```bash
-git clone https://github.com/edwardvaneechoud/Flowfile.git
-cd Flowfile
-
-# Build packaged executable
-make    # Creates platform-specific executable
-
-# Or manually:
-poetry install
-poetry run build_backends
-cd flowfile_frontend
-npm install
-npm run build      # All platforms
-```
-
-#### 2. Docker Setup
-Perfect for quick testing, development or deployment scenarios. Runs all services in containers with proper networking and volume management:
-```bash
-# Clone and start all services
-git clone https://github.com/edwardvaneechoud/Flowfile.git
-cd Flowfile
-docker compose up -d
-
-# Access services:
-Frontend: http://localhost:8080 # main service
-Core API: http://localhost:63578/docs
-Worker API: http://localhost:63579/docs
-```
-Just place your files that you want to transform in the directory in shared_data and you're all set!
-
-Docker Compose is also excellent for development, as it automatically sets up all required services and ensures proper communication between them. Code changes in the mounted volumes will be reflected in the running containers.
-
-#### 3. Manual Setup (Development)
-Ideal for development work when you need direct access to all services and hot-reloading:
-
-```bash
-git clone https://github.com/edwardvaneechoud/Flowfile.git
-cd Flowfile
-
-# Install Python dependencies
-poetry install
-
-# Start backend services
-poetry run flowfile_worker  # Starts worker on :63579
-poetry run flowfile_core   # Starts core on :63578
-
-# Start web frontend
-cd flowfile_frontend
-npm install
-npm run dev:web  # Starts web interface on :8080
-```
-
-## 📋 TODO
-
-### Core Features
-- [ ] Add cloud storage support
-  - S3 integration
-  - Azure Data Lake Storage (ADLS)
-- [x] Multi-flow execution support
-- [ ] Polars code reverse engineering
-  - Generate Polars code from visual flows
-  - Import existing Polars scripts
-
-### Documentation
-- [ ] Add comprehensive docstrings
-- [x] Create detailed node documentation
-- [x] Add architectural documentation
-- [ ] Improve inline code comments
-- [ ] Create user guides and tutorials
-
-### Infrastructure
-- [ ] Implement proper testing
-- [x] Add CI/CD pipeline
-- [x] Improve error handling
-- [x] Add monitoring and logging
-
-## 📝 License
-
-[MIT License](LICENSE)
-
-## Acknowledgments
-
-Built with Polars, Vue.js, FastAPI, Vueflow and Electron.
-

flowfile_frame/__main__.py

@@ -1,12 +0,0 @@
-"""Main entry point for the FlowFrame CLI."""
-
-def main():
-    """Main entry point for the FlowFrame CLI."""
-    print("FlowFrame - A Polars-like API for building ETL graphs")
-    print("Usage: import flowframe as ff")
-    print("       df = ff.from_dict({'a': [1, 2, 3]})")
-    print("       result = df.filter(ff.col('a') > 1)")
-    print("       print(result.collect())")
-
-if __name__ == "__main__":
-    main()