Flowfile 0.3.0__py3-none-any.whl → 0.3.0.2__py3-none-any.whl

flowfile_core/main.py

@@ -54,6 +54,8 @@ origins = [
     "http://localhost:8081",
     "http://localhost:4173",
     "http://localhost:4174",
+    "http://localhost:63578",
+    "http://127.0.0.1:63578"
 ]
 
 app.add_middleware(
@@ -117,7 +119,8 @@ def run(host: str = None, port: int = None):
         app,
         host=host,
         port=port,
-        loop="asyncio"
+        loop="asyncio",
+        log_level="warning",
     )
     server = uvicorn.Server(config)
     server_instance = server  # Store server instance globally

flowfile_core/schemas/input_schema.py

@@ -164,8 +164,7 @@ class OutputSettings(BaseModel):
 
     @model_validator(mode='after')
     def populate_abs_file_path(self):
-        if not self.abs_file_path:
-            self.set_absolute_filepath()
+        self.set_absolute_filepath()
         return self
 
 
@@ -297,12 +296,6 @@ class DatabaseSettings(BaseModel):
     query: Optional[str] = None
     query_mode: Literal['query', 'table', 'reference'] = 'table'
 
-    @model_validator(mode='after')
-    def validate_table_or_query(self):
-        if (not self.table_name and not self.query) and self.query_mode == 'inline':
-            raise ValueError("Either 'table' or 'query' must be provided")
-        return self
-
     @model_validator(mode='after')
     def validate_table_or_query(self):
         # Validate that either table_name or query is provided

flowfile_frame/__init__.py

@@ -26,7 +26,6 @@ from flowfile_frame.flow_frame import (  # noqa: F401
     read_csv, read_parquet, from_dict, concat
 )
 
-# Import Polars data types for user convenience
 from polars.datatypes import (  # noqa: F401
     # Integer types
     Int8, Int16, Int32, Int64, Int128,

flowfile_frame/utils.py

@@ -43,142 +43,3 @@ def create_flow_graph() -> FlowGraph:
     flow_graph = FlowGraph(flow_id=flow_id, flow_settings=flow_settings)
     flow_graph.flow_settings.execution_location = 'local'  # always create a local frame so that the run time does not attempt to use the flowfile_worker process
     return flow_graph
-
-
-def is_flowfile_running() -> bool:
-    """Check if the Flowfile application is running by testing its API endpoint."""
-    try:
-        response = requests.get("http://0.0.0.0:63578/docs", timeout=2)
-        return response.status_code == 200
-    except (requests.ConnectionError, requests.Timeout):
-        return False
-
-
-def start_flowfile_application() -> bool:
-    """Start the Flowfile application on macOS."""
-    try:
-        # Attempt to start the Flowfile application
-        subprocess.Popen(['open', '-a', 'Flowfile'],
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE)
-
-        # Wait for the application to start up (max 10 seconds)
-        start_time = time.time()
-        while time.time() - start_time < 10:
-            if is_flowfile_running():
-                return True
-            time.sleep(0.5)  # Check every half second
-
-        # If we get here, the app didn't start in time
-        return False
-    except Exception as e:
-        print(f"Error starting Flowfile application: {e}")
-        return False
-
-
-def get_auth_token() -> Optional[str]:
-    """Get an authentication token from the Flowfile API."""
-    try:
-        response = requests.post(
-            "http://0.0.0.0:63578/auth/token",
-            json={},  # Empty body as specified
-            timeout=5
-        )
-
-        if response.status_code == 200:
-            token_data = response.json()
-            return token_data.get("access_token")
-        else:
-            print(f"Failed to get auth token: {response.status_code} - {response.text}")
-            return None
-    except Exception as e:
-        print(f"Error getting auth token: {e}")
-        return None
-
-
-def import_flow_to_editor(flow_path: str, auth_token: str) -> Optional[int]:
-    """Import the flow into the Flowfile editor using the API endpoint."""
-    try:
-        flow_path = Path(flow_path).resolve()  # Get absolute path
-        if not flow_path.exists():
-            print(f"Flow file not found: {flow_path}")
-            return None
-
-        # Set authorization header with the token
-        headers = {"Authorization": f"Bearer {auth_token}"}
-
-        # Make a GET request to the import endpoint
-        response = requests.get(
-            "http://0.0.0.0:63578/import_flow/",
-            params={"flow_path": str(flow_path)},
-            headers=headers,
-            timeout=10
-        )
-
-        if response.status_code == 200:
-            flow_id = response.json()
-            print(f"Flow imported successfully with ID: {flow_id}")
-            return flow_id
-        else:
-            print(f"Failed to import flow: {response.status_code} - {response.text}")
-            return None
-    except Exception as e:
-        print(f"Error importing flow: {e}")
-        return None
-
-
-def open_graph_in_editor(flow_graph: FlowGraph, storage_location: str = None) -> bool:
-    """
-    Save the ETL graph and open it in the Flowfile editor.
-
-    Parameters:
-    -----------
-    flow_graph : FlowGraph
-        The graph to save and open
-    storage_location : str, optional
-        Where to save the flowfile. If None, a default name is used.
-
-    Returns:
-    --------
-    bool
-        True if the graph was successfully opened in the editor, False otherwise
-    """
-    # Create a temporary directory if needed
-    temp_dir = None
-    if storage_location is None:
-        temp_dir = TemporaryDirectory()
-        storage_location = os.path.join(temp_dir.name, 'temp_flow.flowfile')
-    else:
-        # Ensure path is absolute
-        storage_location = os.path.abspath(storage_location)
-
-    flow_graph.apply_layout()
-    flow_graph.save_flow(storage_location)
-    print(f"Flow saved to: {storage_location}")
-
-    # Check if Flowfile is running, and start it if not
-    if not is_flowfile_running():
-        print("Flowfile application is not running. Starting it...")
-        if not start_flowfile_application():
-            print("Failed to start Flowfile application")
-            if temp_dir:
-                temp_dir.cleanup()
-            return False
-        print("Flowfile application started successfully")
-
-    # Get authentication token
-    auth_token = get_auth_token()
-    if not auth_token:
-        print("Failed to authenticate with Flowfile API")
-        if temp_dir:
-            temp_dir.cleanup()
-        return False
-
-    # Import the flow into the editor
-    flow_id = import_flow_to_editor(storage_location, auth_token)
-
-    # Clean up temporary directory if we created one
-    if temp_dir:
-        temp_dir.cleanup()
-
-    return flow_id is not None

flowfile-0.3.0.dist-info/METADATA

@@ -1,219 +0,0 @@
-Metadata-Version: 2.3
-Name: Flowfile
-Version: 0.3.0
-Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
-Author: Edward van Eechoud
-Author-email: evaneechoud@gmail.com
-Requires-Python: >=3.10,<3.13
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: XlsxWriter (>=3.2.0,<3.3.0)
-Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
-Requires-Dist: airbyte-cdk (==6.47.2)
-Requires-Dist: bcrypt (>=4.3.0,<5.0.0)
-Requires-Dist: connectorx (>=0.4.2,<0.5.0)
-Requires-Dist: databases (>=0.9.0,<0.10.0)
-Requires-Dist: faker (>=23.1.0,<23.2.0)
-Requires-Dist: fastapi (>=0.115.2,<0.116.0)
-Requires-Dist: fastexcel (>=0.12.0,<0.13.0)
-Requires-Dist: google-api-python-client (>=2.149.0,<2.150.0)
-Requires-Dist: gspread (>=6.1.3,<6.2.0)
-Requires-Dist: loky (>=3.4.1,<3.5.0)
-Requires-Dist: methodtools (>=0.4.7,<0.5.0)
-Requires-Dist: openpyxl (>=3.1.2,<3.2.0)
-Requires-Dist: passlib (>=1.7.4,<1.8.0)
-Requires-Dist: pendulum (==2.1.2) ; python_version < "3.12"
-Requires-Dist: polars (>1.8.2,<=1.25.2)
-Requires-Dist: polars-distance (>=0.4.3,<0.5.0)
-Requires-Dist: polars-ds (>=0.6.0)
-Requires-Dist: polars-expr-transformer (>0.4.7.0)
-Requires-Dist: polars-grouper (>=0.3.0,<0.4.0)
-Requires-Dist: polars_simed (>=0.3.4,<0.4.0)
-Requires-Dist: pyairbyte-flowfile (==0.20.2)
-Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
-Requires-Dist: pydantic (>=2.9.2,<2.10.0)
-Requires-Dist: pyinstaller (>=6.11.0,<7.0.0)
-Requires-Dist: pytest (>=8.3.4,<9.0.0)
-Requires-Dist: python-jose (>=3.4.0,<4.0.0)
-Requires-Dist: python-multipart (>=0.0.12,<0.1.0)
-Requires-Dist: uvicorn (>=0.32.0,<0.33.0)
-Description-Content-Type: text/markdown
-
-<h1 align="center">
-  <img src="https://raw.githubusercontent.com/Edwardvaneechoud/Flowfile/main/.github/images/logo.png" alt="Flowfile Logo" width="100">
-  <br>
-  Flowfile
-</h1>
-
-<p align="center">
-  <b>Main Repository</b>: <a href="https://github.com/Edwardvaneechoud/Flowfile">Edwardvaneechoud/Flowfile</a><br>
-  <b>Documentation</b>: 
-  <a href="https://edwardvaneechoud.github.io/Flowfile/">Website</a> - 
-  <a href="https://github.com/Edwardvaneechoud/Flowfile/blob/main/flowfile_core/README.md">Core</a> - 
-  <a href="https://github.com/Edwardvaneechoud/Flowfile/blob/main/flowfile_worker/README.md">Worker</a> - 
-  <a href="https://github.com/Edwardvaneechoud/Flowfile/blob/main/flowfile_frontend/README.md">Frontend</a> - 
-  <a href="https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c">Technical Architecture</a>
-</p>
-
-<p>
-Flowfile is a visual ETL tool and Python library suite that combines drag-and-drop workflow building with the speed of Polars dataframes. Build data pipelines visually, transform data using powerful nodes, or define data flows programmatically with Python and analyze results - all with high-performance data processing.
-</p>
-
-<div align="center">
-  <img src="https://raw.githubusercontent.com/Edwardvaneechoud/Flowfile/main/.github/images/group_by_screenshot.png" alt="Flowfile Interface" width="800"/>
-</div>
-
-## ⚡ Technical Design
-
-The `Flowfile` PyPI package provides the backend services and the `flowfile_frame` Python library:
-
-- **Core (`flowfile_core`)** (FastAPI): The main ETL engine using Polars for high-performance data transformations. Typically runs on port `:63578`.
-- **Worker (`flowfile_worker`)** (FastAPI): Handles computation-intensive tasks and caching of data operations, supporting the Core service. Typically runs on port `:63579`.
-- **FlowFrame API (`flowfile_frame`)**: A Python library with a Polars-like API for defining data manipulation pipelines programmatically, which also generates an underlying ETL graph compatible with the Flowfile ecosystem.
-
-Each flow is represented as a directed acyclic graph (DAG), where nodes represent data operations and edges represent data flow between operations.
-
-For a deeper dive into the technical architecture, check out [this article](https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c) on how Flowfile leverages Polars for efficient data processing.
-
-## ✨ Introducing FlowFile Frame - A Polars-Like API for ETL
-
-FlowFile Frame is a Python library that provides a familiar Polars-like API for data manipulation, while simultaneously building an ETL (Extract, Transform, Load) graph under the hood. This allows you to:
-
-1. Write data transformation code using a simple, Pandas/Polars-like API
-2. Automatically generate executable ETL workflows compatible with the Flowfile ecosystem
-3. Visualize, save, and share your data pipelines
-4. Get the performance benefits of Polars with the traceability of ETL graphs
-
-### FlowFrame Quick Start
-
-```python
-import flowfile_frame as ff
-from flowfile_frame.utils import open_graph_in_editor
-
-# Create a complex data pipeline
-df = ff.from_dict({
-    "id": [1, 2, 3, 4, 5],
-    "category": ["A", "B", "A", "C", "B"],
-    "value": [100, 200, 150, 300, 250]
-})
-
-open_graph_in_editor(df.flow_graph)
-
-```
-
-### Key FlowFrame Features
-
-- **Familiar API**: Based on Polars, making it easy to learn if you know Pandas or Polars
-- **ETL Graph Generation**: Automatically builds a directed acyclic graph of your data operations
-- **Lazy Evaluation**: Operations are not executed until `collect()` or a write operation
-- **Interoperability**: Saved `.flowfile` graphs can be opened in the visual Flowfile Designer
-- **High Performance**: Leverages Polars for fast data processing
-- **Reproducible**: Save and share your data transformation workflows
-
-### Common FlowFrame Operations
-
-```python
-import flowfile_frame as ff
-from flowfile_frame import col, when
-
-# Create from dictionary
-df = ff.from_dict({
-    "id": [1, 2, 3],
-    "name": ["Alice", "Bob", "Charlie"],
-    "age": [25, 35, 28]
-})
-
-flow_graph = df.flow_graph
-# Reading data
-# df_csv = ff.read_csv("data.csv")
-# df_parquet = ff.read_parquet("data.parquet")
-
-# Filtering
-adults = df.filter(col("age") >= 30)
-
-# Select and transform
-result = df.select(
-    col("name"),
-    (col("age") * 2).alias("double_age")
-)
-
-# Add new columns
-df_with_cols = df.with_columns([
-    (col("age") + 10).alias("future_age"),
-    when(col("age") >= 30).then(ff.lit("Senior")).otherwise(ff.lit("Junior")).alias("status")]
-)
-
-# Group by and aggregate
-df_sales = ff.from_dict({
-    "region": ["North", "South", "North", "South"],
-    "sales": [100, 200, 150, 300]
-})
-sales_by_region = df_sales.group_by("region").agg([
-    col("sales").sum().alias("total_sales"),
-    col("sales").mean().alias("avg_sales")
-])
-
-# Joins
-customers = ff.from_dict({"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}, flow_graph=flow_graph)
-orders = ff.from_dict({"id": [101, 102], "customer_id": [1, 2], "amount": [100, 200]}, flow_graph=flow_graph)
-joined = customers.join(orders, left_on="id", right_on="customer_id")
-
-# Save and visualize ETL graph
-
-result.save_graph("my_pipeline.flowfile")
-# open_graph_in_editor(result.flow_graph, "my_pipeline.flowfile") # Opens in Designer UI if installed
-```
-
-For more detailed information on all available operations, including pivoting, window functions, complex workflows, and more, please refer to the [FlowFrame documentation](https://github.com/Edwardvaneechoud/Flowfile/blob/main/flowfile_frame/README.md).
-
-## 🔥 Example Use Cases
-
-Flowfile is great for:
-
-- **Data Cleaning & Transformation**
-  - Complex joins (fuzzy matching)
-  - Text-to-rows transformations
-  - Advanced filtering and grouping
-  - Custom formulas and expressions
-  - Filter data based on conditions
-
-- **Performance**
-  - Built to scale out of core
-  - Using Polars for data processing
-
-- **Data Integration**
-  - Standardize data formats
-  - Handle messy Excel files 
-
-- **ETL Operations**
-  - Data quality checks
-
-(For more visual examples of these use cases, please see our [main GitHub repository](https://github.com/Edwardvaneechoud/Flowfile#-example-use-cases)).
-
-## 🚀 Getting Started
-
-### Installing the Flowfile Python Package
-
-This package provides the `flowfile_core` and `flowfile_worker` backend services, and the `flowfile_frame` library.
-
-```bash
-pip install Flowfile
-```
-
-Once installed, you can use `flowfile_frame` as a library in your Python scripts (see Quick Start above).
-
-### Full Application with Visual Designer
-
-For the complete visual ETL experience with the Designer UI, please see the [installation instructions in the main repository](https://github.com/Edwardvaneechoud/Flowfile#-getting-started).
-
-Available options include:
-- Desktop application (recommended for most users)
-- Docker setup (backend services + web frontend)
-- Manual setup for development
-
-## 📋 Development Roadmap
-
-For the latest development roadmap and TODO list, please refer to the [main repository](https://github.com/Edwardvaneechoud/Flowfile#-todo).
-

flowfile_frame/__main__.py

@@ -1,12 +0,0 @@
-"""Main entry point for the FlowFrame CLI."""
-
-def main():
-    """Main entry point for the FlowFrame CLI."""
-    print("FlowFrame - A Polars-like API for building ETL graphs")
-    print("Usage: import flowframe as ff")
-    print("       df = ff.from_dict({'a': [1, 2, 3]})")
-    print("       result = df.filter(ff.col('a') > 1)")
-    print("       print(result.collect())")
-
-if __name__ == "__main__":
-    main()