Flowfile 0.3.3.1__py3-none-any.whl → 0.3.4__py3-none-any.whl

flowfile_core/flowfile/flow_data_engine/flow_data_engine.py

@@ -200,7 +200,7 @@ class FlowDataEngine:
         polars_schema = pl.Schema([(flowfile_column.column_name, flowfile_column.get_polars_type().pl_datatype)
                                    for flowfile_column in flowfile_schema])
         try:
-            df = pl.DataFrame(raw_data.data, polars_schema)
+            df = pl.DataFrame(raw_data.data, polars_schema, strict=False)
         except TypeError as e:
             logger.warning(f"Could not parse the data with the schema:\n{e}")
             df = pl.DataFrame(raw_data.data)

flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py

@@ -5,7 +5,7 @@ from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
 from polars import datatypes
 import polars as pl
-
+# TODO: rename flow_file_column to flowfile_column
 DataTypeGroup = Literal['numeric', 'str', 'date']
 
 

flowfile_core/flowfile/{FlowfileFlow.py → flow_graph.py}

@@ -3,7 +3,6 @@ import pickle
 import polars as pl
 import fastexcel
 import copy
-
 from fastapi.exceptions import HTTPException
 from time import time
 from functools import partial
@@ -420,7 +419,7 @@ class FlowGraph:
                            input_columns=[],
                            node_type='unique',
                            setting_input=unique_settings,
-                           input_node_ids=[unique_settings.node_id])
+                           input_node_ids=[unique_settings.depending_on_id])
 
     def add_graph_solver(self, graph_solver_settings: input_schema.NodeGraphSolver):
         def _func(fl: FlowDataEngine) -> FlowDataEngine:
@@ -429,7 +428,8 @@ class FlowGraph:
         self.add_node_step(node_id=graph_solver_settings.node_id,
                            function=_func,
                            node_type='graph_solver',
-                           setting_input=graph_solver_settings)
+                           setting_input=graph_solver_settings,
+                           input_node_ids=[graph_solver_settings.depending_on_id])
 
     def add_formula(self, function_settings: input_schema.NodeFormula):
         error = ""

flowfile_core/flowfile/flow_graph_utils.py

@@ -1,7 +1,7 @@
 from typing import Dict, Tuple, Optional, List, Set, Callable
 from copy import deepcopy
 from flowfile_core.schemas import input_schema, schemas
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph, add_connection
+from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 
 
 def combine_flow_graphs_with_mapping(

flowfile_core/flowfile/handler.py

@@ -10,7 +10,7 @@ import os
 from pathlib import Path
 
 from flowfile_core.flowfile.manage.open_flowfile import open_flow
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas.schemas import FlowSettings
 from flowfile_core.configs import logger
 

flowfile_core/flowfile/manage/open_flowfile.py

@@ -2,7 +2,7 @@ from flowfile_core.schemas import schemas, input_schema
 from typing import List, Tuple
 from flowfile_core.flowfile.manage.compatibility_enhancements import ensure_compatibility
 import pickle
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from pathlib import Path
 
 

flowfile_core/flowfile/util/calculate_layout.py

@@ -5,7 +5,7 @@ from typing import List, Dict, Set, Tuple, TYPE_CHECKING
 
 if TYPE_CHECKING:
     # Make sure this import path is correct for your project structure
-    from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+    from flowfile_core.flowfile.flow_graph import FlowGraph
 
 
 def calculate_layered_layout(

flowfile_core/routes/routes.py

@@ -22,7 +22,8 @@ from flowfile_core.fileExplorer.funcs import (
     FileInfo,
     get_files_from_directory
 )
-from flowfile_core.flowfile.FlowfileFlow import add_connection, delete_connection
+from flowfile_core.flowfile.flow_graph import add_connection, delete_connection
+from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
 from flowfile_core.flowfile.analytics.analytics_processor import AnalyticsProcessor
 from flowfile_core.flowfile.extensions import get_instant_func_results
 # Flow handling
@@ -362,6 +363,15 @@ def get_flow(flow_id: int):
     return result
 
 
+@router.get("/editor/code_to_polars", tags=[], response_model=str)
+def get_generated_code(flow_id: int) -> str:
+    flow_id = int(flow_id)
+    flow = flow_file_handler.get_flow(flow_id)
+    if flow is None:
+        raise HTTPException(404, 'could not find the flow')
+    return export_flow_to_polars(flow)
+
+
 @router.post('/editor/create_flow/', tags=['editor'])
 def create_flow(flow_path: str):
     flow_path = Path(flow_path)

flowfile_core/schemas/input_schema.py

@@ -154,7 +154,7 @@ class OutputSettings(BaseModel):
 
     def set_absolute_filepath(self):
         base_path = Path(self.directory)
-        # Check if the path is relative, resolve it with the current working directory
+
         if not base_path.is_absolute():
             base_path = Path.cwd() / base_path
 
@@ -179,6 +179,7 @@ class NodeBase(BaseModel):
     is_setup: Optional[bool] = True
     description: Optional[str] = ''
     user_id: Optional[int] = None
+    is_flow_output: Optional[bool] = False
 
     @classmethod
     def overridden_hash(cls):

flowfile_frame/adapters.py

@@ -2,7 +2,7 @@
 """Adapters to connect FlowFrame with the flowfile-core library."""
 
 # Import from your existing project
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph, add_connection
+from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 

flowfile_frame/expr.py

@@ -126,6 +126,25 @@ class StringMethods:
         res_expr = self.expr.to_titlecase() if self.expr is not None else None
         return self._create_next_expr(method_name="to_titlecase", result_expr=res_expr, is_complex=True)
 
+    def to_date(self, format: str, *, strict: bool = True, exact: bool = True, cache: bool = True):
+        res_expr = self.expr.to_date(format, strict=strict, exact=exact, cache=cache) if self.expr is not None else None
+        return self._create_next_expr(method_name="to_date", result_expr=res_expr, is_complex=True,
+                                      format=format, strict=strict, exact=exact, cache=cache)
+
+    def to_datetime(self, format: str | None = None,
+                    *,
+                    time_unit: Literal["ns", "us", "ms"] | None = None,
+                    time_zone: str | None = None,
+                    strict: bool = True,
+                    exact: bool = True,
+                    cache: bool = True,
+                    ambiguous: Literal["earliest", "latest", "raise", "null"] | Expr = "raise",):
+        res_expr = self.expr.to_datetime(format, time_unit=time_unit, time_zone=time_zone, strict=strict,
+                                         exact=exact, cache=cache, ambiguous=ambiguous)
+        return self._create_next_expr(method_name="to_datetime", result_expr=res_expr, is_complex=True,
+                                      format=format, time_unit=time_unit, time_zone=time_zone, strict=strict,
+                                      exact=exact, cache=cache, ambiguous=ambiguous)
+
     def __getattr__(self, name):
         if self.expr is None or not hasattr(self.expr, name):
             if self.expr is None:

flowfile_frame/flow_frame.py

@@ -10,7 +10,7 @@ from flowfile_frame.lazy_methods import add_lazyframe_methods
 
 from polars._typing import (FrameInitTypes, SchemaDefinition, SchemaDict, Orientation)
 from collections.abc import Iterator
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph, add_connection
+from flowfile_core.flowfile.flow_graph import FlowGraph, add_connection
 from flowfile_core.flowfile.flow_graph_utils import combine_flow_graphs_with_mapping
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
@@ -1027,7 +1027,6 @@ class FlowFrame:
     ) -> "FlowFrame":
 
         new_node_id = generate_node_id()
-
         is_path_input = isinstance(file, (str, os.PathLike))
         if isinstance(file, os.PathLike):
             file_str = str(file)
@@ -1041,7 +1040,6 @@ class FlowFrame:
         file_name = file_str.split(os.sep)[-1] if is_path_input else "output.csv"
 
         use_polars_code = bool(kwargs) or not is_path_input
-
         output_settings = input_schema.OutputSettings(
             file_type='csv',
             name=file_name,
@@ -1051,7 +1049,6 @@ class FlowFrame:
             output_excel_table=input_schema.OutputExcelTable(),
             output_parquet_table=input_schema.OutputParquetTable()
         )
-
         if is_path_input:
             try:
                 output_settings.set_absolute_filepath()

flowfile_frame/flow_frame.pyi

@@ -19,7 +19,7 @@ from polars.type_aliases import (Schema, IntoExpr, ClosedInterval, Label, StartB
 
 # Local application/library specific imports
 import flowfile_frame
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
 from flowfile_frame import group_frame
 from flowfile_frame.expr import Expr

flowfile_frame/flow_frame_methods.py

@@ -8,7 +8,7 @@ import polars as pl
 from polars._typing import (SchemaDict, IO,PolarsDataType,
                             Sequence, CsvEncoding)
 
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
 from flowfile_core.schemas import input_schema, transform_schema
 

flowfile_frame/lazy.py

@@ -2,7 +2,7 @@ import inspect
 from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
 import polars as pl
 from flowfile_frame.flow_frame import FlowFrame, can_be_expr, generate_node_id
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_frame.expr import Expr
 from flowfile_frame.utils import _get_function_source
 from typing import cast

flowfile_frame/utils.py

@@ -7,7 +7,7 @@ from pathlib import Path
 import polars as pl
 from typing import Iterable, Any, List, Optional
 
-from flowfile_core.flowfile.FlowfileFlow import FlowGraph
+from flowfile_core.flowfile.flow_graph import FlowGraph
 from flowfile_core.schemas import schemas
 from tempfile import TemporaryDirectory
 import inspect

flowfile/readme.md

@@ -1,127 +0,0 @@
-# Flowfile Web UI Documentation
-
-## Overview
-
-Flowfile now supports a web-based user interface that can be launched directly from the pip-installed package. This enhancement allows users to quickly get started with the visual ETL tool without needing to install the desktop application, set up Docker, or manually configure the services.
-
-## Key Features
-
-- **Integrated Web UI**: Launch the Flowfile interface directly in your browser
-- **Unified Service**: Combined API that serves both the web UI and processes worker operations
-- **Easy Installation**: Simple pip installation and startup process
-- **Visual ETL**: Access to all the visual ETL capabilities through a web interface
-
-## Installation
-
-Install Flowfile from PyPI using pip:
-
-```bash
-pip install Flowfile
-```
-
-## Starting the Web UI
-
-You can start the Flowfile web UI using either the Python module or the command-line interface:
-
-### Using the Command-Line Interface
-
-```bash
-# Start the web UI with default settings
-flowfile run ui
-
-# Start without automatically opening a browser window
-flowfile run ui --no-browser
-```
-
-### Using Python
-
-```python
-import flowfile
-
-# Start the web UI with default settings
-flowfile.start_web_ui()
-
-# Customize host, port, and browser launch
-flowfile.start_web_ui(host="0.0.0.0", port=63578, open_browser=False)
-```
-
-## Architecture Overview
-
-The web UI functionality combines multiple components:
-
-1. **Core Service**: The main ETL engine (flowfile_core) that processes data transformations
-2. **Worker Service**: Handles computation and caching of data operations (flowfile_worker)
-3. **Web UI**: A Vue.js frontend that provides the visual interface
-
-When you start the web UI, all these services are launched together in a unified mode, making it simple to get started without configuration.
-
-## Using the Web UI with FlowFrame API
-
-You can create data pipelines programmatically with the FlowFrame API and then visualize them in the web UI:
-
-```python
-import flowfile as ff
-from flowfile import open_graph_in_editor
-
-# Create a data pipeline
-df = ff.from_dict({
-    "id": [1, 2, 3, 4, 5],
-    "category": ["A", "B", "A", "C", "B"],
-    "value": [100, 200, 150, 300, 250]
-})
-
-# Process the data
-result = df.filter(ff.col("value") > 150).with_columns([
-    (ff.col("value") * 2).alias("double_value")
-])
-
-# Open the graph in the web UI (starts the server if it's not running)
-open_graph_in_editor(result.flow_graph)
-```
-
-The `open_graph_in_editor` function automatically:
-1. Saves the flow graph to a temporary file
-2. Starts the Flowfile server if it's not already running
-3. Imports the flow into the editor
-4. Opens a browser tab with the imported flow
-
-## Advanced Server Configuration
-
-For advanced users who need to customize the server behavior:
-
-### Environment Variables
-
-- `FLOWFILE_HOST`: Host to bind the server to (default: "127.0.0.1")
-- `FLOWFILE_PORT`: Port to bind the server to (default: 63578)
-- `FLOWFILE_MODE`: Set to "electron" to enable browser auto-opening behavior
-- `WORKER_URL`: URL for the worker service
-- `SINGLE_FILE_MODE`: Set to "1" to run in unified mode with worker functionality
-- `FLOWFILE_MODULE_NAME`: Module name to run (default: "flowfile")
-
-### Running Individual Components
-
-For development or specialized deployments, you can run the components separately:
-
-```bash
-# Run only the core service
-flowfile run core --host 0.0.0.0 --port 63578
-
-# Run only the worker service
-flowfile run worker --host 0.0.0.0 --port 63579
-```
-
-## Troubleshooting
-
-- If the web UI doesn't open automatically, manually navigate to http://localhost:63578/ui
-- If you encounter connection issues, check if the port is already in use
-- Look for server logs in the terminal where you started the service for error messages
-- For issues with the API, navigate to http://localhost:63578/docs to verify the API is running
-
-## Next Steps
-
-Once you're familiar with the web UI, you might want to explore:
-
-1. The desktop application for a more native experience
-2. Docker deployment for production environments
-3. Advanced ETL operations using the FlowFrame API
-4. Custom node development for specialized transformations