Flowfile 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

flowfile/__init__.py

@@ -70,10 +70,16 @@ from flowfile_frame import (
     concat,
     create_cloud_storage_connection,
     create_cloud_storage_connection_if_not_exists,
+    create_database_connection,
+    create_database_connection_if_not_exists,
     del_cloud_storage_connection,
+    del_database_connection,
     from_dict,
     get_all_available_cloud_storage_connections,
+    get_all_available_database_connections,
+    get_database_connection_by_name,
     read_csv,
+    read_database,
     read_parquet,
     scan_csv,
     scan_csv_from_cloud_storage,
@@ -81,6 +87,7 @@ from flowfile_frame import (
     scan_json_from_cloud_storage,
     scan_parquet,
     scan_parquet_from_cloud_storage,
+    write_database,
 )
 from flowfile_frame.expr import col, column, count, cum_count, len, lit, max, mean, min, sum, when
 from flowfile_frame.flow_frame import FlowFrame
@@ -125,10 +132,19 @@ __all__ = [
     "scan_parquet_from_cloud_storage",
     "scan_json_from_cloud_storage",
     "scan_csv_from_cloud_storage",
+    # Cloud storage connection management
     "get_all_available_cloud_storage_connections",
     "create_cloud_storage_connection",
     "del_cloud_storage_connection",
     "create_cloud_storage_connection_if_not_exists",
+    # Database functions
+    "read_database",
+    "write_database",
+    "create_database_connection",
+    "create_database_connection_if_not_exists",
+    "del_database_connection",
+    "get_all_available_database_connections",
+    "get_database_connection_by_name",
     "FlowGraph",
     "FlowDataEngine",
     "node_interface",

flowfile/__main__.py

@@ -1,5 +1,88 @@
 #  flowfile/__main__.py
 
+import sys
+from pathlib import Path
+
+
+def run_flow(flow_path: str) -> int:
+    """
+    Load and execute a flow from a YAML/JSON file.
+
+    Args:
+        flow_path: Path to the flow file (.yaml, .yml, or .json)
+
+    Returns:
+        Exit code (0 for success, 1 for failure)
+    """
+    # Disable worker communication for CLI execution
+    from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
+
+    OFFLOAD_TO_WORKER.set(False)
+
+    from flowfile_core.flowfile.manage.io_flowfile import open_flow
+
+    path = Path(flow_path)
+    if not path.exists():
+        print(f"Error: File not found: {flow_path}", file=sys.stderr)
+        return 1
+
+    if path.suffix.lower() not in (".yaml", ".yml", ".json"):
+        print(f"Error: Unsupported file format: {path.suffix}", file=sys.stderr)
+        print("Supported formats: .yaml, .yml, .json", file=sys.stderr)
+        return 1
+
+    print(f"Loading flow from: {flow_path}")
+
+    try:
+        flow = open_flow(path)
+    except Exception as e:
+        print(f"Error loading flow: {e}", file=sys.stderr)
+        return 1
+
+    # Force local execution for CLI - no worker service needed
+    flow.execution_location = "local"
+
+    # Remove explore_data nodes - they're UI-only and require a worker service
+    explore_data_nodes = [n.node_id for n in flow.nodes if n.node_type == "explore_data"]
+    for node_id in explore_data_nodes:
+        flow.delete_node(node_id)
+    if explore_data_nodes:
+        print(f"Skipping {len(explore_data_nodes)} explore_data node(s) (UI-only)")
+
+    flow_name = flow.flow_settings.name or f"Flow {flow.flow_id}"
+    print(f"Running flow: {flow_name} (id={flow.flow_id})")
+    print(f"Nodes: {len(flow.nodes)}")
+    flow.print_tree()
+    print("-" * 40)
+
+    try:
+        result = flow.run_graph()
+    except Exception as e:
+        print(f"Error running flow: {e}", file=sys.stderr)
+        return 1
+
+    if result is None:
+        print("Error: Flow execution returned no result", file=sys.stderr)
+        return 1
+
+    # Display results
+    print("-" * 40)
+    if result.success:
+        duration = ""
+        if result.start_time and result.end_time:
+            duration = f" in {(result.end_time - result.start_time).total_seconds():.2f}s"
+        print(f"Flow completed successfully{duration}")
+        print(f"Nodes completed: {result.nodes_completed}/{result.number_of_nodes}")
+    else:
+        print("Flow execution failed", file=sys.stderr)
+        for node_result in result.node_step_result:
+            if not node_result.success and node_result.error:
+                node_name = node_result.node_name or f"Node {node_result.node_id}"
+                print(f"  - {node_name}: {node_result.error}", file=sys.stderr)
+        return 1
+
+    return 0
+
 
 def main():
     """
@@ -12,8 +95,9 @@ def main():
     parser = argparse.ArgumentParser(description="FlowFile: A visual ETL tool with a Polars-like API")
     parser.add_argument("command", nargs="?", choices=["run"], help="Command to execute")
     parser.add_argument(
-        "component", nargs="?", choices=["ui", "core", "worker"], help="Component to run (ui, core, or worker)"
+        "component", nargs="?", choices=["ui", "core", "worker", "flow"], help="Component to run"
     )
+    parser.add_argument("file_path", nargs="?", help="Path to flow file (for 'flow' component)")
     parser.add_argument("--host", default="127.0.0.1", help="Host to bind the server to")
     parser.add_argument("--port", type=int, default=63578, help="Port to bind the server to")
     parser.add_argument("--no-browser", action="store_true", help="Don't open a browser window")
@@ -37,6 +121,12 @@ def main():
             from flowfile_worker.main import run as run_worker
 
             run_worker(host=args.host, port=args.port)
+        elif args.component == "flow":
+            if not args.file_path:
+                print("Error: 'flow' component requires a file path", file=sys.stderr)
+                print("Usage: flowfile run flow <path-to-flow-file>", file=sys.stderr)
+                sys.exit(1)
+            sys.exit(run_flow(args.file_path))
     else:
         # Default action - show info
         print(f"FlowFile v{flowfile.__version__}")
@@ -45,6 +135,9 @@ def main():
         print("  # Start the FlowFile web UI with integrated services")
         print("  flowfile run ui")
         print("")
+        print("  # Run a flow from a file")
+        print("  flowfile run flow my_pipeline.yaml")
+        print("")
         print("  # Advanced: Run individual components")
         print("  flowfile run core  # Start only the core service")
         print("  flowfile run worker  # Start only the worker service")

flowfile/web/static/assets/{AdminView-49392a9a.js → AdminView-c2c7942b.js}

@@ -1,4 +1,4 @@
-import { j as axios, d as defineComponent, a as useAuthStore, A as computed, r as ref, C as onMounted, c as createElementBlock, m as createVNode, p as withCtx, ag as Transition, b as createBaseVNode, w as withModifiers, g as withDirectives, v as vModelText, h as vModelDynamic, n as normalizeClass, q as createTextVNode, f as createCommentVNode, an as vModelCheckbox, t as toDisplayString, F as Fragment, D as renderList, o as openBlock, i as _export_sfc } from "./index-fb6493ae.js";
+import { k as axios, d as defineComponent, h as useAuthStore, G as computed, r as ref, J as onMounted, c as createElementBlock, z as createVNode, A as withCtx, an as Transition, a as createBaseVNode, w as withModifiers, i as withDirectives, v as vModelText, j as vModelDynamic, n as normalizeClass, f as createTextVNode, e as createCommentVNode, au as vModelCheckbox, t as toDisplayString, K as Fragment, L as renderList, o as openBlock, g as _export_sfc } from "./index-ca6799de.js";
 class UserService {
   /**
    * Get all users (admin only)

flowfile/web/static/assets/{CloudConnectionView-f13f202b.js → CloudConnectionView-7a3042c6.js}

@@ -1,5 +1,5 @@
-import { f as fetchCloudStorageConnectionsInterfaces, c as createCloudStorageConnectionApi, d as deleteCloudStorageConnectionApi } from "./api-8b81fa73.js";
-import { d as defineComponent, r as ref, B as watch, A as computed, o as openBlock, c as createElementBlock, b as createBaseVNode, g as withDirectives, v as vModelText, am as vModelSelect, F as Fragment, D as renderList, t as toDisplayString, f as createCommentVNode, h as vModelDynamic, n as normalizeClass, an as vModelCheckbox, w as withModifiers, C as onMounted, q as createTextVNode, m as createVNode, p as withCtx, s as unref, H as ElMessage, ao as ElDialog, ap as ElButton, i as _export_sfc } from "./index-fb6493ae.js";
+import { f as fetchCloudStorageConnectionsInterfaces, c as createCloudStorageConnectionApi, d as deleteCloudStorageConnectionApi } from "./api-ee542cf7.js";
+import { d as defineComponent, r as ref, H as watch, G as computed, o as openBlock, c as createElementBlock, a as createBaseVNode, i as withDirectives, v as vModelText, at as vModelSelect, K as Fragment, L as renderList, t as toDisplayString, e as createCommentVNode, j as vModelDynamic, n as normalizeClass, au as vModelCheckbox, w as withModifiers, J as onMounted, f as createTextVNode, z as createVNode, A as withCtx, B as unref, N as ElMessage, av as ElDialog, aw as ElButton, g as _export_sfc } from "./index-ca6799de.js";
 const _hoisted_1$1 = { class: "form-grid" };
 const _hoisted_2$1 = { class: "form-field" };
 const _hoisted_3$1 = { class: "form-field" };
@@ -776,8 +776,8 @@ const _sfc_main = /* @__PURE__ */ defineComponent({
     };
   }
 });
-const CloudConnectionView_vue_vue_type_style_index_0_scoped_ba78c600_lang = "";
-const CloudConnectionView = /* @__PURE__ */ _export_sfc(_sfc_main, [["__scopeId", "data-v-ba78c600"]]);
+const CloudConnectionView_vue_vue_type_style_index_0_scoped_0a624123_lang = "";
+const CloudConnectionView = /* @__PURE__ */ _export_sfc(_sfc_main, [["__scopeId", "data-v-0a624123"]]);
 export {
   CloudConnectionView as default
 };

flowfile/web/static/assets/{CloudConnectionView-36bcd6df.css → CloudConnectionView-cf85f943.css}

@@ -1,10 +1,10 @@
 
-.description-text[data-v-ba78c600] {
+.description-text[data-v-0a624123] {
   color: var(--color-text-secondary);
   margin-top: var(--spacing-2);
   font-size: var(--font-size-sm);
 }
-.info-box[data-v-ba78c600] {
+.info-box[data-v-0a624123] {
   display: flex;
   gap: var(--spacing-4);
   padding: var(--spacing-4);
@@ -12,28 +12,28 @@
   border-left: 4px solid var(--color-accent);
   border-radius: var(--border-radius-md);
 }
-.info-box i[data-v-ba78c600] {
+.info-box i[data-v-0a624123] {
   color: var(--color-accent);
   font-size: var(--font-size-2xl);
   margin-top: var(--spacing-2);
 }
-.info-box p[data-v-ba78c600] {
+.info-box p[data-v-0a624123] {
   margin: 0;
   margin-bottom: var(--spacing-2);
   font-size: var(--font-size-sm);
   color: var(--color-text-secondary);
 }
-.info-box p[data-v-ba78c600]:last-child {
+.info-box p[data-v-0a624123]:last-child {
   margin-bottom: 0;
 }
-.info-box p strong[data-v-ba78c600] {
+.info-box p strong[data-v-0a624123] {
   color: var(--color-text-primary);
 }
-.modal-description[data-v-ba78c600] {
+.modal-description[data-v-0a624123] {
   color: var(--color-text-secondary);
   font-size: var(--font-size-sm);
 }
-.badge[data-v-ba78c600] {
+.badge[data-v-0a624123] {
   background-color: var(--color-accent-subtle);
   color: var(--color-accent);
   border-radius: var(--border-radius-full);
@@ -42,31 +42,31 @@
   font-weight: var(--font-weight-medium);
   margin-left: var(--spacing-2);
 }
-.auth-badge[data-v-ba78c600] {
+.auth-badge[data-v-0a624123] {
   background-color: var(--color-info-light);
   color: var(--color-info);
 }
-.connection-details[data-v-ba78c600] {
+.connection-details[data-v-0a624123] {
   font-size: var(--font-size-xs);
   color: var(--color-text-tertiary);
   margin-top: var(--spacing-1);
 }
-.separator[data-v-ba78c600] {
+.separator[data-v-0a624123] {
   margin: 0 var(--spacing-2);
 }
-.hint-text[data-v-ba78c600] {
+.hint-text[data-v-0a624123] {
   color: var(--color-text-tertiary);
   font-size: var(--font-size-sm);
   margin-top: var(--spacing-2);
 }
 
 /* Cloud-specific styles */
-.fa-brands[data-v-ba78c600] {
+.fa-brands[data-v-0a624123] {
   font-size: var(--font-size-xl);
 }
-.fa-aws[data-v-ba78c600] {
-  color: #ff9900;
+.fa-aws[data-v-0a624123] {
+  color: var(--color-brand-aws);
 }
-.fa-microsoft[data-v-ba78c600] {
-  color: #0078d4;
+.fa-microsoft[data-v-0a624123] {
+  color: var(--color-brand-microsoft);
 }

flowfile/web/static/assets/{CloudStorageReader-0023d4a5.js → CloudStorageReader-709c4037.js}

@@ -1,11 +1,11 @@
-import { d as defineComponent, r as ref, o as openBlock, c as createElementBlock, m as createVNode, p as withCtx, b as createBaseVNode, g as withDirectives, am as vModelSelect, F as Fragment, D as renderList, t as toDisplayString, q as createTextVNode, f as createCommentVNode, v as vModelText, an as vModelCheckbox, x as createBlock, s as unref, H as ElMessage, i as _export_sfc } from "./index-fb6493ae.js";
-import { C as CodeLoader } from "./vue-content-loader.es-c506ad97.js";
-import { u as useNodeStore } from "./PopOver-b22f049e.js";
-import { f as fetchCloudStorageConnectionsInterfaces } from "./api-8b81fa73.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-99014e1d.js";
-import "./DesignerView-e6f5c0e8.js";
-import "./index-3ba44389.js";
-import "./vue-codemirror.esm-0965f39f.js";
+import { d as defineComponent, l as useNodeStore, r as ref, o as openBlock, c as createElementBlock, z as createVNode, A as withCtx, a as createBaseVNode, i as withDirectives, at as vModelSelect, K as Fragment, L as renderList, t as toDisplayString, f as createTextVNode, e as createCommentVNode, v as vModelText, au as vModelCheckbox, C as createBlock, B as unref, N as ElMessage, g as _export_sfc } from "./index-ca6799de.js";
+import { C as CodeLoader } from "./vue-content-loader.es-808fe33a.js";
+import { f as fetchCloudStorageConnectionsInterfaces } from "./api-ee542cf7.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-0155288b.js";
+import "./DesignerView-a4466dab.js";
+import "./PopOver-ddcfe4f6.js";
+import "./index-057d770d.js";
+import "./vue-codemirror.esm-8f46fb36.js";
 const createNodeCloudStorageReader = (flowId, nodeId) => {
   const cloudStorageReadSettings = {
     auth_mode: "aws-cli",

flowfile/web/static/assets/{CloudStorageWriter-8e781e11.js → CloudStorageWriter-604c51a8.js}

@@ -1,11 +1,11 @@
-import { d as defineComponent, r as ref, o as openBlock, c as createElementBlock, m as createVNode, p as withCtx, b as createBaseVNode, g as withDirectives, am as vModelSelect, F as Fragment, D as renderList, t as toDisplayString, q as createTextVNode, f as createCommentVNode, v as vModelText, x as createBlock, s as unref, H as ElMessage, i as _export_sfc } from "./index-fb6493ae.js";
-import { C as CodeLoader } from "./vue-content-loader.es-c506ad97.js";
-import { u as useNodeStore } from "./PopOver-b22f049e.js";
-import { f as fetchCloudStorageConnectionsInterfaces } from "./api-8b81fa73.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-99014e1d.js";
-import "./DesignerView-e6f5c0e8.js";
-import "./index-3ba44389.js";
-import "./vue-codemirror.esm-0965f39f.js";
+import { d as defineComponent, l as useNodeStore, r as ref, o as openBlock, c as createElementBlock, z as createVNode, A as withCtx, a as createBaseVNode, i as withDirectives, at as vModelSelect, K as Fragment, L as renderList, t as toDisplayString, f as createTextVNode, e as createCommentVNode, v as vModelText, C as createBlock, B as unref, N as ElMessage, g as _export_sfc } from "./index-ca6799de.js";
+import { C as CodeLoader } from "./vue-content-loader.es-808fe33a.js";
+import { f as fetchCloudStorageConnectionsInterfaces } from "./api-ee542cf7.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-0155288b.js";
+import "./DesignerView-a4466dab.js";
+import "./PopOver-ddcfe4f6.js";
+import "./index-057d770d.js";
+import "./vue-codemirror.esm-8f46fb36.js";
 const createNodeCloudStorageWriter = (flowId, nodeId) => {
   const cloudStorageWriteSettings = {
     auth_mode: "aws-cli",

flowfile/web/static/assets/ColumnActionInput-c44b7aee.css

@@ -0,0 +1,159 @@
+
+.component-container[data-v-d3070256] {
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing-4);
+}
+.config-section[data-v-d3070256] {
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing-3);
+  padding: var(--spacing-4);
+  background-color: var(--color-background-secondary);
+  border-radius: var(--border-radius-lg);
+}
+.config-row[data-v-d3070256] {
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing-1);
+}
+.config-label[data-v-d3070256] {
+  font-size: var(--font-size-base);
+  font-weight: var(--font-weight-medium);
+  color: var(--color-text-secondary);
+}
+.column-list-wrapper[data-v-d3070256] {
+  margin-top: var(--spacing-2);
+}
+.listbox-subtitle[data-v-d3070256] {
+  font-size: var(--font-size-base);
+  font-weight: var(--font-weight-semibold);
+  color: var(--color-text-primary);
+  margin-bottom: var(--spacing-2);
+}
+.listbox[data-v-d3070256] {
+  max-height: 200px;
+  overflow-y: auto;
+  border: 1px solid var(--color-border-primary);
+  border-radius: var(--border-radius-lg);
+  padding: 0;
+  margin: 0;
+  list-style: none;
+  background-color: var(--color-background-primary);
+}
+.listbox li[data-v-d3070256] {
+  padding: var(--spacing-2) var(--spacing-3);
+  cursor: pointer;
+  transition: background-color var(--transition-normal) var(--transition-timing);
+  color: var(--color-text-primary);
+}
+.listbox li[data-v-d3070256]:hover {
+  background-color: var(--color-background-hover);
+}
+.listbox li.is-selected[data-v-d3070256] {
+  background-color: var(--color-background-selected);
+  color: var(--color-accent);
+}
+.column-type[data-v-d3070256] {
+  font-size: var(--font-size-sm);
+  color: var(--color-text-tertiary);
+  margin-left: var(--spacing-2);
+}
+.context-menu[data-v-d3070256] {
+  position: fixed;
+  z-index: var(--z-index-dropdown);
+  border: 1px solid var(--color-border-primary);
+  background-color: var(--color-background-primary);
+  padding: var(--spacing-2);
+  box-shadow: var(--shadow-lg);
+  border-radius: var(--border-radius-sm);
+  user-select: none;
+}
+.context-menu button[data-v-d3070256] {
+  display: block;
+  background: none;
+  border: none;
+  padding: var(--spacing-1-5) var(--spacing-3);
+  text-align: left;
+  width: 100%;
+  cursor: pointer;
+  border-radius: var(--border-radius-sm);
+  font-size: var(--font-size-base);
+  color: var(--color-text-primary);
+  transition: background-color var(--transition-fast) var(--transition-timing);
+}
+.context-menu button[data-v-d3070256]:hover {
+  background-color: var(--color-background-hover);
+}
+.table-wrapper[data-v-d3070256] {
+  max-height: 300px;
+  box-shadow: var(--shadow-sm);
+  border-radius: var(--border-radius-lg);
+  overflow: auto;
+  border: 1px solid var(--color-border-primary);
+}
+.styled-table[data-v-d3070256] {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: var(--font-size-base);
+  background-color: var(--color-background-primary);
+}
+.styled-table thead[data-v-d3070256] {
+  background-color: var(--table-header-bg);
+  position: sticky;
+  top: 0;
+}
+.styled-table th[data-v-d3070256],
+.styled-table td[data-v-d3070256] {
+  padding: var(--spacing-2-5) var(--spacing-3);
+  text-align: left;
+  border-bottom: 1px solid var(--table-border);
+}
+.styled-table th[data-v-d3070256] {
+  font-weight: var(--font-weight-semibold);
+  color: var(--color-text-secondary);
+}
+.styled-table td[data-v-d3070256] {
+  color: var(--color-text-primary);
+}
+.styled-table tbody tr[data-v-d3070256]:hover {
+  background-color: var(--table-row-hover);
+}
+.styled-table th[data-v-d3070256]:nth-child(1),
+.styled-table td[data-v-d3070256]:nth-child(1) {
+  width: 25%;
+}
+.styled-table th[data-v-d3070256]:nth-child(2),
+.styled-table td[data-v-d3070256]:nth-child(2) {
+  width: 25%;
+}
+.styled-table th[data-v-d3070256]:nth-child(3),
+.styled-table td[data-v-d3070256]:nth-child(3) {
+  width: 40%;
+}
+.styled-table th[data-v-d3070256]:nth-child(4),
+.styled-table td[data-v-d3070256]:nth-child(4) {
+  width: 10%;
+  text-align: center;
+}
+.action-cell[data-v-d3070256] {
+  text-align: center;
+}
+.action-cell .el-button[data-v-d3070256] {
+  padding: var(--spacing-1);
+}
+.empty-state[data-v-d3070256] {
+  padding: var(--spacing-8);
+  text-align: center;
+  color: var(--color-text-secondary);
+  background-color: var(--color-background-secondary);
+  border-radius: var(--border-radius-lg);
+}
+.empty-state p[data-v-d3070256] {
+  margin: 0;
+}
+.empty-state .hint[data-v-d3070256] {
+  font-size: var(--font-size-base);
+  margin-top: var(--spacing-2);
+  color: var(--color-text-tertiary);
+}