Flowfile 0.3.6__tar.gz → 0.3.7__tar.gz

{flowfile-0.3.6 → flowfile-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Flowfile
-Version: 0.3.6
+Version: 0.3.7
 Summary: Project combining flowfile core (backend) and flowfile_worker (compute offloader) and flowfile_frame (api)
 Author: Edward van Eechoud
 Author-email: evaneechoud@gmail.com
@@ -26,7 +26,7 @@ Requires-Dist: methodtools (>=0.4.7,<0.5.0)
 Requires-Dist: openpyxl (>=3.1.2,<3.2.0)
 Requires-Dist: passlib (>=1.7.4,<1.8.0)
 Requires-Dist: pendulum (==2.1.2) ; python_version < "3.12"
-Requires-Dist: polars (>=1.8.2,<2.0.0) ; sys_platform != "win32"
+Requires-Dist: polars (>=1.8.2,<1.32) ; sys_platform != "win32"
 Requires-Dist: polars (>=1.8.2,<=1.25.2) ; sys_platform == "win32"
 Requires-Dist: polars-distance (>=0.4.3,<0.5.0)
 Requires-Dist: polars-ds (>=0.6.0)

{flowfile-0.3.6 → flowfile-0.3.7}/flowfile/flowfile/__init__.py

@@ -7,20 +7,36 @@ This package ties together the FlowFile ecosystem components:
 - flowfile_worker: Computation engine
 """
 
-__version__ = "0.3.5"
+__version__ = "0.3.7"
 
 import os
 import logging
 
-os.environ['WORKER_PORT'] = "63578"
-os.environ['SINGLE_FILE_MODE'] = "1"
+os.environ['FLOWFILE_WORKER_PORT'] = "63578"
+os.environ['FLOWFILE_SINGLE_FILE_MODE'] = "1"
 
 from flowfile.web import start_server as start_web_ui
 from flowfile.api import (open_graph_in_editor)
 from flowfile_frame.flow_frame import (
     FlowFrame
 )
-from flowfile_frame import read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet
+from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection
+from flowfile_core.flowfile.flow_graph import FlowGraph
+from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
+from flowfile_core.schemas import input_schema as node_interface
+from flowfile_core.schemas.schemas import FlowSettings, FlowInformation
+from flowfile_core.schemas import transform_schema
+from flowfile_frame import (read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet,
+                            scan_delta,
+                            scan_parquet_from_cloud_storage,
+                            scan_json_from_cloud_storage,
+                            scan_csv_from_cloud_storage,
+                            get_all_available_cloud_storage_connections,
+                            create_cloud_storage_connection,
+                            del_cloud_storage_connection,
+                            create_cloud_storage_connection_if_not_exists)
 from flowfile_frame.expr import (
     col, lit, column, cum_count, len,
     sum, min, max, mean, count, when
@@ -47,10 +63,15 @@ from polars.datatypes import (
 
 __all__ = [
     # Core FlowFrame classes
-    'FlowFrame', 'GroupByFrame',
+    'FlowFrame', 'GroupByFrame', 'FullCloudStorageConnection',
 
     # Main creation functions
-    'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet',
+    'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet', 'scan_delta',
+    'scan_parquet_from_cloud_storage', 'scan_json_from_cloud_storage',
+    'scan_csv_from_cloud_storage', 'get_all_available_cloud_storage_connections', 'create_cloud_storage_connection',
+    'del_cloud_storage_connection', 'create_cloud_storage_connection_if_not_exists',
+    'FlowGraph', 'FlowDataEngine', 'node_interface', 'FlowSettings', 'transform_schema',
+    'FlowNode', 'FlowfileColumn', 'FlowInformation',
 
     # Expression API
     'col', 'lit', 'column', 'cum_count', 'len',

{flowfile-0.3.6 → flowfile-0.3.7}/flowfile/flowfile/api.py

@@ -456,6 +456,7 @@ def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str]
         if flow_id is not None:
             if flow_in_single_mode and automatically_open_browser:
                 _open_flow_in_browser(flow_id)
+            logger.info(f"Ui available at: http://127.0.0.1:63578/ui#/main/designer")
             return True
         else:
             return False

{flowfile-0.3.6 → flowfile-0.3.7}/flowfile/flowfile/web/__init__.py

@@ -50,8 +50,8 @@ def extend_app(app: FastAPI):
         return Response(status_code=404)
 
     @app.get("/single_mode")
-    async def in_single_mode():
-        return os.environ.get('SINGLE_FILE_MODE', "0")
+    async def in_single_mode() -> bool:
+        return os.environ.get('FLOWFILE_SINGLE_FILE_MODE', "0") == "1"
 
     @app.get("/ui", include_in_schema=False)
     async def web_ui_root():

flowfile-0.3.6/flowfile/flowfile/web/static/assets/CloudConnectionManager-d004942f.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/CloudConnectionManager-c20a740f.js

@@ -1,5 +1,5 @@
-import { f as fetchCloudStorageConnectionsInterfaces, c as createCloudStorageConnectionApi, d as deleteCloudStorageConnectionApi } from "./api-fb67319c.js";
-import { d as defineComponent, r as ref, m as watch, l as computed, c as openBlock, e as createElementBlock, p as createBaseVNode, a5 as withDirectives, a6 as vModelText, ag as vModelSelect, F as Fragment, q as renderList, t as toDisplayString, i as createCommentVNode, ah as vModelDynamic, s as normalizeClass, ai as vModelCheckbox, v as withModifiers, n as onMounted, g as createTextVNode, f as createVNode, w as withCtx, u as unref, z as ElMessage, aj as ElDialog, ak as ElButton, _ as _export_sfc } from "./index-3a75211d.js";
+import { f as fetchCloudStorageConnectionsInterfaces, c as createCloudStorageConnectionApi, d as deleteCloudStorageConnectionApi } from "./api-6ef0dcef.js";
+import { d as defineComponent, r as ref, m as watch, l as computed, c as openBlock, e as createElementBlock, p as createBaseVNode, a5 as withDirectives, a6 as vModelText, ag as vModelSelect, F as Fragment, q as renderList, t as toDisplayString, i as createCommentVNode, ah as vModelDynamic, s as normalizeClass, ai as vModelCheckbox, v as withModifiers, n as onMounted, g as createTextVNode, f as createVNode, w as withCtx, u as unref, z as ElMessage, aj as ElDialog, ak as ElButton, _ as _export_sfc } from "./index-f6c15e76.js";
 const _hoisted_1$1 = { class: "form-grid" };
 const _hoisted_2$1 = { class: "form-field" };
 const _hoisted_3$1 = { class: "form-field" };
@@ -175,8 +175,7 @@ const _sfc_main$1 = /* @__PURE__ */ defineComponent({
               class: "form-input",
               required: ""
             }, _cache[20] || (_cache[20] = [
-              createBaseVNode("option", { value: "s3" }, "AWS S3", -1),
-              createBaseVNode("option", { value: "adls" }, "Azure Data Lake Storage", -1)
+              createBaseVNode("option", { value: "s3" }, "AWS S3", -1)
             ]), 512), [
               [vModelSelect, connection.value.storageType]
             ])

flowfile-0.3.6/flowfile/flowfile/web/static/assets/CloudStorageReader-eccf9fc2.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/CloudStorageReader-960b400a.js

@@ -1,10 +1,10 @@
-import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, a5 as withDirectives, F as Fragment, q as renderList, t as toDisplayString, ag as vModelSelect, g as createTextVNode, i as createCommentVNode, a6 as vModelText, ai as vModelCheckbox, h as createBlock, u as unref, z as ElMessage, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport } from "./index-3a75211d.js";
-import { C as CodeLoader } from "./vue-content-loader.es-93bd09d7.js";
-import { u as useNodeStore } from "./vue-codemirror.esm-f15a5f87.js";
-import { f as fetchCloudStorageConnectionsInterfaces } from "./api-fb67319c.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-7013cc94.js";
-import { N as NodeButton, a as NodeTitle } from "./nodeTitle-a63d4680.js";
-import "./designer-94a6bf4d.js";
+import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, a5 as withDirectives, F as Fragment, q as renderList, t as toDisplayString, ag as vModelSelect, g as createTextVNode, i as createCommentVNode, a6 as vModelText, ai as vModelCheckbox, h as createBlock, u as unref, z as ElMessage, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport } from "./index-f6c15e76.js";
+import { C as CodeLoader } from "./vue-content-loader.es-0371da73.js";
+import { u as useNodeStore } from "./vue-codemirror.esm-2847001e.js";
+import { f as fetchCloudStorageConnectionsInterfaces } from "./api-6ef0dcef.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-71e11604.js";
+import { N as NodeButton, a as NodeTitle } from "./nodeTitle-988d9efe.js";
+import "./designer-13eabd83.js";
 const createNodeCloudStorageReader = (flowId, nodeId) => {
   const cloudStorageReadSettings = {
     auth_mode: "aws-cli",

flowfile-0.3.6/flowfile/flowfile/web/static/assets/CloudStorageWriter-b1ba6bba.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/CloudStorageWriter-e3decbdd.js

@@ -1,10 +1,10 @@
-import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, a5 as withDirectives, F as Fragment, q as renderList, t as toDisplayString, ag as vModelSelect, g as createTextVNode, i as createCommentVNode, a6 as vModelText, h as createBlock, u as unref, z as ElMessage, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport } from "./index-3a75211d.js";
-import { C as CodeLoader } from "./vue-content-loader.es-93bd09d7.js";
-import { u as useNodeStore } from "./vue-codemirror.esm-f15a5f87.js";
-import { f as fetchCloudStorageConnectionsInterfaces } from "./api-fb67319c.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-7013cc94.js";
-import { N as NodeButton, a as NodeTitle } from "./nodeTitle-a63d4680.js";
-import "./designer-94a6bf4d.js";
+import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, a5 as withDirectives, F as Fragment, q as renderList, t as toDisplayString, ag as vModelSelect, g as createTextVNode, i as createCommentVNode, a6 as vModelText, h as createBlock, u as unref, z as ElMessage, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport } from "./index-f6c15e76.js";
+import { C as CodeLoader } from "./vue-content-loader.es-0371da73.js";
+import { u as useNodeStore } from "./vue-codemirror.esm-2847001e.js";
+import { f as fetchCloudStorageConnectionsInterfaces } from "./api-6ef0dcef.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-71e11604.js";
+import { N as NodeButton, a as NodeTitle } from "./nodeTitle-988d9efe.js";
+import "./designer-13eabd83.js";
 const createNodeCloudStorageWriter = (flowId, nodeId) => {
   const cloudStorageWriteSettings = {
     auth_mode: "aws-cli",

flowfile-0.3.6/flowfile/flowfile/web/static/assets/CrossJoin-68981877.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/CrossJoin-d67e2405.js

@@ -1,11 +1,11 @@
-import { C as CodeLoader } from "./vue-content-loader.es-93bd09d7.js";
-import { u as useNodeStore } from "./vue-codemirror.esm-f15a5f87.js";
-import { s as selectDynamic } from "./selectDynamic-08464729.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-7013cc94.js";
-import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, h as createBlock, u as unref, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport, i as createCommentVNode } from "./index-3a75211d.js";
-import { N as NodeButton, a as NodeTitle } from "./nodeTitle-a63d4680.js";
-import "./UnavailableFields-118f1d20.js";
-import "./designer-94a6bf4d.js";
+import { C as CodeLoader } from "./vue-content-loader.es-0371da73.js";
+import { u as useNodeStore } from "./vue-codemirror.esm-2847001e.js";
+import { s as selectDynamic } from "./selectDynamic-af36165e.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-71e11604.js";
+import { d as defineComponent, r as ref, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, h as createBlock, u as unref, _ as _export_sfc, n as onMounted, R as nextTick, a7 as Teleport, i as createCommentVNode } from "./index-f6c15e76.js";
+import { N as NodeButton, a as NodeTitle } from "./nodeTitle-988d9efe.js";
+import "./UnavailableFields-a1768e52.js";
+import "./designer-13eabd83.js";
 const _hoisted_1$1 = { key: 0 };
 const _hoisted_2 = { class: "listbox-wrapper" };
 const _sfc_main$1 = /* @__PURE__ */ defineComponent({

flowfile-0.3.6/flowfile/flowfile/web/static/assets/DatabaseConnectionSettings-0b06649c.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/DatabaseConnectionSettings-a81e0f7e.js

@@ -1,5 +1,5 @@
-import { f as fetchSecretsApi } from "./secretApi-763aec6e.js";
-import { d as defineComponent, r as ref, n as onMounted, c as openBlock, e as createElementBlock, p as createBaseVNode, t as toDisplayString, F as Fragment, q as renderList, i as createCommentVNode, _ as _export_sfc } from "./index-3a75211d.js";
+import { f as fetchSecretsApi } from "./secretApi-dd636aa2.js";
+import { d as defineComponent, r as ref, n as onMounted, c as openBlock, e as createElementBlock, p as createBaseVNode, t as toDisplayString, F as Fragment, q as renderList, i as createCommentVNode, _ as _export_sfc } from "./index-f6c15e76.js";
 const _hoisted_1 = { class: "connection-settings-container" };
 const _hoisted_2 = { class: "toggle-button" };
 const _hoisted_3 = {

flowfile-0.3.6/flowfile/flowfile/web/static/assets/DatabaseManager-8349a426.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/DatabaseManager-9ea35e84.js

@@ -1,5 +1,5 @@
-import { f as fetchDatabaseConnectionsInterfaces, c as createDatabaseConnectionApi, d as deleteDatabaseConnectionApi } from "./api-602fb95c.js";
-import { d as defineComponent, r as ref, m as watch, l as computed, c as openBlock, e as createElementBlock, p as createBaseVNode, a5 as withDirectives, a6 as vModelText, ag as vModelSelect, ah as vModelDynamic, s as normalizeClass, ai as vModelCheckbox, t as toDisplayString, v as withModifiers, n as onMounted, g as createTextVNode, F as Fragment, q as renderList, f as createVNode, w as withCtx, u as unref, z as ElMessage, aj as ElDialog, ak as ElButton, _ as _export_sfc } from "./index-3a75211d.js";
+import { f as fetchDatabaseConnectionsInterfaces, c as createDatabaseConnectionApi, d as deleteDatabaseConnectionApi } from "./api-a0abbdc7.js";
+import { d as defineComponent, r as ref, m as watch, l as computed, c as openBlock, e as createElementBlock, p as createBaseVNode, a5 as withDirectives, a6 as vModelText, ag as vModelSelect, ah as vModelDynamic, s as normalizeClass, ai as vModelCheckbox, t as toDisplayString, v as withModifiers, n as onMounted, g as createTextVNode, F as Fragment, q as renderList, f as createVNode, w as withCtx, u as unref, z as ElMessage, aj as ElDialog, ak as ElButton, _ as _export_sfc } from "./index-f6c15e76.js";
 const _hoisted_1$1 = { class: "form-grid" };
 const _hoisted_2$1 = { class: "form-field" };
 const _hoisted_3$1 = { class: "form-field" };

flowfile-0.3.6/flowfile/flowfile/web/static/assets/DatabaseReader-905344f8.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/DatabaseReader-9578bfa5.js

@@ -1,12 +1,12 @@
-import { d as defineComponent, c as openBlock, e as createElementBlock, p as createBaseVNode, _ as _export_sfc, r as ref, n as onMounted, b as resolveComponent, f as createVNode, w as withCtx, F as Fragment, q as renderList, h as createBlock, u as unref, al as ElRadio, g as createTextVNode, t as toDisplayString, a5 as withDirectives, ag as vModelSelect, a6 as vModelText, i as createCommentVNode, a as axios, z as ElMessage, R as nextTick, a7 as Teleport } from "./index-3a75211d.js";
-import { C as CodeLoader } from "./vue-content-loader.es-93bd09d7.js";
-import { u as useNodeStore } from "./vue-codemirror.esm-f15a5f87.js";
-import { f as fetchDatabaseConnectionsInterfaces } from "./api-602fb95c.js";
-import { D as DatabaseConnectionSettings } from "./DatabaseConnectionSettings-0b06649c.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-7013cc94.js";
-import { N as NodeButton, a as NodeTitle } from "./nodeTitle-a63d4680.js";
-import "./secretApi-763aec6e.js";
-import "./designer-94a6bf4d.js";
+import { d as defineComponent, c as openBlock, e as createElementBlock, p as createBaseVNode, _ as _export_sfc, r as ref, n as onMounted, b as resolveComponent, f as createVNode, w as withCtx, F as Fragment, q as renderList, h as createBlock, u as unref, al as ElRadio, g as createTextVNode, t as toDisplayString, a5 as withDirectives, ag as vModelSelect, a6 as vModelText, i as createCommentVNode, a as axios, z as ElMessage, R as nextTick, a7 as Teleport } from "./index-f6c15e76.js";
+import { C as CodeLoader } from "./vue-content-loader.es-0371da73.js";
+import { u as useNodeStore } from "./vue-codemirror.esm-2847001e.js";
+import { f as fetchDatabaseConnectionsInterfaces } from "./api-a0abbdc7.js";
+import { D as DatabaseConnectionSettings } from "./DatabaseConnectionSettings-a81e0f7e.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-71e11604.js";
+import { N as NodeButton, a as NodeTitle } from "./nodeTitle-988d9efe.js";
+import "./secretApi-dd636aa2.js";
+import "./designer-13eabd83.js";
 const createNodeDatabaseReader = (flowId, nodeId) => {
   const databaseSettings = {
     query_mode: "table",

flowfile-0.3.6/flowfile/flowfile/web/static/assets/DatabaseWriter-9f5b8638.js → flowfile-0.3.7/flowfile/flowfile/web/static/assets/DatabaseWriter-19531098.js

@@ -1,12 +1,12 @@
-import { d as defineComponent, r as ref, n as onMounted, b as resolveComponent, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, F as Fragment, q as renderList, h as createBlock, u as unref, al as ElRadio, g as createTextVNode, t as toDisplayString, a5 as withDirectives, ag as vModelSelect, a6 as vModelText, z as ElMessage, _ as _export_sfc, R as nextTick, a7 as Teleport, i as createCommentVNode } from "./index-3a75211d.js";
-import { C as CodeLoader } from "./vue-content-loader.es-93bd09d7.js";
-import { u as useNodeStore } from "./vue-codemirror.esm-f15a5f87.js";
-import { f as fetchDatabaseConnectionsInterfaces } from "./api-602fb95c.js";
-import { D as DatabaseConnectionSettings } from "./DatabaseConnectionSettings-0b06649c.js";
-import { G as GenericNodeSettings } from "./genericNodeSettings-7013cc94.js";
-import { N as NodeButton, a as NodeTitle } from "./nodeTitle-a63d4680.js";
-import "./secretApi-763aec6e.js";
-import "./designer-94a6bf4d.js";
+import { d as defineComponent, r as ref, n as onMounted, b as resolveComponent, c as openBlock, e as createElementBlock, f as createVNode, w as withCtx, p as createBaseVNode, F as Fragment, q as renderList, h as createBlock, u as unref, al as ElRadio, g as createTextVNode, t as toDisplayString, a5 as withDirectives, ag as vModelSelect, a6 as vModelText, z as ElMessage, _ as _export_sfc, R as nextTick, a7 as Teleport, i as createCommentVNode } from "./index-f6c15e76.js";
+import { C as CodeLoader } from "./vue-content-loader.es-0371da73.js";
+import { u as useNodeStore } from "./vue-codemirror.esm-2847001e.js";
+import { f as fetchDatabaseConnectionsInterfaces } from "./api-a0abbdc7.js";
+import { D as DatabaseConnectionSettings } from "./DatabaseConnectionSettings-a81e0f7e.js";
+import { G as GenericNodeSettings } from "./genericNodeSettings-71e11604.js";
+import { N as NodeButton, a as NodeTitle } from "./nodeTitle-988d9efe.js";
+import "./secretApi-dd636aa2.js";
+import "./designer-13eabd83.js";
 const createNodeDatabaseWriter = (flowId, nodeId) => {
   const databaseWriteSettings = {
     if_exists: "replace",