Flowfile 0.3.4.1__py3-none-any.whl → 0.3.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of Flowfile might be problematic. Click here for more details.

Files changed (122) hide show
  1. flowfile/__init__.py +3 -3
  2. flowfile/api.py +36 -15
  3. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
  4. flowfile/web/static/assets/CloudConnectionManager-d004942f.js +784 -0
  5. flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
  6. flowfile/web/static/assets/CloudStorageReader-eccf9fc2.js +437 -0
  7. flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
  8. flowfile/web/static/assets/CloudStorageWriter-b1ba6bba.js +430 -0
  9. flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-68981877.js} +8 -8
  10. flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-0b06649c.js} +2 -2
  11. flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-8349a426.js} +2 -2
  12. flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-905344f8.js} +9 -9
  13. flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-9f5b8638.js} +9 -9
  14. flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-131a6d53.js} +5 -5
  15. flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-e3549dcc.js} +6 -6
  16. flowfile/web/static/assets/{Filter-f87bb897.js → Filter-6e0730ae.js} +8 -8
  17. flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-02f033e6.js} +75 -9
  18. flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
  19. flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-54c14036.js} +9 -9
  20. flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-08a3f499.js} +5 -5
  21. flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-2ae38139.js} +6 -6
  22. flowfile/web/static/assets/{Join-eec38203.js → Join-493b9772.js} +23 -15
  23. flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
  24. flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-4373d163.js} +106 -34
  25. flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
  26. flowfile/web/static/assets/{Output-3b2ca045.js → Output-b534f3c7.js} +4 -4
  27. flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-2968ff65.js} +6 -6
  28. flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-65136536.js} +6 -6
  29. flowfile/web/static/assets/{Read-07acdc9a.js → Read-c56339ed.js} +6 -6
  30. flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-1c641a5e.js} +5 -5
  31. flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-df308b8f.js} +6 -6
  32. flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-293e8a64.js} +5 -5
  33. flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-03911655.js} +2 -2
  34. flowfile/web/static/assets/{Select-32b28406.js → Select-3058a13d.js} +8 -8
  35. flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-fbf4fb39.js} +1 -1
  36. flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-a29bbaf7.js} +6 -6
  37. flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-c7d7760e.js} +8 -8
  38. flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-118f1d20.js} +2 -2
  39. flowfile/web/static/assets/{Union-39eecc6c.js → Union-f0589571.js} +5 -5
  40. flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-7329a207.js} +8 -8
  41. flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-30b0be15.js} +5 -5
  42. flowfile/web/static/assets/{api-44ca9e9c.js → api-602fb95c.js} +1 -1
  43. flowfile/web/static/assets/api-fb67319c.js +80 -0
  44. flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
  45. flowfile/web/static/assets/{designer-267d44f1.js → designer-94a6bf4d.js} +36 -34
  46. flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-a224831e.js} +1 -1
  47. flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-c2d2aa97.js} +1 -1
  48. flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-921ac5fd.js} +2 -2
  49. flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-7013cc94.js} +3 -3
  50. flowfile/web/static/assets/{index-e235a8bc.js → index-3a75211d.js} +19 -6
  51. flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-a63d4680.js} +3 -3
  52. flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-763aec6e.js} +1 -1
  53. flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-08464729.js} +3 -3
  54. flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-f15a5f87.js} +2 -1
  55. flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-93bd09d7.js} +1 -1
  56. flowfile/web/static/index.html +1 -1
  57. {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/METADATA +8 -3
  58. {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/RECORD +109 -104
  59. {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/entry_points.txt +2 -0
  60. flowfile_core/__init__.py +2 -0
  61. flowfile_core/configs/node_store/nodes.py +8 -6
  62. flowfile_core/database/connection.py +63 -15
  63. flowfile_core/database/init_db.py +0 -1
  64. flowfile_core/database/models.py +49 -2
  65. flowfile_core/flowfile/code_generator/code_generator.py +402 -18
  66. flowfile_core/flowfile/connection_manager/models.py +1 -1
  67. flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
  68. flowfile_core/flowfile/extensions.py +1 -1
  69. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
  70. flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
  71. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +522 -59
  72. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
  73. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
  74. flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
  75. flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
  76. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
  77. flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
  78. flowfile_core/flowfile/flow_graph.py +119 -82
  79. flowfile_core/flowfile/flow_node/flow_node.py +68 -33
  80. flowfile_core/flowfile/flow_node/models.py +32 -3
  81. flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
  82. flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
  83. flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
  84. flowfile_core/flowfile/utils.py +1 -23
  85. flowfile_core/main.py +3 -2
  86. flowfile_core/routes/cloud_connections.py +81 -0
  87. flowfile_core/routes/logs.py +0 -1
  88. flowfile_core/routes/routes.py +3 -39
  89. flowfile_core/schemas/cloud_storage_schemas.py +215 -0
  90. flowfile_core/schemas/input_schema.py +37 -15
  91. flowfile_core/schemas/schemas.py +7 -2
  92. flowfile_core/schemas/transform_schema.py +97 -22
  93. flowfile_core/utils/utils.py +40 -1
  94. flowfile_core/utils/validate_setup.py +41 -0
  95. flowfile_frame/flow_frame.py +253 -102
  96. flowfile_frame/flow_frame_methods.py +13 -13
  97. flowfile_worker/external_sources/s3_source/main.py +216 -0
  98. flowfile_worker/external_sources/s3_source/models.py +142 -0
  99. flowfile_worker/funcs.py +51 -6
  100. flowfile_worker/models.py +22 -2
  101. flowfile_worker/routes.py +40 -38
  102. flowfile_worker/utils.py +1 -1
  103. test_utils/s3/commands.py +46 -0
  104. test_utils/s3/data_generator.py +291 -0
  105. test_utils/s3/fixtures.py +209 -0
  106. flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
  107. flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
  108. flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
  109. flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
  110. flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
  111. flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
  112. flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
  113. flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
  114. flowfile_worker/external_sources/airbyte_sources/__init__.py +0 -0
  115. flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
  116. flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
  117. flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
  118. flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
  119. {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/LICENSE +0 -0
  120. {flowfile-0.3.4.1.dist-info → flowfile-0.3.6.dist-info}/WHEEL +0 -0
  121. {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
  122. {flowfile_core/schemas/external_sources → test_utils/s3}/__init__.py +0 -0
flowfile/__init__.py CHANGED
@@ -7,7 +7,7 @@ This package ties together the FlowFile ecosystem components:
7
7
  - flowfile_worker: Computation engine
8
8
  """
9
9
 
10
- __version__ = "0.3.3.2"
10
+ __version__ = "0.3.5"
11
11
 
12
12
  import os
13
13
  import logging
@@ -20,7 +20,7 @@ from flowfile.api import (open_graph_in_editor)
20
20
  from flowfile_frame.flow_frame import (
21
21
  FlowFrame
22
22
  )
23
- from flowfile_frame import read_csv, read_parquet, from_dict, concat
23
+ from flowfile_frame import read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet
24
24
  from flowfile_frame.expr import (
25
25
  col, lit, column, cum_count, len,
26
26
  sum, min, max, mean, count, when
@@ -50,7 +50,7 @@ __all__ = [
50
50
  'FlowFrame', 'GroupByFrame',
51
51
 
52
52
  # Main creation functions
53
- 'read_csv', 'read_parquet', 'from_dict', 'concat',
53
+ 'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet',
54
54
 
55
55
  # Expression API
56
56
  'col', 'lit', 'column', 'cum_count', 'len',
flowfile/api.py CHANGED
@@ -147,9 +147,6 @@ def build_server_command(module_name: str) -> List[str]:
147
147
  Build the appropriate command to start the server based on environment detection.
148
148
  Tries Poetry first if in a Poetry environment, falls back to direct module execution.
149
149
  """
150
- command: List[str] = []
151
-
152
- # Case 1: Check if we're in a Poetry environment
153
150
  if is_poetry_environment():
154
151
  logger.info("Poetry environment detected.")
155
152
  if is_command_available(POETRY_PATH):
@@ -166,17 +163,42 @@ def build_server_command(module_name: str) -> List[str]:
166
163
  else:
167
164
  logger.warning(f"Poetry command not found at '{POETRY_PATH}'. Falling back to Python module.")
168
165
 
169
- # Case 2: Try direct module execution
170
- logger.info(f"Using Python module approach with {module_name}")
171
- command = [
172
- sys.executable,
173
- "-m",
174
- module_name,
175
- "run",
176
- "ui",
177
- "--no-browser",
178
- ]
166
+ # Case 2: Fallback to direct script execution
167
+ logger.info("Falling back to direct script execution.")
168
+ python_parent_dir = Path(sys.executable).parent
169
+ command: List[str]
170
+ scripts_dir = Path(sys.executable).parent
171
+
172
+ if platform.system() == "Windows":
173
+ exe_path = scripts_dir / f"{module_name}.exe"
174
+ script_py_path = scripts_dir / f"{module_name}-script.py"
175
+ plain_script_path = scripts_dir / module_name
176
+
177
+ if exe_path.exists():
178
+ logger.info(f"Using .exe wrapper: {exe_path}")
179
+ command = [str(exe_path), "run", "ui", "--no-browser"]
180
+ elif script_py_path.exists():
181
+ logger.info(f"Using '-script.py' with interpreter: {script_py_path}")
182
+ command = [sys.executable, str(script_py_path), "run", "ui", "--no-browser"]
183
+ elif plain_script_path.exists():
184
+ logger.info(f"Using plain script with interpreter: {plain_script_path}")
185
+ command = [sys.executable, str(plain_script_path), "run", "ui", "--no-browser"]
186
+ else:
187
+ raise FileNotFoundError(
188
+ f"Could not find an executable script for '{module_name}' in '{scripts_dir}'. "
189
+ f"Checked for '{exe_path.name}', '{script_py_path.name}', and '{plain_script_path.name}'. "
190
+ "Ensure the package is installed correctly."
191
+ )
192
+ else:
193
+ # On Unix-like systems, the script in 'bin' is directly executable
194
+ script_path = python_parent_dir / "bin" / module_name
195
+ if not script_path.exists():
196
+ script_path = python_parent_dir / module_name # Fallback for different venv structures
197
+
198
+ logger.info(f"Using direct script execution path: {script_path}")
199
+ command = [str(script_path), "run", "ui", "--no-browser"]
179
200
 
201
+ logger.info(f"Built server command: {command}")
180
202
  return command
181
203
 
182
204
 
@@ -210,7 +232,6 @@ def start_flowfile_server_process(module_name: str = DEFAULT_MODULE_NAME) -> Tup
210
232
  # Build command automatically based on environment detection
211
233
  command = build_server_command(module_name)
212
234
  logger.info(f"Starting server with command: {' '.join(command)}")
213
-
214
235
  try:
215
236
  # Windows-specific subprocess creation
216
237
  if platform.system() == "Windows":
@@ -244,7 +265,7 @@ def start_flowfile_server_process(module_name: str = DEFAULT_MODULE_NAME) -> Tup
244
265
  time.sleep(1)
245
266
  else:
246
267
  logger.error("Failed to start server: API did not become responsive within 60 seconds. "
247
- "Try again or try start service by running\n"
268
+ "Try again or start service by running\n"
248
269
  "flowfile run ui")
249
270
  if _server_process and _server_process.stderr:
250
271
  try:
@@ -0,0 +1,86 @@
1
+
2
+ .description-text[data-v-23359c71] {
3
+ color: #6c757d;
4
+ margin-top: 0.5rem;
5
+ font-size: 0.95rem;
6
+ }
7
+ .info-box[data-v-23359c71] {
8
+ display: flex;
9
+ gap: 1rem;
10
+ padding: 1rem;
11
+ background-color: #f8f9fa;
12
+ border-left: 4px solid #17a2b8;
13
+ border-radius: 4px;
14
+ }
15
+ .info-box i[data-v-23359c71] {
16
+ color: #17a2b8;
17
+ font-size: 1.5rem;
18
+ margin-top: 0.5rem;
19
+ }
20
+ .info-box p[data-v-23359c71] {
21
+ margin: 0;
22
+ margin-bottom: 0.5rem;
23
+ }
24
+ .info-box p[data-v-23359c71]:last-child {
25
+ margin-bottom: 0;
26
+ }
27
+ .modal-description[data-v-23359c71] {
28
+ color: #6c757d;
29
+ font-size: 0.9rem;
30
+ }
31
+ .badge[data-v-23359c71] {
32
+ background-color: #e9ecef;
33
+ border-radius: 1rem;
34
+ padding: 0.25rem 0.75rem;
35
+ font-size: 0.75rem;
36
+ margin-left: 0.5rem;
37
+ }
38
+ .auth-badge[data-v-23359c71] {
39
+ background-color: #d1ecf1;
40
+ color: #0c5460;
41
+ }
42
+ .warning-text[data-v-23359c71] {
43
+ color: #dc3545;
44
+ font-size: 0.875rem;
45
+ margin-top: 0.5rem;
46
+ }
47
+ .dialog-footer[data-v-23359c71] {
48
+ display: flex;
49
+ justify-content: flex-end;
50
+ gap: 0.5rem;
51
+ margin-top: 1rem;
52
+ }
53
+ .connection-details[data-v-23359c71] {
54
+ font-size: 0.85rem;
55
+ color: #6c757d;
56
+ margin-top: 0.25rem;
57
+ }
58
+ .separator[data-v-23359c71] {
59
+ margin: 0 0.5rem;
60
+ }
61
+ .hint-text[data-v-23359c71] {
62
+ color: #6c757d;
63
+ font-size: 0.875rem;
64
+ margin-top: 0.5rem;
65
+ }
66
+ .mb-3[data-v-23359c71] {
67
+ margin-bottom: 1rem;
68
+ }
69
+ .flex-col[data-v-23359c71] {
70
+ display: flex;
71
+ flex-direction: column;
72
+ }
73
+ .gap-2[data-v-23359c71] {
74
+ gap: 0.5rem;
75
+ }
76
+
77
+ /* Cloud-specific styles */
78
+ .fa-brands[data-v-23359c71] {
79
+ font-size: 1.2rem;
80
+ }
81
+ .fa-aws[data-v-23359c71] {
82
+ color: #ff9900;
83
+ }
84
+ .fa-microsoft[data-v-23359c71] {
85
+ color: #0078d4;
86
+ }