Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of Flowfile might be problematic. Click here for more details.

Files changed (145) hide show
  1. flowfile/__init__.py +27 -6
  2. flowfile/api.py +1 -0
  3. flowfile/web/__init__.py +2 -2
  4. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
  5. flowfile/web/static/assets/CloudConnectionManager-c20a740f.js +783 -0
  6. flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
  7. flowfile/web/static/assets/CloudStorageReader-960b400a.js +437 -0
  8. flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
  9. flowfile/web/static/assets/CloudStorageWriter-e3decbdd.js +430 -0
  10. flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-d67e2405.js} +8 -8
  11. flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-a81e0f7e.js} +2 -2
  12. flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-9ea35e84.js} +2 -2
  13. flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-9578bfa5.js} +9 -9
  14. flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-19531098.js} +9 -9
  15. flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-40476474.js} +47141 -43697
  16. flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-2297ef96.js} +6 -6
  17. flowfile/web/static/assets/{Filter-f87bb897.js → Filter-f211c03a.js} +8 -8
  18. flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
  19. flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-4207ea31.js} +75 -9
  20. flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-bf120df0.js} +9 -9
  21. flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-5bb7497a.js} +5 -5
  22. flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-92c81b65.js} +6 -6
  23. flowfile/web/static/assets/{Join-eec38203.js → Join-4e49a274.js} +23 -15
  24. flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
  25. flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-90998ae8.js} +106 -34
  26. flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
  27. flowfile/web/static/assets/{Output-3b2ca045.js → Output-81e3e917.js} +4 -4
  28. flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-a3419842.js} +6 -6
  29. flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-72710deb.js} +6 -6
  30. flowfile/web/static/assets/{Read-07acdc9a.js → Read-c4059daf.js} +6 -6
  31. flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-c2b5e095.js} +5 -5
  32. flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-10baf191.js} +6 -6
  33. flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-3ed9a0ae.js} +5 -5
  34. flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-0d49c0e8.js} +2 -2
  35. flowfile/web/static/assets/{Select-32b28406.js → Select-8a02a0b3.js} +8 -8
  36. flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-4c0f45f5.js} +1 -1
  37. flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-f55c9f9d.js} +6 -6
  38. flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-5dbc2145.js} +8 -8
  39. flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-a1768e52.js} +2 -2
  40. flowfile/web/static/assets/{Union-39eecc6c.js → Union-f2aefdc9.js} +5 -5
  41. flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-46b250da.js} +8 -8
  42. flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-25ac84cc.js} +5 -5
  43. flowfile/web/static/assets/api-6ef0dcef.js +80 -0
  44. flowfile/web/static/assets/{api-44ca9e9c.js → api-a0abbdc7.js} +1 -1
  45. flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
  46. flowfile/web/static/assets/{designer-267d44f1.js → designer-13eabd83.js} +36 -34
  47. flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-b87e7f6f.js} +1 -1
  48. flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-13564764.js} +1 -1
  49. flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-fd2cd6f9.js} +2 -2
  50. flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-71e11604.js} +3 -3
  51. flowfile/web/static/assets/{index-e235a8bc.js → index-f6c15e76.js} +59 -22
  52. flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-988d9efe.js} +3 -3
  53. flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-dd636aa2.js} +1 -1
  54. flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-af36165e.js} +3 -3
  55. flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-2847001e.js} +2 -1
  56. flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-0371da73.js} +1 -1
  57. flowfile/web/static/index.html +1 -1
  58. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/METADATA +9 -4
  59. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/RECORD +131 -124
  60. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/entry_points.txt +2 -0
  61. flowfile_core/__init__.py +3 -0
  62. flowfile_core/auth/jwt.py +39 -0
  63. flowfile_core/configs/node_store/nodes.py +9 -6
  64. flowfile_core/configs/settings.py +6 -5
  65. flowfile_core/database/connection.py +63 -15
  66. flowfile_core/database/init_db.py +0 -1
  67. flowfile_core/database/models.py +49 -2
  68. flowfile_core/flowfile/code_generator/code_generator.py +472 -17
  69. flowfile_core/flowfile/connection_manager/models.py +1 -1
  70. flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
  71. flowfile_core/flowfile/extensions.py +1 -1
  72. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
  73. flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
  74. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +1062 -311
  75. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
  76. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
  77. flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
  78. flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
  79. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +3 -1
  80. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
  81. flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
  82. flowfile_core/flowfile/flow_graph.py +718 -253
  83. flowfile_core/flowfile/flow_graph_utils.py +2 -2
  84. flowfile_core/flowfile/flow_node/flow_node.py +563 -117
  85. flowfile_core/flowfile/flow_node/models.py +154 -20
  86. flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
  87. flowfile_core/flowfile/handler.py +2 -33
  88. flowfile_core/flowfile/manage/open_flowfile.py +1 -2
  89. flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
  90. flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
  91. flowfile_core/flowfile/util/calculate_layout.py +0 -2
  92. flowfile_core/flowfile/utils.py +35 -26
  93. flowfile_core/main.py +35 -15
  94. flowfile_core/routes/cloud_connections.py +77 -0
  95. flowfile_core/routes/logs.py +2 -7
  96. flowfile_core/routes/public.py +1 -0
  97. flowfile_core/routes/routes.py +130 -90
  98. flowfile_core/routes/secrets.py +72 -14
  99. flowfile_core/schemas/__init__.py +8 -0
  100. flowfile_core/schemas/cloud_storage_schemas.py +215 -0
  101. flowfile_core/schemas/input_schema.py +121 -71
  102. flowfile_core/schemas/output_model.py +19 -3
  103. flowfile_core/schemas/schemas.py +150 -12
  104. flowfile_core/schemas/transform_schema.py +175 -35
  105. flowfile_core/utils/utils.py +40 -1
  106. flowfile_core/utils/validate_setup.py +41 -0
  107. flowfile_frame/__init__.py +9 -1
  108. flowfile_frame/cloud_storage/frame_helpers.py +39 -0
  109. flowfile_frame/cloud_storage/secret_manager.py +73 -0
  110. flowfile_frame/expr.py +28 -1
  111. flowfile_frame/expr.pyi +76 -61
  112. flowfile_frame/flow_frame.py +481 -208
  113. flowfile_frame/flow_frame.pyi +140 -91
  114. flowfile_frame/flow_frame_methods.py +160 -22
  115. flowfile_frame/group_frame.py +3 -0
  116. flowfile_frame/utils.py +25 -3
  117. flowfile_worker/external_sources/s3_source/main.py +216 -0
  118. flowfile_worker/external_sources/s3_source/models.py +142 -0
  119. flowfile_worker/funcs.py +51 -6
  120. flowfile_worker/models.py +22 -2
  121. flowfile_worker/routes.py +40 -38
  122. flowfile_worker/utils.py +1 -1
  123. test_utils/s3/commands.py +46 -0
  124. test_utils/s3/data_generator.py +292 -0
  125. test_utils/s3/demo_data_generator.py +186 -0
  126. test_utils/s3/fixtures.py +214 -0
  127. flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
  128. flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
  129. flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
  130. flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
  131. flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
  132. flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
  133. flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
  134. flowfile_core/schemas/defaults.py +0 -9
  135. flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
  136. flowfile_core/schemas/models.py +0 -193
  137. flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
  138. flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
  139. flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
  140. flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
  141. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/LICENSE +0 -0
  142. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/WHEEL +0 -0
  143. {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_frame/cloud_storage}/__init__.py +0 -0
  144. {flowfile_core/schemas/external_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
  145. {flowfile_worker/external_sources/airbyte_sources → test_utils/s3}/__init__.py +0 -0
flowfile/__init__.py CHANGED
@@ -7,20 +7,36 @@ This package ties together the FlowFile ecosystem components:
7
7
  - flowfile_worker: Computation engine
8
8
  """
9
9
 
10
- __version__ = "0.3.3.2"
10
+ __version__ = "0.3.7"
11
11
 
12
12
  import os
13
13
  import logging
14
14
 
15
- os.environ['WORKER_PORT'] = "63578"
16
- os.environ['SINGLE_FILE_MODE'] = "1"
15
+ os.environ['FLOWFILE_WORKER_PORT'] = "63578"
16
+ os.environ['FLOWFILE_SINGLE_FILE_MODE'] = "1"
17
17
 
18
18
  from flowfile.web import start_server as start_web_ui
19
19
  from flowfile.api import (open_graph_in_editor)
20
20
  from flowfile_frame.flow_frame import (
21
21
  FlowFrame
22
22
  )
23
- from flowfile_frame import read_csv, read_parquet, from_dict, concat
23
+ from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection
24
+ from flowfile_core.flowfile.flow_graph import FlowGraph
25
+ from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine
26
+ from flowfile_core.flowfile.flow_node.flow_node import FlowNode
27
+ from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
28
+ from flowfile_core.schemas import input_schema as node_interface
29
+ from flowfile_core.schemas.schemas import FlowSettings, FlowInformation
30
+ from flowfile_core.schemas import transform_schema
31
+ from flowfile_frame import (read_csv, read_parquet, from_dict, concat, scan_csv, scan_parquet,
32
+ scan_delta,
33
+ scan_parquet_from_cloud_storage,
34
+ scan_json_from_cloud_storage,
35
+ scan_csv_from_cloud_storage,
36
+ get_all_available_cloud_storage_connections,
37
+ create_cloud_storage_connection,
38
+ del_cloud_storage_connection,
39
+ create_cloud_storage_connection_if_not_exists)
24
40
  from flowfile_frame.expr import (
25
41
  col, lit, column, cum_count, len,
26
42
  sum, min, max, mean, count, when
@@ -47,10 +63,15 @@ from polars.datatypes import (
47
63
 
48
64
  __all__ = [
49
65
  # Core FlowFrame classes
50
- 'FlowFrame', 'GroupByFrame',
66
+ 'FlowFrame', 'GroupByFrame', 'FullCloudStorageConnection',
51
67
 
52
68
  # Main creation functions
53
- 'read_csv', 'read_parquet', 'from_dict', 'concat',
69
+ 'read_csv', 'read_parquet', 'from_dict', 'concat', 'scan_csv', 'scan_parquet', 'scan_delta',
70
+ 'scan_parquet_from_cloud_storage', 'scan_json_from_cloud_storage',
71
+ 'scan_csv_from_cloud_storage', 'get_all_available_cloud_storage_connections', 'create_cloud_storage_connection',
72
+ 'del_cloud_storage_connection', 'create_cloud_storage_connection_if_not_exists',
73
+ 'FlowGraph', 'FlowDataEngine', 'node_interface', 'FlowSettings', 'transform_schema',
74
+ 'FlowNode', 'FlowfileColumn', 'FlowInformation',
54
75
 
55
76
  # Expression API
56
77
  'col', 'lit', 'column', 'cum_count', 'len',
flowfile/api.py CHANGED
@@ -456,6 +456,7 @@ def open_graph_in_editor(flow_graph: FlowGraph, storage_location: Optional[str]
456
456
  if flow_id is not None:
457
457
  if flow_in_single_mode and automatically_open_browser:
458
458
  _open_flow_in_browser(flow_id)
459
+ logger.info(f"Ui available at: http://127.0.0.1:63578/ui#/main/designer")
459
460
  return True
460
461
  else:
461
462
  return False
flowfile/web/__init__.py CHANGED
@@ -50,8 +50,8 @@ def extend_app(app: FastAPI):
50
50
  return Response(status_code=404)
51
51
 
52
52
  @app.get("/single_mode")
53
- async def in_single_mode():
54
- return os.environ.get('SINGLE_FILE_MODE', "0")
53
+ async def in_single_mode() -> bool:
54
+ return os.environ.get('FLOWFILE_SINGLE_FILE_MODE', "0") == "1"
55
55
 
56
56
  @app.get("/ui", include_in_schema=False)
57
57
  async def web_ui_root():
@@ -0,0 +1,86 @@
1
+
2
+ .description-text[data-v-23359c71] {
3
+ color: #6c757d;
4
+ margin-top: 0.5rem;
5
+ font-size: 0.95rem;
6
+ }
7
+ .info-box[data-v-23359c71] {
8
+ display: flex;
9
+ gap: 1rem;
10
+ padding: 1rem;
11
+ background-color: #f8f9fa;
12
+ border-left: 4px solid #17a2b8;
13
+ border-radius: 4px;
14
+ }
15
+ .info-box i[data-v-23359c71] {
16
+ color: #17a2b8;
17
+ font-size: 1.5rem;
18
+ margin-top: 0.5rem;
19
+ }
20
+ .info-box p[data-v-23359c71] {
21
+ margin: 0;
22
+ margin-bottom: 0.5rem;
23
+ }
24
+ .info-box p[data-v-23359c71]:last-child {
25
+ margin-bottom: 0;
26
+ }
27
+ .modal-description[data-v-23359c71] {
28
+ color: #6c757d;
29
+ font-size: 0.9rem;
30
+ }
31
+ .badge[data-v-23359c71] {
32
+ background-color: #e9ecef;
33
+ border-radius: 1rem;
34
+ padding: 0.25rem 0.75rem;
35
+ font-size: 0.75rem;
36
+ margin-left: 0.5rem;
37
+ }
38
+ .auth-badge[data-v-23359c71] {
39
+ background-color: #d1ecf1;
40
+ color: #0c5460;
41
+ }
42
+ .warning-text[data-v-23359c71] {
43
+ color: #dc3545;
44
+ font-size: 0.875rem;
45
+ margin-top: 0.5rem;
46
+ }
47
+ .dialog-footer[data-v-23359c71] {
48
+ display: flex;
49
+ justify-content: flex-end;
50
+ gap: 0.5rem;
51
+ margin-top: 1rem;
52
+ }
53
+ .connection-details[data-v-23359c71] {
54
+ font-size: 0.85rem;
55
+ color: #6c757d;
56
+ margin-top: 0.25rem;
57
+ }
58
+ .separator[data-v-23359c71] {
59
+ margin: 0 0.5rem;
60
+ }
61
+ .hint-text[data-v-23359c71] {
62
+ color: #6c757d;
63
+ font-size: 0.875rem;
64
+ margin-top: 0.5rem;
65
+ }
66
+ .mb-3[data-v-23359c71] {
67
+ margin-bottom: 1rem;
68
+ }
69
+ .flex-col[data-v-23359c71] {
70
+ display: flex;
71
+ flex-direction: column;
72
+ }
73
+ .gap-2[data-v-23359c71] {
74
+ gap: 0.5rem;
75
+ }
76
+
77
+ /* Cloud-specific styles */
78
+ .fa-brands[data-v-23359c71] {
79
+ font-size: 1.2rem;
80
+ }
81
+ .fa-aws[data-v-23359c71] {
82
+ color: #ff9900;
83
+ }
84
+ .fa-microsoft[data-v-23359c71] {
85
+ color: #0078d4;
86
+ }