Flowfile 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of Flowfile might be problematic. Click here for more details.

Files changed (145) hide show
  1. flowfile/__init__.py +27 -6
  2. flowfile/api.py +1 -0
  3. flowfile/web/__init__.py +2 -2
  4. flowfile/web/static/assets/CloudConnectionManager-2dfdce2f.css +86 -0
  5. flowfile/web/static/assets/CloudConnectionManager-c20a740f.js +783 -0
  6. flowfile/web/static/assets/CloudStorageReader-29d14fcc.css +143 -0
  7. flowfile/web/static/assets/CloudStorageReader-960b400a.js +437 -0
  8. flowfile/web/static/assets/CloudStorageWriter-49c9a4b2.css +138 -0
  9. flowfile/web/static/assets/CloudStorageWriter-e3decbdd.js +430 -0
  10. flowfile/web/static/assets/{CrossJoin-dfcf7351.js → CrossJoin-d67e2405.js} +8 -8
  11. flowfile/web/static/assets/{DatabaseConnectionSettings-b2afb1d7.js → DatabaseConnectionSettings-a81e0f7e.js} +2 -2
  12. flowfile/web/static/assets/{DatabaseManager-824a49b2.js → DatabaseManager-9ea35e84.js} +2 -2
  13. flowfile/web/static/assets/{DatabaseReader-a48124d8.js → DatabaseReader-9578bfa5.js} +9 -9
  14. flowfile/web/static/assets/{DatabaseWriter-b47cbae2.js → DatabaseWriter-19531098.js} +9 -9
  15. flowfile/web/static/assets/{ExploreData-fdfc45a4.js → ExploreData-40476474.js} +47141 -43697
  16. flowfile/web/static/assets/{ExternalSource-861b0e71.js → ExternalSource-2297ef96.js} +6 -6
  17. flowfile/web/static/assets/{Filter-f87bb897.js → Filter-f211c03a.js} +8 -8
  18. flowfile/web/static/assets/{Formula-b8cefc31.css → Formula-29f19d21.css} +10 -0
  19. flowfile/web/static/assets/{Formula-1e2ed720.js → Formula-4207ea31.js} +75 -9
  20. flowfile/web/static/assets/{FuzzyMatch-b6cc4fdd.js → FuzzyMatch-bf120df0.js} +9 -9
  21. flowfile/web/static/assets/{GraphSolver-6a371f4c.js → GraphSolver-5bb7497a.js} +5 -5
  22. flowfile/web/static/assets/{GroupBy-f7b7f472.js → GroupBy-92c81b65.js} +6 -6
  23. flowfile/web/static/assets/{Join-eec38203.js → Join-4e49a274.js} +23 -15
  24. flowfile/web/static/assets/{Join-41c0f331.css → Join-f45eff22.css} +20 -20
  25. flowfile/web/static/assets/{ManualInput-9aaa46fb.js → ManualInput-90998ae8.js} +106 -34
  26. flowfile/web/static/assets/{ManualInput-ac7b9972.css → ManualInput-a71b52c6.css} +29 -17
  27. flowfile/web/static/assets/{Output-3b2ca045.js → Output-81e3e917.js} +4 -4
  28. flowfile/web/static/assets/{Pivot-a4f5d88f.js → Pivot-a3419842.js} +6 -6
  29. flowfile/web/static/assets/{PolarsCode-49ce444f.js → PolarsCode-72710deb.js} +6 -6
  30. flowfile/web/static/assets/{Read-07acdc9a.js → Read-c4059daf.js} +6 -6
  31. flowfile/web/static/assets/{RecordCount-6a21da56.js → RecordCount-c2b5e095.js} +5 -5
  32. flowfile/web/static/assets/{RecordId-949bdc17.js → RecordId-10baf191.js} +6 -6
  33. flowfile/web/static/assets/{Sample-7afca6e1.js → Sample-3ed9a0ae.js} +5 -5
  34. flowfile/web/static/assets/{SecretManager-b41c029d.js → SecretManager-0d49c0e8.js} +2 -2
  35. flowfile/web/static/assets/{Select-32b28406.js → Select-8a02a0b3.js} +8 -8
  36. flowfile/web/static/assets/{SettingsSection-a0f15a05.js → SettingsSection-4c0f45f5.js} +1 -1
  37. flowfile/web/static/assets/{Sort-fc6ba0e2.js → Sort-f55c9f9d.js} +6 -6
  38. flowfile/web/static/assets/{TextToRows-23127596.js → TextToRows-5dbc2145.js} +8 -8
  39. flowfile/web/static/assets/{UnavailableFields-c42880a3.js → UnavailableFields-a1768e52.js} +2 -2
  40. flowfile/web/static/assets/{Union-39eecc6c.js → Union-f2aefdc9.js} +5 -5
  41. flowfile/web/static/assets/{Unique-a0e8fe61.js → Unique-46b250da.js} +8 -8
  42. flowfile/web/static/assets/{Unpivot-1e2d43f0.js → Unpivot-25ac84cc.js} +5 -5
  43. flowfile/web/static/assets/api-6ef0dcef.js +80 -0
  44. flowfile/web/static/assets/{api-44ca9e9c.js → api-a0abbdc7.js} +1 -1
  45. flowfile/web/static/assets/cloud_storage_reader-aa1415d6.png +0 -0
  46. flowfile/web/static/assets/{designer-267d44f1.js → designer-13eabd83.js} +36 -34
  47. flowfile/web/static/assets/{documentation-6c0810a2.js → documentation-b87e7f6f.js} +1 -1
  48. flowfile/web/static/assets/{dropDown-52790b15.js → dropDown-13564764.js} +1 -1
  49. flowfile/web/static/assets/{fullEditor-e272b506.js → fullEditor-fd2cd6f9.js} +2 -2
  50. flowfile/web/static/assets/{genericNodeSettings-4bdcf98e.js → genericNodeSettings-71e11604.js} +3 -3
  51. flowfile/web/static/assets/{index-e235a8bc.js → index-f6c15e76.js} +59 -22
  52. flowfile/web/static/assets/{nodeTitle-fc3fc4b7.js → nodeTitle-988d9efe.js} +3 -3
  53. flowfile/web/static/assets/{secretApi-cdc2a3fd.js → secretApi-dd636aa2.js} +1 -1
  54. flowfile/web/static/assets/{selectDynamic-96aa82cd.js → selectDynamic-af36165e.js} +3 -3
  55. flowfile/web/static/assets/{vue-codemirror.esm-25e75a08.js → vue-codemirror.esm-2847001e.js} +2 -1
  56. flowfile/web/static/assets/{vue-content-loader.es-6c4b1c24.js → vue-content-loader.es-0371da73.js} +1 -1
  57. flowfile/web/static/index.html +1 -1
  58. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/METADATA +9 -4
  59. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/RECORD +131 -124
  60. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/entry_points.txt +2 -0
  61. flowfile_core/__init__.py +3 -0
  62. flowfile_core/auth/jwt.py +39 -0
  63. flowfile_core/configs/node_store/nodes.py +9 -6
  64. flowfile_core/configs/settings.py +6 -5
  65. flowfile_core/database/connection.py +63 -15
  66. flowfile_core/database/init_db.py +0 -1
  67. flowfile_core/database/models.py +49 -2
  68. flowfile_core/flowfile/code_generator/code_generator.py +472 -17
  69. flowfile_core/flowfile/connection_manager/models.py +1 -1
  70. flowfile_core/flowfile/database_connection_manager/db_connections.py +216 -2
  71. flowfile_core/flowfile/extensions.py +1 -1
  72. flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +259 -0
  73. flowfile_core/flowfile/flow_data_engine/create/funcs.py +19 -8
  74. flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +1062 -311
  75. flowfile_core/flowfile/flow_data_engine/flow_file_column/main.py +12 -2
  76. flowfile_core/flowfile/flow_data_engine/fuzzy_matching/settings_validator.py +1 -1
  77. flowfile_core/flowfile/flow_data_engine/join/__init__.py +2 -1
  78. flowfile_core/flowfile/flow_data_engine/join/utils.py +25 -0
  79. flowfile_core/flowfile/flow_data_engine/polars_code_parser.py +3 -1
  80. flowfile_core/flowfile/flow_data_engine/subprocess_operations/subprocess_operations.py +29 -22
  81. flowfile_core/flowfile/flow_data_engine/utils.py +1 -40
  82. flowfile_core/flowfile/flow_graph.py +718 -253
  83. flowfile_core/flowfile/flow_graph_utils.py +2 -2
  84. flowfile_core/flowfile/flow_node/flow_node.py +563 -117
  85. flowfile_core/flowfile/flow_node/models.py +154 -20
  86. flowfile_core/flowfile/flow_node/schema_callback.py +3 -2
  87. flowfile_core/flowfile/handler.py +2 -33
  88. flowfile_core/flowfile/manage/open_flowfile.py +1 -2
  89. flowfile_core/flowfile/sources/external_sources/__init__.py +0 -2
  90. flowfile_core/flowfile/sources/external_sources/factory.py +4 -7
  91. flowfile_core/flowfile/util/calculate_layout.py +0 -2
  92. flowfile_core/flowfile/utils.py +35 -26
  93. flowfile_core/main.py +35 -15
  94. flowfile_core/routes/cloud_connections.py +77 -0
  95. flowfile_core/routes/logs.py +2 -7
  96. flowfile_core/routes/public.py +1 -0
  97. flowfile_core/routes/routes.py +130 -90
  98. flowfile_core/routes/secrets.py +72 -14
  99. flowfile_core/schemas/__init__.py +8 -0
  100. flowfile_core/schemas/cloud_storage_schemas.py +215 -0
  101. flowfile_core/schemas/input_schema.py +121 -71
  102. flowfile_core/schemas/output_model.py +19 -3
  103. flowfile_core/schemas/schemas.py +150 -12
  104. flowfile_core/schemas/transform_schema.py +175 -35
  105. flowfile_core/utils/utils.py +40 -1
  106. flowfile_core/utils/validate_setup.py +41 -0
  107. flowfile_frame/__init__.py +9 -1
  108. flowfile_frame/cloud_storage/frame_helpers.py +39 -0
  109. flowfile_frame/cloud_storage/secret_manager.py +73 -0
  110. flowfile_frame/expr.py +28 -1
  111. flowfile_frame/expr.pyi +76 -61
  112. flowfile_frame/flow_frame.py +481 -208
  113. flowfile_frame/flow_frame.pyi +140 -91
  114. flowfile_frame/flow_frame_methods.py +160 -22
  115. flowfile_frame/group_frame.py +3 -0
  116. flowfile_frame/utils.py +25 -3
  117. flowfile_worker/external_sources/s3_source/main.py +216 -0
  118. flowfile_worker/external_sources/s3_source/models.py +142 -0
  119. flowfile_worker/funcs.py +51 -6
  120. flowfile_worker/models.py +22 -2
  121. flowfile_worker/routes.py +40 -38
  122. flowfile_worker/utils.py +1 -1
  123. test_utils/s3/commands.py +46 -0
  124. test_utils/s3/data_generator.py +292 -0
  125. test_utils/s3/demo_data_generator.py +186 -0
  126. test_utils/s3/fixtures.py +214 -0
  127. flowfile/web/static/assets/AirbyteReader-1ac35765.css +0 -314
  128. flowfile/web/static/assets/AirbyteReader-e08044e5.js +0 -922
  129. flowfile/web/static/assets/dropDownGeneric-60f56a8a.js +0 -72
  130. flowfile/web/static/assets/dropDownGeneric-895680d6.css +0 -10
  131. flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py +0 -159
  132. flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py +0 -172
  133. flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py +0 -173
  134. flowfile_core/schemas/defaults.py +0 -9
  135. flowfile_core/schemas/external_sources/airbyte_schemas.py +0 -20
  136. flowfile_core/schemas/models.py +0 -193
  137. flowfile_worker/external_sources/airbyte_sources/cache_manager.py +0 -161
  138. flowfile_worker/external_sources/airbyte_sources/main.py +0 -89
  139. flowfile_worker/external_sources/airbyte_sources/models.py +0 -133
  140. flowfile_worker/external_sources/airbyte_sources/settings.py +0 -0
  141. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/LICENSE +0 -0
  142. {flowfile-0.3.5.dist-info → flowfile-0.3.7.dist-info}/WHEEL +0 -0
  143. {flowfile_core/flowfile/sources/external_sources/airbyte_sources → flowfile_frame/cloud_storage}/__init__.py +0 -0
  144. {flowfile_core/schemas/external_sources → flowfile_worker/external_sources/s3_source}/__init__.py +0 -0
  145. {flowfile_worker/external_sources/airbyte_sources → test_utils/s3}/__init__.py +0 -0
@@ -0,0 +1,186 @@
1
+ import logging
2
+ import io
3
+ import os
4
+ import tempfile
5
+ import shutil
6
+ import random
7
+ from datetime import datetime, timedelta
8
+
9
+ # Third-party libraries
10
+ import boto3
11
+ from botocore.client import Config
12
+ import polars as pl
13
+ import pyarrow as pa
14
+ from pyarrow import parquet as pq
15
+
16
+ # Configure logging
17
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
18
+ logger = logging.getLogger(__name__)
19
+
20
+ # --- MinIO/S3 Configuration ---
21
+ MINIO_HOST = os.environ.get("TEST_MINIO_HOST", "localhost")
22
+ MINIO_PORT = int(os.environ.get("TEST_MINIO_PORT", 9000))
23
+ MINIO_ACCESS_KEY = os.environ.get("TEST_MINIO_ACCESS_KEY", "minioadmin")
24
+ MINIO_SECRET_KEY = os.environ.get("TEST_MINIO_SECRET_KEY", "minioadmin")
25
+ MINIO_ENDPOINT_URL = f"http://{MINIO_HOST}:{MINIO_PORT}"
26
+
27
+ # --- Data Generation Functions ---
28
+
29
+ def _create_sales_data(s3_client, df: pl.DataFrame, bucket_name: str):
30
+ """
31
+ Creates partitioned Parquet files for the sales data based on year and month.
32
+ s3://data-lake/sales/year=YYYY/month=MM/
33
+ """
34
+ logger.info("Writing partitioned sales data...")
35
+ # Use Polars' built-in partitioning
36
+ # A temporary local directory is needed to stage the partitioned files before uploading
37
+ with tempfile.TemporaryDirectory() as temp_dir:
38
+ df.write_parquet(
39
+ temp_dir,
40
+ use_pyarrow=True,
41
+ pyarrow_options={"partition_cols": ["year", "month"]}
42
+ )
43
+ # Walk through the local directory and upload files to S3
44
+ for root, _, files in os.walk(temp_dir):
45
+ for file in files:
46
+ if file.endswith(".parquet"):
47
+ local_path = os.path.join(root, file)
48
+ # Construct the S3 key to match the desired structure
49
+ relative_path = os.path.relpath(local_path, temp_dir)
50
+ s3_key = f"data-lake/sales/{relative_path.replace(os.path.sep, '/')}"
51
+ s3_client.upload_file(local_path, bucket_name, s3_key)
52
+ logger.info(f"Finished writing sales data to s3://{bucket_name}/data-lake/sales/")
53
+
54
+ def _create_customers_data(s3_client, df: pl.DataFrame, bucket_name: str):
55
+ """
56
+ Creates a Parquet file for the customers data.
57
+ s3://data-lake/customers/
58
+ """
59
+ logger.info("Writing customers Parquet data...")
60
+ parquet_buffer = io.BytesIO()
61
+ df.write_parquet(parquet_buffer)
62
+ parquet_buffer.seek(0)
63
+ s3_client.put_object(
64
+ Bucket=bucket_name,
65
+ Key='data-lake/customers/customers.parquet',
66
+ Body=parquet_buffer.getvalue()
67
+ )
68
+ logger.info(f"Finished writing customers data to s3://{bucket_name}/data-lake/customers/")
69
+
70
+
71
+ def _create_orders_data(s3_client, df: pl.DataFrame, bucket_name: str):
72
+ """
73
+ Creates a pipe-delimited CSV file for the orders data.
74
+ s3://raw-data/orders/
75
+ """
76
+ logger.info("Writing orders CSV data...")
77
+ csv_buffer = io.BytesIO()
78
+ # Write with pipe delimiter and header
79
+ df.write_csv(csv_buffer, separator="|")
80
+ csv_buffer.seek(0)
81
+ s3_client.put_object(
82
+ Bucket=bucket_name,
83
+ Key='raw-data/orders/orders.csv',
84
+ Body=csv_buffer.getvalue()
85
+ )
86
+ logger.info(f"Finished writing orders data to s3://{bucket_name}/raw-data/orders/")
87
+
88
+ def _create_products_data(df: pl.DataFrame):
89
+ """
90
+ Creates a local Parquet file for the products data.
91
+ """
92
+ logger.info("Writing local products Parquet data...")
93
+ # Create a directory for local data if it doesn't exist
94
+ local_data_dir = "local_data"
95
+ os.makedirs(local_data_dir, exist_ok=True)
96
+ file_path = os.path.join(local_data_dir, "local_products.parquet")
97
+ df.write_parquet(file_path)
98
+ logger.info(f"Finished writing products data to {file_path}")
99
+
100
+
101
+ def create_demo_data(endpoint_url: str, access_key: str, secret_key: str, bucket_name: str):
102
+ """
103
+ Populates a MinIO bucket with test data matching the schemas from the examples.
104
+ """
105
+ logger.info("🚀 Starting data population for flowfile examples...")
106
+ s3_client = boto3.client(
107
+ 's3',
108
+ endpoint_url=endpoint_url,
109
+ aws_access_key_id=access_key,
110
+ aws_secret_access_key=secret_key,
111
+ config=Config(signature_version='s3v4'),
112
+ region_name='us-east-1'
113
+ )
114
+
115
+ # --- Generate Core DataFrames ---
116
+ DATA_SIZE = 15_000 # Increased data size for more variety
117
+ START_DATE = datetime(2022, 1, 1)
118
+ END_DATE = datetime(2024, 12, 31)
119
+ TOTAL_DAYS = (END_DATE - START_DATE).days
120
+
121
+ # States for region mapping
122
+ states = ["CA", "OR", "WA", "NY", "NJ", "PA", "TX", "FL", "GA", "IL", "OH", "MI"]
123
+
124
+ # Generate base sales data across multiple years
125
+ sales_data = {
126
+ "order_id": range(1, DATA_SIZE + 1),
127
+ "customer_id": [random.randint(100, 299) for _ in range(DATA_SIZE)],
128
+ "product_id": [random.randint(1, 100) for _ in range(DATA_SIZE)],
129
+ "order_date": [START_DATE + timedelta(days=random.randint(0, TOTAL_DAYS)) for _ in range(DATA_SIZE)],
130
+ "quantity": [random.randint(1, 5) for _ in range(DATA_SIZE)],
131
+ "unit_price": [round(random.uniform(10.0, 500.0), 2) for _ in range(DATA_SIZE)],
132
+ "discount_rate": [random.choice([0.0, 0.1, 0.15, 0.2, None]) for _ in range(DATA_SIZE)],
133
+ "status": [random.choice(["completed", "pending", "cancelled"]) for _ in range(DATA_SIZE)],
134
+ "customer_lifetime_value": [random.uniform(500, 20000) for _ in range(DATA_SIZE)],
135
+ "state": [random.choice(states) for _ in range(DATA_SIZE)],
136
+ }
137
+ sales_df = pl.from_dict(sales_data).with_columns([
138
+ pl.col("order_date").dt.year().alias("year"),
139
+ pl.col("order_date").dt.month().alias("month"),
140
+ # The 'amount' column in the example seems to be the price before discount
141
+ pl.col("unit_price").alias("amount")
142
+ ])
143
+
144
+ # Generate customers DataFrame
145
+ unique_customer_ids = sales_df["customer_id"].unique().to_list()
146
+ customers_df = pl.DataFrame({
147
+ "customer_id": unique_customer_ids,
148
+ "customer_segment": [random.choice(["VIP", "Regular", "New"]) for _ in unique_customer_ids]
149
+ })
150
+
151
+ # Generate products DataFrame
152
+ unique_product_ids = sales_df["product_id"].unique().to_list()
153
+ # Create a map of product_id to unit_price from the first occurrence in sales_df
154
+ product_price_map = sales_df.group_by("product_id").agg(pl.first("unit_price")).to_dict(as_series=False)
155
+ price_dict = dict(zip(product_price_map['product_id'], product_price_map['unit_price']))
156
+
157
+ products_df = pl.DataFrame({
158
+ "product_id": unique_product_ids,
159
+ "product_category": [random.choice(["Electronics", "Books", "Clothing", "Home Goods"]) for _ in unique_product_ids],
160
+ "unit_price": [price_dict.get(pid) for pid in unique_product_ids]
161
+ })
162
+
163
+ # Generate orders DataFrame for the CSV file (subset of sales)
164
+ orders_df = sales_df.select(["customer_id", "product_id", "quantity", "discount_rate"])
165
+
166
+ logger.info(f"Generated {len(sales_df)} sales records across {sales_df['year'].n_unique()} years, for {len(customers_df)} customers, and {len(products_df)} products.")
167
+
168
+ # --- Write Data to S3 and Local Filesystem ---
169
+ _create_sales_data(s3_client, sales_df, bucket_name)
170
+ _create_customers_data(s3_client, customers_df, bucket_name)
171
+ _create_orders_data(s3_client, orders_df, bucket_name)
172
+ _create_products_data(products_df)
173
+
174
+ logger.info("✅ All test data populated successfully.")
175
+
176
+
177
+ if __name__ == '__main__':
178
+ # The bucket that will be created and populated
179
+ BUCKET = "flowfile-demo-data"
180
+
181
+ create_demo_data(
182
+ endpoint_url=MINIO_ENDPOINT_URL,
183
+ access_key=MINIO_ACCESS_KEY,
184
+ secret_key=MINIO_SECRET_KEY,
185
+ bucket_name=BUCKET
186
+ )
@@ -0,0 +1,214 @@
1
+ import os
2
+ import time
3
+ import subprocess
4
+ import logging
5
+ from contextlib import contextmanager
6
+ from typing import Dict, Generator
7
+ import shutil
8
+ import boto3
9
+ from botocore.client import Config
10
+ from test_utils.s3.data_generator import populate_test_data
11
+ from test_utils.s3.demo_data_generator import create_demo_data
12
+
13
+ logger = logging.getLogger("s3_fixture")
14
+
15
+ MINIO_HOST = os.environ.get("TEST_MINIO_HOST", "localhost")
16
+ MINIO_PORT = int(os.environ.get("TEST_MINIO_PORT", 9000))
17
+ MINIO_CONSOLE_PORT = int(os.environ.get("TEST_MINIO_CONSOLE_PORT", 9001))
18
+ MINIO_ACCESS_KEY = os.environ.get("TEST_MINIO_ACCESS_KEY", "minioadmin")
19
+ MINIO_SECRET_KEY = os.environ.get("TEST_MINIO_SECRET_KEY", "minioadmin")
20
+ MINIO_CONTAINER_NAME = os.environ.get("TEST_MINIO_CONTAINER", "test-minio-s3")
21
+ MINIO_ENDPOINT_URL = f"http://{MINIO_HOST}:{MINIO_PORT}"
22
+
23
+ # Operating system detection
24
+ IS_MACOS = os.uname().sysname == 'Darwin' if hasattr(os, 'uname') else False
25
+ IS_WINDOWS = os.name == 'nt'
26
+
27
+ def get_minio_client():
28
+ """Get boto3 client for MinIO"""
29
+ return boto3.client(
30
+ 's3',
31
+ endpoint_url=MINIO_ENDPOINT_URL,
32
+ aws_access_key_id=MINIO_ACCESS_KEY,
33
+ aws_secret_access_key=MINIO_SECRET_KEY,
34
+ config=Config(signature_version='s3v4'),
35
+ region_name='us-east-1'
36
+ )
37
+
38
+
39
+ def wait_for_minio(max_retries=30, interval=1):
40
+ """Wait for MinIO to be ready"""
41
+ for i in range(max_retries):
42
+ try:
43
+ client = get_minio_client()
44
+ client.list_buckets()
45
+ logger.info("MinIO is ready")
46
+ return True
47
+ except Exception:
48
+ if i < max_retries - 1:
49
+ time.sleep(interval)
50
+ continue
51
+ return False
52
+
53
+ def is_container_running(container_name: str) -> bool:
54
+ """Check if MinIO container is already running"""
55
+ try:
56
+ result = subprocess.run(
57
+ ["docker", "ps", "--filter", f"name={container_name}", "--format", "{{.Names}}"],
58
+ capture_output=True,
59
+ text=True,
60
+ check=True
61
+ )
62
+ return container_name in result.stdout.strip()
63
+ except subprocess.CalledProcessError:
64
+ return False
65
+
66
+
67
+ def stop_minio_container() -> bool:
68
+ """Stop the MinIO container and remove its data volume for a clean shutdown."""
69
+ container_name = MINIO_CONTAINER_NAME
70
+ volume_name = f"{container_name}-data"
71
+
72
+ if not is_container_running(container_name):
73
+ logger.info(f"Container '{container_name}' is not running.")
74
+ # Attempt to remove the volume in case it was left orphaned
75
+ try:
76
+ subprocess.run(["docker", "volume", "rm", volume_name], check=False, capture_output=True)
77
+ except Exception:
78
+ pass # Ignore errors if volume doesn't exist
79
+ return True
80
+
81
+ logger.info(f"Stopping and cleaning up container '{container_name}' and volume '{volume_name}'...")
82
+ try:
83
+ # Stop and remove the container
84
+ subprocess.run(["docker", "stop", container_name], check=True, capture_output=True)
85
+ subprocess.run(["docker", "rm", container_name], check=True, capture_output=True)
86
+
87
+ # Remove the associated volume to clear all data
88
+ subprocess.run(["docker", "volume", "rm", volume_name], check=True, capture_output=True)
89
+
90
+ logger.info("✅ MinIO container and data volume successfully removed.")
91
+ return True
92
+ except subprocess.CalledProcessError as e:
93
+ stderr = e.stderr.decode()
94
+ if "no such volume" in stderr:
95
+ logger.info("Volume was already removed or never created.")
96
+ return True
97
+ logger.error(f"❌ Failed to clean up MinIO resources: {stderr}")
98
+ return False
99
+
100
+
101
+ def create_test_buckets():
102
+ """Create test buckets and populate with sample data"""
103
+ client = get_minio_client()
104
+
105
+ # Create test buckets
106
+ buckets = ['test-bucket', 'flowfile-test', 'sample-data', 'worker-test-bucket', 'demo-bucket']
107
+ for bucket in buckets:
108
+ try:
109
+ client.create_bucket(Bucket=bucket)
110
+ logger.info(f"Created bucket: {bucket}")
111
+ except client.exceptions.BucketAlreadyExists:
112
+ logger.info(f"Bucket already exists: {bucket}")
113
+ except client.exceptions.BucketAlreadyOwnedByYou:
114
+ logger.info(f"Bucket already owned: {bucket}")
115
+
116
+
117
+ def is_docker_available() -> bool:
118
+ """
119
+ Check if Docker is available on the system.
120
+
121
+ Returns:
122
+ bool: True if Docker is available and working, False otherwise
123
+ """
124
+ # Skip Docker on macOS and Windows in CI
125
+ if (IS_MACOS or IS_WINDOWS) and os.environ.get('CI', '').lower() in ('true', '1', 'yes'):
126
+ logger.info("Skipping Docker on macOS/Windows in CI environment")
127
+ return False
128
+
129
+ # If docker executable is not in PATH
130
+ if shutil.which("docker") is None:
131
+ logger.warning("Docker executable not found in PATH")
132
+ return False
133
+
134
+ # Try a simple docker command
135
+ try:
136
+ result = subprocess.run(
137
+ ["docker", "info"],
138
+ stdout=subprocess.PIPE,
139
+ stderr=subprocess.PIPE,
140
+ timeout=5,
141
+ check=False # Don't raise exception on non-zero return code
142
+ )
143
+
144
+ if result.returncode != 0:
145
+ logger.warning("Docker is not operational")
146
+ return False
147
+
148
+ return True
149
+ except (subprocess.SubprocessError, OSError):
150
+ logger.warning("Error running Docker command")
151
+ return False
152
+
153
+
154
+ def start_minio_container() -> bool:
155
+ """Start MinIO container with initialization"""
156
+ if is_container_running(MINIO_CONTAINER_NAME):
157
+ logger.info(f"Container {MINIO_CONTAINER_NAME} is already running")
158
+ return True
159
+
160
+ try:
161
+ # Start MinIO with volume for persistence
162
+ subprocess.run([
163
+ "docker", "run", "-d",
164
+ "--name", MINIO_CONTAINER_NAME,
165
+ "-p", f"{MINIO_PORT}:9000",
166
+ "-p", f"{MINIO_CONSOLE_PORT}:9001",
167
+ "-e", f"MINIO_ROOT_USER={MINIO_ACCESS_KEY}",
168
+ "-e", f"MINIO_ROOT_PASSWORD={MINIO_SECRET_KEY}",
169
+ "-v", f"{MINIO_CONTAINER_NAME}-data:/data",
170
+ "minio/minio", "server", "/data", "--console-address", ":9001"
171
+ ], check=True)
172
+
173
+ # Wait for MinIO to be ready
174
+ if wait_for_minio():
175
+ create_test_buckets()
176
+ populate_test_data(endpoint_url=MINIO_ENDPOINT_URL,
177
+ access_key=MINIO_ACCESS_KEY,
178
+ secret_key=MINIO_SECRET_KEY,
179
+ bucket_name="test-bucket")
180
+ create_demo_data(endpoint_url=MINIO_ENDPOINT_URL,
181
+ access_key=MINIO_ACCESS_KEY,
182
+ secret_key=MINIO_SECRET_KEY,
183
+ bucket_name="demo-bucket")
184
+ return True
185
+ return False
186
+
187
+ except Exception as e:
188
+ logger.error(f"Failed to start MinIO: {e}")
189
+ stop_minio_container()
190
+ return False
191
+
192
+
193
+ @contextmanager
194
+ def managed_minio() -> Generator[Dict[str, any], None, None]:
195
+ """Context manager for MinIO container with full connection info"""
196
+ if not start_minio_container():
197
+ yield {}
198
+ return
199
+
200
+ try:
201
+ connection_info = {
202
+ "endpoint_url": MINIO_ENDPOINT_URL,
203
+ "access_key": MINIO_ACCESS_KEY,
204
+ "secret_key": MINIO_SECRET_KEY,
205
+ "host": MINIO_HOST,
206
+ "port": MINIO_PORT,
207
+ "console_port": MINIO_CONSOLE_PORT,
208
+ "connection_string": f"s3://{MINIO_ACCESS_KEY}:{MINIO_SECRET_KEY}@{MINIO_HOST}:{MINIO_PORT}"
209
+ }
210
+ yield connection_info
211
+ finally:
212
+ # Optionally keep container running for debugging
213
+ if os.environ.get("KEEP_MINIO_RUNNING", "false").lower() != "true":
214
+ stop_minio_container()
@@ -1,314 +0,0 @@
1
-
2
- /* Array input styles */
3
- .array-input-section[data-v-b2f2d704] {
4
- display: flex;
5
- flex-direction: column;
6
- gap: 8px;
7
- }
8
- .input-with-button[data-v-b2f2d704] {
9
- display: flex;
10
- gap: 8px;
11
- align-items: center;
12
- }
13
- .items-container[data-v-b2f2d704] {
14
- display: flex;
15
- flex-wrap: wrap;
16
- gap: 10px; /* Space between items */
17
- }
18
- .item-box[data-v-b2f2d704] {
19
- display: flex;
20
- align-items: center;
21
- padding: 5px 10px;
22
- background-color: #f0f0f0;
23
- border-radius: 4px;
24
- font-size: 12px;
25
- position: relative;
26
- }
27
- .remove-btn[data-v-b2f2d704] {
28
- margin-left: 8px;
29
- cursor: pointer;
30
- color: #100f0f72;
31
- font-weight: bold;
32
- }
33
- .add-btn[data-v-b2f2d704] {
34
- padding: 4px 12px;
35
- background: #7878ff5b;
36
- border: none;
37
- border-radius: 3px;
38
- cursor: pointer;
39
- font-size: 12px;
40
- transition: background-color 0.2s;
41
- white-space: nowrap;
42
- }
43
- .add-btn[data-v-b2f2d704]:hover {
44
- background: #6363ff5b;
45
- }
46
-
47
- /* Your existing styles */
48
- .form-container[data-v-b2f2d704] {
49
- font-family: -apple-system, BlinkMacSystemFont, sans-serif;
50
- font-size: 13px;
51
- color: #333;
52
- }
53
- .form-grid[data-v-b2f2d704] {
54
- display: grid;
55
- gap: 8px;
56
- padding: 12px;
57
- background: #fff;
58
- border: 1px solid #eee;
59
- border-radius: 4px;
60
- }
61
- .form-item-wrapper[data-v-b2f2d704] {
62
- margin-bottom: 2px;
63
- }
64
- .single-item[data-v-b2f2d704] {
65
- margin-bottom: 4px;
66
- }
67
- .compact-header[data-v-b2f2d704] {
68
- font-size: 12px;
69
- color: #666;
70
- margin-bottom: 2px;
71
- display: flex;
72
- align-items: center;
73
- gap: 4px;
74
- }
75
- .minimal-header[data-v-b2f2d704] {
76
- width: 100%;
77
- text-align: left;
78
- padding: 6px 8px;
79
- background: #f5f5f5;
80
- border: 1px solid #eee;
81
- border-radius: 3px;
82
- font-size: 12px;
83
- display: flex;
84
- align-items: center;
85
- gap: 6px;
86
- cursor: pointer;
87
- transition: background 0.2s;
88
- }
89
- .minimal-header[data-v-b2f2d704]:hover {
90
- background: #f0f0f0;
91
- }
92
- .minimal-header.is-open[data-v-b2f2d704] {
93
- border-bottom-left-radius: 0;
94
- border-bottom-right-radius: 0;
95
- }
96
- .minimal-chevron[data-v-b2f2d704] {
97
- font-size: 14px;
98
- color: #999;
99
- width: 12px;
100
- }
101
- .tag[data-v-b2f2d704] {
102
- color: #ff4757;
103
- font-size: 14px;
104
- }
105
- .type-indicator[data-v-b2f2d704] {
106
- color: #999;
107
- font-size: 11px;
108
- }
109
- .nested-content[data-v-b2f2d704] {
110
- padding: 8px;
111
- border: 1px solid #eee;
112
- border-top: none;
113
- background: #fff;
114
- border-bottom-left-radius: 3px;
115
- border-bottom-right-radius: 3px;
116
- }
117
- .nested-item[data-v-b2f2d704] {
118
- margin-bottom: 6px;
119
- }
120
- .nested-item[data-v-b2f2d704]:last-child {
121
- margin-bottom: 0;
122
- }
123
- .minimal-input[data-v-b2f2d704] {
124
- box-sizing: border-box; /* Add this to include padding in width calculation */
125
- width: 100%;
126
- padding: 4px 8px;
127
- border: 1px solid #ddd;
128
- border-radius: 3px;
129
- font-size: 12px;
130
- background: #fff;
131
- transition: border 0.2s;
132
- }
133
- .minimal-input[data-v-b2f2d704]:focus {
134
- outline: none;
135
- border-color: #666;
136
- }
137
- .minimal-input[data-v-b2f2d704]::placeholder {
138
- color: #ccc;
139
- }
140
- .minimal-popover[data-v-b2f2d704] {
141
- position: fixed;
142
- background: rgba(0, 0, 0, 0.8);
143
- color: #fff;
144
- padding: 4px 8px;
145
- border-radius: 3px;
146
- font-size: 12px;
147
- max-width: 250px;
148
- z-index: 100;
149
- pointer-events: none;
150
- }
151
- @media (max-width: 640px) {
152
- .form-grid[data-v-b2f2d704] {
153
- padding: 8px;
154
- gap: 6px;
155
- }
156
- .minimal-header[data-v-b2f2d704] {
157
- padding: 4px 6px;
158
- }
159
- .minimal-input[data-v-b2f2d704] {
160
- padding: 3px 6px;
161
- }
162
- }
163
-
164
- .to-front[data-v-9dcbd94f] {
165
- z-index: 1000;
166
- }
167
- .config-section[data-v-9dcbd94f] {
168
- margin-top: 16px;
169
- }
170
- .stream-section[data-v-9dcbd94f] {
171
- margin-top: 16px;
172
- }
173
- .stream-select[data-v-9dcbd94f] {
174
- width: 100%;
175
- max-width: 400px;
176
- }
177
- .icon-button[data-v-9dcbd94f] {
178
- padding: 2px;
179
- border: none;
180
- background: none;
181
- cursor: pointer;
182
- color: #666;
183
- }
184
- .icon-button[data-v-9dcbd94f]:hover {
185
- color: #333;
186
- }
187
- .primary-button[data-v-9dcbd94f] {
188
- display: inline-flex;
189
- align-items: center;
190
- justify-content: center;
191
- cursor: pointer;
192
- padding: 8px 16px;
193
- background-color: #7878ff5b;
194
- border: none;
195
- border-radius: 4px;
196
- font-size: 13px;
197
- transition: background-color 0.3s ease;
198
- }
199
- .primary-button[data-v-9dcbd94f]:hover:not(:disabled) {
200
- background-color: #b3b5ba;
201
- }
202
- .primary-button[data-v-9dcbd94f]:disabled {
203
- opacity: 0.6;
204
- cursor: not-allowed;
205
- }
206
- .secondary-button[data-v-9dcbd94f] {
207
- display: inline-flex;
208
- align-items: center;
209
- gap: 4px;
210
- padding: 4px 8px;
211
- background-color: #f1f1f1;
212
- border: 1px solid #ddd;
213
- border-radius: 4px;
214
- font-size: 12px;
215
- color: #666;
216
- transition: all 0.2s ease;
217
- }
218
- .secondary-button[data-v-9dcbd94f]:hover {
219
- background-color: #e4e4e4;
220
- color: #333;
221
- }
222
- .validation-banner[data-v-9dcbd94f] {
223
- display: flex;
224
- align-items: center;
225
- gap: 8px;
226
- padding: 12px;
227
- margin-top: 12px;
228
- border-radius: 4px;
229
- font-size: 14px;
230
- }
231
- .validation-banner.success[data-v-9dcbd94f] {
232
- background-color: #ecfdf5;
233
- color: #047857;
234
- }
235
- .validation-banner.error[data-v-9dcbd94f] {
236
- background-color: #fef2f2;
237
- color: #dc2626;
238
- }
239
- .spin[data-v-9dcbd94f] {
240
- animation: spin-9dcbd94f 1s linear infinite;
241
- }
242
- @keyframes spin-9dcbd94f {
243
- from {
244
- transform: rotate(0deg);
245
- }
246
- to {
247
- transform: rotate(360deg);
248
- }
249
- }
250
- .file-upload-label[data-v-9dcbd94f] {
251
- display: flex;
252
- align-items: center;
253
- justify-content: flex-start;
254
- background-color: #f5f5f5;
255
- border: 1px solid #ddd;
256
- border-radius: 4px;
257
- padding: 10px 15px;
258
- color: #333;
259
- font-size: 16px;
260
- font-weight: 500;
261
- text-align: left;
262
- user-select: none;
263
- cursor: pointer;
264
- transition: background-color 0.3s ease;
265
- }
266
- .file-upload-label[data-v-9dcbd94f]:hover {
267
- background-color: #e4e4e4;
268
- }
269
- .file-icon[data-v-9dcbd94f] {
270
- margin-right: 10px;
271
- font-size: 20px;
272
- width: 24px;
273
- height: auto;
274
- }
275
- .attention-notice[data-v-9dcbd94f] {
276
- display: flex;
277
- align-items: center;
278
- gap: 6px;
279
- padding: 4px 8px;
280
- border-radius: 4px;
281
- width: fit-content;
282
- margin-top: 4px;
283
- }
284
- .docker-notice[data-v-9dcbd94f] {
285
- font-size: 12px;
286
- font-weight: 600;
287
- }
288
- .warning-icon[data-v-9dcbd94f] {
289
- font-size: 14px;
290
- animation: pulse-9dcbd94f 2s infinite;
291
- }
292
- .flex[data-v-9dcbd94f] {
293
- display: flex;
294
- }
295
- .justify-between[data-v-9dcbd94f] {
296
- justify-content: space-between;
297
- }
298
- .items-center[data-v-9dcbd94f] {
299
- align-items: center;
300
- }
301
- .gap-2[data-v-9dcbd94f] {
302
- gap: 8px;
303
- }
304
- @keyframes pulse-9dcbd94f {
305
- 0% {
306
- opacity: 1;
307
- }
308
- 50% {
309
- opacity: 0.5;
310
- }
311
- 100% {
312
- opacity: 1;
313
- }
314
- }