lfx-nightly 0.2.0.dev0__py3-none-any.whl → 0.2.0.dev26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lfx/_assets/component_index.json +1 -1
- lfx/base/agents/agent.py +13 -1
- lfx/base/agents/altk_base_agent.py +380 -0
- lfx/base/agents/altk_tool_wrappers.py +565 -0
- lfx/base/agents/events.py +2 -1
- lfx/base/composio/composio_base.py +159 -224
- lfx/base/data/base_file.py +88 -21
- lfx/base/data/storage_utils.py +192 -0
- lfx/base/data/utils.py +178 -14
- lfx/base/embeddings/embeddings_class.py +113 -0
- lfx/base/models/groq_constants.py +74 -58
- lfx/base/models/groq_model_discovery.py +265 -0
- lfx/base/models/model.py +1 -1
- lfx/base/models/model_utils.py +100 -0
- lfx/base/models/openai_constants.py +7 -0
- lfx/base/models/watsonx_constants.py +32 -8
- lfx/base/tools/run_flow.py +601 -129
- lfx/cli/commands.py +6 -3
- lfx/cli/common.py +2 -2
- lfx/cli/run.py +1 -1
- lfx/cli/script_loader.py +53 -11
- lfx/components/Notion/create_page.py +1 -1
- lfx/components/Notion/list_database_properties.py +1 -1
- lfx/components/Notion/list_pages.py +1 -1
- lfx/components/Notion/list_users.py +1 -1
- lfx/components/Notion/page_content_viewer.py +1 -1
- lfx/components/Notion/search.py +1 -1
- lfx/components/Notion/update_page_property.py +1 -1
- lfx/components/__init__.py +19 -5
- lfx/components/{agents → altk}/__init__.py +5 -9
- lfx/components/altk/altk_agent.py +193 -0
- lfx/components/apify/apify_actor.py +1 -1
- lfx/components/composio/__init__.py +70 -18
- lfx/components/composio/apollo_composio.py +11 -0
- lfx/components/composio/bitbucket_composio.py +11 -0
- lfx/components/composio/canva_composio.py +11 -0
- lfx/components/composio/coda_composio.py +11 -0
- lfx/components/composio/composio_api.py +10 -0
- lfx/components/composio/discord_composio.py +1 -1
- lfx/components/composio/elevenlabs_composio.py +11 -0
- lfx/components/composio/exa_composio.py +11 -0
- lfx/components/composio/firecrawl_composio.py +11 -0
- lfx/components/composio/fireflies_composio.py +11 -0
- lfx/components/composio/gmail_composio.py +1 -1
- lfx/components/composio/googlebigquery_composio.py +11 -0
- lfx/components/composio/googlecalendar_composio.py +1 -1
- lfx/components/composio/googledocs_composio.py +1 -1
- lfx/components/composio/googlemeet_composio.py +1 -1
- lfx/components/composio/googlesheets_composio.py +1 -1
- lfx/components/composio/googletasks_composio.py +1 -1
- lfx/components/composio/heygen_composio.py +11 -0
- lfx/components/composio/mem0_composio.py +11 -0
- lfx/components/composio/peopledatalabs_composio.py +11 -0
- lfx/components/composio/perplexityai_composio.py +11 -0
- lfx/components/composio/serpapi_composio.py +11 -0
- lfx/components/composio/slack_composio.py +3 -574
- lfx/components/composio/slackbot_composio.py +1 -1
- lfx/components/composio/snowflake_composio.py +11 -0
- lfx/components/composio/tavily_composio.py +11 -0
- lfx/components/composio/youtube_composio.py +2 -2
- lfx/components/cuga/__init__.py +34 -0
- lfx/components/cuga/cuga_agent.py +730 -0
- lfx/components/data/__init__.py +78 -28
- lfx/components/data_source/__init__.py +58 -0
- lfx/components/{data → data_source}/api_request.py +26 -3
- lfx/components/{data → data_source}/csv_to_data.py +15 -10
- lfx/components/{data → data_source}/json_to_data.py +15 -8
- lfx/components/{data → data_source}/news_search.py +1 -1
- lfx/components/{data → data_source}/rss.py +1 -1
- lfx/components/{data → data_source}/sql_executor.py +1 -1
- lfx/components/{data → data_source}/url.py +1 -1
- lfx/components/{data → data_source}/web_search.py +1 -1
- lfx/components/datastax/astradb_cql.py +1 -1
- lfx/components/datastax/astradb_graph.py +1 -1
- lfx/components/datastax/astradb_tool.py +1 -1
- lfx/components/datastax/astradb_vectorstore.py +1 -1
- lfx/components/datastax/hcd.py +1 -1
- lfx/components/deactivated/json_document_builder.py +1 -1
- lfx/components/docling/__init__.py +0 -3
- lfx/components/elastic/elasticsearch.py +1 -1
- lfx/components/elastic/opensearch_multimodal.py +1575 -0
- lfx/components/files_and_knowledge/__init__.py +47 -0
- lfx/components/{data → files_and_knowledge}/directory.py +1 -1
- lfx/components/{data → files_and_knowledge}/file.py +246 -18
- lfx/components/{knowledge_bases → files_and_knowledge}/retrieval.py +2 -2
- lfx/components/{data → files_and_knowledge}/save_file.py +142 -22
- lfx/components/flow_controls/__init__.py +58 -0
- lfx/components/{logic → flow_controls}/conditional_router.py +1 -1
- lfx/components/{logic → flow_controls}/loop.py +43 -9
- lfx/components/flow_controls/run_flow.py +108 -0
- lfx/components/glean/glean_search_api.py +1 -1
- lfx/components/groq/groq.py +35 -28
- lfx/components/helpers/__init__.py +102 -0
- lfx/components/input_output/__init__.py +3 -1
- lfx/components/input_output/chat.py +4 -3
- lfx/components/input_output/chat_output.py +4 -4
- lfx/components/input_output/text.py +1 -1
- lfx/components/input_output/text_output.py +1 -1
- lfx/components/{data → input_output}/webhook.py +1 -1
- lfx/components/knowledge_bases/__init__.py +59 -4
- lfx/components/langchain_utilities/character.py +1 -1
- lfx/components/langchain_utilities/csv_agent.py +84 -16
- lfx/components/langchain_utilities/json_agent.py +67 -12
- lfx/components/langchain_utilities/language_recursive.py +1 -1
- lfx/components/llm_operations/__init__.py +46 -0
- lfx/components/{processing → llm_operations}/batch_run.py +1 -1
- lfx/components/{processing → llm_operations}/lambda_filter.py +1 -1
- lfx/components/{logic → llm_operations}/llm_conditional_router.py +1 -1
- lfx/components/{processing/llm_router.py → llm_operations/llm_selector.py} +3 -3
- lfx/components/{processing → llm_operations}/structured_output.py +1 -1
- lfx/components/logic/__init__.py +126 -0
- lfx/components/mem0/mem0_chat_memory.py +11 -0
- lfx/components/models/__init__.py +64 -9
- lfx/components/models_and_agents/__init__.py +49 -0
- lfx/components/{agents → models_and_agents}/agent.py +2 -2
- lfx/components/models_and_agents/embedding_model.py +423 -0
- lfx/components/models_and_agents/language_model.py +398 -0
- lfx/components/{agents → models_and_agents}/mcp_component.py +53 -44
- lfx/components/{helpers → models_and_agents}/memory.py +1 -1
- lfx/components/nvidia/system_assist.py +1 -1
- lfx/components/olivya/olivya.py +1 -1
- lfx/components/ollama/ollama.py +17 -3
- lfx/components/processing/__init__.py +9 -57
- lfx/components/processing/converter.py +1 -1
- lfx/components/processing/dataframe_operations.py +1 -1
- lfx/components/processing/parse_json_data.py +2 -2
- lfx/components/processing/parser.py +1 -1
- lfx/components/processing/split_text.py +1 -1
- lfx/components/qdrant/qdrant.py +1 -1
- lfx/components/redis/redis.py +1 -1
- lfx/components/twelvelabs/split_video.py +10 -0
- lfx/components/twelvelabs/video_file.py +12 -0
- lfx/components/utilities/__init__.py +43 -0
- lfx/components/{helpers → utilities}/calculator_core.py +1 -1
- lfx/components/{helpers → utilities}/current_date.py +1 -1
- lfx/components/{processing → utilities}/python_repl_core.py +1 -1
- lfx/components/vectorstores/local_db.py +9 -0
- lfx/components/youtube/youtube_transcripts.py +118 -30
- lfx/custom/custom_component/component.py +57 -1
- lfx/custom/custom_component/custom_component.py +68 -6
- lfx/graph/edge/base.py +43 -20
- lfx/graph/graph/base.py +4 -1
- lfx/graph/state/model.py +15 -2
- lfx/graph/utils.py +6 -0
- lfx/graph/vertex/base.py +4 -1
- lfx/graph/vertex/param_handler.py +10 -7
- lfx/helpers/__init__.py +12 -0
- lfx/helpers/flow.py +117 -0
- lfx/inputs/input_mixin.py +24 -1
- lfx/inputs/inputs.py +13 -1
- lfx/interface/components.py +161 -83
- lfx/log/logger.py +5 -3
- lfx/services/database/__init__.py +5 -0
- lfx/services/database/service.py +25 -0
- lfx/services/deps.py +87 -22
- lfx/services/manager.py +19 -6
- lfx/services/mcp_composer/service.py +998 -157
- lfx/services/session.py +5 -0
- lfx/services/settings/base.py +51 -7
- lfx/services/settings/constants.py +8 -0
- lfx/services/storage/local.py +76 -46
- lfx/services/storage/service.py +152 -29
- lfx/template/field/base.py +3 -0
- lfx/utils/ssrf_protection.py +384 -0
- lfx/utils/validate_cloud.py +26 -0
- {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/METADATA +38 -22
- {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/RECORD +182 -150
- {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/WHEEL +1 -1
- lfx/components/agents/altk_agent.py +0 -366
- lfx/components/agents/cuga_agent.py +0 -1013
- lfx/components/docling/docling_remote_vlm.py +0 -284
- lfx/components/logic/run_flow.py +0 -71
- lfx/components/models/embedding_model.py +0 -195
- lfx/components/models/language_model.py +0 -144
- /lfx/components/{data → data_source}/mock_data.py +0 -0
- /lfx/components/{knowledge_bases → files_and_knowledge}/ingestion.py +0 -0
- /lfx/components/{logic → flow_controls}/data_conditional_router.py +0 -0
- /lfx/components/{logic → flow_controls}/flow_tool.py +0 -0
- /lfx/components/{logic → flow_controls}/listen.py +0 -0
- /lfx/components/{logic → flow_controls}/notify.py +0 -0
- /lfx/components/{logic → flow_controls}/pass_message.py +0 -0
- /lfx/components/{logic → flow_controls}/sub_flow.py +0 -0
- /lfx/components/{processing → models_and_agents}/prompt.py +0 -0
- /lfx/components/{helpers → processing}/create_list.py +0 -0
- /lfx/components/{helpers → processing}/output_parser.py +0 -0
- /lfx/components/{helpers → processing}/store_message.py +0 -0
- /lfx/components/{helpers → utilities}/id_generator.py +0 -0
- {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/entry_points.txt +0 -0
lfx/services/session.py
CHANGED
|
@@ -75,6 +75,11 @@ class NoopSession:
|
|
|
75
75
|
"""Context manager that disables autoflush (no-op implementation)."""
|
|
76
76
|
return self
|
|
77
77
|
|
|
78
|
+
@property
|
|
79
|
+
def is_active(self):
|
|
80
|
+
"""Check if session is active (always True for NoopSession)."""
|
|
81
|
+
return True
|
|
82
|
+
|
|
78
83
|
def __enter__(self):
|
|
79
84
|
return self
|
|
80
85
|
|
lfx/services/settings/base.py
CHANGED
|
@@ -17,7 +17,7 @@ from typing_extensions import override
|
|
|
17
17
|
from lfx.constants import BASE_COMPONENTS_PATH
|
|
18
18
|
from lfx.log.logger import logger
|
|
19
19
|
from lfx.serialization.constants import MAX_ITEMS_LENGTH, MAX_TEXT_LENGTH
|
|
20
|
-
from lfx.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
|
|
20
|
+
from lfx.services.settings.constants import AGENTIC_VARIABLES, VARIABLES_TO_GET_FROM_ENVIRONMENT
|
|
21
21
|
from lfx.utils.util_strings import is_valid_database_url
|
|
22
22
|
|
|
23
23
|
|
|
@@ -87,6 +87,10 @@ class Settings(BaseSettings):
|
|
|
87
87
|
db_connect_timeout: int = 30
|
|
88
88
|
"""The number of seconds to wait before giving up on a lock to released or establishing a connection to the
|
|
89
89
|
database."""
|
|
90
|
+
migration_lock_namespace: str | None = None
|
|
91
|
+
"""Optional namespace identifier for PostgreSQL advisory lock during migrations.
|
|
92
|
+
If not provided, a hash of the database URL will be used. Useful when multiple Langflow
|
|
93
|
+
instances share the same database and need coordinated migration locking."""
|
|
90
94
|
|
|
91
95
|
mcp_server_timeout: int = 20
|
|
92
96
|
"""The number of seconds to wait before giving up on a lock to released or establishing a connection to the
|
|
@@ -109,7 +113,7 @@ class Settings(BaseSettings):
|
|
|
109
113
|
reap idle sessions."""
|
|
110
114
|
|
|
111
115
|
# sqlite configuration
|
|
112
|
-
sqlite_pragmas: dict | None = {"synchronous": "NORMAL", "journal_mode": "WAL"}
|
|
116
|
+
sqlite_pragmas: dict | None = {"synchronous": "NORMAL", "journal_mode": "WAL", "busy_timeout": 30000}
|
|
113
117
|
"""SQLite pragmas to use when connecting to the database."""
|
|
114
118
|
|
|
115
119
|
db_driver_connection_settings: dict | None = None
|
|
@@ -187,6 +191,13 @@ class Settings(BaseSettings):
|
|
|
187
191
|
like_webhook_url: str | None = "https://api.langflow.store/flows/trigger/64275852-ec00-45c1-984e-3bff814732da"
|
|
188
192
|
|
|
189
193
|
storage_type: str = "local"
|
|
194
|
+
"""Storage type for file storage. Defaults to 'local'. Supports 'local' and 's3'."""
|
|
195
|
+
object_storage_bucket_name: str | None = "langflow-bucket"
|
|
196
|
+
"""Object storage bucket name for file storage. Defaults to 'langflow-bucket'."""
|
|
197
|
+
object_storage_prefix: str | None = "files"
|
|
198
|
+
"""Object storage prefix for file storage. Defaults to 'files'."""
|
|
199
|
+
object_storage_tags: dict[str, str] | None = None
|
|
200
|
+
"""Object storage tags for file storage."""
|
|
190
201
|
|
|
191
202
|
celery_enabled: bool = False
|
|
192
203
|
|
|
@@ -245,6 +256,8 @@ class Settings(BaseSettings):
|
|
|
245
256
|
"""The path to log file for Langflow."""
|
|
246
257
|
alembic_log_file: str = "alembic/alembic.log"
|
|
247
258
|
"""The path to log file for Alembic for SQLAlchemy."""
|
|
259
|
+
alembic_log_to_stdout: bool = False
|
|
260
|
+
"""If set to True, the log file will be ignored and Alembic will log to stdout."""
|
|
248
261
|
frontend_path: str | None = None
|
|
249
262
|
"""The path to the frontend directory containing build files. This is for development purposes only.."""
|
|
250
263
|
open_browser: bool = False
|
|
@@ -292,9 +305,13 @@ class Settings(BaseSettings):
|
|
|
292
305
|
# MCP Composer
|
|
293
306
|
mcp_composer_enabled: bool = True
|
|
294
307
|
"""If set to False, Langflow will not start the MCP Composer service."""
|
|
295
|
-
mcp_composer_version: str = "
|
|
296
|
-
"""Version constraint for mcp-composer when using uvx. Uses PEP 440 syntax.
|
|
297
|
-
|
|
308
|
+
mcp_composer_version: str = "==0.1.0.8.10"
|
|
309
|
+
"""Version constraint for mcp-composer when using uvx. Uses PEP 440 syntax."""
|
|
310
|
+
|
|
311
|
+
# Agentic Experience
|
|
312
|
+
agentic_experience: bool = False
|
|
313
|
+
"""If set to True, Langflow will start the agentic MCP server that provides tools for
|
|
314
|
+
flow/component operations, template search, and graph visualization."""
|
|
298
315
|
|
|
299
316
|
# Public Flow Settings
|
|
300
317
|
public_flow_cleanup_interval: int = Field(default=3600, gt=600)
|
|
@@ -317,6 +334,23 @@ class Settings(BaseSettings):
|
|
|
317
334
|
update_starter_projects: bool = True
|
|
318
335
|
"""If set to True, Langflow will update starter projects."""
|
|
319
336
|
|
|
337
|
+
# SSRF Protection
|
|
338
|
+
ssrf_protection_enabled: bool = False
|
|
339
|
+
"""If set to True, Langflow will enable SSRF (Server-Side Request Forgery) protection.
|
|
340
|
+
When enabled, blocks requests to private IP ranges, localhost, and cloud metadata endpoints.
|
|
341
|
+
When False (default), no URL validation is performed, allowing requests to any destination
|
|
342
|
+
including internal services, private networks, and cloud metadata endpoints.
|
|
343
|
+
Default is False for backward compatibility. In v2.0, this will be changed to True.
|
|
344
|
+
|
|
345
|
+
Note: When ssrf_protection_enabled is disabled, the ssrf_allowed_hosts setting is ignored and has no effect."""
|
|
346
|
+
ssrf_allowed_hosts: list[str] = []
|
|
347
|
+
"""Comma-separated list of hosts/IPs/CIDR ranges to allow despite SSRF protection.
|
|
348
|
+
Examples: 'internal-api.company.local,192.168.1.0/24,10.0.0.5,*.dev.internal'
|
|
349
|
+
Supports exact hostnames, wildcard domains (*.example.com), exact IPs, and CIDR ranges.
|
|
350
|
+
|
|
351
|
+
Note: This setting only takes effect when ssrf_protection_enabled is True.
|
|
352
|
+
When protection is disabled, all hosts are allowed regardless of this setting."""
|
|
353
|
+
|
|
320
354
|
@field_validator("cors_origins", mode="before")
|
|
321
355
|
@classmethod
|
|
322
356
|
def validate_cors_origins(cls, value):
|
|
@@ -367,7 +401,7 @@ class Settings(BaseSettings):
|
|
|
367
401
|
Supports PEP 440 specifiers: ==, !=, <=, >=, <, >, ~=, ===
|
|
368
402
|
"""
|
|
369
403
|
if not value:
|
|
370
|
-
return "
|
|
404
|
+
return "==0.1.0.8.10" # Default
|
|
371
405
|
|
|
372
406
|
# Check if it already has a version specifier
|
|
373
407
|
# Order matters: check longer specifiers first to avoid false matches
|
|
@@ -389,9 +423,19 @@ class Settings(BaseSettings):
|
|
|
389
423
|
@field_validator("variables_to_get_from_environment", mode="before")
|
|
390
424
|
@classmethod
|
|
391
425
|
def set_variables_to_get_from_environment(cls, value):
|
|
426
|
+
import os
|
|
427
|
+
|
|
392
428
|
if isinstance(value, str):
|
|
393
429
|
value = value.split(",")
|
|
394
|
-
|
|
430
|
+
|
|
431
|
+
result = list(set(VARIABLES_TO_GET_FROM_ENVIRONMENT + value))
|
|
432
|
+
|
|
433
|
+
# Add agentic variables if agentic_experience is enabled
|
|
434
|
+
# Check env var directly since we can't access instance attributes in validator
|
|
435
|
+
if os.getenv("LANGFLOW_AGENTIC_EXPERIENCE", "true").lower() == "true":
|
|
436
|
+
result.extend(AGENTIC_VARIABLES)
|
|
437
|
+
|
|
438
|
+
return list(set(result))
|
|
395
439
|
|
|
396
440
|
@field_validator("log_file", mode="before")
|
|
397
441
|
@classmethod
|
lfx/services/storage/local.py
CHANGED
|
@@ -1,30 +1,70 @@
|
|
|
1
1
|
"""Local file-based storage service for lfx package."""
|
|
2
2
|
|
|
3
|
-
from
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from aiofile import async_open
|
|
4
8
|
|
|
5
9
|
from lfx.log.logger import logger
|
|
6
10
|
from lfx.services.storage.service import StorageService
|
|
7
11
|
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from langflow.services.session.service import SessionService
|
|
14
|
+
|
|
15
|
+
from lfx.services.settings.service import SettingsService
|
|
16
|
+
|
|
17
|
+
# Constants for path parsing
|
|
18
|
+
EXPECTED_PATH_PARTS = 2 # Path format: "flow_id/filename"
|
|
19
|
+
|
|
8
20
|
|
|
9
21
|
class LocalStorageService(StorageService):
|
|
10
22
|
"""A service class for handling local file storage operations."""
|
|
11
23
|
|
|
12
|
-
def __init__(
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
session_service: SessionService,
|
|
27
|
+
settings_service: SettingsService,
|
|
28
|
+
) -> None:
|
|
29
|
+
"""Initialize the local storage service.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
session_service: Session service instance
|
|
33
|
+
settings_service: Settings service instance containing configuration
|
|
34
|
+
"""
|
|
35
|
+
# Initialize base class with services
|
|
36
|
+
super().__init__(session_service, settings_service)
|
|
37
|
+
# Base class already sets self.data_dir as anyio.Path from settings_service.settings.config_dir
|
|
38
|
+
|
|
39
|
+
def resolve_component_path(self, logical_path: str) -> str:
|
|
40
|
+
"""Convert logical path to absolute filesystem path for local storage.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
logical_path: Path in format "flow_id/filename"
|
|
44
|
+
Returns:
|
|
45
|
+
str: Absolute filesystem path
|
|
46
|
+
"""
|
|
47
|
+
# Split the logical path into flow_id and filename
|
|
48
|
+
parts = logical_path.split("/", 1)
|
|
49
|
+
if len(parts) != EXPECTED_PATH_PARTS:
|
|
50
|
+
# Handle edge case - return as-is if format is unexpected
|
|
51
|
+
return logical_path
|
|
52
|
+
|
|
53
|
+
flow_id, file_name = parts
|
|
54
|
+
return self.build_full_path(flow_id, file_name)
|
|
16
55
|
|
|
17
56
|
def build_full_path(self, flow_id: str, file_name: str) -> str:
|
|
18
57
|
"""Build the full path of a file in the local storage."""
|
|
19
58
|
return str(self.data_dir / flow_id / file_name)
|
|
20
59
|
|
|
21
|
-
async def save_file(self, flow_id: str, file_name: str, data: bytes) -> None:
|
|
60
|
+
async def save_file(self, flow_id: str, file_name: str, data: bytes, *, append: bool = False) -> None:
|
|
22
61
|
"""Save a file in the local storage.
|
|
23
62
|
|
|
24
63
|
Args:
|
|
25
64
|
flow_id: The identifier for the flow.
|
|
26
65
|
file_name: The name of the file to be saved.
|
|
27
66
|
data: The byte content of the file.
|
|
67
|
+
append: If True, append to existing file; if False, overwrite.
|
|
28
68
|
|
|
29
69
|
Raises:
|
|
30
70
|
FileNotFoundError: If the specified flow does not exist.
|
|
@@ -32,17 +72,18 @@ class LocalStorageService(StorageService):
|
|
|
32
72
|
PermissionError: If there is no permission to write the file.
|
|
33
73
|
"""
|
|
34
74
|
folder_path = self.data_dir / flow_id
|
|
35
|
-
folder_path.mkdir(parents=True, exist_ok=True)
|
|
75
|
+
await folder_path.mkdir(parents=True, exist_ok=True)
|
|
36
76
|
file_path = folder_path / file_name
|
|
37
77
|
|
|
38
78
|
try:
|
|
39
|
-
|
|
40
|
-
|
|
79
|
+
mode = "ab" if append else "wb"
|
|
80
|
+
async with async_open(str(file_path), mode) as f:
|
|
81
|
+
await f.write(data)
|
|
82
|
+
action = "appended to" if append else "saved"
|
|
83
|
+
await logger.ainfo(f"File {file_name} {action} successfully in flow {flow_id}.")
|
|
41
84
|
except Exception:
|
|
42
85
|
logger.exception(f"Error saving file {file_name} in flow {flow_id}")
|
|
43
86
|
raise
|
|
44
|
-
else:
|
|
45
|
-
logger.info(f"File {file_name} saved successfully in flow {flow_id}.")
|
|
46
87
|
|
|
47
88
|
async def get_file(self, flow_id: str, file_name: str) -> bytes:
|
|
48
89
|
"""Retrieve a file from the local storage.
|
|
@@ -58,20 +99,16 @@ class LocalStorageService(StorageService):
|
|
|
58
99
|
FileNotFoundError: If the file does not exist.
|
|
59
100
|
"""
|
|
60
101
|
file_path = self.data_dir / flow_id / file_name
|
|
61
|
-
if not file_path.exists():
|
|
62
|
-
logger.
|
|
102
|
+
if not await file_path.exists():
|
|
103
|
+
await logger.awarning(f"File {file_name} not found in flow {flow_id}.")
|
|
63
104
|
msg = f"File {file_name} not found in flow {flow_id}"
|
|
64
105
|
raise FileNotFoundError(msg)
|
|
65
106
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
raise
|
|
72
|
-
else:
|
|
73
|
-
logger.debug(f"File {file_name} retrieved successfully from flow {flow_id}.")
|
|
74
|
-
return content
|
|
107
|
+
async with async_open(str(file_path), "rb") as f:
|
|
108
|
+
content = await f.read()
|
|
109
|
+
|
|
110
|
+
logger.debug(f"File {file_name} retrieved successfully from flow {flow_id}.")
|
|
111
|
+
return content
|
|
75
112
|
|
|
76
113
|
async def list_files(self, flow_id: str) -> list[str]:
|
|
77
114
|
"""List all files in a specific flow directory.
|
|
@@ -86,21 +123,17 @@ class LocalStorageService(StorageService):
|
|
|
86
123
|
flow_id = str(flow_id)
|
|
87
124
|
|
|
88
125
|
folder_path = self.data_dir / flow_id
|
|
89
|
-
if not folder_path.exists():
|
|
90
|
-
logger.
|
|
91
|
-
return []
|
|
92
|
-
|
|
93
|
-
if not folder_path.is_dir():
|
|
94
|
-
logger.warning(f"Flow path {flow_id} is not a directory.")
|
|
126
|
+
if not await folder_path.exists() or not await folder_path.is_dir():
|
|
127
|
+
await logger.awarning(f"Flow {flow_id} directory does not exist.")
|
|
95
128
|
return []
|
|
96
129
|
|
|
97
130
|
try:
|
|
98
|
-
files = [
|
|
131
|
+
files = [p.name async for p in folder_path.iterdir() if await p.is_file()]
|
|
99
132
|
except Exception: # noqa: BLE001
|
|
100
133
|
logger.exception(f"Error listing files in flow {flow_id}")
|
|
101
134
|
return []
|
|
102
135
|
else:
|
|
103
|
-
logger.
|
|
136
|
+
await logger.ainfo(f"Listed {len(files)} files in flow {flow_id}.")
|
|
104
137
|
return files
|
|
105
138
|
|
|
106
139
|
async def delete_file(self, flow_id: str, file_name: str) -> None:
|
|
@@ -114,17 +147,11 @@ class LocalStorageService(StorageService):
|
|
|
114
147
|
FileNotFoundError: If the file does not exist.
|
|
115
148
|
"""
|
|
116
149
|
file_path = self.data_dir / flow_id / file_name
|
|
117
|
-
if
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
try:
|
|
123
|
-
file_path.unlink()
|
|
124
|
-
logger.info(f"File {file_name} deleted successfully from flow {flow_id}.")
|
|
125
|
-
except Exception:
|
|
126
|
-
logger.exception(f"Error deleting file {file_name} in flow {flow_id}")
|
|
127
|
-
raise
|
|
150
|
+
if await file_path.exists():
|
|
151
|
+
await file_path.unlink()
|
|
152
|
+
await logger.ainfo(f"File {file_name} deleted successfully from flow {flow_id}.")
|
|
153
|
+
else:
|
|
154
|
+
await logger.awarning(f"Attempted to delete non-existent file {file_name} in flow {flow_id}.")
|
|
128
155
|
|
|
129
156
|
async def get_file_size(self, flow_id: str, file_name: str) -> int:
|
|
130
157
|
"""Get the size of a file in bytes.
|
|
@@ -140,16 +167,19 @@ class LocalStorageService(StorageService):
|
|
|
140
167
|
FileNotFoundError: If the file does not exist.
|
|
141
168
|
"""
|
|
142
169
|
file_path = self.data_dir / flow_id / file_name
|
|
143
|
-
if not file_path.exists():
|
|
144
|
-
logger.
|
|
170
|
+
if not await file_path.exists():
|
|
171
|
+
await logger.awarning(f"File {file_name} not found in flow {flow_id}.")
|
|
145
172
|
msg = f"File {file_name} not found in flow {flow_id}"
|
|
146
173
|
raise FileNotFoundError(msg)
|
|
147
174
|
|
|
148
175
|
try:
|
|
149
|
-
|
|
176
|
+
file_size_stat = await file_path.stat()
|
|
150
177
|
except Exception:
|
|
151
178
|
logger.exception(f"Error getting size of file {file_name} in flow {flow_id}")
|
|
152
179
|
raise
|
|
153
180
|
else:
|
|
154
|
-
|
|
155
|
-
|
|
181
|
+
return file_size_stat.st_size
|
|
182
|
+
|
|
183
|
+
async def teardown(self) -> None:
|
|
184
|
+
"""Perform any cleanup operations when the service is being torn down."""
|
|
185
|
+
# No specific teardown actions required for local
|
lfx/services/storage/service.py
CHANGED
|
@@ -1,54 +1,177 @@
|
|
|
1
|
-
|
|
1
|
+
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from abc import
|
|
4
|
-
from
|
|
3
|
+
from abc import abstractmethod
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
5
|
|
|
6
|
+
import anyio
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
"""Abstract base class for storage services."""
|
|
8
|
+
from lfx.services.base import Service
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from collections.abc import AsyncIterator
|
|
12
|
+
|
|
13
|
+
from lfx.services.settings.service import SettingsService
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class StorageService(Service):
|
|
17
|
+
"""Abstract base class for file storage services.
|
|
18
|
+
|
|
19
|
+
This class defines the interface for file storage operations that can be
|
|
20
|
+
implemented by different backends (local filesystem, S3, etc.).
|
|
21
|
+
|
|
22
|
+
All file operations are namespaced by flow_id to isolate files between
|
|
23
|
+
different flows or users.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
name = "storage_service"
|
|
27
|
+
|
|
28
|
+
def __init__(self, session_service, settings_service: SettingsService):
|
|
11
29
|
"""Initialize the storage service.
|
|
12
30
|
|
|
13
31
|
Args:
|
|
14
|
-
|
|
32
|
+
session_service: The session service instance
|
|
33
|
+
settings_service: The settings service instance containing configuration
|
|
34
|
+
"""
|
|
35
|
+
self.settings_service = settings_service
|
|
36
|
+
self.session_service = session_service
|
|
37
|
+
self.data_dir: anyio.Path = anyio.Path(settings_service.settings.config_dir)
|
|
38
|
+
self.set_ready()
|
|
39
|
+
|
|
40
|
+
def build_full_path(self, flow_id: str, file_name: str) -> str:
|
|
41
|
+
"""Build the full path/key for a file.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
flow_id: The flow/user identifier for namespacing
|
|
45
|
+
file_name: The name of the file
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
str: The full path or key for the file
|
|
49
|
+
"""
|
|
50
|
+
raise NotImplementedError
|
|
51
|
+
|
|
52
|
+
def resolve_component_path(self, logical_path: str) -> str:
|
|
53
|
+
"""Convert a logical path to a format that components can use directly.
|
|
54
|
+
|
|
55
|
+
Logical paths are in the format "{flow_id}/{filename}" as stored in the database.
|
|
56
|
+
This method converts them to a format appropriate for the storage backend:
|
|
57
|
+
- Local storage: Absolute filesystem path (/data_dir/flow_id/filename)
|
|
58
|
+
- S3 storage: Logical path as-is (flow_id/filename)
|
|
59
|
+
|
|
60
|
+
Components receive this resolved path and can use it without knowing the
|
|
61
|
+
storage implementation details.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
logical_path: Path in the format "flow_id/filename"
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
str: A path that components can use directly
|
|
15
68
|
"""
|
|
16
|
-
|
|
17
|
-
data_dir = Path.home() / ".lfx" / "data"
|
|
18
|
-
self.data_dir = Path(data_dir)
|
|
19
|
-
self._ready = False
|
|
69
|
+
raise NotImplementedError
|
|
20
70
|
|
|
21
71
|
def set_ready(self) -> None:
|
|
22
72
|
"""Mark the service as ready."""
|
|
23
73
|
self._ready = True
|
|
24
|
-
# Ensure the data directory exists
|
|
25
|
-
self.data_dir.mkdir(parents=True, exist_ok=True)
|
|
26
|
-
|
|
27
|
-
@property
|
|
28
|
-
def ready(self) -> bool:
|
|
29
|
-
"""Check if the service is ready."""
|
|
30
|
-
return self._ready
|
|
31
74
|
|
|
32
75
|
@abstractmethod
|
|
33
|
-
def
|
|
34
|
-
"""
|
|
76
|
+
async def save_file(self, flow_id: str, file_name: str, data: bytes, *, append: bool = False) -> None:
|
|
77
|
+
"""Save a file to storage.
|
|
35
78
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
79
|
+
Args:
|
|
80
|
+
flow_id: The flow/user identifier for namespacing
|
|
81
|
+
file_name: The name of the file to save
|
|
82
|
+
data: The file content as bytes
|
|
83
|
+
append: If True, append to existing file instead of overwriting.
|
|
84
|
+
|
|
85
|
+
Raises:
|
|
86
|
+
Exception: If the file cannot be saved
|
|
87
|
+
"""
|
|
88
|
+
raise NotImplementedError
|
|
39
89
|
|
|
40
90
|
@abstractmethod
|
|
41
91
|
async def get_file(self, flow_id: str, file_name: str) -> bytes:
|
|
42
|
-
"""Retrieve a file.
|
|
92
|
+
"""Retrieve a file from storage.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
flow_id: The flow/user identifier for namespacing
|
|
96
|
+
file_name: The name of the file to retrieve
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
bytes: The file content
|
|
100
|
+
|
|
101
|
+
Raises:
|
|
102
|
+
FileNotFoundError: If the file does not exist
|
|
103
|
+
"""
|
|
104
|
+
raise NotImplementedError
|
|
105
|
+
|
|
106
|
+
async def get_file_stream(self, flow_id: str, file_name: str, chunk_size: int = 8192) -> AsyncIterator[bytes]:
|
|
107
|
+
"""Retrieve a file from storage as a stream.
|
|
108
|
+
|
|
109
|
+
Default implementation loads the entire file and yields it in chunks.
|
|
110
|
+
Subclasses can override this for more efficient streaming.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
flow_id: The flow/user identifier for namespacing
|
|
114
|
+
file_name: The name of the file to retrieve
|
|
115
|
+
chunk_size: Size of chunks to yield (default: 8192 bytes)
|
|
116
|
+
|
|
117
|
+
Yields:
|
|
118
|
+
bytes: Chunks of the file content
|
|
119
|
+
|
|
120
|
+
Raises:
|
|
121
|
+
FileNotFoundError: If the file does not exist
|
|
122
|
+
"""
|
|
123
|
+
# Default implementation - subclasses can override for true streaming
|
|
124
|
+
content = await self.get_file(flow_id, file_name)
|
|
125
|
+
for i in range(0, len(content), chunk_size):
|
|
126
|
+
yield content[i : i + chunk_size]
|
|
43
127
|
|
|
44
128
|
@abstractmethod
|
|
45
129
|
async def list_files(self, flow_id: str) -> list[str]:
|
|
46
|
-
"""List files in a flow.
|
|
130
|
+
"""List all files in a flow's storage namespace.
|
|
47
131
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
132
|
+
Args:
|
|
133
|
+
flow_id: The flow/user identifier for namespacing
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
list[str]: List of file names in the namespace
|
|
137
|
+
|
|
138
|
+
Raises:
|
|
139
|
+
FileNotFoundError: If the namespace directory does not exist
|
|
140
|
+
"""
|
|
141
|
+
raise NotImplementedError
|
|
51
142
|
|
|
52
143
|
@abstractmethod
|
|
53
144
|
async def get_file_size(self, flow_id: str, file_name: str) -> int:
|
|
54
|
-
"""Get the size of a file.
|
|
145
|
+
"""Get the size of a file in bytes.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
flow_id: The flow/user identifier for namespacing
|
|
149
|
+
file_name: The name of the file
|
|
150
|
+
|
|
151
|
+
Returns:
|
|
152
|
+
int: Size of the file in bytes
|
|
153
|
+
|
|
154
|
+
Raises:
|
|
155
|
+
FileNotFoundError: If the file does not exist
|
|
156
|
+
"""
|
|
157
|
+
raise NotImplementedError
|
|
158
|
+
|
|
159
|
+
@abstractmethod
|
|
160
|
+
async def delete_file(self, flow_id: str, file_name: str) -> None:
|
|
161
|
+
"""Delete a file from storage.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
flow_id: The flow/user identifier for namespacing
|
|
165
|
+
file_name: The name of the file to delete
|
|
166
|
+
|
|
167
|
+
Note:
|
|
168
|
+
Should not raise an error if the file doesn't exist
|
|
169
|
+
"""
|
|
170
|
+
raise NotImplementedError
|
|
171
|
+
|
|
172
|
+
async def teardown(self) -> None:
|
|
173
|
+
"""Perform cleanup operations when the service is being shut down.
|
|
174
|
+
|
|
175
|
+
Subclasses should override this to clean up any resources (connections, etc.)
|
|
176
|
+
"""
|
|
177
|
+
raise NotImplementedError
|
lfx/template/field/base.py
CHANGED
|
@@ -208,6 +208,9 @@ class Output(BaseModel):
|
|
|
208
208
|
allows_loop: bool = Field(default=False)
|
|
209
209
|
"""Specifies if the output allows looping."""
|
|
210
210
|
|
|
211
|
+
loop_types: list[str] | None = Field(default=None)
|
|
212
|
+
"""List of additional types to include for loop inputs when allows_loop is True."""
|
|
213
|
+
|
|
211
214
|
group_outputs: bool = Field(default=False)
|
|
212
215
|
"""Specifies if all outputs should be grouped and shown without dropdowns."""
|
|
213
216
|
|