lfx-nightly 0.2.0.dev0__py3-none-any.whl → 0.2.0.dev26__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. lfx/_assets/component_index.json +1 -1
  2. lfx/base/agents/agent.py +13 -1
  3. lfx/base/agents/altk_base_agent.py +380 -0
  4. lfx/base/agents/altk_tool_wrappers.py +565 -0
  5. lfx/base/agents/events.py +2 -1
  6. lfx/base/composio/composio_base.py +159 -224
  7. lfx/base/data/base_file.py +88 -21
  8. lfx/base/data/storage_utils.py +192 -0
  9. lfx/base/data/utils.py +178 -14
  10. lfx/base/embeddings/embeddings_class.py +113 -0
  11. lfx/base/models/groq_constants.py +74 -58
  12. lfx/base/models/groq_model_discovery.py +265 -0
  13. lfx/base/models/model.py +1 -1
  14. lfx/base/models/model_utils.py +100 -0
  15. lfx/base/models/openai_constants.py +7 -0
  16. lfx/base/models/watsonx_constants.py +32 -8
  17. lfx/base/tools/run_flow.py +601 -129
  18. lfx/cli/commands.py +6 -3
  19. lfx/cli/common.py +2 -2
  20. lfx/cli/run.py +1 -1
  21. lfx/cli/script_loader.py +53 -11
  22. lfx/components/Notion/create_page.py +1 -1
  23. lfx/components/Notion/list_database_properties.py +1 -1
  24. lfx/components/Notion/list_pages.py +1 -1
  25. lfx/components/Notion/list_users.py +1 -1
  26. lfx/components/Notion/page_content_viewer.py +1 -1
  27. lfx/components/Notion/search.py +1 -1
  28. lfx/components/Notion/update_page_property.py +1 -1
  29. lfx/components/__init__.py +19 -5
  30. lfx/components/{agents → altk}/__init__.py +5 -9
  31. lfx/components/altk/altk_agent.py +193 -0
  32. lfx/components/apify/apify_actor.py +1 -1
  33. lfx/components/composio/__init__.py +70 -18
  34. lfx/components/composio/apollo_composio.py +11 -0
  35. lfx/components/composio/bitbucket_composio.py +11 -0
  36. lfx/components/composio/canva_composio.py +11 -0
  37. lfx/components/composio/coda_composio.py +11 -0
  38. lfx/components/composio/composio_api.py +10 -0
  39. lfx/components/composio/discord_composio.py +1 -1
  40. lfx/components/composio/elevenlabs_composio.py +11 -0
  41. lfx/components/composio/exa_composio.py +11 -0
  42. lfx/components/composio/firecrawl_composio.py +11 -0
  43. lfx/components/composio/fireflies_composio.py +11 -0
  44. lfx/components/composio/gmail_composio.py +1 -1
  45. lfx/components/composio/googlebigquery_composio.py +11 -0
  46. lfx/components/composio/googlecalendar_composio.py +1 -1
  47. lfx/components/composio/googledocs_composio.py +1 -1
  48. lfx/components/composio/googlemeet_composio.py +1 -1
  49. lfx/components/composio/googlesheets_composio.py +1 -1
  50. lfx/components/composio/googletasks_composio.py +1 -1
  51. lfx/components/composio/heygen_composio.py +11 -0
  52. lfx/components/composio/mem0_composio.py +11 -0
  53. lfx/components/composio/peopledatalabs_composio.py +11 -0
  54. lfx/components/composio/perplexityai_composio.py +11 -0
  55. lfx/components/composio/serpapi_composio.py +11 -0
  56. lfx/components/composio/slack_composio.py +3 -574
  57. lfx/components/composio/slackbot_composio.py +1 -1
  58. lfx/components/composio/snowflake_composio.py +11 -0
  59. lfx/components/composio/tavily_composio.py +11 -0
  60. lfx/components/composio/youtube_composio.py +2 -2
  61. lfx/components/cuga/__init__.py +34 -0
  62. lfx/components/cuga/cuga_agent.py +730 -0
  63. lfx/components/data/__init__.py +78 -28
  64. lfx/components/data_source/__init__.py +58 -0
  65. lfx/components/{data → data_source}/api_request.py +26 -3
  66. lfx/components/{data → data_source}/csv_to_data.py +15 -10
  67. lfx/components/{data → data_source}/json_to_data.py +15 -8
  68. lfx/components/{data → data_source}/news_search.py +1 -1
  69. lfx/components/{data → data_source}/rss.py +1 -1
  70. lfx/components/{data → data_source}/sql_executor.py +1 -1
  71. lfx/components/{data → data_source}/url.py +1 -1
  72. lfx/components/{data → data_source}/web_search.py +1 -1
  73. lfx/components/datastax/astradb_cql.py +1 -1
  74. lfx/components/datastax/astradb_graph.py +1 -1
  75. lfx/components/datastax/astradb_tool.py +1 -1
  76. lfx/components/datastax/astradb_vectorstore.py +1 -1
  77. lfx/components/datastax/hcd.py +1 -1
  78. lfx/components/deactivated/json_document_builder.py +1 -1
  79. lfx/components/docling/__init__.py +0 -3
  80. lfx/components/elastic/elasticsearch.py +1 -1
  81. lfx/components/elastic/opensearch_multimodal.py +1575 -0
  82. lfx/components/files_and_knowledge/__init__.py +47 -0
  83. lfx/components/{data → files_and_knowledge}/directory.py +1 -1
  84. lfx/components/{data → files_and_knowledge}/file.py +246 -18
  85. lfx/components/{knowledge_bases → files_and_knowledge}/retrieval.py +2 -2
  86. lfx/components/{data → files_and_knowledge}/save_file.py +142 -22
  87. lfx/components/flow_controls/__init__.py +58 -0
  88. lfx/components/{logic → flow_controls}/conditional_router.py +1 -1
  89. lfx/components/{logic → flow_controls}/loop.py +43 -9
  90. lfx/components/flow_controls/run_flow.py +108 -0
  91. lfx/components/glean/glean_search_api.py +1 -1
  92. lfx/components/groq/groq.py +35 -28
  93. lfx/components/helpers/__init__.py +102 -0
  94. lfx/components/input_output/__init__.py +3 -1
  95. lfx/components/input_output/chat.py +4 -3
  96. lfx/components/input_output/chat_output.py +4 -4
  97. lfx/components/input_output/text.py +1 -1
  98. lfx/components/input_output/text_output.py +1 -1
  99. lfx/components/{data → input_output}/webhook.py +1 -1
  100. lfx/components/knowledge_bases/__init__.py +59 -4
  101. lfx/components/langchain_utilities/character.py +1 -1
  102. lfx/components/langchain_utilities/csv_agent.py +84 -16
  103. lfx/components/langchain_utilities/json_agent.py +67 -12
  104. lfx/components/langchain_utilities/language_recursive.py +1 -1
  105. lfx/components/llm_operations/__init__.py +46 -0
  106. lfx/components/{processing → llm_operations}/batch_run.py +1 -1
  107. lfx/components/{processing → llm_operations}/lambda_filter.py +1 -1
  108. lfx/components/{logic → llm_operations}/llm_conditional_router.py +1 -1
  109. lfx/components/{processing/llm_router.py → llm_operations/llm_selector.py} +3 -3
  110. lfx/components/{processing → llm_operations}/structured_output.py +1 -1
  111. lfx/components/logic/__init__.py +126 -0
  112. lfx/components/mem0/mem0_chat_memory.py +11 -0
  113. lfx/components/models/__init__.py +64 -9
  114. lfx/components/models_and_agents/__init__.py +49 -0
  115. lfx/components/{agents → models_and_agents}/agent.py +2 -2
  116. lfx/components/models_and_agents/embedding_model.py +423 -0
  117. lfx/components/models_and_agents/language_model.py +398 -0
  118. lfx/components/{agents → models_and_agents}/mcp_component.py +53 -44
  119. lfx/components/{helpers → models_and_agents}/memory.py +1 -1
  120. lfx/components/nvidia/system_assist.py +1 -1
  121. lfx/components/olivya/olivya.py +1 -1
  122. lfx/components/ollama/ollama.py +17 -3
  123. lfx/components/processing/__init__.py +9 -57
  124. lfx/components/processing/converter.py +1 -1
  125. lfx/components/processing/dataframe_operations.py +1 -1
  126. lfx/components/processing/parse_json_data.py +2 -2
  127. lfx/components/processing/parser.py +1 -1
  128. lfx/components/processing/split_text.py +1 -1
  129. lfx/components/qdrant/qdrant.py +1 -1
  130. lfx/components/redis/redis.py +1 -1
  131. lfx/components/twelvelabs/split_video.py +10 -0
  132. lfx/components/twelvelabs/video_file.py +12 -0
  133. lfx/components/utilities/__init__.py +43 -0
  134. lfx/components/{helpers → utilities}/calculator_core.py +1 -1
  135. lfx/components/{helpers → utilities}/current_date.py +1 -1
  136. lfx/components/{processing → utilities}/python_repl_core.py +1 -1
  137. lfx/components/vectorstores/local_db.py +9 -0
  138. lfx/components/youtube/youtube_transcripts.py +118 -30
  139. lfx/custom/custom_component/component.py +57 -1
  140. lfx/custom/custom_component/custom_component.py +68 -6
  141. lfx/graph/edge/base.py +43 -20
  142. lfx/graph/graph/base.py +4 -1
  143. lfx/graph/state/model.py +15 -2
  144. lfx/graph/utils.py +6 -0
  145. lfx/graph/vertex/base.py +4 -1
  146. lfx/graph/vertex/param_handler.py +10 -7
  147. lfx/helpers/__init__.py +12 -0
  148. lfx/helpers/flow.py +117 -0
  149. lfx/inputs/input_mixin.py +24 -1
  150. lfx/inputs/inputs.py +13 -1
  151. lfx/interface/components.py +161 -83
  152. lfx/log/logger.py +5 -3
  153. lfx/services/database/__init__.py +5 -0
  154. lfx/services/database/service.py +25 -0
  155. lfx/services/deps.py +87 -22
  156. lfx/services/manager.py +19 -6
  157. lfx/services/mcp_composer/service.py +998 -157
  158. lfx/services/session.py +5 -0
  159. lfx/services/settings/base.py +51 -7
  160. lfx/services/settings/constants.py +8 -0
  161. lfx/services/storage/local.py +76 -46
  162. lfx/services/storage/service.py +152 -29
  163. lfx/template/field/base.py +3 -0
  164. lfx/utils/ssrf_protection.py +384 -0
  165. lfx/utils/validate_cloud.py +26 -0
  166. {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/METADATA +38 -22
  167. {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/RECORD +182 -150
  168. {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/WHEEL +1 -1
  169. lfx/components/agents/altk_agent.py +0 -366
  170. lfx/components/agents/cuga_agent.py +0 -1013
  171. lfx/components/docling/docling_remote_vlm.py +0 -284
  172. lfx/components/logic/run_flow.py +0 -71
  173. lfx/components/models/embedding_model.py +0 -195
  174. lfx/components/models/language_model.py +0 -144
  175. /lfx/components/{data → data_source}/mock_data.py +0 -0
  176. /lfx/components/{knowledge_bases → files_and_knowledge}/ingestion.py +0 -0
  177. /lfx/components/{logic → flow_controls}/data_conditional_router.py +0 -0
  178. /lfx/components/{logic → flow_controls}/flow_tool.py +0 -0
  179. /lfx/components/{logic → flow_controls}/listen.py +0 -0
  180. /lfx/components/{logic → flow_controls}/notify.py +0 -0
  181. /lfx/components/{logic → flow_controls}/pass_message.py +0 -0
  182. /lfx/components/{logic → flow_controls}/sub_flow.py +0 -0
  183. /lfx/components/{processing → models_and_agents}/prompt.py +0 -0
  184. /lfx/components/{helpers → processing}/create_list.py +0 -0
  185. /lfx/components/{helpers → processing}/output_parser.py +0 -0
  186. /lfx/components/{helpers → processing}/store_message.py +0 -0
  187. /lfx/components/{helpers → utilities}/id_generator.py +0 -0
  188. {lfx_nightly-0.2.0.dev0.dist-info → lfx_nightly-0.2.0.dev26.dist-info}/entry_points.txt +0 -0
lfx/services/session.py CHANGED
@@ -75,6 +75,11 @@ class NoopSession:
75
75
  """Context manager that disables autoflush (no-op implementation)."""
76
76
  return self
77
77
 
78
+ @property
79
+ def is_active(self):
80
+ """Check if session is active (always True for NoopSession)."""
81
+ return True
82
+
78
83
  def __enter__(self):
79
84
  return self
80
85
 
@@ -17,7 +17,7 @@ from typing_extensions import override
17
17
  from lfx.constants import BASE_COMPONENTS_PATH
18
18
  from lfx.log.logger import logger
19
19
  from lfx.serialization.constants import MAX_ITEMS_LENGTH, MAX_TEXT_LENGTH
20
- from lfx.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
20
+ from lfx.services.settings.constants import AGENTIC_VARIABLES, VARIABLES_TO_GET_FROM_ENVIRONMENT
21
21
  from lfx.utils.util_strings import is_valid_database_url
22
22
 
23
23
 
@@ -87,6 +87,10 @@ class Settings(BaseSettings):
87
87
  db_connect_timeout: int = 30
88
88
  """The number of seconds to wait before giving up on a lock to released or establishing a connection to the
89
89
  database."""
90
+ migration_lock_namespace: str | None = None
91
+ """Optional namespace identifier for PostgreSQL advisory lock during migrations.
92
+ If not provided, a hash of the database URL will be used. Useful when multiple Langflow
93
+ instances share the same database and need coordinated migration locking."""
90
94
 
91
95
  mcp_server_timeout: int = 20
92
96
  """The number of seconds to wait before giving up on a lock to released or establishing a connection to the
@@ -109,7 +113,7 @@ class Settings(BaseSettings):
109
113
  reap idle sessions."""
110
114
 
111
115
  # sqlite configuration
112
- sqlite_pragmas: dict | None = {"synchronous": "NORMAL", "journal_mode": "WAL"}
116
+ sqlite_pragmas: dict | None = {"synchronous": "NORMAL", "journal_mode": "WAL", "busy_timeout": 30000}
113
117
  """SQLite pragmas to use when connecting to the database."""
114
118
 
115
119
  db_driver_connection_settings: dict | None = None
@@ -187,6 +191,13 @@ class Settings(BaseSettings):
187
191
  like_webhook_url: str | None = "https://api.langflow.store/flows/trigger/64275852-ec00-45c1-984e-3bff814732da"
188
192
 
189
193
  storage_type: str = "local"
194
+ """Storage type for file storage. Defaults to 'local'. Supports 'local' and 's3'."""
195
+ object_storage_bucket_name: str | None = "langflow-bucket"
196
+ """Object storage bucket name for file storage. Defaults to 'langflow-bucket'."""
197
+ object_storage_prefix: str | None = "files"
198
+ """Object storage prefix for file storage. Defaults to 'files'."""
199
+ object_storage_tags: dict[str, str] | None = None
200
+ """Object storage tags for file storage."""
190
201
 
191
202
  celery_enabled: bool = False
192
203
 
@@ -245,6 +256,8 @@ class Settings(BaseSettings):
245
256
  """The path to log file for Langflow."""
246
257
  alembic_log_file: str = "alembic/alembic.log"
247
258
  """The path to log file for Alembic for SQLAlchemy."""
259
+ alembic_log_to_stdout: bool = False
260
+ """If set to True, the log file will be ignored and Alembic will log to stdout."""
248
261
  frontend_path: str | None = None
249
262
  """The path to the frontend directory containing build files. This is for development purposes only.."""
250
263
  open_browser: bool = False
@@ -292,9 +305,13 @@ class Settings(BaseSettings):
292
305
  # MCP Composer
293
306
  mcp_composer_enabled: bool = True
294
307
  """If set to False, Langflow will not start the MCP Composer service."""
295
- mcp_composer_version: str = "~=0.1.0.7"
296
- """Version constraint for mcp-composer when using uvx. Uses PEP 440 syntax.
297
- ~=0.1.0.7 allows patch updates (0.1.0.x) but prevents minor/major version changes."""
308
+ mcp_composer_version: str = "==0.1.0.8.10"
309
+ """Version constraint for mcp-composer when using uvx. Uses PEP 440 syntax."""
310
+
311
+ # Agentic Experience
312
+ agentic_experience: bool = False
313
+ """If set to True, Langflow will start the agentic MCP server that provides tools for
314
+ flow/component operations, template search, and graph visualization."""
298
315
 
299
316
  # Public Flow Settings
300
317
  public_flow_cleanup_interval: int = Field(default=3600, gt=600)
@@ -317,6 +334,23 @@ class Settings(BaseSettings):
317
334
  update_starter_projects: bool = True
318
335
  """If set to True, Langflow will update starter projects."""
319
336
 
337
+ # SSRF Protection
338
+ ssrf_protection_enabled: bool = False
339
+ """If set to True, Langflow will enable SSRF (Server-Side Request Forgery) protection.
340
+ When enabled, blocks requests to private IP ranges, localhost, and cloud metadata endpoints.
341
+ When False (default), no URL validation is performed, allowing requests to any destination
342
+ including internal services, private networks, and cloud metadata endpoints.
343
+ Default is False for backward compatibility. In v2.0, this will be changed to True.
344
+
345
+ Note: When ssrf_protection_enabled is disabled, the ssrf_allowed_hosts setting is ignored and has no effect."""
346
+ ssrf_allowed_hosts: list[str] = []
347
+ """Comma-separated list of hosts/IPs/CIDR ranges to allow despite SSRF protection.
348
+ Examples: 'internal-api.company.local,192.168.1.0/24,10.0.0.5,*.dev.internal'
349
+ Supports exact hostnames, wildcard domains (*.example.com), exact IPs, and CIDR ranges.
350
+
351
+ Note: This setting only takes effect when ssrf_protection_enabled is True.
352
+ When protection is disabled, all hosts are allowed regardless of this setting."""
353
+
320
354
  @field_validator("cors_origins", mode="before")
321
355
  @classmethod
322
356
  def validate_cors_origins(cls, value):
@@ -367,7 +401,7 @@ class Settings(BaseSettings):
367
401
  Supports PEP 440 specifiers: ==, !=, <=, >=, <, >, ~=, ===
368
402
  """
369
403
  if not value:
370
- return "~=0.1.0.7" # Default
404
+ return "==0.1.0.8.10" # Default
371
405
 
372
406
  # Check if it already has a version specifier
373
407
  # Order matters: check longer specifiers first to avoid false matches
@@ -389,9 +423,19 @@ class Settings(BaseSettings):
389
423
  @field_validator("variables_to_get_from_environment", mode="before")
390
424
  @classmethod
391
425
  def set_variables_to_get_from_environment(cls, value):
426
+ import os
427
+
392
428
  if isinstance(value, str):
393
429
  value = value.split(",")
394
- return list(set(VARIABLES_TO_GET_FROM_ENVIRONMENT + value))
430
+
431
+ result = list(set(VARIABLES_TO_GET_FROM_ENVIRONMENT + value))
432
+
433
+ # Add agentic variables if agentic_experience is enabled
434
+ # Check env var directly since we can't access instance attributes in validator
435
+ if os.getenv("LANGFLOW_AGENTIC_EXPERIENCE", "true").lower() == "true":
436
+ result.extend(AGENTIC_VARIABLES)
437
+
438
+ return list(set(result))
395
439
 
396
440
  @field_validator("log_file", mode="before")
397
441
  @classmethod
@@ -33,3 +33,11 @@ VARIABLES_TO_GET_FROM_ENVIRONMENT = [
33
33
  "TAVILY_API_KEY",
34
34
  "COMETAPI_KEY",
35
35
  ]
36
+
37
+ # Agentic experience specific variables
38
+ AGENTIC_VARIABLES = [
39
+ "FLOW_ID",
40
+ "COMPONENT_ID",
41
+ "FIELD_NAME",
42
+ "ASTRA_TOKEN",
43
+ ]
@@ -1,30 +1,70 @@
1
1
  """Local file-based storage service for lfx package."""
2
2
 
3
- from pathlib import Path
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from aiofile import async_open
4
8
 
5
9
  from lfx.log.logger import logger
6
10
  from lfx.services.storage.service import StorageService
7
11
 
12
+ if TYPE_CHECKING:
13
+ from langflow.services.session.service import SessionService
14
+
15
+ from lfx.services.settings.service import SettingsService
16
+
17
+ # Constants for path parsing
18
+ EXPECTED_PATH_PARTS = 2 # Path format: "flow_id/filename"
19
+
8
20
 
9
21
  class LocalStorageService(StorageService):
10
22
  """A service class for handling local file storage operations."""
11
23
 
12
- def __init__(self, data_dir: str | Path | None = None) -> None:
13
- """Initialize the local storage service."""
14
- super().__init__(data_dir)
15
- self.set_ready()
24
+ def __init__(
25
+ self,
26
+ session_service: SessionService,
27
+ settings_service: SettingsService,
28
+ ) -> None:
29
+ """Initialize the local storage service.
30
+
31
+ Args:
32
+ session_service: Session service instance
33
+ settings_service: Settings service instance containing configuration
34
+ """
35
+ # Initialize base class with services
36
+ super().__init__(session_service, settings_service)
37
+ # Base class already sets self.data_dir as anyio.Path from settings_service.settings.config_dir
38
+
39
+ def resolve_component_path(self, logical_path: str) -> str:
40
+ """Convert logical path to absolute filesystem path for local storage.
41
+
42
+ Args:
43
+ logical_path: Path in format "flow_id/filename"
44
+ Returns:
45
+ str: Absolute filesystem path
46
+ """
47
+ # Split the logical path into flow_id and filename
48
+ parts = logical_path.split("/", 1)
49
+ if len(parts) != EXPECTED_PATH_PARTS:
50
+ # Handle edge case - return as-is if format is unexpected
51
+ return logical_path
52
+
53
+ flow_id, file_name = parts
54
+ return self.build_full_path(flow_id, file_name)
16
55
 
17
56
  def build_full_path(self, flow_id: str, file_name: str) -> str:
18
57
  """Build the full path of a file in the local storage."""
19
58
  return str(self.data_dir / flow_id / file_name)
20
59
 
21
- async def save_file(self, flow_id: str, file_name: str, data: bytes) -> None:
60
+ async def save_file(self, flow_id: str, file_name: str, data: bytes, *, append: bool = False) -> None:
22
61
  """Save a file in the local storage.
23
62
 
24
63
  Args:
25
64
  flow_id: The identifier for the flow.
26
65
  file_name: The name of the file to be saved.
27
66
  data: The byte content of the file.
67
+ append: If True, append to existing file; if False, overwrite.
28
68
 
29
69
  Raises:
30
70
  FileNotFoundError: If the specified flow does not exist.
@@ -32,17 +72,18 @@ class LocalStorageService(StorageService):
32
72
  PermissionError: If there is no permission to write the file.
33
73
  """
34
74
  folder_path = self.data_dir / flow_id
35
- folder_path.mkdir(parents=True, exist_ok=True)
75
+ await folder_path.mkdir(parents=True, exist_ok=True)
36
76
  file_path = folder_path / file_name
37
77
 
38
78
  try:
39
- with file_path.open("wb") as f:
40
- f.write(data)
79
+ mode = "ab" if append else "wb"
80
+ async with async_open(str(file_path), mode) as f:
81
+ await f.write(data)
82
+ action = "appended to" if append else "saved"
83
+ await logger.ainfo(f"File {file_name} {action} successfully in flow {flow_id}.")
41
84
  except Exception:
42
85
  logger.exception(f"Error saving file {file_name} in flow {flow_id}")
43
86
  raise
44
- else:
45
- logger.info(f"File {file_name} saved successfully in flow {flow_id}.")
46
87
 
47
88
  async def get_file(self, flow_id: str, file_name: str) -> bytes:
48
89
  """Retrieve a file from the local storage.
@@ -58,20 +99,16 @@ class LocalStorageService(StorageService):
58
99
  FileNotFoundError: If the file does not exist.
59
100
  """
60
101
  file_path = self.data_dir / flow_id / file_name
61
- if not file_path.exists():
62
- logger.warning(f"File {file_name} not found in flow {flow_id}.")
102
+ if not await file_path.exists():
103
+ await logger.awarning(f"File {file_name} not found in flow {flow_id}.")
63
104
  msg = f"File {file_name} not found in flow {flow_id}"
64
105
  raise FileNotFoundError(msg)
65
106
 
66
- try:
67
- with file_path.open("rb") as f:
68
- content = f.read()
69
- except Exception:
70
- logger.exception(f"Error reading file {file_name} in flow {flow_id}")
71
- raise
72
- else:
73
- logger.debug(f"File {file_name} retrieved successfully from flow {flow_id}.")
74
- return content
107
+ async with async_open(str(file_path), "rb") as f:
108
+ content = await f.read()
109
+
110
+ logger.debug(f"File {file_name} retrieved successfully from flow {flow_id}.")
111
+ return content
75
112
 
76
113
  async def list_files(self, flow_id: str) -> list[str]:
77
114
  """List all files in a specific flow directory.
@@ -86,21 +123,17 @@ class LocalStorageService(StorageService):
86
123
  flow_id = str(flow_id)
87
124
 
88
125
  folder_path = self.data_dir / flow_id
89
- if not folder_path.exists():
90
- logger.debug(f"Flow folder {flow_id} does not exist.")
91
- return []
92
-
93
- if not folder_path.is_dir():
94
- logger.warning(f"Flow path {flow_id} is not a directory.")
126
+ if not await folder_path.exists() or not await folder_path.is_dir():
127
+ await logger.awarning(f"Flow {flow_id} directory does not exist.")
95
128
  return []
96
129
 
97
130
  try:
98
- files = [item.name for item in folder_path.iterdir() if item.is_file()]
131
+ files = [p.name async for p in folder_path.iterdir() if await p.is_file()]
99
132
  except Exception: # noqa: BLE001
100
133
  logger.exception(f"Error listing files in flow {flow_id}")
101
134
  return []
102
135
  else:
103
- logger.debug(f"Listed {len(files)} files in flow {flow_id}.")
136
+ await logger.ainfo(f"Listed {len(files)} files in flow {flow_id}.")
104
137
  return files
105
138
 
106
139
  async def delete_file(self, flow_id: str, file_name: str) -> None:
@@ -114,17 +147,11 @@ class LocalStorageService(StorageService):
114
147
  FileNotFoundError: If the file does not exist.
115
148
  """
116
149
  file_path = self.data_dir / flow_id / file_name
117
- if not file_path.exists():
118
- logger.warning(f"File {file_name} not found in flow {flow_id}.")
119
- msg = f"File {file_name} not found in flow {flow_id}"
120
- raise FileNotFoundError(msg)
121
-
122
- try:
123
- file_path.unlink()
124
- logger.info(f"File {file_name} deleted successfully from flow {flow_id}.")
125
- except Exception:
126
- logger.exception(f"Error deleting file {file_name} in flow {flow_id}")
127
- raise
150
+ if await file_path.exists():
151
+ await file_path.unlink()
152
+ await logger.ainfo(f"File {file_name} deleted successfully from flow {flow_id}.")
153
+ else:
154
+ await logger.awarning(f"Attempted to delete non-existent file {file_name} in flow {flow_id}.")
128
155
 
129
156
  async def get_file_size(self, flow_id: str, file_name: str) -> int:
130
157
  """Get the size of a file in bytes.
@@ -140,16 +167,19 @@ class LocalStorageService(StorageService):
140
167
  FileNotFoundError: If the file does not exist.
141
168
  """
142
169
  file_path = self.data_dir / flow_id / file_name
143
- if not file_path.exists():
144
- logger.warning(f"File {file_name} not found in flow {flow_id}.")
170
+ if not await file_path.exists():
171
+ await logger.awarning(f"File {file_name} not found in flow {flow_id}.")
145
172
  msg = f"File {file_name} not found in flow {flow_id}"
146
173
  raise FileNotFoundError(msg)
147
174
 
148
175
  try:
149
- size = file_path.stat().st_size
176
+ file_size_stat = await file_path.stat()
150
177
  except Exception:
151
178
  logger.exception(f"Error getting size of file {file_name} in flow {flow_id}")
152
179
  raise
153
180
  else:
154
- logger.debug(f"File {file_name} size: {size} bytes in flow {flow_id}.")
155
- return size
181
+ return file_size_stat.st_size
182
+
183
+ async def teardown(self) -> None:
184
+ """Perform any cleanup operations when the service is being torn down."""
185
+ # No specific teardown actions required for local
@@ -1,54 +1,177 @@
1
- """Base storage service for lfx package."""
1
+ from __future__ import annotations
2
2
 
3
- from abc import ABC, abstractmethod
4
- from pathlib import Path
3
+ from abc import abstractmethod
4
+ from typing import TYPE_CHECKING
5
5
 
6
+ import anyio
6
7
 
7
- class StorageService(ABC):
8
- """Abstract base class for storage services."""
8
+ from lfx.services.base import Service
9
9
 
10
- def __init__(self, data_dir: str | Path | None = None) -> None:
10
+ if TYPE_CHECKING:
11
+ from collections.abc import AsyncIterator
12
+
13
+ from lfx.services.settings.service import SettingsService
14
+
15
+
16
+ class StorageService(Service):
17
+ """Abstract base class for file storage services.
18
+
19
+ This class defines the interface for file storage operations that can be
20
+ implemented by different backends (local filesystem, S3, etc.).
21
+
22
+ All file operations are namespaced by flow_id to isolate files between
23
+ different flows or users.
24
+ """
25
+
26
+ name = "storage_service"
27
+
28
+ def __init__(self, session_service, settings_service: SettingsService):
11
29
  """Initialize the storage service.
12
30
 
13
31
  Args:
14
- data_dir: Directory path for storing data. Defaults to ~/.lfx/data
32
+ session_service: The session service instance
33
+ settings_service: The settings service instance containing configuration
34
+ """
35
+ self.settings_service = settings_service
36
+ self.session_service = session_service
37
+ self.data_dir: anyio.Path = anyio.Path(settings_service.settings.config_dir)
38
+ self.set_ready()
39
+
40
+ def build_full_path(self, flow_id: str, file_name: str) -> str:
41
+ """Build the full path/key for a file.
42
+
43
+ Args:
44
+ flow_id: The flow/user identifier for namespacing
45
+ file_name: The name of the file
46
+
47
+ Returns:
48
+ str: The full path or key for the file
49
+ """
50
+ raise NotImplementedError
51
+
52
+ def resolve_component_path(self, logical_path: str) -> str:
53
+ """Convert a logical path to a format that components can use directly.
54
+
55
+ Logical paths are in the format "{flow_id}/{filename}" as stored in the database.
56
+ This method converts them to a format appropriate for the storage backend:
57
+ - Local storage: Absolute filesystem path (/data_dir/flow_id/filename)
58
+ - S3 storage: Logical path as-is (flow_id/filename)
59
+
60
+ Components receive this resolved path and can use it without knowing the
61
+ storage implementation details.
62
+
63
+ Args:
64
+ logical_path: Path in the format "flow_id/filename"
65
+
66
+ Returns:
67
+ str: A path that components can use directly
15
68
  """
16
- if data_dir is None:
17
- data_dir = Path.home() / ".lfx" / "data"
18
- self.data_dir = Path(data_dir)
19
- self._ready = False
69
+ raise NotImplementedError
20
70
 
21
71
  def set_ready(self) -> None:
22
72
  """Mark the service as ready."""
23
73
  self._ready = True
24
- # Ensure the data directory exists
25
- self.data_dir.mkdir(parents=True, exist_ok=True)
26
-
27
- @property
28
- def ready(self) -> bool:
29
- """Check if the service is ready."""
30
- return self._ready
31
74
 
32
75
  @abstractmethod
33
- def build_full_path(self, flow_id: str, file_name: str) -> str:
34
- """Build the full path for a file."""
76
+ async def save_file(self, flow_id: str, file_name: str, data: bytes, *, append: bool = False) -> None:
77
+ """Save a file to storage.
35
78
 
36
- @abstractmethod
37
- async def save_file(self, flow_id: str, file_name: str, data: bytes) -> None:
38
- """Save a file."""
79
+ Args:
80
+ flow_id: The flow/user identifier for namespacing
81
+ file_name: The name of the file to save
82
+ data: The file content as bytes
83
+ append: If True, append to existing file instead of overwriting.
84
+
85
+ Raises:
86
+ Exception: If the file cannot be saved
87
+ """
88
+ raise NotImplementedError
39
89
 
40
90
  @abstractmethod
41
91
  async def get_file(self, flow_id: str, file_name: str) -> bytes:
42
- """Retrieve a file."""
92
+ """Retrieve a file from storage.
93
+
94
+ Args:
95
+ flow_id: The flow/user identifier for namespacing
96
+ file_name: The name of the file to retrieve
97
+
98
+ Returns:
99
+ bytes: The file content
100
+
101
+ Raises:
102
+ FileNotFoundError: If the file does not exist
103
+ """
104
+ raise NotImplementedError
105
+
106
+ async def get_file_stream(self, flow_id: str, file_name: str, chunk_size: int = 8192) -> AsyncIterator[bytes]:
107
+ """Retrieve a file from storage as a stream.
108
+
109
+ Default implementation loads the entire file and yields it in chunks.
110
+ Subclasses can override this for more efficient streaming.
111
+
112
+ Args:
113
+ flow_id: The flow/user identifier for namespacing
114
+ file_name: The name of the file to retrieve
115
+ chunk_size: Size of chunks to yield (default: 8192 bytes)
116
+
117
+ Yields:
118
+ bytes: Chunks of the file content
119
+
120
+ Raises:
121
+ FileNotFoundError: If the file does not exist
122
+ """
123
+ # Default implementation - subclasses can override for true streaming
124
+ content = await self.get_file(flow_id, file_name)
125
+ for i in range(0, len(content), chunk_size):
126
+ yield content[i : i + chunk_size]
43
127
 
44
128
  @abstractmethod
45
129
  async def list_files(self, flow_id: str) -> list[str]:
46
- """List files in a flow."""
130
+ """List all files in a flow's storage namespace.
47
131
 
48
- @abstractmethod
49
- async def delete_file(self, flow_id: str, file_name: str) -> None:
50
- """Delete a file."""
132
+ Args:
133
+ flow_id: The flow/user identifier for namespacing
134
+
135
+ Returns:
136
+ list[str]: List of file names in the namespace
137
+
138
+ Raises:
139
+ FileNotFoundError: If the namespace directory does not exist
140
+ """
141
+ raise NotImplementedError
51
142
 
52
143
  @abstractmethod
53
144
  async def get_file_size(self, flow_id: str, file_name: str) -> int:
54
- """Get the size of a file."""
145
+ """Get the size of a file in bytes.
146
+
147
+ Args:
148
+ flow_id: The flow/user identifier for namespacing
149
+ file_name: The name of the file
150
+
151
+ Returns:
152
+ int: Size of the file in bytes
153
+
154
+ Raises:
155
+ FileNotFoundError: If the file does not exist
156
+ """
157
+ raise NotImplementedError
158
+
159
+ @abstractmethod
160
+ async def delete_file(self, flow_id: str, file_name: str) -> None:
161
+ """Delete a file from storage.
162
+
163
+ Args:
164
+ flow_id: The flow/user identifier for namespacing
165
+ file_name: The name of the file to delete
166
+
167
+ Note:
168
+ Should not raise an error if the file doesn't exist
169
+ """
170
+ raise NotImplementedError
171
+
172
+ async def teardown(self) -> None:
173
+ """Perform cleanup operations when the service is being shut down.
174
+
175
+ Subclasses should override this to clean up any resources (connections, etc.)
176
+ """
177
+ raise NotImplementedError
@@ -208,6 +208,9 @@ class Output(BaseModel):
208
208
  allows_loop: bool = Field(default=False)
209
209
  """Specifies if the output allows looping."""
210
210
 
211
+ loop_types: list[str] | None = Field(default=None)
212
+ """List of additional types to include for loop inputs when allows_loop is True."""
213
+
211
214
  group_outputs: bool = Field(default=False)
212
215
  """Specifies if all outputs should be grouped and shown without dropdowns."""
213
216