model-compose 0.4.20__tar.gz → 0.4.22__tar.gz

{model_compose-0.4.20/src/model_compose.egg-info → model_compose-0.4.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model-compose
-Version: 0.4.20
+Version: 0.4.22
 Summary: model-compose: Declarative AI Workflow Orchestrator
 Author-email: Hanyeol Cho <hanyeol.cho@gmail.com>
 License-Expression: MIT
@@ -18,8 +18,10 @@ Requires-Dist: fastapi
 Requires-Dist: uvicorn
 Requires-Dist: mcp>=1.10.1
 Requires-Dist: ulid
-Requires-Dist: gradio
+Requires-Dist: gradio>=6.0.0
 Requires-Dist: Pillow
+Requires-Dist: beautifulsoup4
+Requires-Dist: lxml
 Dynamic: license-file
 
 <div align="center">
@@ -69,14 +71,14 @@ Built-in SSE (Server-Sent Events) streaming for real-time AI responses. Stream f
 ### 🔄 **Advanced Workflow Composition**
 Build multi-step pipelines with conditional logic, data transformation, and parallel execution. Pass data between jobs with powerful variable binding—`${input}`, `${response}`, `${env}`, with type conversion and defaults.
 
-### 🚀 **Production-Ready Controllers**
+### 🚀 **Production-Ready Deployment**
 Deploy as HTTP REST API or MCP (Model Context Protocol) server by changing one line. Includes concurrency control, health checks, and automatic API documentation.
 
 ### 🎯 **Event-Driven Architecture**
 HTTP Callback listeners for async workflows (image generation, video processing). HTTP Trigger listeners for webhooks and external events. Build reactive AI systems that respond to real-world events.
 
 ### 🌐 **Smart Tunneling & Gateways**
-Expose local services to the internet instantly with ngrok, Cloudflare, or SSH tunnels. Perfect for webhook integration, Slack bots, and public API deployment without complex networking.
+Expose local services to the internet instantly with ngrok, Cloudflare, or SSH tunnels. Perfect for webhook integration and public API deployment without complex networking.
 
 ### 🐳 **Container-Native Deployment**
 First-class Docker support with runtime configuration, volume mounting, and environment management. Deploy to any cloud provider or Kubernetes cluster with minimal configuration.
@@ -174,6 +176,12 @@ controller:
   type: mcp-server
 ```
 
+### 🔄 Run Components in Separate Processes
+```yaml
+component:
+  runtime: process
+```
+
 ### 🐳 Deploy in Docker with 1 Line
 ```yaml
 controller:

{model_compose-0.4.20 → model_compose-0.4.22}/README.md

@@ -45,14 +45,14 @@ Built-in SSE (Server-Sent Events) streaming for real-time AI responses. Stream f
 ### 🔄 **Advanced Workflow Composition**
 Build multi-step pipelines with conditional logic, data transformation, and parallel execution. Pass data between jobs with powerful variable binding—`${input}`, `${response}`, `${env}`, with type conversion and defaults.
 
-### 🚀 **Production-Ready Controllers**
+### 🚀 **Production-Ready Deployment**
 Deploy as HTTP REST API or MCP (Model Context Protocol) server by changing one line. Includes concurrency control, health checks, and automatic API documentation.
 
 ### 🎯 **Event-Driven Architecture**
 HTTP Callback listeners for async workflows (image generation, video processing). HTTP Trigger listeners for webhooks and external events. Build reactive AI systems that respond to real-world events.
 
 ### 🌐 **Smart Tunneling & Gateways**
-Expose local services to the internet instantly with ngrok, Cloudflare, or SSH tunnels. Perfect for webhook integration, Slack bots, and public API deployment without complex networking.
+Expose local services to the internet instantly with ngrok, Cloudflare, or SSH tunnels. Perfect for webhook integration and public API deployment without complex networking.
 
 ### 🐳 **Container-Native Deployment**
 First-class Docker support with runtime configuration, volume mounting, and environment management. Deploy to any cloud provider or Kubernetes cluster with minimal configuration.
@@ -150,6 +150,12 @@ controller:
   type: mcp-server
 ```
 
+### 🔄 Run Components in Separate Processes
+```yaml
+component:
+  runtime: process
+```
+
 ### 🐳 Deploy in Docker with 1 Line
 ```yaml
 controller:

{model_compose-0.4.20 → model_compose-0.4.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "model-compose"
-version = "0.4.20"
+version = "0.4.22"
 description = "model-compose: Declarative AI Workflow Orchestrator"
 authors = [
     { name = "Hanyeol Cho", email = "hanyeol.cho@gmail.com" }
@@ -18,9 +18,11 @@ dependencies = [
     "fastapi",
     "uvicorn",
     "mcp>=1.10.1",
-    "ulid", 
-    "gradio",
-    "Pillow"
+    "ulid",
+    "gradio>=6.0.0",
+    "Pillow",
+    "beautifulsoup4",
+    "lxml"
 ]
 
 [project.scripts]

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/runtime/process_manager.py

@@ -1,7 +1,9 @@
 from typing import Any, Dict
 from mindor.core.foundation import ProcessRuntimeManager
+from mindor.core.foundation.process_worker import ProcessWorkerParams
 from mindor.core.component.base import ComponentGlobalConfigs
 from mindor.core.component.runtime.process_worker import ComponentProcessWorker
+from mindor.core.utils.time import parse_duration
 from mindor.dsl.schema.component import ComponentConfig
 from mindor.dsl.schema.runtime import ProcessRuntimeConfig
 from multiprocessing import Queue
@@ -30,10 +32,22 @@ class ComponentProcessRuntimeManager(ProcessRuntimeManager):
             global_configs=global_configs
         )
 
+        # Convert ProcessRuntimeConfig to ProcessWorkerParams
+        worker_params = self._convert_runtime_config(config.runtime)
+
         super().__init__(
             worker_id=component_id,
-            runtime_config=config.runtime,
-            worker_factory=worker_factory
+            worker_factory=worker_factory,
+            worker_params=worker_params
+        )
+
+    @staticmethod
+    def _convert_runtime_config(config: ProcessRuntimeConfig) -> ProcessWorkerParams:
+        """Convert DSL ProcessRuntimeConfig to foundation ProcessWorkerParams"""
+        return ProcessWorkerParams(
+            env=config.env,
+            start_timeout=parse_duration(config.start_timeout).total_seconds(),
+            stop_timeout=parse_duration(config.stop_timeout).total_seconds()
         )
 
     def _component_worker_factory(

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/runtime/process_worker.py

@@ -46,8 +46,7 @@ class ComponentProcessWorker(ProcessWorker):
         run_id = payload["run_id"]
         input_data = payload["input"]
 
-        output = await self.component.run(action_id, run_id, input_data)
-        return output
+        return await self.component.run(action_id, run_id, input_data)
 
     async def _cleanup(self) -> None:
         """Clean up the component"""

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/services/__init__.py

@@ -9,3 +9,4 @@ from .workflow import *
 from .shell import *
 from .text_splitter import *
 from .image_processor import *
+from .web_scraper import *

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/services/model/tasks/image_to_text/huggingface.py

@@ -78,15 +78,12 @@ class HuggingfaceImageToTextTaskAction:
             async def _stream_output_generator():
                 async for chunk in AsyncStreamer(streamer, loop):
                     if chunk:
-                        context.register_source("result[]", chunk)
-                        yield (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else chunk
+                        yield await self._render_output_chunk(context, chunk)
 
             return _stream_output_generator()
         else:
             result = results[0] if is_single_input else results
-            context.register_source("result", result)
-
-            return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+            return await self._render_output(context, result)
 
     async def _prepare_input(self, context: ComponentActionContext) -> Tuple[Union[PILImage.Image, List[PILImage.Image]], Optional[Union[str, List[str]]]]:
         image = await context.render_image(self.config.image)
@@ -94,6 +91,14 @@ class HuggingfaceImageToTextTaskAction:
         
         return image, text
 
+    async def _render_output_chunk(self, context: ComponentActionContext, chunk: str) -> Any:
+        context.register_source("result[]", chunk)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else chunk
+
+    async def _render_output(self, context: ComponentActionContext, result: Union[str, List[str]]) -> Any:
+        context.register_source("result", result)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+
     async def _resolve_processor_params(self, context: ComponentActionContext) -> Dict[str, Any]:
         max_input_length = await context.render_variable(self.config.max_input_length)
 

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/services/model/tasks/text_classification/huggingface.py

@@ -26,7 +26,7 @@ class HuggingfaceTextClassificationTaskAction:
 
         text = await self._prepare_input(context)
         is_single_input: bool = bool(not isinstance(text, list))
-        is_output_array_mode: bool = context.contains_variable_reference("result[]", self.config.output)
+        uses_array_output: bool = context.contains_variable_reference("result[]", self.config.output)
         texts: List[str] = [ text ] if is_single_input else text
         results = []
 
@@ -59,12 +59,10 @@ class HuggingfaceTextClassificationTaskAction:
                         for predicted_index in predicted_indices:
                             predictions.append(labels[predicted_index] if labels else predicted_index)
 
-                if self.config.output and is_output_array_mode:
+                if uses_array_output:
                     rendered_outputs = []
                     for prediction in predictions:
-                        context.register_source("result[]", prediction)
-                        output = await context.render_variable(self.config.output, ignore_files=True)
-                        rendered_outputs.append(output)
+                        rendered_outputs.append(await self._render_output_item(context, prediction))
                     yield rendered_outputs
                 else:
                     yield predictions
@@ -72,10 +70,9 @@ class HuggingfaceTextClassificationTaskAction:
         if streaming:
             async def _stream_output_generator():
                 async for predictions in _predict():
-                    if not is_output_array_mode:
+                    if not uses_array_output:
                         for prediction in predictions:
-                            context.register_source("result", prediction)
-                            yield (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+                            yield await self._render_output(context, prediction)
                     else:
                         for prediction in predictions:
                             yield prediction
@@ -85,17 +82,23 @@ class HuggingfaceTextClassificationTaskAction:
             async for predictions in _predict():
                 results.extend(predictions)
 
-            if not is_output_array_mode:
+            if not uses_array_output:
                 result = results[0] if is_single_input else results
-                context.register_source("result", result)
-
-                return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+                return await self._render_output(context, result)
             else:
                 return results
 
     async def _prepare_input(self, context: ComponentActionContext) -> Union[str, List[str]]:
         return await context.render_variable(self.config.text)
 
+    async def _render_output_item(self, context: ComponentActionContext, prediction: Union[Dict[str, Any], str, int]) -> Any:
+        context.register_source("result[]", prediction)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else prediction
+
+    async def _render_output(self, context: ComponentActionContext, result: Union[Dict[str, Any], str, int, List[Union[Dict[str, Any], str, int]]]) -> Any:
+        context.register_source("result", result)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+
     async def _resolve_tokenizer_params(self, context: ComponentActionContext) -> Dict[str, Any]:
         max_input_length = await context.render_variable(self.config.max_input_length)
 

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/services/model/tasks/text_embedding/huggingface.py

@@ -26,7 +26,7 @@ class HuggingfaceTextEmbeddingTaskAction:
 
         text = await self._prepare_input(context)
         is_single_input: bool = bool(not isinstance(text, list))
-        is_output_array_mode: bool = context.contains_variable_reference("result[]", self.config.output)
+        uses_array_output: bool = context.contains_variable_reference("result[]", self.config.output)
         texts: List[str] = [ text ] if is_single_input else text
         results = []
 
@@ -54,12 +54,10 @@ class HuggingfaceTextEmbeddingTaskAction:
 
                 embeddings = embeddings.cpu().tolist()
 
-                if self.config.output and is_output_array_mode:
+                if uses_array_output:
                     rendered_outputs = []
                     for embedding in embeddings:
-                        context.register_source("result[]", embedding)
-                        output = await context.render_variable(self.config.output, ignore_files=True)
-                        rendered_outputs.append(output)
+                        rendered_outputs.append(await self._render_output_item(context, embedding))
                     yield rendered_outputs
                 else:
                     yield embeddings
@@ -67,10 +65,9 @@ class HuggingfaceTextEmbeddingTaskAction:
         if streaming:
             async def _stream_output_generator():
                 async for embeddings in _embed():
-                    if not is_output_array_mode:
+                    if not uses_array_output:
                         for embedding in embeddings:
-                            context.register_source("result", embedding)
-                            yield (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+                            yield await self._render_output(context, embedding)
                     else:
                         for embedding in embeddings:
                             yield embedding
@@ -80,17 +77,23 @@ class HuggingfaceTextEmbeddingTaskAction:
             async for embeddings in _embed():
                 results.extend(embeddings)
 
-            if not is_output_array_mode:
+            if not uses_array_output:
                 result = results[0] if is_single_input else results
-                context.register_source("result", result)
-
-                return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+                return await self._render_output(context, result)
             else:
                 return results
 
     async def _prepare_input(self, context: ComponentActionContext) -> Union[str, List[str]]:
         return await context.render_variable(self.config.text)
 
+    async def _render_output_item(self, context: ComponentActionContext, embedding: List[float]) -> Any:
+        context.register_source("result[]", embedding)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else embedding
+
+    async def _render_output(self, context: ComponentActionContext, result: Union[List[float], List[List[float]]]) -> Any:
+        context.register_source("result", result)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+
     async def _resolve_tokenizer_params(self, context: ComponentActionContext) -> Dict[str, Any]:
         max_input_length = await context.render_variable(self.config.max_input_length)
 

{model_compose-0.4.20 → model_compose-0.4.22}/src/mindor/core/component/services/model/tasks/text_generation/huggingface.py

@@ -72,19 +72,24 @@ class HuggingfaceTextGenerationTaskAction:
             async def _stream_output_generator():
                 async for chunk in AsyncStreamer(streamer, loop):
                     if chunk:
-                        context.register_source("result[]", chunk)
-                        yield (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else chunk
+                        yield await self._render_output_chunk(context, chunk)
 
             return _stream_output_generator()
         else:
             result = results[0] if is_single_input else results
-            context.register_source("result", result)
-
-            return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+            return await self._render_output(context, result)
 
     async def _prepare_input(self, context: ComponentActionContext) -> Union[str, List[str]]:
         return await context.render_variable(self.config.text)
 
+    async def _render_output_chunk(self, context: ComponentActionContext, chunk: str) -> Any:
+        context.register_source("result[]", chunk)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else chunk
+
+    async def _render_output(self, context: ComponentActionContext, result: Union[str, List[str]]) -> Any:
+        context.register_source("result", result)
+        return (await context.render_variable(self.config.output, ignore_files=True)) if self.config.output else result
+
     async def _resolve_tokenizer_params(self, context: ComponentActionContext) -> Dict[str, Any]:
         max_input_length = await context.render_variable(self.config.max_input_length)
 

model_compose-0.4.22/src/mindor/core/component/services/text_splitter.py

@@ -0,0 +1,188 @@
+from typing import Type, Union, Literal, Optional, Dict, List, Tuple, Set, Annotated, Any
+import re
+from mindor.dsl.schema.component import TextSplitterComponentConfig
+from mindor.dsl.schema.action import ActionConfig, TextSplitterActionConfig
+from ..base import ComponentService, ComponentType, ComponentGlobalConfigs, register_component
+from ..context import ComponentActionContext
+
+class TextSplitterAction:
+    def __init__(self, config: TextSplitterActionConfig):
+        self.config: TextSplitterActionConfig = config
+
+    async def run(self, context: ComponentActionContext) -> Any:
+        text           = await context.render_variable(self.config.text)
+        separators     = await context.render_variable(self.config.separators)
+        chunk_size     = await context.render_variable(self.config.chunk_size)
+        chunk_overlap  = await context.render_variable(self.config.chunk_overlap)
+
+        return self._split_text(text, separators or ["\n\n", "\n", " ", ""], chunk_size, chunk_overlap)
+
+    def _split_text(self, text: str, separators: List[str], chunk_size: int, chunk_overlap: int) -> List[str]:
+        """Split text using recursive character text splitting."""
+        if not text:
+            return []
+
+        if chunk_overlap > chunk_size:
+            raise ValueError(f"Got a larger chunk overlap ({chunk_overlap}) than chunk size ({chunk_size}), should be smaller.")
+
+        return self._split_text_recursive(text, separators, chunk_size, chunk_overlap)
+
+    def _split_text_recursive(self, text: str, separators: List[str], chunk_size: int, chunk_overlap: int) -> List[str]:
+        """Recursively split text by separators."""
+        final_chunks = []
+
+        # Find the first separator that exists in the text
+        separator = separators[-1] if separators else ""
+        remaining_separators = []
+
+        for i, sep in enumerate(separators):
+            if not sep:
+                separator = sep
+                break
+            if sep in text:
+                separator = sep
+                remaining_separators = separators[i + 1:]
+                break
+
+        # Split using the separator with keep_separator=True logic
+        splits = self._split_text_with_separator(text, separator, keep_separator=True)
+
+        # Now go merging things, recursively splitting longer texts.
+        good_splits = []
+        # Use empty separator when merging because separator is already in splits
+        merge_separator = ""
+
+        for s in splits:
+            if len(s) < chunk_size:
+                good_splits.append(s)
+            else:
+                if good_splits:
+                    merged_text = self._merge_splits(good_splits, chunk_size, chunk_overlap, merge_separator)
+                    final_chunks.extend(merged_text)
+                    good_splits = []
+                if not remaining_separators:
+                    final_chunks.append(s)
+                else:
+                    other_info = self._split_text_recursive(s, remaining_separators, chunk_size, chunk_overlap)
+                    final_chunks.extend(other_info)
+
+        if good_splits:
+            merged_text = self._merge_splits(good_splits, chunk_size, chunk_overlap, merge_separator)
+            final_chunks.extend(merged_text)
+
+        return final_chunks
+
+    def _split_text_with_separator(self, text: str, separator: str, keep_separator: bool) -> List[str]:
+        """Split text with regex and optionally keep separator."""
+        if separator:
+            if keep_separator:
+                # The parentheses in the pattern keep the delimiters in the result.
+                # keep_separator=True means "start" mode: attach separator to the start of next chunk
+                separator_pattern = re.escape(separator)
+                splits_ = re.split(f"({separator_pattern})", text)
+
+                # Combine separator with following chunk (start mode)
+                # For splits_ = ['First', '. ', 'Second', '. ', 'Third.']
+                # We want: ['First', '. Second', '. Third.']
+                splits = [splits_[i] + splits_[i + 1] for i in range(1, len(splits_), 2)]
+
+                # Handle even number of elements (last chunk has separator)
+                if len(splits_) % 2 == 0:
+                    splits += splits_[-1:]
+
+                # Add first element at the beginning
+                splits = [splits_[0], *splits]
+            else:
+                splits = re.split(re.escape(separator), text)
+        else:
+            # Empty separator means split into characters
+            splits = list(text)
+
+        # Filter out empty strings
+        return [s for s in splits if s]
+
+    def _merge_splits(self, splits: List[str], chunk_size: int, chunk_overlap: int, separator: str) -> List[str]:
+        """Merge splits into chunks with overlap, matching langchain's algorithm exactly."""
+        separator_len = len(separator)
+        chunks = []
+        current_chunk_parts: List[str] = []
+        current_chunk_length = 0
+
+        def _calculate_separator_contribution(num_parts: int) -> int:
+            """Calculate the total length added by separators between parts."""
+            return separator_len if num_parts > 0 else 0
+
+        def _would_exceed_chunk_size(new_part_length: int) -> bool:
+            """Check if adding a new part would exceed the chunk size."""
+            separator_contribution = _calculate_separator_contribution(len(current_chunk_parts))
+            return current_chunk_length + new_part_length + separator_contribution > chunk_size
+
+        def _save_current_chunk() -> None:
+            """Save the current chunk to the result list."""
+            if current_chunk_parts:
+                chunk_text = self._join_docs(current_chunk_parts, separator)
+                if chunk_text is not None:
+                    chunks.append(chunk_text)
+
+        def _should_remove_oldest_part(new_part_length: int) -> bool:
+            """Determine if we should remove the oldest part to make room for overlap."""
+            # Remove if current chunk exceeds the overlap limit
+            if current_chunk_length > chunk_overlap:
+                return True
+
+            # Or if adding the new part would still exceed chunk size and we have content
+            if current_chunk_length > 0:
+                separator_contribution = _calculate_separator_contribution(len(current_chunk_parts))
+                return current_chunk_length + new_part_length + separator_contribution > chunk_size
+
+            return False
+
+        def _remove_oldest_part() -> None:
+            """Remove the oldest part from the current chunk and update the length."""
+            nonlocal current_chunk_length
+            oldest_part_length = len(current_chunk_parts[0])
+            separator_contribution = separator_len if len(current_chunk_parts) > 1 else 0
+            current_chunk_length -= oldest_part_length + separator_contribution
+            current_chunk_parts.pop(0)
+
+        def _add_part(part: str) -> None:
+            """Add a new part to the current chunk and update the length."""
+            nonlocal current_chunk_length
+            part_length = len(part)
+            separator_contribution = separator_len if len(current_chunk_parts) > 1 else 0
+            current_chunk_parts.append(part)
+            current_chunk_length += part_length + separator_contribution
+
+        # Process each split
+        for split in splits:
+            split_length = len(split)
+
+            # If adding this split would exceed chunk size, finalize current chunk
+            if _would_exceed_chunk_size(split_length):
+                _save_current_chunk()
+
+                # Trim the current chunk to fit within overlap constraints
+                while _should_remove_oldest_part(split_length):
+                    _remove_oldest_part()
+
+            # Add the split to the current chunk
+            _add_part(split)
+
+        # Save any remaining chunk
+        _save_current_chunk()
+
+        return chunks
+
+    def _join_docs(self, docs: List[str], separator: str) -> Optional[str]:
+        """Join documents with separator and strip whitespace."""
+        text = separator.join(docs)
+        text = text.strip()
+        return text or None
+
+@register_component(ComponentType.TEXT_SPLITTER)
+class TextSplitterComponent(ComponentService):
+    def __init__(self, id: str, config: TextSplitterComponentConfig, global_configs: ComponentGlobalConfigs, daemon: bool):
+        super().__init__(id, config, global_configs, daemon)
+
+    async def _run(self, action: ActionConfig, context: ComponentActionContext) -> Any:
+        return await TextSplitterAction(action).run(context)