model-compose 0.4.53__tar.gz → 0.4.55__tar.gz

{model_compose-0.4.53/src/model_compose.egg-info → model_compose-0.4.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model-compose
-Version: 0.4.53
+Version: 0.4.55
 Summary: model-compose: Declarative AI Workflow Orchestrator
 Author-email: Hanyeol Cho <hanyeol.cho@gmail.com>
 License-Expression: MIT
@@ -14,6 +14,7 @@ Requires-Dist: pydantic
 Requires-Dist: python-dotenv
 Requires-Dist: docker
 Requires-Dist: aiohttp
+Requires-Dist: brotlicffi
 Requires-Dist: requests
 Requires-Dist: fastapi
 Requires-Dist: websockets

{model_compose-0.4.53 → model_compose-0.4.55}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "model-compose"
-version = "0.4.53"
+version = "0.4.55"
 description = "model-compose: Declarative AI Workflow Orchestrator"
 authors = [
     { name = "Hanyeol Cho", email = "hanyeol.cho@gmail.com" }
@@ -15,6 +15,7 @@ dependencies = [
     "python-dotenv",
     "docker",
     "aiohttp",
+    "brotlicffi",
     "requests",
     "fastapi",
     "websockets",

{model_compose-0.4.53 → model_compose-0.4.55}/src/mindor/cli/compose.py

@@ -206,6 +206,12 @@ def stop_command(
     required=False,
     help="JSON metadata string to pass through the workflow and echo back in the response.",
 )
+@click.option(
+    "--session-id", "session_id",
+    type=str,
+    required=False,
+    help="Session ID to group multiple workflow runs.",
+)
 @click.option("--auto-resume", is_flag=True, help="Automatically resume interrupts without prompting.")
 @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output.")
 @click.pass_context
@@ -217,6 +223,7 @@ def run_command(
     env_data: List[str],
     output_path: Optional[Path],
     metadata_json: Optional[str],
+    session_id: Optional[str],
     auto_resume: bool,
     verbose: bool
 ) -> None:
@@ -232,7 +239,7 @@ def run_command(
             is_tty = sys.stdin.isatty()
 
             manager = ComposeManager(config, daemon=False)
-            state = await manager.run_workflow(workflow_id or "__default__", input, output_path, verbose, metadata=metadata)
+            state = await manager.run_workflow(workflow_id or "__default__", input, output_path, verbose, session_id=session_id, metadata=metadata)
 
             while state.status == TaskStatus.INTERRUPTED:
                 if not auto_resume and not is_tty:
@@ -310,6 +317,8 @@ def validate_command(
                 summary_parts.append(f"{len(config.systems)} system(s)")
             if config.loggers:
                 summary_parts.append(f"{len(config.loggers)} logger(s)")
+            if config.tracers:
+                summary_parts.append(f"{len(config.tracers)} tracer(s)")
             click.echo(f"✅ Configuration is valid ({', '.join(summary_parts)})")
     except Exception as e:
         if verbose:

{model_compose-0.4.53 → model_compose-0.4.55}/src/mindor/core/component/services/__init__.py

@@ -20,3 +20,4 @@ from .audio_extractor import *
 from .audio_converter import *
 from .key_value_store import *
 from .graph_store import *
+from .tokenizer import *

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/__init__.py

@@ -0,0 +1 @@
+from .tokenizer import *

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/base/__init__.py

@@ -0,0 +1,2 @@
+from .common import *
+from .huggingface import *

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/base/common.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from typing import Type, Union, Literal, Optional, Dict, List, Tuple, Set, Annotated, Any
+from abc import ABC, abstractmethod
+from mindor.dsl.schema.component import TokenizerComponentConfig, TokenizerTaskType, TokenizerDriver
+from mindor.dsl.schema.component.impl.model.impl.common import HuggingfaceModelConfig, LocalModelConfig
+from mindor.dsl.schema.action import TokenizerActionConfig
+from mindor.core.logger import logging
+from ....context import ComponentActionContext
+
+class TokenizerTaskService:
+    def __init__(self, id: str, config: TokenizerComponentConfig):
+        self.id: str = id
+        self.config: TokenizerComponentConfig = config
+        self._tokenizer = None
+
+    def get_setup_requirements(self) -> Optional[List[str]]:
+        return None
+
+    def load(self) -> None:
+        if self._tokenizer is None:
+            logging.info(f"Component '{self.id}': loading tokenizer...")
+            self._load_tokenizer()
+            logging.info(f"Component '{self.id}': tokenizer loaded.")
+
+    @abstractmethod
+    def _load_tokenizer(self) -> None:
+        pass
+
+    @abstractmethod
+    async def run(self, action: TokenizerActionConfig, context: ComponentActionContext) -> Any:
+        pass
+
+    def _get_model_path(self) -> str:
+        if isinstance(self.config.model, HuggingfaceModelConfig):
+            return self.config.model.repository
+
+        if isinstance(self.config.model, LocalModelConfig):
+            return self.config.model.path
+
+        if isinstance(self.config.model, str):
+            return self.config.model
+
+        raise ValueError(f"Unknown model config type: {type(self.config.model)}")
+
+def register_tokenizer_task_service(task: TokenizerTaskType, driver: TokenizerDriver):
+    def decorator(cls: Type[TokenizerTaskService]) -> Type[TokenizerTaskService]:
+        if task not in TokenizerTaskServiceRegistry:
+            TokenizerTaskServiceRegistry[task] = {}
+        TokenizerTaskServiceRegistry[task][driver] = cls
+        return cls
+    return decorator
+
+TokenizerTaskServiceRegistry: Dict[TokenizerTaskType, Dict[TokenizerDriver, Type[TokenizerTaskService]]] = {}

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/base/huggingface.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from typing import Type, Union, Literal, Optional, Dict, List, Tuple, Set, Annotated, Any
+from mindor.dsl.schema.component import TokenizerComponentConfig, HuggingfaceModelConfig
+from mindor.core.logger import logging
+from .common import TokenizerTaskService
+
+class HuggingfaceTokenizerTaskService(TokenizerTaskService):
+    def __init__(self, id: str, config: TokenizerComponentConfig):
+        super().__init__(id, config)
+
+    def get_setup_requirements(self) -> Optional[List[str]]:
+        return [ "transformers" ]
+
+    def _load_tokenizer(self) -> None:
+        self._tokenizer = self._load_pretrained_tokenizer()
+
+    def _load_pretrained_tokenizer(self) -> Any:
+        tokenizer_cls = self._get_tokenizer_class()
+        tokenizer = tokenizer_cls.from_pretrained(self._get_model_path(), **self._get_tokenizer_params())
+
+        if tokenizer.pad_token is None:
+            logging.info("Tokenizer does not have a pad_token defined. Configuring pad_token automatically.")
+            self._configure_missing_pad_token(tokenizer)
+
+        return tokenizer
+
+    def _configure_missing_pad_token(self, tokenizer: Any) -> None:
+        if tokenizer.eos_token is not None:
+            tokenizer.pad_token = tokenizer.eos_token
+            logging.info(f"Set pad_token to eos_token: {tokenizer.eos_token}")
+        else:
+            tokenizer.add_special_tokens({ "pad_token": "[PAD]" })
+            logging.info("Added new pad_token: [PAD]")
+
+    def _get_tokenizer_class(self) -> Type:
+        from transformers import AutoTokenizer
+        return AutoTokenizer
+
+    def _get_tokenizer_params(self) -> Dict[str, Any]:
+        params: Dict[str, Any] = {}
+
+        if isinstance(self.config.model, HuggingfaceModelConfig):
+            if self.config.model.revision:
+                params["revision"] = self.config.model.revision
+
+            if self.config.model.cache_dir:
+                params["cache_dir"] = self.config.model.cache_dir
+
+            if self.config.model.local_files_only:
+                params["local_files_only"] = True
+
+            if self.config.model.token:
+                params["token"] = self.config.model.token
+
+        if not self.config.use_fast:
+            params["use_fast"] = False
+
+        return params

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/tasks/__init__.py

@@ -0,0 +1 @@
+from .text import *

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/tasks/text/__init__.py

@@ -0,0 +1 @@
+from .huggingface import *

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/tasks/text/huggingface.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from typing import Type, Union, Literal, Optional, Dict, List, Tuple, Set, Annotated, Any
+from mindor.dsl.schema.action import TokenizerActionConfig
+from mindor.dsl.schema.action.impl.tokenizer.impl.common import TokenizerMethod
+from ...base import TokenizerTaskType, TokenizerDriver, register_tokenizer_task_service
+from ...base import HuggingfaceTokenizerTaskService, ComponentActionContext
+
+class HuggingfaceTextTokenizerTaskAction:
+    def __init__(self, config: TokenizerActionConfig, tokenizer: Any):
+        self.config: TokenizerActionConfig = config
+        self.tokenizer = tokenizer
+
+    async def run(self, context: ComponentActionContext) -> Any:
+        if self.config.method == TokenizerMethod.ENCODE:
+            return await self._encode(context)
+
+        if self.config.method == TokenizerMethod.DECODE:
+            return await self._decode(context)
+
+        if self.config.method == TokenizerMethod.COUNT:
+            return await self._count(context)
+
+        raise ValueError(f"Unsupported tokenizer method: {self.config.method}")
+
+    async def _encode(self, context: ComponentActionContext) -> Dict[str, Any]:
+        text       = await context.render_variable(self.config.text)
+        max_length = await context.render_variable(self.config.max_length)
+        padding    = await context.render_variable(self.config.padding)
+        truncation = await context.render_variable(self.config.truncation)
+
+        encode_params = {}
+        if max_length is not None:
+            encode_params["max_length"] = int(max_length)
+        if padding:
+            encode_params["padding"] = "max_length" if max_length else True
+        if truncation:
+            encode_params["truncation"] = True
+
+        encoded = self.tokenizer(text, **encode_params)
+
+        return {
+            "input_ids": encoded["input_ids"],
+            "attention_mask": encoded["attention_mask"]
+        }
+
+    async def _decode(self, context: ComponentActionContext) -> Dict[str, str]:
+        token_ids           = await context.render_variable(self.config.token_ids)
+        skip_special_tokens = await context.render_variable(self.config.skip_special_tokens)
+
+        text = self.tokenizer.decode(token_ids, skip_special_tokens=bool(skip_special_tokens))
+        
+        return { "text": text }
+
+    async def _count(self, context: ComponentActionContext) -> Dict[str, int]:
+        text = await context.render_variable(self.config.text)
+
+        token_ids = self.tokenizer.encode(text)
+
+        return { "count": len(token_ids) }
+
+@register_tokenizer_task_service(TokenizerTaskType.TEXT, TokenizerDriver.HUGGINGFACE)
+class HuggingfaceTextTokenizerTaskService(HuggingfaceTokenizerTaskService):
+    async def run(self, action: TokenizerActionConfig, context: ComponentActionContext) -> Any:
+        return await HuggingfaceTextTokenizerTaskAction(action, self._tokenizer).run(context)

model_compose-0.4.55/src/mindor/core/component/services/tokenizer/tokenizer.py

@@ -0,0 +1,41 @@
+from typing import Type, Union, Literal, Optional, Dict, List, Tuple, Set, Annotated, Any
+from mindor.dsl.schema.component import TokenizerComponentConfig, TokenizerTaskType, TokenizerDriver
+from mindor.dsl.schema.action import ActionConfig, TokenizerActionConfig
+from ...base import ComponentService, ComponentType, ComponentGlobalConfigs, register_component
+from ...context import ComponentActionContext
+from .base import TokenizerTaskService, TokenizerTaskServiceRegistry
+
+class TokenizerAction:
+    def __init__(self, config: TokenizerActionConfig):
+        self.config: TokenizerActionConfig = config
+
+    async def run(self, context: ComponentActionContext, task_service: TokenizerTaskService) -> Any:
+        return await task_service.run(self.config, context)
+
+@register_component(ComponentType.TOKENIZER)
+class TokenizerComponent(ComponentService):
+    def __init__(
+        self,
+        id: str,
+        config: TokenizerComponentConfig,
+        global_configs: ComponentGlobalConfigs,
+        daemon: bool
+    ):
+        super().__init__(id, config, global_configs, daemon)
+
+        self.task_service: TokenizerTaskService = self._create_task_service(self.config.task, self.config.driver)
+
+    def _create_task_service(self, task: TokenizerTaskType, driver: TokenizerDriver) -> TokenizerTaskService:
+        try:
+            if not TokenizerTaskServiceRegistry:
+                from . import tasks
+            return TokenizerTaskServiceRegistry[task][driver](self.id, self.config)
+        except KeyError:
+            raise ValueError(f"Unsupported tokenizer task type: {task} on {driver}")
+
+    def _get_setup_requirements(self) -> Optional[List[str]]:
+        return self.task_service.get_setup_requirements()
+
+    async def _run(self, action: ActionConfig, context: ComponentActionContext) -> Any:
+        self.task_service.load()
+        return await TokenizerAction(action).run(context, self.task_service)

{model_compose-0.4.53 → model_compose-0.4.55}/src/mindor/core/component/services/web_browser/base.py

@@ -39,7 +39,7 @@ class WebBrowserSession(ABC):
         pass
 
     @abstractmethod
-    async def scroll(self, selector: Optional[str], x: int, y: int) -> Dict[str, Any]:
+    async def scroll(self, selector: Optional[str], xpath: Optional[str], x: Optional[int], y: Optional[int]) -> Dict[str, Any]:
         pass
 
     # ---- Query ----
@@ -108,6 +108,10 @@ class WebBrowserService(AsyncService):
     async def create_session(self) -> WebBrowserSession:
         pass
 
+    async def close_browser(self) -> None:
+        """Release browser-level resources. Override in drivers that manage a browser process (e.g. Playwright)."""
+        pass
+
 def register_web_browser_service(driver: WebBrowserDriver):
     def decorator(cls: Type[WebBrowserService]) -> Type[WebBrowserService]:
         WebBrowserServiceRegistry[driver] = cls