service-forge 0.1.11__py3-none-any.whl → 0.1.24__py3-none-any.whl

service_forge/service.py

@@ -4,54 +4,43 @@ import os
 import asyncio
 import threading
 import uuid
-from omegaconf import OmegaConf
+from loguru import logger
+from typing import Callable, AsyncIterator, Awaitable, Any, TYPE_CHECKING
 from service_forge.workflow.node import node_register
-from service_forge.workflow.workflow_factory import create_workflows
+from service_forge.workflow.workflow_factory import create_workflow_group
 from service_forge.api.http_api import start_fastapi_server
 from service_forge.api.kafka_api import start_kafka_server
 from service_forge.db.database import DatabaseManager
-from loguru import logger
-from typing import Callable, AsyncIterator, Awaitable, Any, TYPE_CHECKING
 from service_forge.api.http_api_doc import generate_service_http_api_doc
-from service_forge.api.routers.service.service_router import set_service
 from service_forge.sft.config.sf_metadata import SfMetadata
+from service_forge.service_config import ServiceConfig
+from service_forge.current_service import set_service
 
 if TYPE_CHECKING:
     from service_forge.workflow.workflow_group import WorkflowGroup
+    from service_forge.workflow.workflow import Workflow
 
 class Service:
     def __init__(
         self,
         metadata: SfMetadata,
+        config: ServiceConfig,
         config_path: str,
-        workflow_config_paths: list[str],
-        _handle_stream_output: Callable[[str, AsyncIterator[str]], Awaitable[None]] = None,
-        _handle_query_user: Callable[[str, str], Awaitable[str]] = None,
-        enable_http: bool = True,
-        http_host: str = "0.0.0.0",
-        http_port: int = 8000,
-        enable_kafka: bool = True,
-        kafka_host: str = "localhost",
-        kafka_port: int = 9092,
         service_env: dict[str, Any] = None,
         database_manager: DatabaseManager = None,
+        _handle_stream_output: Callable[[str, AsyncIterator[str]], Awaitable[None]] = None,
+        _handle_query_user: Callable[[str, str], Awaitable[str]] = None,
     ) -> None:
         self.metadata = metadata
+        self.config = config
         self.config_path = config_path
-        self.workflow_config_paths = workflow_config_paths
         self._handle_stream_output = _handle_stream_output
         self._handle_query_user = _handle_query_user
-        self.enable_http = enable_http
-        self.http_host = http_host
-        self.http_port = http_port
-        self.enable_kafka = enable_kafka
-        self.kafka_host = kafka_host
-        self.kafka_port = kafka_port
         self.service_env = {} if service_env is None else service_env
         self.database_manager = database_manager
         self.workflow_groups: list[WorkflowGroup] = []
         self.workflow_tasks: dict[str, asyncio.Task] = {}  # workflow_name -> task mapping
-        self.workflow_config_map: dict[str, str] = {}  # workflow_name -> config_path mapping
+        self.workflow_config_map: dict[uuid.UUID, str] = {}  # workflow_id -> config_path mapping
         self.fastapi_thread: threading.Thread | None = None
         self.fastapi_loop: asyncio.AbstractEventLoop | None = None
 
@@ -70,22 +59,24 @@ class Service:
     async def start(self):
         set_service(self)
         
-        if self.enable_http:
-            fastapi_task = asyncio.create_task(start_fastapi_server(self.http_host, self.http_port))
+        if self.config.enable_http:
+            fastapi_task = asyncio.create_task(start_fastapi_server(self.config.http_host, self.config.http_port))
             doc_task = asyncio.create_task(generate_service_http_api_doc(self))
         else:
             fastapi_task = None
             doc_task = None
-        if self.enable_kafka:
-            kafka_task = asyncio.create_task(start_kafka_server(f"{self.kafka_host}:{self.kafka_port}"))
+
+        if self.config.enable_kafka:
+            kafka_task = asyncio.create_task(start_kafka_server(f"{self.config.kafka_host}:{self.config.kafka_port}"))
         else:
             kafka_task = None
 
         workflow_tasks: list[asyncio.Task] = []
 
-        for workflow_config_path in self.workflow_config_paths:
-            workflow_group = create_workflows(
-                self.parse_workflow_path(workflow_config_path),
+        for workflow_config_path in self.config.workflows:
+            logger.info(f"Loading workflow from {self.parse_workflow_path(workflow_config_path)}")
+            workflow_group = create_workflow_group(
+                config_path=self.parse_workflow_path(workflow_config_path),
                 service_env=self.service_env,
                 _handle_stream_output=self._handle_stream_output, 
                 _handle_query_user=self._handle_query_user,
@@ -95,7 +86,7 @@ class Service:
             main_workflow = workflow_group.get_main_workflow()
             task = asyncio.create_task(workflow_group.run())
             workflow_tasks.append(task)
-            self.workflow_tasks[main_workflow.name] = task
+            self.workflow_tasks[main_workflow.id] = task
             self.workflow_config_map[main_workflow.name] = workflow_config_path
 
         try:
@@ -138,120 +129,123 @@ class Service:
         else:
             return os.path.join(os.path.dirname(self.config_path), workflow_config_path)
     
-    def get_workflow_group_by_name(self, workflow_name: str) -> WorkflowGroup | None:
+    def get_workflow_group_by_name(self, workflow_name: str, workflow_version: str, allow_none: bool = True) -> WorkflowGroup | None:
         for workflow_group in self.workflow_groups:
-            if workflow_group.get_workflow(workflow_name) is not None:
+            if workflow_group.get_workflow_by_name(workflow_name, workflow_version) is not None:
                 return workflow_group
+        if not allow_none:
+            raise ValueError(f"Workflow group with name {workflow_name} and version {workflow_version} not found in service {self.name}")
+        return None
+
+    def get_workflow_group_by_id(self, workflow_id: str, allow_none: bool = True) -> WorkflowGroup | None:
+        for workflow_group in self.workflow_groups:
+            if workflow_group.get_workflow_by_id(workflow_id) is not None:
+                return workflow_group
+        if not allow_none:
+            raise ValueError(f"Workflow group with id {workflow_id} not found in service {self.name}")
         return None
-    
-    def trigger_workflow(self, workflow_name: str, trigger_name: str, **kwargs) -> uuid.UUID:
-        workflow_group = self.get_workflow_group_by_name(workflow_name)
-        if workflow_group is None:
-            logger.error(f"Workflow {workflow_name} not found")
-            return False
-        
-        workflow = workflow_group.get_main_workflow()
-        if workflow is None:
-            logger.error(f"Workflow {workflow_name} not found")
-            return False
 
+    def trigger_workflow(self, workflow_group: WorkflowGroup, trigger_name: str, **kwargs) -> uuid.UUID:
+        workflow = workflow_group.get_main_workflow(allow_none=False)
         return workflow.trigger(trigger_name, **kwargs)
 
-    async def start_workflow(self, workflow_name: str) -> bool:
-        if workflow_name in self.workflow_tasks:
-            task = self.workflow_tasks[workflow_name]
+    def trigger_workflow_by_name(self, workflow_name: str, workflow_version: str, trigger_name: str, **kwargs) -> uuid.UUID:
+        workflow_group = self.get_workflow_group_by_name(workflow_name, workflow_version, allow_none=False)
+        return self.trigger_workflow(workflow_group, trigger_name, **kwargs)
+
+    def trigger_workflow_by_id(self, workflow_id: str, trigger_name: str, **kwargs) -> uuid.UUID:
+        workflow_group = self.get_workflow_group_by_id(workflow_id, allow_none=False)
+        return self.trigger_workflow(workflow_group, trigger_name, **kwargs)
+
+    def start_workflow(self, workflow_group: WorkflowGroup) -> bool:
+        workflow = workflow_group.get_main_workflow(allow_none=False)
+        if workflow.id in self.workflow_tasks:
+            task = self.workflow_tasks[workflow.id]
             if not task.done():
-                logger.warning(f"Workflow {workflow_name} is already running")
+                logger.warning(f"Workflow {workflow.id} is already running")
                 return False
-            del self.workflow_tasks[workflow_name]
-        
-        workflow_group = self.get_workflow_group_by_name(workflow_name)
-        if workflow_group is None:
-            logger.error(f"Workflow {workflow_name} not found")
-            return False
+            del self.workflow_tasks[workflow.id]
         
-        task = asyncio.create_task(workflow_group.run(workflow_name))
-        self.workflow_tasks[workflow_name] = task
-        logger.info(f"Started workflow {workflow_name}")
+        task = asyncio.create_task(workflow_group.run())
+        self.workflow_tasks[workflow.id] = task
+        logger.info(f"Started workflow {workflow.id}")
         return True
+
+    def start_workflow_by_name(self, workflow_name: str, workflow_version: str) -> bool:
+        workflow_group = self.get_workflow_group_by_name(workflow_name, workflow_version, allow_none=False)
+        return self.start_workflow(workflow_group)
     
-    async def stop_workflow(self, workflow_name: str) -> bool:
-        if workflow_name not in self.workflow_tasks:
-            logger.warning(f"Workflow {workflow_name} is not running")
+    def start_workflow_by_id(self, workflow_id: str) -> bool:
+        workflow_group = self.get_workflow_group_by_id(workflow_id, allow_none=False)
+        return self.start_workflow(workflow_group)
+
+    async def stop_workflow(self, workflow_group: WorkflowGroup) -> bool:
+        workflow = workflow_group.get_main_workflow(allow_none=False)
+        if workflow.id not in self.workflow_tasks:
+            logger.warning(f"Workflow {workflow.id} is not running")
             return False
-        
-        task = self.workflow_tasks[workflow_name]
+        task = self.workflow_tasks[workflow.id]
         if task.done():
-            logger.warning(f"Workflow {workflow_name} is already stopped")
-            del self.workflow_tasks[workflow_name]
+            logger.warning(f"Workflow {workflow.id} is already stopped")
+            del self.workflow_tasks[workflow.id]
             return False
-        
         task.cancel()
+        await workflow.stop()
         try:
             await task
         except asyncio.CancelledError:
             pass
-        
-        del self.workflow_tasks[workflow_name]
-        logger.info(f"Stopped workflow {workflow_name}")
+        del self.workflow_tasks[workflow.id]
+        logger.info(f"Stopped workflow {workflow.id}")
         return True
+
+    async def stop_workflow_by_name(self, workflow_name: str, workflow_version: str) -> bool:
+        workflow_group = self.get_workflow_group_by_name(workflow_name, workflow_version, allow_none=False)
+        return await self.stop_workflow(workflow_group)
     
-    async def load_workflow_from_config(self, config_path: str = None, config: dict = None, workflow_name: str = None) -> bool:
-        try:
-            if config is None:
-                if config_path is None:
-                    raise ValueError("Either config_path or config must be provided")
-                if os.path.isabs(config_path):
-                    full_path = config_path
-                else:
-                    full_path = self.parse_workflow_path(config_path)
-                workflow_group = create_workflows(
-                    config_path=full_path,
-                    service_env=self.service_env,
-                    _handle_stream_output=self._handle_stream_output,
-                    _handle_query_user=self._handle_query_user,
-                    database_manager=self.database_manager,
-                )
-                config_identifier = config_path
-            else:
-                workflow_group = create_workflows(
-                    config=config,
-                    service_env=self.service_env,
-                    _handle_stream_output=self._handle_stream_output,
-                    _handle_query_user=self._handle_query_user,
-                    database_manager=self.database_manager,
-                )
-                config_identifier = config_path if config_path else "config_dict"
-            
-            self.workflow_groups.append(workflow_group)
-            main_workflow = workflow_group.get_main_workflow()
-            actual_name = workflow_name if workflow_name else main_workflow.name
-            
-            if workflow_name and workflow_name != main_workflow.name:
-                actual_name = main_workflow.name
-            
-            if actual_name in self.workflow_tasks:
-                await self.stop_workflow(actual_name)
-            
-            task = asyncio.create_task(workflow_group.run(actual_name))
-            self.workflow_tasks[actual_name] = task
-            self.workflow_config_map[actual_name] = config_identifier
-            
-            logger.info(f"Loaded and started workflow {actual_name} from {config_identifier}")
-            return True
-        except Exception as e:
-            logger.error(f"Failed to load workflow from {config_path or 'config_dict'}: {e}")
-            return False
+    async def stop_workflow_by_id(self, workflow_id: str) -> bool:
+        workflow_group = self.get_workflow_group_by_id(workflow_id, allow_none=False)
+        return await self.stop_workflow(workflow_group)
+    
+    async def load_workflow_from_config(self, config_path: str = None, config: dict = None) -> uuid.UUID:
+        workflow_group = create_workflow_group(
+            config_path=config_path,
+            config=config,
+            service_env=self.service_env,
+            _handle_stream_output=self._handle_stream_output,
+            _handle_query_user=self._handle_query_user,
+            database_manager=self.database_manager,
+        )
+
+        for workflow in workflow_group.workflows:
+            existing_workflow_group = self.get_workflow_group_by_name(workflow.name, workflow.version)
+            if existing_workflow_group is not None:
+                raise ValueError(f"Workflow group with name {workflow.name} and version {workflow.version} already exists")
+
+        self.workflow_groups.append(workflow_group)
+        main_workflow = workflow_group.get_main_workflow()
+        
+        if main_workflow.id in self.workflow_tasks:
+            await self.stop_workflow(workflow_group)
+
+        self.start_workflow(workflow_group)
+        return main_workflow.id
     
     def get_service_status(self) -> dict[str, Any]:
         workflow_statuses = []
         for workflow_group in self.workflow_groups:
             for workflow in workflow_group.workflows:
+                workflow_id = workflow.id
+                workflow_version = workflow.version
+                workflow_config = workflow.config
                 workflow_name = workflow.name
-                is_running = workflow_name in self.workflow_tasks and not self.workflow_tasks[workflow_name].done()
+                is_running = workflow_id in self.workflow_tasks and not self.workflow_tasks[workflow_id].done()
                 config_path = self.workflow_config_map.get(workflow_name, "unknown")
                 workflow_statuses.append({
                     "name": workflow_name,
+                    "id": workflow_id,
+                    "version": workflow_version,
+                    "config": workflow_config,
                     "description": workflow.description,
                     "status": "running" if is_running else "stopped",
                     "config_path": config_path,
@@ -265,23 +259,21 @@ class Service:
         }
     
     @staticmethod
-    def from_config(metadata, service_env: dict[str, Any] = None) -> Service:
-        config = OmegaConf.to_object(OmegaConf.load(metadata.service_config))
+    def from_config(metadata: SfMetadata, service_env: dict[str, Any] = None, config: ServiceConfig = None) -> Service:
+        if config is not None:
+            config_path = None
+        else:
+            config_path = metadata.service_config
+            config = ServiceConfig.from_yaml_file(config_path)
         database_manager = DatabaseManager.from_config(config=config)
         return Service(
             metadata=metadata,
-            config_path=metadata.service_config,
-            workflow_config_paths=config.get('workflows', []),
-            _handle_stream_output=None,
-            _handle_query_user=None,
-            enable_http=config.get('enable_http', True),
-            http_host=config.get('http_host', '0.0.0.0'),
-            http_port=config.get('http_port', 8000),
-            enable_kafka=config.get('enable_kafka', True),
-            kafka_host=config.get('kafka_host', 'localhost'),
-            kafka_port=config.get('kafka_port', 9092),
+            config_path=config_path,
+            config=config,
             service_env=service_env,
             database_manager=database_manager,
+            _handle_stream_output=None,
+            _handle_query_user=None,
         )
 
 def create_service(config_path: str, name: str, version: str, service_env: dict[str, Any] = None) -> Service:

service_forge/service_config.py

@@ -1,158 +1,44 @@
 from __future__ import annotations
 
-class ServiceDatabaseConfig:
-    def __init__(
-        self,
-        name: str,
-
-        postgres_user: str,
-        postgres_password: str,
-        postgres_host: str,
-        postgres_port: int,
-        postgres_db: str,
-
-        mongo_host: str,
-        mongo_port: int,
-        mongo_user: str,
-        mongo_password: str,
-        mongo_db: str,
-
-        redis_host: str,
-        redis_port: int,
-        redis_password: str,
-    ) -> None:
-        self.name = name
-
-        self.postgres_user = postgres_user
-        self.postgres_password = postgres_password
-        self.postgres_host = postgres_host
-        self.postgres_port = postgres_port
-        self.postgres_db = postgres_db
-
-        self.mongo_host = mongo_host
-        self.mongo_port = mongo_port
-        self.mongo_user = mongo_user
-        self.mongo_password = mongo_password
-        self.mongo_db = mongo_db
-
-        self.redis_host = redis_host
-        self.redis_port = redis_port
-        self.redis_password = redis_password
-    
-    @staticmethod
-    def from_dict(config: dict) -> ServiceDatabaseConfig:
-        return ServiceDatabaseConfig(
-            name=config['name'],
-            postgres_user=config.get('postgres_user', None),
-            postgres_password=config.get('postgres_password', None),
-            postgres_host=config.get('postgres_host', None),
-            postgres_port=config.get('postgres_port', None),
-            postgres_db=config.get('postgres_db', None),
-
-            mongo_host=config.get('mongo_host', None),
-            mongo_port=config.get('mongo_port', None),
-            mongo_user=config.get('mongo_user', None),
-            mongo_password=config.get('mongo_password', None),
-            mongo_db=config.get('mongo_db', None),
-
-            redis_host=config.get('redis_host', None),
-            redis_port=config.get('redis_port', None),
-            redis_password=config.get('redis_password', None),
-        )
-
-    def to_dict(self) -> dict:
-        return {
-            'name': self.name,
-
-            'postgres_user': self.postgres_user,
-            'postgres_password': self.postgres_password,
-            'postgres_host': self.postgres_host,
-            'postgres_port': self.postgres_port,
-            'postgres_db': self.postgres_db,
-
-            'mongo_host': self.mongo_host,
-            'mongo_port': self.mongo_port,
-            'mongo_user': self.mongo_user,
-            'mongo_password': self.mongo_password,
-            'mongo_db': self.mongo_db,
-
-            'redis_host': self.redis_host,
-            'redis_port': self.redis_port,
-            'redis_password': self.redis_password,
-        }
-
-class ServiceConfig:
-    def __init__(
-        self,
-        name: str,
-        workflows: list[str],
-        enable_http: bool,
-        http_host: str,
-        http_port: int,
-        enable_kafka: bool,
-        kafka_host: str,
-        kafka_port: int,
-        databases: list[ServiceDatabaseConfig],
-    ) -> None:
-        self.name = name
-        self.workflows = workflows
-        self.enable_http = enable_http
-        self.http_host = http_host
-        self.http_port = http_port
-        self.enable_kafka = enable_kafka
-        self.kafka_host = kafka_host
-        self.kafka_port = kafka_port
-        self.databases = databases
-
-    @staticmethod
-    def from_dict(config: dict) -> ServiceConfig:
-        return ServiceConfig(
-            name=config['name'],
-            workflows=config['workflows'],
-            enable_http=config['enable_http'],
-            http_host=config['http_host'],
-            http_port=config['http_port'],
-            enable_kafka=config['enable_kafka'],
-            kafka_host=config['kafka_host'],
-            kafka_port=config['kafka_port'],
-            databases=[ServiceDatabaseConfig.from_dict(database) for database in config['databases']],
-        )
-
-    def to_dict(self) -> dict:
-        return {
-            'name': self.name,
-            'workflows': self.workflows,
-            'enable_http': self.enable_http,
-            'http_host': self.http_host,
-            'http_port': self.http_port,
-            'enable_kafka': self.enable_kafka,
-            'kafka_host': self.kafka_host,
-            'kafka_port': self.kafka_port,
-            'databases': [database.to_dict() for database in self.databases],
-        }
-
-# name: tag_service
-# workflows:
-#   # - ./workflow/kafka_workflow.yaml
-#   - ./workflow/query_tags_workflow.yaml
-#   - ./workflow/create_tag_workflow.yaml
-#   - ./workflow/update_tag_workflow.yaml
-#   - ./workflow/delete_tag_workflow.yaml
-#   - ./workflow/get_tags_from_record.yaml
-
-# enable_http: true
-# enable_kafka: false
-
-# # Following configs will be auto-injected by sft.
-# http_host: 0.0.0.0
-# http_port: 37200
-# kafka_host: localhost
-# kafka_port: 9092
-
-# databases:
-#   - name: tag
-#     postgres_user: postgres
-#     postgres_password: "gnBGWg7aL4"
-#     postgres_host: second-brain-postgres-postgresql
-#     postgres_port: 5432
-#     postgres_db: tag-service-tag
+import yaml
+from pydantic import BaseModel
+
+class ServiceFeedbackConfig(BaseModel):
+    api_url: str
+    api_timeout: int = 5
+
+class ServiceDatabaseConfig(BaseModel):
+    name: str
+    postgres_user: str | None = None
+    postgres_password: str | None = None
+    postgres_host: str | None = None
+    postgres_port: int | None = None
+    postgres_db: str | None = None
+
+    mongo_host: str | None = None
+    mongo_port: int | None = None
+    mongo_user: str | None = None
+    mongo_password: str | None = None
+    mongo_db: str | None = None
+
+    redis_host: str | None = None
+    redis_port: int | None = None
+    redis_password: str | None = None
+
+class ServiceConfig(BaseModel):
+    name: str
+    workflows: list[str]
+    enable_http: bool
+    http_host: str | None = None
+    http_port: int | None = None
+    enable_kafka: bool
+    kafka_host: str | None = None
+    kafka_port: int | None = None
+    databases: list[ServiceDatabaseConfig] | None = None
+    feedback: ServiceFeedbackConfig | None = None
+
+    @classmethod
+    def from_yaml_file(cls, filepath: str) -> ServiceConfig:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data = yaml.safe_load(f)
+        return cls(**data)

service_forge/sft/cli.py

@@ -9,6 +9,8 @@ from service_forge.sft.cmd.upload_service import upload_service
 from service_forge.sft.cmd.deploy_service import deploy_service
 from service_forge.sft.cmd.config_command import list_config, get_config, set_config
 from service_forge.sft.cmd.service_command import list_services, delete_service, show_service_logs
+from service_forge.sft.cmd.remote_list_tars import remote_list_tars
+from service_forge.sft.cmd.remote_deploy import remote_deploy_tar, remote_list_and_deploy
 
 app = typer.Typer(
     name="sft",
@@ -33,6 +35,43 @@ def list_tars_command() -> None:
 def deploy_service_command(name: str, version: str) -> None:
     deploy_service(name, version)
 
+@app.command(name="remote-list")
+def remote_list_tars_command(
+    url: str = typer.Option(
+        None,
+        "--url",
+        "-u",
+        help="Service Center URL (default: http://localhost:5000 or from service_center_address config)"
+    )
+) -> None:
+    """List tar packages and their status on remote server"""
+    remote_list_tars(url)
+
+@app.command(name="remote-deploy")
+def remote_deploy_command(
+    filename: str = typer.Argument(help="Filename of the tar package to deploy"),
+    url: str = typer.Option(
+        None,
+        "--url",
+        "-u",
+        help="Service Center URL (default: http://localhost:5000 or from service_center_address config)"
+    )
+) -> None:
+    """Remote deploy specified tar package"""
+    remote_deploy_tar(filename, url)
+
+@app.command(name="remote-deploy-interactive")
+def remote_deploy_interactive_command(
+    url: str = typer.Option(
+        None,
+        "--url",
+        "-u",
+        help="Service Center URL (default: http://localhost:5000 or from service_center_address config)"
+    )
+) -> None:
+    """Interactive remote deployment of tar packages (list available packages first, then select for deployment)"""
+    remote_list_and_deploy(url)
+
 config_app = typer.Typer(
     name="config",
     help="Configuration management commands",