service-forge 0.1.18__py3-none-any.whl

service_forge/service.py

@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import os
+import asyncio
+import threading
+import uuid
+from loguru import logger
+from typing import Callable, AsyncIterator, Awaitable, Any, TYPE_CHECKING
+from service_forge.workflow.node import node_register
+from service_forge.workflow.workflow_factory import create_workflow_group
+from service_forge.api.http_api import start_fastapi_server
+from service_forge.api.kafka_api import start_kafka_server
+from service_forge.db.database import DatabaseManager
+from service_forge.api.http_api_doc import generate_service_http_api_doc
+from service_forge.sft.config.sf_metadata import SfMetadata
+from service_forge.service_config import ServiceConfig
+from service_forge.current_service import set_service
+
+if TYPE_CHECKING:
+    from service_forge.workflow.workflow_group import WorkflowGroup
+    from service_forge.workflow.workflow import Workflow
+
+class Service:
+    def __init__(
+        self,
+        metadata: SfMetadata,
+        config: ServiceConfig,
+        config_path: str,
+        service_env: dict[str, Any] = None,
+        database_manager: DatabaseManager = None,
+        _handle_stream_output: Callable[[str, AsyncIterator[str]], Awaitable[None]] = None,
+        _handle_query_user: Callable[[str, str], Awaitable[str]] = None,
+    ) -> None:
+        self.metadata = metadata
+        self.config = config
+        self.config_path = config_path
+        self._handle_stream_output = _handle_stream_output
+        self._handle_query_user = _handle_query_user
+        self.service_env = {} if service_env is None else service_env
+        self.database_manager = database_manager
+        self.workflow_groups: list[WorkflowGroup] = []
+        self.workflow_tasks: dict[str, asyncio.Task] = {}  # workflow_name -> task mapping
+        self.workflow_config_map: dict[uuid.UUID, str] = {}  # workflow_id -> config_path mapping
+        self.fastapi_thread: threading.Thread | None = None
+        self.fastapi_loop: asyncio.AbstractEventLoop | None = None
+
+    @property
+    def name(self) -> str:
+        return self.metadata.name
+    
+    @property
+    def version(self) -> str:
+        return self.metadata.version
+    
+    @property
+    def description(self) -> str:
+        return self.metadata.description
+
+    async def start(self):
+        set_service(self)
+        
+        if self.config.enable_http:
+            fastapi_task = asyncio.create_task(start_fastapi_server(self.config.http_host, self.config.http_port))
+            doc_task = asyncio.create_task(generate_service_http_api_doc(self))
+        else:
+            fastapi_task = None
+            doc_task = None
+
+        if self.config.enable_kafka:
+            kafka_task = asyncio.create_task(start_kafka_server(f"{self.config.kafka_host}:{self.config.kafka_port}"))
+        else:
+            kafka_task = None
+
+        workflow_tasks: list[asyncio.Task] = []
+
+        for workflow_config_path in self.config.workflows:
+            logger.info(f"Loading workflow from {self.parse_workflow_path(workflow_config_path)}")
+            workflow_group = create_workflow_group(
+                config_path=self.parse_workflow_path(workflow_config_path),
+                service_env=self.service_env,
+                _handle_stream_output=self._handle_stream_output, 
+                _handle_query_user=self._handle_query_user,
+                database_manager=self.database_manager,
+            )
+            self.workflow_groups.append(workflow_group)
+            main_workflow = workflow_group.get_main_workflow()
+            task = asyncio.create_task(workflow_group.run())
+            workflow_tasks.append(task)
+            self.workflow_tasks[main_workflow.id] = task
+            self.workflow_config_map[main_workflow.name] = workflow_config_path
+
+        try:
+            core_tasks = []
+            if fastapi_task:
+                core_tasks.append(fastapi_task)
+            if doc_task:
+                core_tasks.append(doc_task)
+            if kafka_task:
+                core_tasks.append(kafka_task)
+            
+            all_tasks = core_tasks + workflow_tasks
+            results = await asyncio.gather(*all_tasks, return_exceptions=True)
+            
+            # Check core tasks
+            for i, result in enumerate(results[:len(core_tasks)]):
+                if isinstance(result, Exception):
+                    logger.error(f"Error in service {self.name} core task {i}: {result}")
+                    raise result
+            
+            # Check workflow tasks
+            for i, result in enumerate(results[len(core_tasks):], start=len(core_tasks)):
+                if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
+                    # Workflow task exception should not stop the service
+                    logger.error(f"Error in service {self.name} workflow task {i}: {result}")
+
+        except Exception as e:
+            logger.error(f"Error in service {self.name}: {e}")
+            if fastapi_task:
+                fastapi_task.cancel()
+            if kafka_task:
+                kafka_task.cancel()
+            for workflow_task in workflow_tasks:
+                workflow_task.cancel()
+            raise
+
+    def parse_workflow_path(self, workflow_config_path: str) -> str:
+        if os.path.isabs(workflow_config_path):
+            return workflow_config_path
+        else:
+            return os.path.join(os.path.dirname(self.config_path), workflow_config_path)
+    
+    def get_workflow_group_by_name(self, workflow_name: str, workflow_version: str, allow_none: bool = True) -> WorkflowGroup | None:
+        for workflow_group in self.workflow_groups:
+            if workflow_group.get_workflow_by_name(workflow_name, workflow_version) is not None:
+                return workflow_group
+        if not allow_none:
+            raise ValueError(f"Workflow group with name {workflow_name} and version {workflow_version} not found in service {self.name}")
+        return None
+
+    def get_workflow_group_by_id(self, workflow_id: str, allow_none: bool = True) -> WorkflowGroup | None:
+        for workflow_group in self.workflow_groups:
+            if workflow_group.get_workflow_by_id(workflow_id) is not None:
+                return workflow_group
+        if not allow_none:
+            raise ValueError(f"Workflow group with id {workflow_id} not found in service {self.name}")
+        return None
+
+    def trigger_workflow(self, workflow_group: WorkflowGroup, trigger_name: str, **kwargs) -> uuid.UUID:
+        workflow = workflow_group.get_main_workflow(allow_none=False)
+        return workflow.trigger(trigger_name, **kwargs)
+
+    def trigger_workflow_by_name(self, workflow_name: str, workflow_version: str, trigger_name: str, **kwargs) -> uuid.UUID:
+        workflow_group = self.get_workflow_group_by_name(workflow_name, workflow_version, allow_none=False)
+        return self.trigger_workflow(workflow_group, trigger_name, **kwargs)
+
+    def trigger_workflow_by_id(self, workflow_id: str, trigger_name: str, **kwargs) -> uuid.UUID:
+        workflow_group = self.get_workflow_group_by_id(workflow_id, allow_none=False)
+        return self.trigger_workflow(workflow_group, trigger_name, **kwargs)
+
+    def start_workflow(self, workflow_group: WorkflowGroup) -> bool:
+        workflow = workflow_group.get_main_workflow(allow_none=False)
+        if workflow.id in self.workflow_tasks:
+            task = self.workflow_tasks[workflow.id]
+            if not task.done():
+                logger.warning(f"Workflow {workflow.id} is already running")
+                return False
+            del self.workflow_tasks[workflow.id]
+        
+        task = asyncio.create_task(workflow_group.run())
+        self.workflow_tasks[workflow.id] = task
+        logger.info(f"Started workflow {workflow.id}")
+        return True
+
+    def start_workflow_by_name(self, workflow_name: str, workflow_version: str) -> bool:
+        workflow_group = self.get_workflow_group_by_name(workflow_name, workflow_version, allow_none=False)
+        return self.start_workflow(workflow_group)
+    
+    def start_workflow_by_id(self, workflow_id: str) -> bool:
+        workflow_group = self.get_workflow_group_by_id(workflow_id, allow_none=False)
+        return self.start_workflow(workflow_group)
+
+    async def stop_workflow(self, workflow_group: WorkflowGroup) -> bool:
+        workflow = workflow_group.get_main_workflow(allow_none=False)
+        if workflow.id not in self.workflow_tasks:
+            logger.warning(f"Workflow {workflow.id} is not running")
+            return False
+        task = self.workflow_tasks[workflow.id]
+        if task.done():
+            logger.warning(f"Workflow {workflow.id} is already stopped")
+            del self.workflow_tasks[workflow.id]
+            return False
+        task.cancel()
+        await workflow.stop()
+        try:
+            await task
+        except asyncio.CancelledError:
+            pass
+        del self.workflow_tasks[workflow.id]
+        logger.info(f"Stopped workflow {workflow.id}")
+        return True
+
+    async def stop_workflow_by_name(self, workflow_name: str, workflow_version: str) -> bool:
+        workflow_group = self.get_workflow_group_by_name(workflow_name, workflow_version, allow_none=False)
+        return await self.stop_workflow(workflow_group)
+    
+    async def stop_workflow_by_id(self, workflow_id: str) -> bool:
+        workflow_group = self.get_workflow_group_by_id(workflow_id, allow_none=False)
+        return await self.stop_workflow(workflow_group)
+    
+    async def load_workflow_from_config(self, config_path: str = None, config: dict = None) -> uuid.UUID:
+        workflow_group = create_workflow_group(
+            config_path=config_path,
+            config=config,
+            service_env=self.service_env,
+            _handle_stream_output=self._handle_stream_output,
+            _handle_query_user=self._handle_query_user,
+            database_manager=self.database_manager,
+        )
+
+        for workflow in workflow_group.workflows:
+            existing_workflow_group = self.get_workflow_group_by_name(workflow.name, workflow.version)
+            if existing_workflow_group is not None:
+                raise ValueError(f"Workflow group with name {workflow.name} and version {workflow.version} already exists")
+
+        self.workflow_groups.append(workflow_group)
+        main_workflow = workflow_group.get_main_workflow()
+        
+        if main_workflow.id in self.workflow_tasks:
+            await self.stop_workflow(workflow_group)
+
+        self.start_workflow(workflow_group)
+        return main_workflow.id
+    
+    def get_service_status(self) -> dict[str, Any]:
+        workflow_statuses = []
+        for workflow_group in self.workflow_groups:
+            for workflow in workflow_group.workflows:
+                workflow_id = workflow.id
+                workflow_version = workflow.version
+                workflow_config = workflow.config
+                workflow_name = workflow.name
+                is_running = workflow_id in self.workflow_tasks and not self.workflow_tasks[workflow_id].done()
+                config_path = self.workflow_config_map.get(workflow_name, "unknown")
+                workflow_statuses.append({
+                    "name": workflow_name,
+                    "id": workflow_id,
+                    "version": workflow_version,
+                    "config": workflow_config,
+                    "description": workflow.description,
+                    "status": "running" if is_running else "stopped",
+                    "config_path": config_path,
+                })
+        
+        return {
+            "name": self.name,
+            "version": self.version,
+            "description": self.description,
+            "workflows": workflow_statuses,
+        }
+    
+    @staticmethod
+    def from_config(metadata: SfMetadata, service_env: dict[str, Any] = None, config: ServiceConfig = None) -> Service:
+        if config is not None:
+            config_path = None
+        else:
+            config_path = metadata.service_config
+            config = ServiceConfig.from_yaml_file(config_path)
+        database_manager = DatabaseManager.from_config(config=config)
+        return Service(
+            metadata=metadata,
+            config_path=config_path,
+            config=config,
+            service_env=service_env,
+            database_manager=database_manager,
+            _handle_stream_output=None,
+            _handle_query_user=None,
+        )
+
+def create_service(config_path: str, name: str, version: str, service_env: dict[str, Any] = None) -> Service:
+    return Service.from_config(config_path, name, version, service_env)

service_forge/service_config.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import yaml
+from pydantic import BaseModel
+
+class ServiceFeedbackConfig(BaseModel):
+    api_url: str
+    api_timeout: int = 5
+
+class ServiceDatabaseConfig(BaseModel):
+    name: str
+    postgres_user: str | None = None
+    postgres_password: str | None = None
+    postgres_host: str | None = None
+    postgres_port: int | None = None
+    postgres_db: str | None = None
+
+    mongo_host: str | None = None
+    mongo_port: int | None = None
+    mongo_user: str | None = None
+    mongo_password: str | None = None
+    mongo_db: str | None = None
+
+    redis_host: str | None = None
+    redis_port: int | None = None
+    redis_password: str | None = None
+
+class ServiceConfig(BaseModel):
+    name: str
+    workflows: list[str]
+    enable_http: bool
+    http_host: str | None = None
+    http_port: int | None = None
+    enable_kafka: bool
+    kafka_host: str | None = None
+    kafka_port: int | None = None
+    databases: list[ServiceDatabaseConfig] | None = None
+    feedback: ServiceFeedbackConfig | None = None
+
+    @classmethod
+    def from_yaml_file(cls, filepath: str) -> ServiceConfig:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data = yaml.safe_load(f)
+        return cls(**data)

service_forge/sft/cli.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import sys
+from typing import Optional
+
+import typer
+from service_forge.sft.util.logger import log_error, log_info
+from service_forge.sft.cmd.list_tars import list_tars
+from service_forge.sft.cmd.upload_service import upload_service
+from service_forge.sft.cmd.deploy_service import deploy_service
+from service_forge.sft.cmd.config_command import list_config, get_config, set_config
+from service_forge.sft.cmd.service_command import list_services, delete_service, show_service_logs
+
+app = typer.Typer(
+    name="sft",
+    help="Service Forge CLI - Service management tool",
+    add_completion=False,
+)
+
+@app.command(name="upload")
+def upload_service_command(
+    project_path: Optional[str] = typer.Argument(
+        default=".",
+        help="Project path, default is the current directory"
+    )
+) -> None:
+    upload_service(project_path)
+
+@app.command(name="list")
+def list_tars_command() -> None:
+    list_tars()
+
+@app.command(name="deploy")
+def deploy_service_command(name: str, version: str) -> None:
+    deploy_service(name, version)
+
+config_app = typer.Typer(
+    name="config",
+    help="Configuration management commands",
+    add_completion=False,
+)
+
+@config_app.command(name="list")
+def config_list_command() -> None:
+    list_config()
+
+@config_app.command(name="get")
+def config_get_command(
+    key: str = typer.Argument(help="Configuration item key")
+) -> None:
+    get_config(key)
+
+@config_app.command(name="set")
+def config_set_command(
+    key: str = typer.Argument(help="Configuration item key"),
+    value: str = typer.Argument(help="Configuration item value")
+) -> None:
+    set_config(key, value)
+
+app.add_typer(config_app)
+
+service_app = typer.Typer(
+    name="service",
+    help="Kubernetes service management commands",
+    add_completion=False,
+)
+
+@service_app.command(name="list")
+def service_list_command() -> None:
+    list_services()
+
+@service_app.command(name="delete")
+def service_delete_command(
+    service_name: str = typer.Argument(help="Service name to delete (must start with sf-)"),
+    force: bool = typer.Option(False, "--force", "-f", help="Force delete")
+) -> None:
+    delete_service(service_name, force)
+
+@service_app.command(name="logs")
+def service_logs_command(
+    service_name: str = typer.Argument(help="Service name to view logs for (must start with sf-)"),
+    container: Optional[str] = typer.Option(None, "--container", "-c", help="Container name (if pod has multiple containers)"),
+    tail: int = typer.Option(100, "--tail", "-n", help="Number of lines to show from the end of logs"),
+    follow: bool = typer.Option(False, "--follow", "-f", help="Follow log output"),
+    previous: bool = typer.Option(False, "--previous", "-p", help="Get logs from previous instance of container")
+) -> None:
+    show_service_logs(service_name, container, tail, follow, previous)
+
+app.add_typer(service_app)
+
+def main() -> None:
+    app()

service_forge/sft/cmd/config_command.py

@@ -0,0 +1,67 @@
+from typing import Optional
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from service_forge.sft.config.sft_config import SftConfig
+from service_forge.sft.util.logger import log_error, log_info, log_success, log_warning
+from service_forge.sft.config.sft_config import sft_config
+
+def list_config() -> None:
+    try:
+        console = Console()
+        
+        table = Table(title="SFT Configuration", show_header=True, header_style="bold magenta")
+        table.add_column("Key", style="cyan", no_wrap=True)
+        table.add_column("Value", style="green")
+        table.add_column("Description", style="yellow")
+        
+        # Automatically add rows for all config items
+        config_dict = sft_config.to_dict()
+        for key, value in sorted(config_dict.items()):
+            description = SftConfig.CONFIG_DESCRIPTIONS.get(key, "No description available")
+            table.add_row(key, str(value), description)
+        
+        console.print(table)
+        console.print(f"\n[dim]Config file location: {sft_config.config_file_path}[/dim]")
+    except Exception as e:
+        log_error(f"Failed to load config: {e}")
+        raise typer.Exit(1)
+
+def get_config(key: str) -> None:
+    try:
+        value = sft_config.get(key)
+        
+        if value is None:
+            log_error(f"Config key '{key}' not found")
+            log_info("Available keys: config_root, sft_file_root, k8s_namespace")
+            raise typer.Exit(1)
+        
+        log_info(f"{key} = {value}")
+    except ValueError as e:
+        log_error(str(e))
+        raise typer.Exit(1)
+    except Exception as e:
+        log_error(f"Failed to get config: {e}")
+        raise typer.Exit(1)
+
+def set_config(key: str, value: str) -> None:
+    try:
+        current_value = sft_config.get(key)
+        if current_value is None:
+            log_error(f"Unknown config key: {key}")
+            log_info("Available keys: config_root, sft_file_root, k8s_namespace")
+            raise typer.Exit(1)
+        
+        sft_config.set(key, value)
+        sft_config.save()
+        
+        log_success(f"Updated {key} = {value}")
+        log_info(f"Config saved to {sft_config.config_file_path}")
+    except ValueError as e:
+        log_error(str(e))
+        raise typer.Exit(1)
+    except Exception as e:
+        log_error(f"Failed to set config: {e}")
+        raise typer.Exit(1)
+

service_forge/sft/cmd/deploy_service.py

@@ -0,0 +1,123 @@
+import os
+import shutil
+import subprocess
+import tarfile
+import tempfile
+from pathlib import Path
+
+import typer
+from omegaconf import OmegaConf
+from service_forge.sft.util.logger import log_error, log_info, log_success, log_warning
+from service_forge.sft.file.sft_file_manager import sft_file_manager
+from service_forge.sft.config.sft_config import sft_config
+from service_forge.sft.util.assert_util import assert_file_exists, assert_dir_exists
+from service_forge.sft.config.sf_metadata import load_metadata
+from service_forge.sft.kubernetes.kubernetes_manager import KubernetesManager
+from service_forge.sft.config.injector import Injector
+from service_forge.sft.util.name_util import get_service_name
+
+def _extract_tar_file(tar_file: Path, temp_path: Path) -> None:
+    log_info(f"Extracting tar file to: {temp_path}")
+    
+    try:
+        with tarfile.open(tar_file, 'r') as tar:
+            tar.extractall(temp_path)
+    except Exception as e:
+        log_error(f"Failed to extract tar file: {e}")
+        raise typer.Exit(1)
+    
+    log_success("Tar file extracted successfully")
+
+def _build_docker_image(project_dir: Path, name: str, version: str) -> None:
+    image_name = f"sf-{name}:{version}"
+    full_image_name = sft_config.registry_address + "/" + image_name
+    log_info(f"Building Docker image: {image_name}")
+    try:
+        # build docker image
+        build_result = subprocess.run(
+            ["docker", "build", "-t", full_image_name, str(project_dir)],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        log_success(f"Docker image built successfully: {image_name}")
+        if build_result.stdout:
+            log_info(build_result.stdout)
+
+        # push docker image to registry
+        log_info(f"Pushing Docker image to registry: {full_image_name}")
+        push_result = subprocess.run(
+            ["docker", "push", full_image_name],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        log_success(f"Docker image pushed successfully: {full_image_name}")
+        if push_result.stdout:
+            log_info(push_result.stdout)
+
+    except subprocess.CalledProcessError as e:
+        log_error(f"Docker operation failed: {e}")
+        if e.stderr:
+            log_error(e.stderr)
+        raise typer.Exit(1)
+    except FileNotFoundError:
+        log_error("Docker command not found. Please install Docker.")
+        raise typer.Exit(1)
+
+def _apply_k8s_deployment(deployment_yaml: Path, ingress_yaml: Path, name: str, version: str) -> None:
+    log_info("Applying k8s deployment...")
+    
+    try:
+        k8s_manager = KubernetesManager()
+        k8s_manager.delete_service(sft_config.k8s_namespace, get_service_name(name, version), force=True)
+        k8s_manager.apply_deployment_yaml(deployment_yaml, sft_config.k8s_namespace)
+        k8s_manager.apply_deployment_yaml(ingress_yaml, sft_config.k8s_namespace)
+        log_success("K8s deployment applied successfully")
+    except Exception as e:
+        log_error(f"K8s deployment failed: {e}")
+        raise typer.Exit(1)
+    
+    log_success(f"Deployment process completed for {name}:{version}")
+
+def _inject_config(project_dir: Path) -> None:
+    injector = Injector(project_dir)
+    injector.inject()
+
+def deploy_service(name: str, version: str) -> None:
+    tar_file = sft_file_manager.tar_path / f"sf_{name}_{version}.tar"
+
+    assert_file_exists(tar_file)
+    
+    temp_parent = os.path.join(tempfile.gettempdir(), "sft")
+    os.makedirs(temp_parent, exist_ok=True)
+    
+    with tempfile.TemporaryDirectory(prefix=f"deploy_{name}_{version}", dir=temp_parent) as temp_dir:
+        temp_path = Path(temp_dir)
+
+        _extract_tar_file(tar_file, temp_path)
+
+        project_dir = temp_path / f"{name}_{version}"
+
+        _inject_config(project_dir)
+
+        dockerfile_path = project_dir / "Dockerfile"
+        metadata_path = project_dir / "sf-meta.yaml"
+        deployment_yaml = project_dir / "deployment.yaml"
+        ingress_yaml = project_dir / "ingress.yaml"
+        
+        assert_dir_exists(project_dir)
+        assert_file_exists(dockerfile_path)
+        assert_file_exists(metadata_path)
+        assert_file_exists(deployment_yaml)
+        assert_file_exists(ingress_yaml)
+
+        try:
+            meta_data = load_metadata(metadata_path)
+        except Exception as e:
+            log_error(f"Failed to read sf-meta.yaml: {e}")
+            raise typer.Exit(1)
+        
+        _build_docker_image(project_dir, meta_data.name, meta_data.version)
+        # TODO: create new user in mongodb and redis
+        _apply_k8s_deployment(deployment_yaml, ingress_yaml, meta_data.name, meta_data.version)

service_forge/sft/cmd/list_tars.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from service_forge.sft.util.logger import log_error, log_info
+from service_forge.sft.file.sft_file_manager import sft_file_manager
+
+def list_tars() -> None:
+    tar_files = sft_file_manager.load_tars()
+    
+    if not tar_files:
+        log_info("No tar files found.")
+        return
+    
+    console = Console()
+    table = Table(title="Service Tar Files", show_header=True, header_style="bold magenta")
+    table.add_column("Project", style="cyan", no_wrap=True)
+    table.add_column("Version", style="cyan", no_wrap=True)
+    table.add_column("File Name", style="cyan", no_wrap=True)
+    table.add_column("Size", justify="right", style="green")
+    table.add_column("Modified Time", style="yellow")
+    
+    for tar_file in tar_files:
+        table.add_row(tar_file.project_name, tar_file.version, tar_file.path.name, tar_file._format_size(), tar_file._format_modified_time())
+    
+    console.print(table)
+
+
+def _format_size(size_bytes: int) -> str:
+    for unit in ['B', 'KB', 'MB', 'GB']:
+        if size_bytes < 1024.0:
+            return f"{size_bytes:.2f} {unit}"
+        size_bytes /= 1024.0
+    return f"{size_bytes:.2f} TB"
+
+
+def _format_time(timestamp: float) -> str:
+    from datetime import datetime
+    return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S")
+