service-forge 0.1.0__py3-none-any.whl

service_forge/main.py

@@ -0,0 +1,121 @@
+default_service_config = "configs/service/test_service_websocket.yaml"
+#!/usr/bin/env python3
+"""
+Service Forge 主入口文件
+用于启动 Service Forge 服务
+"""
+
+import os
+import sys
+import argparse
+from pathlib import Path
+from loguru import logger
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from service_forge.service import create_service
+
+
+def parse_args():
+    """解析命令行参数"""
+    parser = argparse.ArgumentParser(description="Service Forge - 工作流服务框架")
+
+    # 配置文件参数
+    parser.add_argument(
+        "-c", "--config",
+        type=str,
+        default="config.yaml",
+        help="服务配置文件路径 (默认: config.yaml)"
+    )
+
+    # 工作流配置参数
+    parser.add_argument(
+        "-w", "--workflows",
+        type=str,
+        nargs="+",
+        help="工作流配置文件路径列表"
+    )
+
+    # 调试模式
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="启用调试模式"
+    )
+
+    # 端口参数
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=8000,
+        help="HTTP服务端口 (默认: 8000)"
+    )
+
+    # 主机参数
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="0.0.0.0",
+        help="HTTP服务主机 (默认: 0.0.0.0)"
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    """主函数"""
+    args = parse_args()
+
+    # 设置日志级别
+    if args.debug:
+        logger.remove()
+        logger.add(
+            sys.stderr,
+            level="DEBUG",
+            format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
+        )
+
+    # 检查配置文件是否存在
+    config_path = Path(args.config)
+    if not config_path.exists():
+        config_path = default_service_config
+
+    # 创建服务
+    try:
+        logger.info(f"正在加载配置文件: {config_path}")
+        
+        # 导入WebSocketMessage模型
+        from service_forge.model.websocket import WebSocketMessage
+        
+        # 创建服务并添加WEBSOCKET_MESSAGE到service_env
+        service = create_service(str(config_path), service_env={"WEBSOCKET_MESSAGE": WebSocketMessage})
+
+        # 如果指定了工作流配置，则添加到服务中
+        if args.workflows:
+            for workflow_path in args.workflows:
+                if not Path(workflow_path).exists():
+                    logger.error(f"工作流配置文件不存在: {workflow_path}")
+                    continue
+                logger.info(f"添加工作流配置: {workflow_path}")
+                # 这里需要根据实际情况调整
+
+        # 启动服务
+        logger.info("正在启动 Service Forge 服务...")
+        logger.info(f"HTTP服务器: http://{args.host}:{args.port}")
+        logger.info("WebSocket端点: ws://{}:{}/ws/connect/{{client_id}}".format(args.host, args.port))
+
+        # 运行服务
+        import asyncio
+        asyncio.run(service.start())
+
+    except KeyboardInterrupt:
+        logger.info("服务已停止")
+    except Exception as e:
+        logger.error(f"服务启动失败: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

service_forge/model/websocket.py

@@ -0,0 +1,13 @@
+
+from pydantic import BaseModel
+
+class WebSocketMessage(BaseModel):
+    """WebSocket消息模型"""
+    message: str = None
+    client_id: str
+
+class WebSocketResponse(BaseModel):
+    """WebSocket响应模型"""
+    status: str
+    message: str
+    data: dict = None

service_forge/proto/foo_input.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+class FooInput(BaseModel):
+    user_id: int
+    data: str

service_forge/service.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import os
+import asyncio
+from omegaconf import OmegaConf
+from service_forge.workflow.workflow_factory import create_workflows
+from service_forge.api.http_api import start_fastapi_server
+from service_forge.api.kafka_api import start_kafka_server
+from service_forge.db.database import DatabaseManager
+from loguru import logger
+from typing import Callable, AsyncIterator, Awaitable, Any
+
+class Service:
+    def __init__(
+        self,
+        name: str,
+        config_path: str,
+        workflow_config_paths: list[str],
+        _handle_stream_output: Callable[[str, AsyncIterator[str]], Awaitable[None]] = None,
+        _handle_query_user: Callable[[str, str], Awaitable[str]] = None,
+        enable_http: bool = True,
+        http_host: str = "0.0.0.0",
+        http_port: int = 8000,
+        enable_kafka: bool = True,
+        kafka_host: str = "localhost",
+        kafka_port: int = 9092,
+        service_env: dict[str, Any] = None,
+        database_manager: DatabaseManager = None,
+    ) -> None:
+        self.name = name
+        self.config_path = config_path
+        self.workflow_config_paths = workflow_config_paths
+        self._handle_stream_output = _handle_stream_output
+        self._handle_query_user = _handle_query_user
+        self.enable_http = enable_http
+        self.http_host = http_host
+        self.http_port = http_port
+        self.enable_kafka = enable_kafka
+        self.kafka_host = kafka_host
+        self.kafka_port = kafka_port
+        self.service_env = {} if service_env is None else service_env
+        self.database_manager = database_manager
+
+    async def start(self):
+        if self.enable_http:
+            fastapi_task = asyncio.create_task(start_fastapi_server(self.http_host, self.http_port))
+        else:
+            fastapi_task = None
+        if self.enable_kafka:
+            kafka_task = asyncio.create_task(start_kafka_server(f"{self.kafka_host}:{self.kafka_port}"))
+        else:
+            kafka_task = None
+
+        workflow_tasks = []
+
+        for workflow_config_path in self.workflow_config_paths:
+            workflow = create_workflows(
+                self.parse_workflow_path(workflow_config_path),
+                service_env=self.service_env,
+                _handle_stream_output=self._handle_stream_output, 
+                _handle_query_user=self._handle_query_user,
+                database_manager=self.database_manager,
+            )
+            workflow_tasks.append(asyncio.create_task(workflow.run()))
+        
+        try:
+            tasks = []
+            if fastapi_task:
+                tasks.append(fastapi_task)
+            if kafka_task:
+                tasks.append(kafka_task)
+            tasks.extend(workflow_tasks)
+            await asyncio.gather(*tasks)
+        except Exception as e:
+            logger.error(f"Error in service {self.name}: {e}")
+            if fastapi_task:
+                fastapi_task.cancel()
+            if kafka_task:
+                kafka_task.cancel()
+            for workflow_task in workflow_tasks:
+                workflow_task.cancel()
+            raise
+
+    def parse_workflow_path(self, workflow_config_path: str) -> str:
+        if os.path.isabs(workflow_config_path):
+            return workflow_config_path
+        else:
+            return os.path.join(os.path.dirname(self.config_path), workflow_config_path)
+    
+    @staticmethod
+    def from_config(config_path: str, service_env: dict[str, Any] = None) -> Service:
+        config = OmegaConf.to_object(OmegaConf.load(config_path))
+        database_manager = DatabaseManager.from_config(config=config)
+        return Service(
+            name=config['name'],
+            config_path=config_path,
+            workflow_config_paths=config.get('workflows', []),
+            _handle_stream_output=None,
+            _handle_query_user=None,
+            enable_http=config.get('enable_http', True),
+            http_host=config.get('http_host', '0.0.0.0'),
+            http_port=config.get('http_port', 8000),
+            enable_kafka=config.get('enable_kafka', True),
+            kafka_host=config.get('kafka_host', 'localhost'),
+            kafka_port=config.get('kafka_port', 9092),
+            service_env=service_env,
+            database_manager=database_manager,
+        )
+
+def create_service(config_path: str, service_env: dict[str, Any] = None) -> Service:
+    return Service.from_config(config_path, service_env)

service_forge/service_config.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+class ServiceDatabaseConfig:
+    def __init__(
+        self,
+        name: str,
+        postgres_user: str,
+        postgres_password: str,
+        postgres_host: str,
+        postgres_port: int,
+        postgres_db: str,
+    ) -> None:
+        self.name = name
+        self.postgres_user = postgres_user
+        self.postgres_password = postgres_password
+        self.postgres_host = postgres_host
+        self.postgres_port = postgres_port
+        self.postgres_db = postgres_db
+    
+    @staticmethod
+    def from_dict(config: dict) -> ServiceDatabaseConfig:
+        return ServiceDatabaseConfig(
+            name=config['name'],
+            postgres_user=config['postgres_user'],
+            postgres_password=config['postgres_password'],
+            postgres_host=config['postgres_host'],
+            postgres_port=config['postgres_port'],
+            postgres_db=config['postgres_db'],
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            'name': self.name,
+            'postgres_user': self.postgres_user,
+            'postgres_password': self.postgres_password,
+            'postgres_host': self.postgres_host,
+            'postgres_port': self.postgres_port,
+            'postgres_db': self.postgres_db,
+        }
+
+class ServiceConfig:
+    def __init__(
+        self,
+        name: str,
+        workflows: list[str],
+        enable_http: bool,
+        http_host: str,
+        http_port: int,
+        enable_kafka: bool,
+        kafka_host: str,
+        kafka_port: int,
+        databases: list[ServiceDatabaseConfig],
+    ) -> None:
+        self.name = name
+        self.workflows = workflows
+        self.enable_http = enable_http
+        self.http_host = http_host
+        self.http_port = http_port
+        self.enable_kafka = enable_kafka
+        self.kafka_host = kafka_host
+        self.kafka_port = kafka_port
+        self.databases = databases
+
+    @staticmethod
+    def from_dict(config: dict) -> ServiceConfig:
+        return ServiceConfig(
+            name=config['name'],
+            workflows=config['workflows'],
+            enable_http=config['enable_http'],
+            http_host=config['http_host'],
+            http_port=config['http_port'],
+            enable_kafka=config['enable_kafka'],
+            kafka_host=config['kafka_host'],
+            kafka_port=config['kafka_port'],
+            databases=[ServiceDatabaseConfig.from_dict(database) for database in config['databases']],
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            'name': self.name,
+            'workflows': self.workflows,
+            'enable_http': self.enable_http,
+            'http_host': self.http_host,
+            'http_port': self.http_port,
+            'enable_kafka': self.enable_kafka,
+            'kafka_host': self.kafka_host,
+            'kafka_port': self.kafka_port,
+            'databases': [database.to_dict() for database in self.databases],
+        }
+
+# name: tag_service
+# workflows:
+#   # - ./workflow/kafka_workflow.yaml
+#   - ./workflow/query_tags_workflow.yaml
+#   - ./workflow/create_tag_workflow.yaml
+#   - ./workflow/update_tag_workflow.yaml
+#   - ./workflow/delete_tag_workflow.yaml
+#   - ./workflow/get_tags_from_record.yaml
+
+# enable_http: true
+# enable_kafka: false
+
+# # Following configs will be auto-injected by sft.
+# http_host: 0.0.0.0
+# http_port: 37200
+# kafka_host: localhost
+# kafka_port: 9092
+
+# databases:
+#   - name: tag
+#     postgres_user: postgres
+#     postgres_password: "gnBGWg7aL4"
+#     postgres_host: second-brain-postgres-postgresql
+#     postgres_port: 5432
+#     postgres_db: tag-service-tag

service_forge/sft/cli.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import sys
+from typing import Optional
+
+import typer
+from service_forge.sft.util.logger import log_error, log_info
+from service_forge.sft.cmd.list_tars import list_tars
+from service_forge.sft.cmd.upload_service import upload_service
+from service_forge.sft.cmd.deploy_service import deploy_service
+from service_forge.sft.cmd.config_command import list_config, get_config, set_config
+from service_forge.sft.cmd.service_command import list_services, delete_service, show_service_logs
+
+app = typer.Typer(
+    name="sft",
+    help="Service Forge CLI - Service management tool",
+    add_completion=False,
+)
+
+@app.command(name="upload")
+def upload_service_command(
+    project_path: Optional[str] = typer.Argument(
+        default=".",
+        help="Project path, default is the current directory"
+    )
+) -> None:
+    upload_service(project_path)
+
+@app.command(name="list")
+def list_tars_command() -> None:
+    list_tars()
+
+@app.command(name="deploy")
+def deploy_service_command(name: str, version: str) -> None:
+    deploy_service(name, version)
+
+config_app = typer.Typer(
+    name="config",
+    help="Configuration management commands",
+    add_completion=False,
+)
+
+@config_app.command(name="list")
+def config_list_command() -> None:
+    list_config()
+
+@config_app.command(name="get")
+def config_get_command(
+    key: str = typer.Argument(help="Configuration item key")
+) -> None:
+    get_config(key)
+
+@config_app.command(name="set")
+def config_set_command(
+    key: str = typer.Argument(help="Configuration item key"),
+    value: str = typer.Argument(help="Configuration item value")
+) -> None:
+    set_config(key, value)
+
+app.add_typer(config_app)
+
+service_app = typer.Typer(
+    name="service",
+    help="Kubernetes service management commands",
+    add_completion=False,
+)
+
+@service_app.command(name="list")
+def service_list_command() -> None:
+    list_services()
+
+@service_app.command(name="delete")
+def service_delete_command(
+    service_name: str = typer.Argument(help="Service name to delete (must start with sf-)"),
+    force: bool = typer.Option(False, "--force", "-f", help="Force delete")
+) -> None:
+    delete_service(service_name, force)
+
+@service_app.command(name="logs")
+def service_logs_command(
+    service_name: str = typer.Argument(help="Service name to view logs for (must start with sf-)"),
+    container: Optional[str] = typer.Option(None, "--container", "-c", help="Container name (if pod has multiple containers)"),
+    tail: int = typer.Option(100, "--tail", "-n", help="Number of lines to show from the end of logs"),
+    follow: bool = typer.Option(False, "--follow", "-f", help="Follow log output"),
+    previous: bool = typer.Option(False, "--previous", "-p", help="Get logs from previous instance of container")
+) -> None:
+    show_service_logs(service_name, container, tail, follow, previous)
+
+app.add_typer(service_app)
+
+def main() -> None:
+    app()

service_forge/sft/cmd/config_command.py

@@ -0,0 +1,67 @@
+from typing import Optional
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from service_forge.sft.config.sft_config import SftConfig
+from service_forge.sft.util.logger import log_error, log_info, log_success, log_warning
+from service_forge.sft.config.sft_config import sft_config
+
+def list_config() -> None:
+    try:
+        console = Console()
+        
+        table = Table(title="SFT Configuration", show_header=True, header_style="bold magenta")
+        table.add_column("Key", style="cyan", no_wrap=True)
+        table.add_column("Value", style="green")
+        table.add_column("Description", style="yellow")
+        
+        # Automatically add rows for all config items
+        config_dict = sft_config.to_dict()
+        for key, value in sorted(config_dict.items()):
+            description = SftConfig.CONFIG_DESCRIPTIONS.get(key, "No description available")
+            table.add_row(key, str(value), description)
+        
+        console.print(table)
+        console.print(f"\n[dim]Config file location: {sft_config.config_file_path}[/dim]")
+    except Exception as e:
+        log_error(f"Failed to load config: {e}")
+        raise typer.Exit(1)
+
+def get_config(key: str) -> None:
+    try:
+        value = sft_config.get(key)
+        
+        if value is None:
+            log_error(f"Config key '{key}' not found")
+            log_info("Available keys: config_root, sft_file_root, k8s_namespace")
+            raise typer.Exit(1)
+        
+        log_info(f"{key} = {value}")
+    except ValueError as e:
+        log_error(str(e))
+        raise typer.Exit(1)
+    except Exception as e:
+        log_error(f"Failed to get config: {e}")
+        raise typer.Exit(1)
+
+def set_config(key: str, value: str) -> None:
+    try:
+        current_value = sft_config.get(key)
+        if current_value is None:
+            log_error(f"Unknown config key: {key}")
+            log_info("Available keys: config_root, sft_file_root, k8s_namespace")
+            raise typer.Exit(1)
+        
+        sft_config.set(key, value)
+        sft_config.save()
+        
+        log_success(f"Updated {key} = {value}")
+        log_info(f"Config saved to {sft_config.config_file_path}")
+    except ValueError as e:
+        log_error(str(e))
+        raise typer.Exit(1)
+    except Exception as e:
+        log_error(f"Failed to set config: {e}")
+        raise typer.Exit(1)
+

service_forge/sft/cmd/deploy_service.py

@@ -0,0 +1,124 @@
+import os
+import shutil
+import subprocess
+import tarfile
+import tempfile
+from pathlib import Path
+
+import typer
+from omegaconf import OmegaConf
+from service_forge.sft.util.logger import log_error, log_info, log_success, log_warning
+from service_forge.sft.file.sft_file_manager import sft_file_manager
+from service_forge.sft.config.sft_config import sft_config
+from service_forge.sft.util.assert_util import assert_file_exists, assert_dir_exists
+from service_forge.sft.config.sf_metadata import load_metadata
+from service_forge.sft.kubernetes.kubernetes_manager import KubernetesManager
+from service_forge.sft.config.injector import Injector
+from service_forge.sft.util.name_util import get_service_name
+
+def _extract_tar_file(tar_file: Path, temp_path: Path) -> None:
+    log_info(f"Extracting tar file to: {temp_path}")
+    
+    try:
+        with tarfile.open(tar_file, 'r') as tar:
+            tar.extractall(temp_path)
+    except Exception as e:
+        log_error(f"Failed to extract tar file: {e}")
+        raise typer.Exit(1)
+    
+    log_success("Tar file extracted successfully")
+
+def _build_docker_image(project_dir: Path, name: str, version: str) -> None:
+    image_name = f"sf-{name}:{version}"
+    full_image_name = sft_config.registry_address + "/" + image_name
+    log_info(f"Building Docker image: {image_name}")
+    try:
+        # build docker image
+        build_result = subprocess.run(
+            ["docker", "build", "-t", full_image_name, str(project_dir)],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        log_success(f"Docker image built successfully: {image_name}")
+        if build_result.stdout:
+            log_info(build_result.stdout)
+
+        # push docker image to registry
+        log_info(f"Pushing Docker image to registry: {full_image_name}")
+        push_result = subprocess.run(
+            ["docker", "push", full_image_name],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        log_success(f"Docker image pushed successfully: {full_image_name}")
+        if push_result.stdout:
+            log_info(push_result.stdout)
+
+    except subprocess.CalledProcessError as e:
+        log_error(f"Docker operation failed: {e}")
+        if e.stderr:
+            log_error(e.stderr)
+        raise typer.Exit(1)
+    except FileNotFoundError:
+        log_error("Docker command not found. Please install Docker.")
+        raise typer.Exit(1)
+
+def _apply_k8s_deployment(deployment_yaml: Path, ingress_yaml: Path, name: str, version: str) -> None:
+    log_info("Applying k8s deployment...")
+    
+    try:
+        k8s_manager = KubernetesManager()
+        k8s_manager.delete_service(sft_config.k8s_namespace, get_service_name(name, version), force=True)
+        k8s_manager.apply_deployment_yaml(deployment_yaml, sft_config.k8s_namespace)
+        k8s_manager.apply_deployment_yaml(ingress_yaml, sft_config.k8s_namespace)
+        log_success("K8s deployment applied successfully")
+    except Exception as e:
+        log_error(f"K8s deployment failed: {e}")
+        raise typer.Exit(1)
+    
+    log_success(f"Deployment process completed for {name}:{version}")
+
+def _inject_config(project_dir: Path) -> None:
+    injector = Injector(project_dir)
+    injector.inject()
+
+def deploy_service(name: str, version: str) -> None:
+    tar_file = sft_file_manager.tar_path / f"sf_{name}_{version}.tar"
+
+    assert_file_exists(tar_file)
+    
+    import os
+    # 确保临时父目录存在
+    temp_parent = os.path.join(tempfile.gettempdir(), "sft")
+    os.makedirs(temp_parent, exist_ok=True)
+    
+    with tempfile.TemporaryDirectory(prefix=f"deploy_{name}_{version}", dir=temp_parent) as temp_dir:
+        temp_path = Path(temp_dir)
+
+        _extract_tar_file(tar_file, temp_path)
+
+        project_dir = temp_path / f"{name}_{version}"
+
+        _inject_config(project_dir)
+
+        dockerfile_path = project_dir / "Dockerfile"
+        metadata_path = project_dir / "sf-meta.yaml"
+        deployment_yaml = project_dir / "deployment.yaml"
+        ingress_yaml = project_dir / "ingress.yaml"
+        
+        assert_dir_exists(project_dir)
+        assert_file_exists(dockerfile_path)
+        assert_file_exists(metadata_path)
+        assert_file_exists(deployment_yaml)
+        assert_file_exists(ingress_yaml)
+
+        try:
+            meta_data = load_metadata(metadata_path)
+        except Exception as e:
+            log_error(f"Failed to read sf-meta.yaml: {e}")
+            raise typer.Exit(1)
+        
+        _build_docker_image(project_dir, meta_data.name, meta_data.version)
+        _apply_k8s_deployment(deployment_yaml, ingress_yaml, meta_data.name, meta_data.version)

service_forge/sft/cmd/list_tars.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from service_forge.sft.util.logger import log_error, log_info
+from service_forge.sft.file.sft_file_manager import sft_file_manager
+
+def list_tars() -> None:
+    tar_files = sft_file_manager.load_tars()
+    
+    if not tar_files:
+        log_info("No tar files found.")
+        return
+    
+    console = Console()
+    table = Table(title="Service Tar Files", show_header=True, header_style="bold magenta")
+    table.add_column("Project", style="cyan", no_wrap=True)
+    table.add_column("Version", style="cyan", no_wrap=True)
+    table.add_column("File Name", style="cyan", no_wrap=True)
+    table.add_column("Size", justify="right", style="green")
+    table.add_column("Modified Time", style="yellow")
+    
+    for tar_file in tar_files:
+        table.add_row(tar_file.project_name, tar_file.version, tar_file.path.name, tar_file._format_size(), tar_file._format_modified_time())
+    
+    console.print(table)
+
+
+def _format_size(size_bytes: int) -> str:
+    for unit in ['B', 'KB', 'MB', 'GB']:
+        if size_bytes < 1024.0:
+            return f"{size_bytes:.2f} {unit}"
+        size_bytes /= 1024.0
+    return f"{size_bytes:.2f} TB"
+
+
+def _format_time(timestamp: float) -> str:
+    from datetime import datetime
+    return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S")
+