service-forge 0.1.18__py3-none-any.whl

service_forge/sft/cmd/service_command.py

@@ -0,0 +1,149 @@
+from typing import List, Optional
+import typer
+from rich.console import Console
+from rich.table import Table
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+
+from service_forge.sft.config.sft_config import sft_config
+from service_forge.sft.util.logger import log_error, log_info, log_success, log_warning
+from service_forge.sft.kubernetes.kubernetes_manager import KubernetesManager
+
+def list_services() -> None:
+    namespace = sft_config.k8s_namespace
+    kubernetes_manager = KubernetesManager()
+    services = kubernetes_manager.get_services_in_namespace(namespace)
+    
+    if not services:
+        log_warning(f"No services starting with 'sf-' found in namespace '{namespace}'")
+        return
+    
+    console = Console()
+    table = Table(title=f"Services in namespace '{namespace}' (sf-*)", show_header=True, header_style="bold magenta")
+    table.add_column("Name", style="cyan", no_wrap=True)
+    table.add_column("Type", style="green")
+    table.add_column("Port", style="yellow")
+    table.add_column("Target Port", style="yellow")
+    
+    for service_name in sorted(services):
+        details = kubernetes_manager.get_service_details(namespace, service_name)
+        table.add_row(
+            details.name,
+            details.type or "-",
+            str(details.port or "-"),
+            str(details.target_port or "-")
+        )
+    
+    console.print(table)
+    log_info(f"Found {len(services)} service(s)")
+
+def show_logs(
+    namespace: str,
+    pod_name: str,
+    container_name: str,
+    tail: int,
+    follow: bool,
+    previous: bool
+) -> None:
+    kubernetes_manager = KubernetesManager()
+    try:
+        if not follow:
+            log_info(f"Fetching logs from pod '{pod_name}' (container: {container_name})...")
+        
+        logs = kubernetes_manager.get_pod_logs(
+            namespace=namespace,
+            pod_name=pod_name,
+            container_name=container_name,
+            tail=tail,
+            follow=follow,
+            previous=previous
+        )
+        
+        if follow:
+            try:
+                for line in logs:
+                    log_info(line, end="")
+            except KeyboardInterrupt:
+                log_warning("Log streaming interrupted")
+                raise typer.Exit(0)
+        else:
+            if logs:
+                log_info(logs)
+            else:
+                log_warning(f"No logs available for pod '{pod_name}' container '{container_name}'")
+                
+    except ApiException as e:
+        if e.status == 404:
+            log_error(f"Pod '{pod_name}' or container '{container_name}' not found")
+        elif e.status == 400:
+            log_error(f"Bad request: {e.reason}")
+            if "previous" in str(e.body).lower():
+                log_info("Note: 'previous' flag only works for stopped containers")
+        else:
+            log_error(f"Failed to get logs: {e.reason}")
+            if e.body:
+                log_error(f"Error details: {e.body}")
+    except Exception as e:
+        log_error(f"Failed to get logs: {e}")
+
+
+def show_service_logs(
+    service_name: str,
+    container: Optional[str] = None,
+    tail: int = 100,
+    follow: bool = False,
+    previous: bool = False
+) -> None:
+    namespace = sft_config.k8s_namespace
+    kubernetes_manager = KubernetesManager()
+    
+    if not service_name.startswith("sf-"):
+        log_warning(f"Service name '{service_name}' does not start with 'sf-'. Proceeding anyway...")
+    
+    services = kubernetes_manager.get_services_in_namespace(namespace)
+    if service_name not in services:
+        log_error(f"Service '{service_name}' not found in namespace '{namespace}'")
+        log_info(f"Available services: {', '.join(services) if services else 'None'}")
+        raise typer.Exit(1)
+    
+    pod_names = kubernetes_manager.get_pods_for_service(namespace, service_name)
+    if not pod_names:
+        log_error(f"No pods found for service '{service_name}' in namespace '{namespace}'")
+        raise typer.Exit(1)
+    
+    log_info(f"Found {len(pod_names)} pod(s) for service '{service_name}'")
+    
+    for pod_name in pod_names:
+        containers = kubernetes_manager.get_pod_containers(namespace, pod_name)
+        
+        if not containers:
+            log_warning(f"No containers found in pod '{pod_name}'")
+            continue
+
+        if container and container not in containers:
+            log_error(f"Container '{container}' not found in pod '{pod_name}'")
+            log_info(f"Available containers: {', '.join(containers)}")
+            continue
+        
+        target_containers = [container] if container else containers
+        
+        for container_name in target_containers:
+            show_logs(namespace, pod_name, container_name, tail, follow, previous)
+
+def delete_service(service_name: str, force: bool = False) -> None:
+    namespace = sft_config.k8s_namespace
+    
+    if not service_name.startswith("sf-"):
+        log_error(f"Service name '{service_name}' does not start with 'sf-'")
+        raise typer.Exit(1)
+    
+    kubernetes_manager = KubernetesManager()
+
+    services = kubernetes_manager.get_services_in_namespace(namespace)
+    if service_name not in services:
+        log_warning(f"Service '{service_name}' not found in namespace '{namespace}'")
+        log_info(f"Available services: {', '.join(services) if services else 'None'}")
+    
+    log_info(f"Deleting service '{service_name}' from namespace '{namespace}'...")
+    kubernetes_manager.delete_service(namespace, service_name, force)
+    

service_forge/sft/cmd/upload_service.py

@@ -0,0 +1,36 @@
+import typer
+from pathlib import Path
+from service_forge.sft.util.logger import log_error, log_success
+from service_forge.sft.file.sft_file_manager import sft_file_manager
+from service_forge.sft.util.assert_util import assert_dir_exists, assert_file_exists
+from service_forge.sft.config.sf_metadata import load_metadata
+
+def upload_service(project_path: str) -> None:
+    project_dir = Path(project_path).resolve()
+    assert_dir_exists(project_dir)
+    
+    metadata_path = project_dir / "sf-meta.yaml"
+    assert_file_exists(metadata_path)
+    
+    try:
+        meta_data = load_metadata(metadata_path)
+    except Exception as e:
+        log_error(f"Failed to read sf-meta.yaml: {e}")
+        raise typer.Exit(1)
+        
+    try:
+        tar_file = sft_file_manager.create_tar(project_dir, meta_data.name, meta_data.version)
+    except Exception as e:
+        log_error(f"Failed to create tar file: {e}")
+        raise typer.Exit(1)
+
+    log_success(f"Packaging successful: {tar_file}")
+
+    # upload to the service
+    try:
+        sft_file_manager.upload_tar(tar_file)
+    except Exception as e:
+        log_error(f"Failed to upload tar file: {e}")
+        raise typer.Exit(1)
+
+    log_success(f"Upload successful: {tar_file}")

service_forge/sft/config/injector.py

@@ -0,0 +1,129 @@
+import yaml
+from pathlib import Path
+from service_forge.sft.util.logger import log_info, log_error
+from service_forge.sft.config.injector_default_files import *
+from service_forge.sft.config.sf_metadata import load_metadata
+from service_forge.sft.config.sft_config import sft_config
+from service_forge.service_config import ServiceConfig, ServiceFeedbackConfig
+from service_forge.sft.util.name_util import get_service_name
+from service_forge.sft.util.yaml_utils import load_sf_metadata_as_string
+
+class Injector:
+    def __init__(self, project_dir: Path):
+        self.project_dir = project_dir
+        self.deployment_yaml_path = project_dir / "deployment.yaml"
+        self.metadata_path = project_dir / "sf-meta.yaml"
+        self.ingress_yaml_path = project_dir / "ingress.yaml"
+        self.dockerfile_path = project_dir / "Dockerfile"
+        self.pyproject_toml_path = project_dir / "pyproject.toml"
+        self.metadata = load_metadata(self.metadata_path)
+        self.name = self.metadata.name
+        self.version = self.metadata.version
+        self.namespace = sft_config.k8s_namespace
+
+        try:
+            self.sf_metadata_string = load_sf_metadata_as_string(self.metadata_path)
+        except Exception as e:
+            log_error(f"Failed to load sf-metadata as string: {e}")
+            self.sf_metadata_string = ""
+
+    @property
+    def service_name(self) -> str:
+        return get_service_name(self.name, self.version)
+
+    def inject_deployment(self) -> None:
+        single_line_metadata = self.sf_metadata_string.replace('\n', '\\n').replace('"', '\\"')
+
+        envs = {
+            "DEEPSEEK_API_KEY": sft_config.deepseek_api_key,
+            "DEEPSEEK_BASE_URL": sft_config.deepseek_base_url,
+        }
+
+        for env in self.metadata.env:
+            envs[env['name']] = env['value']
+
+        env_str = ""
+        for key, value in envs.items():
+            env_str += f"          - name: {key}\n            value: {value}\n"
+
+        deployment_yaml = DEFAULT_DEPLOYMENT_YAML.format(
+            service_name=self.service_name,
+            name=self.name,
+            version=self.version,
+            namespace=self.namespace,
+            sf_metadata=f'"{single_line_metadata}"',
+            env=env_str,
+        )
+        with open(self.deployment_yaml_path, "w") as f:
+            f.write(deployment_yaml)
+        print("deployment_yaml_path: ", self.deployment_yaml_path)
+
+    def inject_service_config(self) -> None:
+        service_config_path = self.project_dir / Path(self.metadata.service_config)
+
+        config = ServiceConfig.from_yaml_file(service_config_path)
+
+        config.http_port = sft_config.inject_http_port
+        config.kafka_host = sft_config.inject_kafka_host
+        config.kafka_port = sft_config.inject_kafka_port
+        if config.databases is not None:
+            for database in config.databases:
+                if database.postgres_host is not None:
+                    database.postgres_host = sft_config.inject_postgres_host
+                    database.postgres_port = sft_config.inject_postgres_port
+                    database.postgres_user = sft_config.inject_postgres_user
+                    database.postgres_password = sft_config.inject_postgres_password
+                    database.postgres_db = self.service_name
+                if database.mongo_host is not None:
+                    database.mongo_host = sft_config.inject_mongo_host
+                    database.mongo_port = sft_config.inject_mongo_port
+                    database.mongo_user = sft_config.inject_mongo_user
+                    database.mongo_password = sft_config.inject_mongo_password
+                    database.mongo_db = sft_config.inject_mongo_db
+                if database.redis_host is not None:
+                    database.redis_host = sft_config.inject_redis_host
+                    database.redis_port = sft_config.inject_redis_port
+                    database.redis_password = sft_config.inject_redis_password
+        if config.feedback is not None:
+            config.feedback.api_url = sft_config.inject_feedback_api_url
+            config.feedback.api_timeout = sft_config.inject_feedback_api_timeout
+        else:
+            config.feedback = ServiceFeedbackConfig(
+                api_url=sft_config.inject_feedback_api_url,
+                api_timeout=sft_config.inject_feedback_api_timeout,
+            )
+
+        with open(service_config_path, "w", encoding="utf-8") as f:
+            yaml.dump(config.model_dump(), f, allow_unicode=True, indent=2)
+
+    def inject_ingress(self) -> None:
+        ingress_yaml = DEFAULT_TRAEFIK_INGRESS_YAML.format(
+            name=self.name,
+            version=self.version.replace(".", "-"),
+            namespace=self.namespace,
+        )
+        with open(self.ingress_yaml_path, "w") as f:
+            f.write(ingress_yaml)
+        print("ingress_yaml_path: ", self.ingress_yaml_path)
+
+    def inject_dockerfile(self) -> None:
+        dockerfile = DEFAULT_DOCKERFILE
+        with open(self.dockerfile_path, "w") as f:
+            f.write(dockerfile)
+        print("dockerfile_path: ", self.dockerfile_path)
+
+    def inject_pyproject_toml(self) -> None:
+        pyproject_toml = DEFAULT_PYPROJECT_TOML
+        with open(self.pyproject_toml_path, "r") as f:
+            existing_pyproject_toml = f.read()
+        if pyproject_toml.strip() not in existing_pyproject_toml.strip():
+            with open(self.pyproject_toml_path, "a") as f:
+                f.write(pyproject_toml)
+        print("pyproject_toml_path: ", self.pyproject_toml_path)
+
+    def inject(self) -> None:
+        self.inject_deployment()
+        self.inject_service_config()
+        self.inject_ingress()
+        self.inject_dockerfile()
+        self.inject_pyproject_toml()

service_forge/sft/config/injector_default_files.py

@@ -0,0 +1,131 @@
+DEFAULT_DEPLOYMENT_YAML = """
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {service_name}
+  namespace: {namespace}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {service_name}
+  template:
+    metadata:
+      labels:
+        app: {service_name}
+    spec:
+      imagePullSecrets:
+        - name: aliyun-regcred
+      containers:
+      - name: {service_name}
+        image: crpi-cev6qq28wwgwwj0y.cn-beijing.personal.cr.aliyuncs.com/nexthci/sf-{name}:{version}
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8000
+        env:
+{env}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {service_name}
+  namespace: {namespace}
+  annotations:
+    metadata: {sf_metadata}
+spec:
+  type: ClusterIP
+  selector:
+    app: {service_name}
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8000
+"""
+
+DEFAULT_TRAEFIK_INGRESS_YAML = """
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: strip-prefix-sf-{name}-{version}v
+  namespace: {namespace}
+spec:
+  stripPrefix:
+    prefixes:
+      - /api/v1/{name}-{version}
+
+---
+
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: sf-{name}-{version}v
+  namespace: {namespace}
+spec:
+  entryPoints:
+    - web
+  routes:
+    - match: PathPrefix(`/api/v1/{name}-{version}/openapi.json`)
+      kind: Rule
+      services:
+        - name: sf-{name}-{version}v
+          namespace: {namespace}
+          port: 80
+      middlewares:
+        - name: strip-prefix-sf-{name}-{version}v
+          namespace: {namespace}
+        - name: cors
+          namespace: {namespace}
+
+    - match: PathPrefix(`/api/v1/{name}-{version}/docs`)
+      kind: Rule
+      services:
+        - name: sf-{name}-{version}v
+          namespace: {namespace}
+          port: 80
+      middlewares:
+        - name: strip-prefix-sf-{name}-{version}v
+          namespace: {namespace}
+        - name: cors
+          namespace: {namespace}
+
+    - match: PathPrefix(`/api/v1/{name}-{version}`)
+      kind: Rule
+      services:
+        - name: sf-{name}-{version}v
+          namespace: {namespace}
+          port: 80
+      middlewares:
+        - name: strip-prefix-sf-{name}-{version}v
+          namespace: {namespace}
+        - name: cors
+          namespace: {namespace}
+        - name: jwt-auth
+          namespace: {namespace}
+"""
+
+DEFAULT_DOCKERFILE = """
+FROM crpi-cev6qq28wwgwwj0y.cn-beijing.personal.cr.aliyuncs.com/nexthci/service-forge:latest
+
+WORKDIR /app
+
+COPY . ./service
+
+WORKDIR /app
+RUN uv sync
+
+ENV PYTHONPATH=/app/service:/app:/app/src
+ENV PATH="/app/.venv/bin:$PATH"
+
+WORKDIR /app/service
+
+RUN chmod +x start.sh
+
+CMD ["./start.sh"]
+"""
+
+DEFAULT_PYPROJECT_TOML = """
+
+[tool.uv.sources]
+service-forge = { workspace = true }
+"""

service_forge/sft/config/sf_metadata.py

@@ -0,0 +1,30 @@
+from omegaconf import OmegaConf
+
+class SfMetadata:
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        description: str,
+        service_config: str,
+        config_only: bool,
+        env: list[dict],
+    ) -> None:
+        self.name = name
+        self.version = version
+        self.description = description
+        self.service_config = service_config
+        self.config_only = config_only
+        self.env = env
+
+def load_metadata(path: str) -> SfMetadata:
+    with open(path, 'r') as file:
+        data = OmegaConf.load(file)
+        return SfMetadata(
+            name=data.get('name'),
+            version=data.get('version'),
+            description=data.get('description'),
+            service_config=data.get('service_config'),
+            config_only=data.get('config_only'),
+            env=data.get('env', []),
+        )

service_forge/sft/config/sft_config.py

@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+import inspect
+from pathlib import Path
+from typing import Optional
+from omegaconf import OmegaConf
+
+class SftConfig:
+    CONFIG_ROOT = Path.home() / ".sft"
+    
+    # Configuration descriptions mapping
+    CONFIG_DESCRIPTIONS = {
+        "sft_file_root": "SFT file storage root directory",
+        "service_center_address": "Service center address",
+        "k8s_namespace": "Kubernetes namespace",
+        "registry_address": "Registry address",
+        "inject_http_port": "HTTP port for services",
+        "inject_kafka_host": "Kafka host for services",
+        "inject_kafka_port": "Kafka port for services",
+        "inject_postgres_host": "Postgres host for services",
+        "inject_postgres_port": "Postgres port for services",
+        "inject_postgres_user": "Postgres user for services",
+        "inject_postgres_password": "Postgres password for services",
+        "inject_feedback_api_url": "Feedback API URL for services",
+        "inject_feedback_api_timeout": "Feedback API timeout for services",
+        "deepseek_api_key": "DeepSeek API key",
+        "deepseek_base_url": "DeepSeek base URL",
+    }
+
+    def __init__(
+        self,
+        sft_file_root: str = "/tmp/sft",
+        service_center_address: str = "http://vps.shiweinan.com:37919/service_center",
+        k8s_namespace: str = "secondbrain",
+        registry_address: str = "crpi-cev6qq28wwgwwj0y.cn-beijing.personal.cr.aliyuncs.com/nexthci",
+
+        inject_http_port: int = 8000,
+
+        inject_kafka_host: str = "localhost",
+        inject_kafka_port: int = 9092,
+
+        inject_postgres_host: str = "second-brain-postgres-postgresql",
+        inject_postgres_port: int = 5432,
+        inject_postgres_user: str = "postgres",
+        inject_postgres_password: str = "gnBGWg7aL4",
+
+        inject_mongo_host: str = "mongo-mongodb",
+        inject_mongo_port: int = 27017,
+        inject_mongo_user: str = "secondbrain",
+        inject_mongo_password: str = "secondbrain",
+        inject_mongo_db: str = "secondbrain",
+
+        inject_redis_host: str = "redis-master",
+        inject_redis_port: int = 6379,
+        inject_redis_password: str = "rDdM2Y2gX9",
+
+        inject_feedback_api_url: str = "http://vps.shiweinan.com:37919/api/v1/feedback",
+        inject_feedback_api_timeout: int = 5,
+
+        deepseek_api_key: str = "82c9df22-f6ed-411e-90d7-c5255376b7ca",
+        deepseek_base_url: str = "https://ark.cn-beijing.volces.com/api/v3",
+    ):
+        self.sft_file_root = sft_file_root
+        self.service_center_address = service_center_address
+        self.k8s_namespace = k8s_namespace
+        self.registry_address = registry_address
+
+        self.inject_http_port = inject_http_port
+
+        self.inject_kafka_host = inject_kafka_host
+        self.inject_kafka_port = inject_kafka_port
+
+        self.inject_postgres_host = inject_postgres_host
+        self.inject_postgres_port = inject_postgres_port
+        self.inject_postgres_user = inject_postgres_user
+        self.inject_postgres_password = inject_postgres_password
+
+        self.inject_mongo_host = inject_mongo_host
+        self.inject_mongo_port = inject_mongo_port
+        self.inject_mongo_user = inject_mongo_user
+        self.inject_mongo_password = inject_mongo_password
+        self.inject_mongo_db = inject_mongo_db
+
+        self.inject_redis_host = inject_redis_host
+        self.inject_redis_port = inject_redis_port
+        self.inject_redis_password = inject_redis_password
+
+        self.inject_feedback_api_url = inject_feedback_api_url
+        self.inject_feedback_api_timeout = inject_feedback_api_timeout
+
+        self.deepseek_api_key = deepseek_api_key
+        self.deepseek_base_url = deepseek_base_url
+
+    @property
+    def server_url(self) -> str:
+        return self.service_center_address
+
+    @property
+    def upload_timeout(self) -> int:
+        return 300  # 5 minutes default timeout
+
+    def get_config_keys(self) -> list[str]:
+        # Get initial configuration parameters from __init__ method
+        sig = inspect.signature(self.__class__.__init__)
+        init_keys = [param for param in sig.parameters.keys() if param != 'self']
+        
+        # Get all instance attributes (including dynamically added configurations)
+        instance_keys = []
+        for attr_name in dir(self):
+            # Exclude special methods, private attributes, class attributes, and methods
+            if (not attr_name.startswith('_') and
+                not callable(getattr(self, attr_name)) and
+                attr_name not in ['CONFIG_ROOT', 'CONFIG_DESCRIPTIONS']):
+                instance_keys.append(attr_name)
+        
+        # Merge and deduplicate, maintaining order (initial configs first, then dynamically added)
+        all_keys = list(dict.fromkeys(init_keys + instance_keys))
+        return all_keys
+    
+    @property
+    def config_file_path(self) -> Path:
+        return self.CONFIG_ROOT / "config.yaml"
+    
+    def ensure_config_dir(self) -> None:
+        self.CONFIG_ROOT.mkdir(parents=True, exist_ok=True)
+    
+    def to_dict(self) -> dict:
+        config_keys = self.get_config_keys()
+        result = {}
+        for key in config_keys:
+            value = getattr(self, key)
+            # Convert Path objects to strings for JSON serialization
+            if isinstance(value, Path):
+                value = str(value)
+            result[key] = value
+        return result
+    
+    def from_dict(self, data: dict) -> None:
+        # Get initial configuration parameters from __init__ method
+        sig = inspect.signature(self.__class__.__init__)
+        init_keys = [param for param in sig.parameters.keys() if param != 'self']
+        
+        # First, set all initial configuration parameters
+        for key in init_keys:
+            if key in data:
+                setattr(self, key, data[key])
+        
+        # Then, handle any additional keys that might be dynamically added configurations
+        for key, value in data.items():
+            if key not in init_keys and key not in ['CONFIG_ROOT', 'CONFIG_DESCRIPTIONS']:
+                # This might be a dynamically added configuration
+                setattr(self, key, value)
+    
+    def save(self) -> None:
+        self.ensure_config_dir()
+        config_dict = self.to_dict()
+        OmegaConf.save(config_dict, self.config_file_path)
+    
+    def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
+        return getattr(self, key, default)
+    
+    def set(self, key: str, value: str, description: Optional[str] = None) -> None:
+        if key in ["config_root"]:
+            raise ValueError(f"{key} is read-only")
+        if hasattr(self, key):
+            setattr(self, key, value)
+            if description:
+                self.CONFIG_DESCRIPTIONS[key] = description
+        else:
+            raise ValueError(f"Unknown config key: {key}")
+        
+    def add(self, key: str, value: str, description: Optional[str] = None) -> None:
+        if hasattr(self, key):
+            raise ValueError(f"{key} already exists")
+        setattr(self, key, value)
+        if description:
+            self.CONFIG_DESCRIPTIONS[key] = description
+    
+    def update(self, updates: dict) -> None:
+        for key, value in updates.items():
+            self.set(key, value)
+
+
+def load_config() -> SftConfig:
+    # config = SftConfig()
+    config = SftConfig()
+    config_file = config.config_file_path
+    
+    if config_file.exists():
+        try:
+            data = OmegaConf.load(config_file)
+            config = SftConfig(**OmegaConf.to_container(data, resolve=True))
+        except Exception as e:
+            ...
+
+    config.save()
+    
+    return config
+
+sft_config = load_config()