service-forge 0.1.0__py3-none-any.whl

service_forge/sft/file/sft_file_manager.py

@@ -0,0 +1,107 @@
+import os
+import tarfile
+from pathlib import Path
+import requests
+
+from service_forge.sft.file.ignore_pattern import load_ignore_patterns
+from service_forge.sft.config.sft_config import sft_config
+from service_forge.sft.util.logger import log_success, log_info, log_error
+
+class SftTarFile:
+    # example: sf_tag_service_0.0.1.tar
+    def __init__(self, path: Path):
+        self.path = path
+        self.name = path.name
+        self.size = path.stat().st_size
+        self.modified_time = path.stat().st_mtime
+        self.project_name = '_'.join(path.name.split('_')[1:-1])
+        self.version = path.name.split('_')[-1][:-4]
+
+    def _format_size(self) -> str:
+        for unit in ['B', 'KB', 'MB', 'GB']:
+            if self.size < 1024.0:
+                return f"{self.size:.2f} {unit}"
+            self.size /= 1024.0
+        return f"{self.size:.2f} TB"
+
+    def _format_modified_time(self) -> str:
+        from datetime import datetime
+        return datetime.fromtimestamp(self.modified_time).strftime("%Y-%m-%d %H:%M:%S")
+
+    @staticmethod
+    def is_valid_path(path: Path) -> bool:
+        return path.is_file() and path.suffix == '.tar' and path.name.startswith('sf_')
+
+
+class SftFileManager:
+    def __init__(self):
+        self.tar_path = Path(sft_config.sft_file_root) / "service-tar"
+        self.tars: list[SftTarFile] = []
+        os.makedirs(self.tar_path, exist_ok=True)
+        self.load_tars()
+
+    def create_tar(self, project_path: Path, name: str, version: str) -> Path:
+        project_path = Path(project_path).resolve()
+        tar_path = Path(self.tar_path) / f"sf_{name}_{version}.tar"
+        
+        ignore_pattern = load_ignore_patterns(project_path)
+        
+        with tarfile.open(tar_path, 'w') as tar:
+            for root, dirs, files in os.walk(project_path):
+                root_path = Path(root)
+                
+                dirs[:] = [
+                    d for d in dirs 
+                    if not ignore_pattern.should_ignore(root_path / d)
+                ]
+                
+                for file in files:
+                    file_path = root_path / file
+                    if ignore_pattern.should_ignore(file_path):
+                        continue
+                    
+                    arcname = file_path.relative_to(project_path)
+                    tar.add(file_path, arcname=Path(f"{name}_{version}") / arcname)
+        self.load_tars()
+        return tar_path
+
+    def load_tars(self) -> list[SftTarFile]:
+        self.tars = [SftTarFile(p) for p in self.tar_path.iterdir() if SftTarFile.is_valid_path(p)]
+        return self.tars
+    
+    def upload_tar(self, tar_path: Path) -> None:
+        if not tar_path.exists():
+            raise FileNotFoundError(f"File not found: {tar_path}")
+        
+        upload_url = f"{sft_config.server_url}/api/v1/services/upload-tar"
+        
+        try:
+            with open(tar_path, 'rb') as file:
+                files = {'file': (tar_path.name, file)}
+                
+                response = requests.post(
+                    upload_url,
+                    files=files,
+                    timeout=sft_config.upload_timeout
+                )
+                
+                if response.status_code == 200:
+                    result = response.json()
+                    if result.get('code') == 200:
+                        log_success(f"Upload successful: {result.get('message')}")
+                        log_info(f"File saved path: {result.get('data', {}).get('file_path')}")
+                    else:
+                        raise Exception(f"Upload failed: {result.get('message')}")
+                else:
+                    try:
+                        error_detail = response.json()
+                        error_message = error_detail.get('message', f"HTTP错误: {response.status_code}")
+                        if 'debug' in error_detail and error_detail['debug']:
+                            log_error(f"Error details: {error_detail['debug']}")
+                        raise Exception(error_message)
+                    except ValueError:
+                        raise Exception(f"Server returned error status code: {response.status_code}")
+        except requests.exceptions.RequestException as e:
+            raise Exception(f"Upload request failed: {str(e)}")
+
+sft_file_manager = SftFileManager()

service_forge/sft/kubernetes/kubernetes_manager.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+from kubernetes.utils.create_from_yaml import FailToCreateError
+import threading
+from pathlib import Path
+import yaml
+from kubernetes import client, config, utils
+from kubernetes.client.rest import ApiException
+from kubernetes.dynamic import DynamicClient
+from kubernetes.dynamic.exceptions import NotFoundError
+from service_forge.sft.util.logger import log_error, log_info, log_success, log_warning
+
+class KubernetesServiceDetails:
+    def __init__(self, name: str, type: str | None = None, port: int | None = None, target_port: int | None = None):
+        self.name = name
+        self.type = type
+        self.port = port
+        self.target_port = target_port
+
+class KubernetesManager:
+    _instance_lock = threading.Lock()
+
+    def __init__(self):
+        try:
+            config.load_incluster_config()
+            # 使用InCluster配置创建DynamicClient
+            self.dynamic_client = DynamicClient(client.ApiClient())
+        except config.ConfigException:
+            config.load_kube_config()
+            # 如果InCluster配置失败，使用kubeconfig文件创建DynamicClient
+            self.dynamic_client = DynamicClient(config.new_client_from_config())
+
+        self.k8s_client = client.CoreV1Api()
+        self.k8s_apps_client = client.AppsV1Api()
+        self.k8s_batch_client = client.BatchV1Api()
+        self.k8s_rbac_client = client.RbacAuthorizationV1Api()
+        self.k8s_networking_client = client.NetworkingV1Api()
+        self.k8s_apiextensions_client = client.ApiextensionsV1Api()
+
+        self.api_mapping = {
+            "v1": self.k8s_client,
+            "apps/v1": self.k8s_apps_client,
+            "batch/v1": self.k8s_batch_client,
+            "rbac.authorization.k8s.io/v1": self.k8s_rbac_client,
+            "networking.k8s.io/v1": self.k8s_networking_client,
+            "apiextensions.k8s.io/v1": self.k8s_apiextensions_client,
+        }
+
+    def __new__(cls) -> KubernetesManager:
+        if not hasattr(cls, '_instance'):
+            with KubernetesManager._instance_lock:
+                if not hasattr(cls, '_instance'):
+                    KubernetesManager._instance = super().__new__(cls)
+        return KubernetesManager._instance
+
+    def get_services_in_namespace(self, namespace: str) -> list[str]:
+        try:
+            services = self.k8s_client.list_namespaced_service(namespace=namespace)
+            return [svc.metadata.name for svc in services.items if svc.metadata.name.startswith("sf-")]
+        except ApiException as e:
+            log_error(f"Failed to get services: {e.reason}")
+            return []
+        except Exception as e:
+            log_error(f"Failed to get services: {e}")
+            return []
+
+    def get_service_details(self, namespace: str, service_name: str) -> KubernetesServiceDetails:
+        try:
+            service = self.k8s_client.read_namespaced_service(name=service_name, namespace=namespace)
+            return KubernetesServiceDetails(
+                name=service.metadata.name,
+                type=service.spec.type,
+                port=service.spec.ports[0].port,
+                target_port=service.spec.ports[0].target_port
+            )
+        except ApiException as e:
+            log_error(f"Failed to get service details: {e.reason}")
+            return KubernetesServiceDetails(name=service_name)
+        except Exception as e:
+            log_error(f"Failed to get service details: {e}")
+            return KubernetesServiceDetails(name=service_name)
+
+    def get_pods_for_service(self, namespace: str, service_name: str) -> list[str]:
+        try:
+            service = self.k8s_client.read_namespaced_service(name=service_name, namespace=namespace)
+            selector = service.spec.selector
+            if not selector:
+                log_error(f"Service '{service_name}' has no selector")
+                return []
+            label_selector = ",".join([f"{k}={v}" for k, v in selector.items()])
+            pods = self.k8s_client.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
+            return [pod.metadata.name for pod in pods.items]
+        except ApiException as e:
+            log_error(f"Failed to get pods for service: {e.reason}")
+            return []
+        except Exception as e:
+            log_error(f"Failed to get pods for service: {e}")
+            return []
+
+    def get_pod_containers(self, namespace: str, pod_name: str) -> list[str]:
+        try:
+            pod = self.k8s_client.read_namespaced_pod(name=pod_name, namespace=namespace)
+            containers = []
+            if pod.spec.containers:
+                containers.extend([c.name for c in pod.spec.containers])
+            if pod.spec.init_containers:
+                containers.extend([c.name for c in pod.spec.init_containers])
+            return containers
+        except ApiException as e:
+            log_error(f"Failed to get pod containers: {e.reason}")
+            return []
+        except Exception as e:
+            log_error(f"Failed to get pod containers: {e}")
+            return []
+
+    def get_pod_logs(self, namespace: str, pod_name: str, container_name: str, tail: int, follow: bool, previous: bool) -> str:
+        try:
+            logs = self.k8s_client.read_namespaced_pod_log(
+                name=pod_name,
+                namespace=namespace,
+                container=container_name,
+                tail_lines=tail if not follow else None,
+                previous=previous,
+                follow=follow,
+                _preload_content=not follow
+            )
+            return logs
+        except ApiException as e:
+            log_error(f"Failed to get pod logs: {e.reason}")
+            return ""
+        except Exception as e:
+            log_error(f"Failed to get pod logs: {e}")
+
+    def apply_dynamic_yaml(self, obj: dict, namespace: str) -> None:
+        api_version = obj["apiVersion"]
+        kind = obj["kind"]
+        metadata = obj["metadata"]
+        name = metadata["name"]
+
+        resource = self.dynamic_client.resources.get(api_version=api_version, kind=kind)
+
+        try:
+            resource.get(name=name, namespace=namespace)
+            print(f"{kind}/{name} exists → patching...")
+            resource.patch(name=name, namespace=namespace, body=obj, content_type="application/merge-patch+json")
+
+        except NotFoundError:
+            print(f"{kind}/{name} not found → creating...")
+            resource.create(body=obj, namespace=namespace)
+
+
+    def apply_deployment_yaml(self, deployment_yaml: Path, namespace: str) -> None:
+        with open(deployment_yaml, 'r') as f:
+            objs = yaml.safe_load_all(f)
+            for obj in objs:
+                api_version = obj["apiVersion"]
+                kind = obj["kind"]
+                metadata = obj["metadata"]
+
+                name = metadata["name"]
+
+                api_client = self.api_mapping.get(api_version)
+                if not api_client:
+                    self.apply_dynamic_yaml(obj, namespace)
+                    continue
+
+                read_fn = getattr(api_client, f"read_namespaced_{kind.lower()}", None)
+                create_fn = getattr(api_client, f"create_namespaced_{kind.lower()}", None)
+                patch_fn = getattr(api_client, f"patch_namespaced_{kind.lower()}", None)
+
+                if not read_fn:
+                    raise Exception(f"Unsupported resource type: {kind}")
+
+                try:
+                    read_fn(name=name, namespace=namespace)
+                    print(f"{kind}/{name} exists → patching...")
+                    patch_fn(name=name, namespace=namespace, body=obj)
+
+                except ApiException as e:
+                    if e.status == 404:
+                        print(f"{kind}/{name} not found → creating...")
+                        create_fn(namespace=namespace, body=obj)
+                    else:
+                        raise
+
+    def delete_service(self, namespace: str, service_name: str, force: bool = False) -> None:
+        delete_options = client.V1DeleteOptions()
+        if force:
+            delete_options.grace_period_seconds = 0
+            delete_options.propagation_policy = "Background"
+        
+        # Delete deployment
+        try:
+            log_info(f"Attempting to delete deployment '{service_name}'...")
+            self.k8s_apps_client.delete_namespaced_deployment(
+                name=service_name,
+                namespace=namespace,
+                body=delete_options
+            )
+            log_success(f"Deployment '{service_name}' deleted successfully")
+        except ApiException as e:
+            if e.status == 404:
+                log_warning(f"Deployment '{service_name}' not found, skipping...")
+            else:
+                log_warning(f"Failed to delete deployment '{service_name}': {e.reason}")
+                log_warning("Continuing with service deletion...")
+        except Exception as e:
+            log_warning(f"Failed to delete deployment '{service_name}': {e}")
+            log_warning("Continuing with service deletion...")
+        
+        # Delete service
+        try:
+            log_info(f"Attempting to delete service '{service_name}'...")
+            self.k8s_client.delete_namespaced_service(
+                name=service_name,
+                namespace=namespace,
+                body=delete_options
+            )
+            log_success(f"Service '{service_name}' deleted successfully")
+        except ApiException as e:
+            if e.status == 404:
+                log_warning(f"Service '{service_name}' not found, skipping...")
+            else:
+                log_error(f"Failed to delete service '{service_name}': {e.reason}")
+                if e.body:
+                    log_error(f"Error details: {e.body}")
+        except Exception as e:
+            log_error(f"Failed to delete service '{service_name}': {e}")
+        
+        # Delete IngressRoute (Traefik CRD)
+        try:
+            log_info(f"Attempting to delete IngressRoute '{service_name}'...")
+            ingressroute_resource = self.dynamic_client.resources.get(
+                api_version="traefik.io/v1alpha1",
+                kind="IngressRoute"
+            )
+            ingressroute_resource.delete(name=service_name, namespace=namespace)
+            log_success(f"IngressRoute '{service_name}' deleted successfully")
+        except NotFoundError:
+            log_warning(f"IngressRoute '{service_name}' not found, skipping...")
+        except Exception as e:
+            log_warning(f"Failed to delete IngressRoute '{service_name}': {e}")
+        
+        # Delete Middleware (Traefik CRD)
+        middleware_name = f"strip-prefix-{service_name}"
+        try:
+            log_info(f"Attempting to delete Middleware '{middleware_name}'...")
+            middleware_resource = self.dynamic_client.resources.get(
+                api_version="traefik.io/v1alpha1",
+                kind="Middleware"
+            )
+            middleware_resource.delete(name=middleware_name, namespace=namespace)
+            log_success(f"Middleware '{middleware_name}' deleted successfully")
+        except NotFoundError:
+            log_warning(f"Middleware '{middleware_name}' not found, skipping...")
+        except Exception as e:
+            log_warning(f"Failed to delete Middleware '{middleware_name}': {e}")
+        

service_forge/sft/util/assert_util.py

@@ -0,0 +1,25 @@
+import typer
+from pathlib import Path
+from typing import Callable, TypeVar, Any
+from service_forge.sft.util.logger import log_error, log_info
+from service_forge.sft.config.sf_metadata import load_metadata, SfMetadata
+
+T = TypeVar('T')
+
+def assert_dir_exists(path: Path) -> None:
+    if not path.exists():
+        log_error(f"Directory does not exist: {path}")
+        raise typer.Exit(1)
+    if not path.is_dir():
+        log_error(f"Path is not a directory: {path}")
+        raise typer.Exit(1)
+    log_info(f"Directory exists: {path}")
+
+def assert_file_exists(path: Path) -> None:
+    if not path.exists():
+        log_error(f"File does not exist: {path}")
+        raise typer.Exit(1)
+    if not path.is_file():
+        log_error(f"Path is not a file: {path}")
+        raise typer.Exit(1)
+    log_info(f"File exists: {path}")

service_forge/sft/util/logger.py

@@ -0,0 +1,16 @@
+from rich.console import Console
+from typing import Any
+
+console = Console()
+
+def log_error(message: str, **kwargs: Any) -> None:
+    console.print(f"[red]{message}[/red]", **kwargs)
+
+def log_info(message: str, **kwargs: Any) -> None:
+    console.print(f"{message}", **kwargs)
+
+def log_success(message: str, **kwargs: Any) -> None:
+    console.print(f"[green]{message}[/green]", **kwargs)
+
+def log_warning(message: str, **kwargs: Any) -> None:
+    console.print(f"[yellow]{message}[/yellow]", **kwargs)

service_forge/sft/util/name_util.py

@@ -0,0 +1,2 @@
+def get_service_name(name: str, version: str) -> str:
+    return f"sf-{name}-{version.replace('.', '-')}v"

service_forge/utils/default_type_converter.py

@@ -0,0 +1,12 @@
+from ..utils.type_converter import TypeConverter
+from ..workflow.workflow import Workflow
+from ..api.http_api import fastapi_app
+from ..api.kafka_api import KafkaApp, kafka_app
+from fastapi import FastAPI
+from ..workflow.workflow_type import WorkflowType, workflow_type_register
+
+type_converter = TypeConverter()
+type_converter.register(str, Workflow, lambda s, node: node.sub_workflows.get_workflow(s))
+type_converter.register(str, FastAPI, lambda s, node: fastapi_app)
+type_converter.register(str, KafkaApp, lambda s, node: kafka_app)
+type_converter.register(str, type, lambda s, node: workflow_type_register.items[s].type)

service_forge/utils/register.py

@@ -0,0 +1,39 @@
+from typing import TypeVar, Generic, Any
+from loguru import logger
+
+T = TypeVar('T')
+
+class Register(Generic[T]):
+  def __init__(self) -> None:
+    self.items:dict[str, T] = {}
+
+  def register(
+      self,
+      name: str,
+      item: T,
+      show_info_log: bool = False,
+  ) -> None:
+    if name not in self.items:
+      self.items[name] = item
+      if show_info_log:
+        logger.info(f'Register {name}.')
+    else:
+      logger.warning(f'{name} has been registered.')
+
+  def instance(
+      self,
+      name: str,
+      kwargs: dict[str, Any] = {},
+      ignore_keys: list[str] = []
+  ) -> None:
+    for key in ignore_keys:
+      try:
+        kwargs.pop(key)
+      except:
+        pass
+    if name not in self.items:
+      logger.error(f'{name} has not been registered.')
+    return self.items[name](**kwargs)
+  
+  def __len__(self) -> int:
+    return len(self.items)

service_forge/utils/type_converter.py

@@ -0,0 +1,74 @@
+from typing import Any, Callable, Type, Dict, Tuple, Set, List
+from collections import deque
+import inspect
+
+class TypeConverter:
+    def __init__(self):
+        self._registry: Dict[Tuple[Type, Type], Callable[..., Any]] = {}
+
+    def register(self, src_type: Type, dst_type: Type, func: Callable[..., Any]):
+        self._registry[(src_type, dst_type)] = func
+
+    def can_convert(self, src_type: Type, dst_type: Type) -> bool:
+        return self._find_path(src_type, dst_type) is not None
+
+    def convert(self, value: Any, dst_type: Type, **kwargs) -> Any:
+        if value is None:
+            return None
+        
+        if dst_type == Any:
+            return value
+
+        src_type = type(value)
+
+        if isinstance(value, dst_type):
+            return value
+
+        if (src_type, dst_type) in self._registry:
+            return self._call_func(self._registry[(src_type, dst_type)], value, **kwargs)
+
+        try:
+            return dst_type(value)
+        except Exception:
+            pass
+
+        path = self._find_path(src_type, dst_type)
+        if not path:
+            raise TypeError(f"No conversion path found from {src_type.__name__} to {dst_type.__name__}.")
+
+        result = value
+        for i in range(len(path) - 1):
+            func = self._registry[(path[i], path[i + 1])]
+            result = self._call_func(func, result, **kwargs)
+        return result
+
+    def _call_func(self, func: Callable[..., Any], value: Any, **kwargs) -> Any:
+        sig = inspect.signature(func)
+        if len(sig.parameters) == 1:
+            return func(value)
+        else:
+            return func(value, **kwargs)
+
+    def _find_path(self, src_type: Type, dst_type: Type) -> List[Type] | None:
+        if src_type == dst_type:
+            return [src_type]
+
+        graph: Dict[Type, Set[Type]] = {}
+        for (s, d) in self._registry.keys():
+            graph.setdefault(s, set()).add(d)
+
+        queue = deque([[src_type]])
+        visited = {src_type}
+
+        while queue:
+            path = queue.popleft()
+            current = path[-1]
+            for neighbor in graph.get(current, []):
+                if neighbor in visited:
+                    continue
+                new_path = path + [neighbor]
+                if neighbor == dst_type:
+                    return new_path
+                queue.append(new_path)
+                visited.add(neighbor)
+        return None

service_forge/workflow/__init__.py

@@ -0,0 +1 @@
+from .workflow_type import workflow_type_register

service_forge/workflow/context.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+class Context():
+    def __init__(
+        self,
+        variables = dict(),
+    ) -> None:
+        self.variables = variables
+
+    def _clone(self) -> Context:
+        return Context(
+            variables={key: value for key, value in self.variables.items()},
+        )

service_forge/workflow/edge.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .node import Node
+    from .port import Port
+
+class Edge:
+    def __init__(
+        self,
+        start_node: Node,
+        end_node: Node,
+        start_port: Port,
+        end_port: Port,
+    ) -> None:
+        self.start_node = start_node
+        self.end_node = end_node
+        self.start_port = start_port
+        self.end_port = end_port
+
+    def _simple_clone(self, node_map: dict[Node, Node], port_map: dict[Port, Port]) -> Edge:
+        # 检查节点是否为None，避免KeyError
+        start_node = node_map[self.start_node] if self.start_node is not None else None
+        end_node = node_map[self.end_node] if self.end_node is not None else None
+        return Edge(
+            start_node=start_node,
+            end_node=end_node,
+            start_port=port_map[self.start_port],
+            end_port=port_map[self.end_port],
+        )