cancan-microstack 0.0.1__py3-none-any.whl

cancan_microstack/services/infrasrv/application/workflow/workflow_queue.py

@@ -0,0 +1,302 @@
+"""Workflow task dispatch helpers for Celery.
+
+提供封装后的 Celery 调度工具，避免业务层到处硬编码 task 名称。
+This module centralizes Celery helper functions so higher layers can enqueue work
+without importing Celery primitives directly.
+"""
+import asyncio
+import threading
+import time
+from dataclasses import dataclass
+from typing import (
+    Awaitable,
+    Callable,
+    Dict,
+    Optional,
+    Tuple,
+)
+
+from linglong_web.utils import logger
+
+from cancan_microstack.public.const.workflow_consts import WorkflowTask
+from linglong_web import LinglongConfig
+from linglong_web import Rmanager
+
+_WORKER_STATUS = {
+    "checked_at": 0.0,
+    "has_worker": False,
+}
+_WORKER_STATUS_LOCK = threading.Lock()
+
+
+def _missing_inline_orchestrator(run_id: str, node_id: str, loop_index: int) -> Awaitable[None]:
+    """当未注册 orchestrator 时，给出明确错误。
+    Raise a clear error when inline orchestrator hasn't been registered.
+    """
+
+    async def _raise() -> None:
+        raise RuntimeError(
+            "Workflow inline orchestrator is not registered. "
+            "Ensure workflow_app registers it via register_inline_orchestrator()."
+        )
+
+    return _raise()
+
+
+_INLINE_ORCHESTRATOR: Callable[[str, str, int], Awaitable[None]] = _missing_inline_orchestrator
+_INLINE_ORCHESTRATOR_LOCK = threading.Lock()
+
+
+def register_inline_orchestrator(func: Callable[[str, str, int], Awaitable[None]]) -> None:
+    """注册工作流节点内联执行 orchestrator。
+    Register workflow node inline orchestrator.
+
+    Note:
+        This avoids a circular import between workflow_app and workflow_queue.
+    """
+
+    if not callable(func):
+        raise TypeError("Inline orchestrator must be callable")
+
+    global _INLINE_ORCHESTRATOR
+    with _INLINE_ORCHESTRATOR_LOCK:
+        _INLINE_ORCHESTRATOR = func
+
+
+def _get_inline_orchestrator() -> Callable[[str, str, int], Awaitable[None]]:
+    with _INLINE_ORCHESTRATOR_LOCK:
+        return _INLINE_ORCHESTRATOR
+
+
+def _to_positive_int(value, default: int) -> int:
+    """将配置值转换为正整数 / Ensure config-driven integers stay positive."""
+
+    try:
+        candidate = int(value)
+    except (TypeError, ValueError):
+        return default
+    return candidate if candidate > 0 else default
+
+
+@dataclass(slots=True)
+class _InlineJob:
+    """表示一次内联执行任务 / Represents a single inline execution request."""
+
+    run_id: str
+    node_id: str
+    loop_index: int
+
+
+class _InlineFallbackExecutor:
+    """受限并发的内联执行器，防止 Celery 降级时无限制堆积。
+    Bounded-concurrency inline executor that prevents unbounded fan-out when Celery is unavailable.
+    """
+
+    def __init__(self, *, max_concurrency: int, queue_limit: int) -> None:
+        self._max_concurrency = max_concurrency
+        self._queue_limit = queue_limit
+        self._queues: Dict[int, asyncio.Queue[_InlineJob]] = {}
+
+    async def _run_job(self, job: _InlineJob) -> None:
+        orchestrator = _get_inline_orchestrator()
+        await orchestrator(job.run_id, job.node_id, job.loop_index)
+
+    async def _worker(self, queue: asyncio.Queue[_InlineJob], worker_idx: int) -> None:
+        """后台 worker，串行消费队列任务 / Background worker that drains inline queue with bounded concurrency."""
+
+        while True:
+            job = await queue.get()
+            try:
+                await self._run_job(job)
+            except Exception as exc:  # noqa: BLE001 - inline fallback必须兜底 / inline fallback must never crash the orchestrator
+                logger.error(
+                    "Inline workflow worker %s failed for run=%s node=%s: %s",
+                    worker_idx,
+                    job.run_id,
+                    job.node_id,
+                    exc,
+                    exc_info=True,
+                )
+            finally:
+                queue.task_done()
+
+    async def _enqueue_with_backpressure(self, queue: asyncio.Queue[_InlineJob], job: _InlineJob) -> None:
+        """等待队列空位，避免一次性放入过多任务 / Wait for free slot instead of flooding the loop."""
+
+        await queue.put(job)
+
+    def _ensure_queue(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue[_InlineJob]:
+        """按事件循环初始化队列和 worker / Ensure queue + workers exist for the active loop."""
+
+        loop_id = id(loop)
+        queue = self._queues.get(loop_id)
+        if queue:
+            return queue
+
+        queue = asyncio.Queue(maxsize=self._queue_limit)
+        self._queues[loop_id] = queue
+
+        for idx in range(self._max_concurrency):
+            loop.create_task(
+                self._worker(queue, idx + 1),
+                name=f"workflow-inline-worker-{idx + 1}",
+            )
+
+        return queue
+
+    def submit(self, run_id: str, node_id: str, loop_index: int) -> None:
+        """投递内联执行任务，自动施加并发限流。
+        Submit inline execution with bounded concurrency to keep fallback predictable.
+        """
+
+        job = _InlineJob(run_id=run_id, node_id=node_id, loop_index=loop_index)
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            logger.warning(
+                "No running event loop detected while executing workflow node inline; creating a temporary loop",
+            )
+            asyncio.run(self._run_job(job))
+            return
+
+        queue = self._ensure_queue(loop)
+
+        try:
+            queue.put_nowait(job)
+        except asyncio.QueueFull:
+            logger.warning(
+                "Inline workflow queue saturated (size=%s limit=%s); applying async backpressure",
+                queue.qsize(),
+                self._queue_limit,
+            )
+            loop.create_task(self._enqueue_with_backpressure(queue, job))
+
+    async def wait_for_idle(self, loop: asyncio.AbstractEventLoop | None = None) -> None:
+        """等待队列清空（测试辅助） / Wait for inline queue to drain (useful for tests)."""
+
+        target_loop = loop or asyncio.get_running_loop()
+        queue = self._queues.get(id(target_loop))
+        if not queue:
+            return
+        await queue.join()
+
+
+_INLINE_EXECUTOR: Optional[_InlineFallbackExecutor] = None
+_INLINE_EXECUTOR_CONFIG: Optional[Tuple[int, int]] = None
+_INLINE_EXECUTOR_OVERRIDE: Optional[_InlineFallbackExecutor] = None
+_INLINE_EXECUTOR_LOCK = threading.Lock()
+
+
+def _read_inline_executor_config() -> Tuple[int, int]:
+    """读取并规范化内联执行器配置 / Read + normalize inline executor config."""
+
+    max_concurrency = _to_positive_int(LinglongConfig.WORKFLOW_INLINE_MAX_CONCURRENCY, 4)
+    queue_limit_hint = max(max_concurrency * 4, 1024)
+    queue_limit = _to_positive_int(LinglongConfig.WORKFLOW_INLINE_QUEUE_LIMIT, queue_limit_hint)
+    queue_limit = max(queue_limit, max_concurrency)
+    return max_concurrency, queue_limit
+
+
+def _get_inline_executor() -> _InlineFallbackExecutor:
+    """根据当前 Config 获取内联执行器，支持热更新。
+    Resolve inline executor lazily so Config updates take effect without process restarts.
+    """
+
+    override = _INLINE_EXECUTOR_OVERRIDE
+    if override is not None:
+        return override
+
+    global _INLINE_EXECUTOR, _INLINE_EXECUTOR_CONFIG
+    with _INLINE_EXECUTOR_LOCK:
+        desired = _read_inline_executor_config()
+        if _INLINE_EXECUTOR and _INLINE_EXECUTOR_CONFIG == desired:
+            return _INLINE_EXECUTOR
+
+        max_concurrency, queue_limit = desired
+        logger.info(
+            "Rebuilding inline executor due to config change: concurrency=%s queue_limit=%s",
+            max_concurrency,
+            queue_limit,
+        )
+        _INLINE_EXECUTOR = _InlineFallbackExecutor(
+            max_concurrency=max_concurrency,
+            queue_limit=queue_limit,
+        )
+        _INLINE_EXECUTOR_CONFIG = desired
+        return _INLINE_EXECUTOR
+
+
+async def wait_for_inline_queue_idle() -> None:
+    """等待内联队列清空（测试辅助） / Wait for inline queue to drain (test helper)."""
+
+    executor = _get_inline_executor()
+    await executor.wait_for_idle()
+
+
+def _celery_has_workers(celery_app, cache_ttl: float = 15.0) -> bool:
+    """检测 Celery worker 是否在线，避免任务无限排队。
+    Check if at least one Celery worker is alive so we can decide whether to enqueue or run inline."""
+
+    now = time.monotonic()
+    with _WORKER_STATUS_LOCK:
+        checked_at = _WORKER_STATUS["checked_at"]
+        cached_result = _WORKER_STATUS["has_worker"]
+
+    if now - checked_at < cache_ttl:
+        return cached_result
+
+    try:
+        responses = celery_app.control.ping(timeout=1.0) or []
+        has_worker = len(responses) > 0
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("Failed to ping Celery workers: %s", exc)
+        has_worker = False
+
+    with _WORKER_STATUS_LOCK:
+        _WORKER_STATUS["checked_at"] = now
+        _WORKER_STATUS["has_worker"] = has_worker
+    return has_worker
+
+
+def _execute_inline_node(run_id: str, node_id: str, loop_index: int) -> None:
+    """直接在当前进程执行节点，但通过队列控制瞬时并发。
+    Execute node orchestrator inline while honoring bounded inline concurrency.
+    """
+
+    executor = _get_inline_executor()
+    executor.submit(run_id, node_id, loop_index)
+
+
+def enqueue_node_execution(run_id: str, node_id: str, loop_index: int) -> None:
+    """将节点执行请求投递到 Celery 队列 / Send a node execution request to Celery."""
+
+    celery_app = Rmanager.CeleryApp
+    if not celery_app:
+        logger.warning(
+            "Celery is not initialized, executing workflow node inline: run=%s node=%s",
+            run_id,
+            node_id,
+        )
+        _execute_inline_node(run_id, node_id, loop_index)
+        return
+
+    if not _celery_has_workers(celery_app):
+        logger.warning(
+            "Celery has no active workers, executing workflow node inline: run=%s node=%s",
+            run_id,
+            node_id,
+        )
+        _execute_inline_node(run_id, node_id, loop_index)
+        return
+
+    try:
+        celery_app.send_task(WorkflowTask.EXECUTE_NODE, args=(run_id, node_id, loop_index))
+    except Exception as exc:  # noqa: BLE001 - we need to capture all Celery backend issues
+        logger.error(
+            "Failed to enqueue workflow node via Celery (run=%s node=%s). Falling back to inline execution. Error: %s",
+            run_id,
+            node_id,
+            exc,
+            exc_info=True,
+        )
+        _execute_inline_node(run_id, node_id, loop_index)

cancan_microstack/services/infrasrv/application/workflow/workflow_tasks.py

@@ -0,0 +1,46 @@
+"""Workflow Celery task registration helpers.
+
+按照项目规范集中注册 Celery 任务，便于在 cmd/infrasrv/run.py 中一眼看出
+服务暴露了哪些任务；同时也避免循环导入。
+This module centralizes Celery task declarations so run.py can register them
+explicitly, making task discovery and testing easier.
+"""
+import asyncio
+import threading
+
+from celery import Celery
+
+from linglong_web import Rmanager
+from cancan_microstack.public.const.workflow_consts import WorkflowTask
+from cancan_microstack.services.infrasrv.application.workflow.workflow_app import workflow_app
+from cancan_microstack.services.infrasrv.interface.schedule.workflow_scheduler import scan_and_trigger_workflows
+
+_TASKS_REGISTERED = False
+_REGISTER_LOCK = threading.Lock()
+
+
+def register_workflow_tasks() -> None:
+    """注册工作流相关的 Celery 任务 / Register workflow Celery tasks only once."""
+    global _TASKS_REGISTERED
+    if _TASKS_REGISTERED:
+        return
+
+    # 双重检查锁，避免并发调用重复注册任务
+    # Double-checked locking to keep concurrent registrations idempotent
+    with _REGISTER_LOCK:
+        if _TASKS_REGISTERED:
+            return
+
+        celery_app: Celery = Rmanager.CeleryApp
+
+        @celery_app.task(name=WorkflowTask.EXECUTE_NODE, bind=True, max_retries=3, acks_late=True)
+        def execute_node_task(self, run_id: str, node_id: str, loop_index: int = 1):
+            """调用执行协调器 / Run the async node orchestrator inside Celery."""
+            asyncio.run(workflow_app._execute_node_orchestrator(run_id, node_id, loop_index))
+
+        @celery_app.task(name=WorkflowTask.SCAN_SCHEDULED, bind=True, autoretry_for=(Exception,), retry_backoff=True)
+        def scan_scheduled_task(self):
+            """周期扫描数据库查找待执行的工作流 / Periodically scan DB for due workflows."""
+            asyncio.run(scan_and_trigger_workflows())
+
+        _TASKS_REGISTERED = True

cancan_microstack/services/infrasrv/application/workflow/workflow_worker_runtime.py

@@ -0,0 +1,122 @@
+"""Embedded Celery worker runtime for infrasrv workflow tasks.
+
+在单实例 infrasrv 内启动/停止 Celery worker，避免额外容器或进程。
+The worker runs inside a dedicated daemon thread so the service can keep a
+single deployment unit while still benefiting from Celery's queue semantics.
+"""
+import os
+import threading
+from typing import Optional
+
+from celery import Celery
+
+from linglong_web import LinglongConfig
+from linglong_web import Rmanager
+from linglong_web.utils import logger
+
+_worker_thread: Optional[threading.Thread] = None
+_worker_hostname: Optional[str] = None
+_worker_shutdown_lock = threading.Lock()
+
+
+def _build_worker_hostname() -> str:
+    suffix = LinglongConfig.CELERY_WORKER_HOSTNAME_SUFFIX
+    base = (
+            os.environ.get("SERVICE_WORKER_HOSTNAME")
+            or os.environ.get("HOSTNAME")
+            or "infrasrv"
+    )
+    base = base.replace("/", "").split(".")[0]
+    suffix = suffix.lstrip(".@") or "inline"
+    return f"{base}@{suffix}"
+
+
+def _build_worker_argv(hostname: str) -> list[str]:
+    log_level = LinglongConfig.CELERY_WORKER_LOG_LEVEL
+    concurrency = LinglongConfig.CELERY_WORKER_CONCURRENCY
+    pool = LinglongConfig.CELERY_WORKER_POOL
+    enable_beat = LinglongConfig.CELERY_WORKER_ENABLE_BEAT
+
+    argv = [
+        "worker",
+        f"--hostname={hostname}",
+        f"--loglevel={log_level}",
+        "--concurrency",
+        str(concurrency),
+    ]
+
+    if pool:
+        argv.extend(["--pool", str(pool)])
+
+    if enable_beat:
+        argv.append("--beat")
+
+    return argv
+
+
+def start_inline_worker() -> None:
+    """启动内嵌 Celery worker / Start embedded Celery worker."""
+
+    if not LinglongConfig.CELERY_WORKER_AUTOSTART:
+        logger.info("Embedded Celery worker autostart disabled via config")
+        return
+
+    app: Celery | None = Rmanager.CeleryApp
+    if not app:
+        logger.warning("Celery is not initialized; embedded worker skipped")
+        return
+
+    global _worker_thread, _worker_hostname
+    with _worker_shutdown_lock:
+        if _worker_thread and _worker_thread.is_alive():
+            logger.info("Embedded Celery worker already running: hostname=%s", _worker_hostname)
+            return
+
+        hostname = _build_worker_hostname()
+        argv = _build_worker_argv(hostname)
+
+        def _run_worker():
+            logger.info(
+                "Starting embedded Celery worker hostname=%s argv=%s",
+                hostname,
+                " ".join(argv),
+            )
+            try:
+                app.worker_main(argv=argv)
+            except SystemExit as exc:
+                logger.info("Embedded Celery worker exited: %s", exc)
+            except Exception as exc:  # noqa: BLE001
+                logger.error("Embedded Celery worker crashed: %s", exc, exc_info=True)
+            finally:
+                logger.info("Embedded Celery worker stopped")
+
+        worker_thread = threading.Thread(target=_run_worker, name="embedded-celery-worker", daemon=True)
+        worker_thread.start()
+        _worker_thread = worker_thread
+        _worker_hostname = hostname
+
+
+def stop_inline_worker(timeout: float = 30.0) -> None:
+    """停止内嵌 Celery worker / Stop embedded Celery worker."""
+
+    global _worker_thread
+    with _worker_shutdown_lock:
+        thread = _worker_thread
+        if not thread or not thread.is_alive():
+            _worker_thread = None
+            return
+
+        app: Celery | None = Rmanager.CeleryApp
+        if app and _worker_hostname:
+            try:
+                logger.info("Sending shutdown signal to embedded Celery worker: %s", _worker_hostname)
+                app.control.shutdown(destination=[_worker_hostname])
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("Failed to send shutdown command to worker: %s", exc)
+
+        thread.join(timeout=timeout)
+        if thread.is_alive():
+            logger.warning("Embedded Celery worker did not stop within %.1fs", timeout)
+        else:
+            logger.info("Embedded Celery worker terminated cleanly")
+        _worker_thread = None

cancan_microstack/services/infrasrv/conf/config.py

@@ -0,0 +1,98 @@
+import logging
+import os
+
+from linglong_web import LinglongConfigBase
+from cancan_microstack.public.const.workflow_consts import WorkflowTask
+
+from cancan_microstack.runtime.resources import resolve_workspace_or_asset
+from cancan_microstack.runtime.workspace import configure_workspace, ensure_server_log_dir
+
+_WORKSPACE_ROOT = str(configure_workspace())
+ensure_server_log_dir()
+
+
+class _BaseConfig(LinglongConfigBase):
+    """基础配置"""
+    DEBUG = False
+
+    # 项目根目录（动态获取）
+    _PROJECT_ROOT = _WORKSPACE_ROOT
+
+    INFRASRV_HOST = "http://infrasrv.service:8080"
+    CONTROLLERSRV_HOST = os.getenv("CONTROLLERSRV_HOST", "http://host.containers.internal:22100")
+
+    SKIP_SERVICE_REGISTRY = True
+
+    # DDL 自管理配置
+    ENABLE_DDL_AUTO_INIT = True  # 启动时自动创建 infra 表
+    # DDL 脚本路径使用项目根目录
+    # - In Pod: /app/ddl/infra (Dockerfile COPY)
+    # - Out Pod: {project_root}/ddl/infra
+    DDL_SCRIPT_PATH = str(resolve_workspace_or_asset("ddl/infra", "ddl/infra"))
+
+    # 数据库配置
+    PGSQL_DB = "infra"  # infrasrv 使用 infra 数据库
+
+    # PostgreSQL 连接池配置（兼顾并发与连接上限）
+    # PostgreSQL pool settings (balance concurrency and connection cap)
+    # 说明：infrasrv 需要同时处理操作轮询、健康检查、日志写入和服务管理写操作。
+    # Note: infrasrv handles operation polling, health checks, log ingestion, and management writes concurrently.
+    PGSQL_POOL_SIZE = 5
+    PGSQL_MAX_OVERFLOW = 5
+
+    # 服务注册清理配置 / Service registry cleanup settings
+    INSTANCE_HEARTBEAT_TIMEOUT_SECONDS = 90  # 僵尸实例的超时阈值（秒） / Stale instance threshold (seconds)
+
+    # 业务日志消费配置 / Business log ingestion settings
+    LOG_PIPELINE_CONSUMER_ENABLED = True
+    RABBITMQ_LOG_CONNECTION_NAME = "infrasrv-log-consumer"
+
+    # RabbitMQ 连接配置（日志消费 + 工作流 broker）
+    # RabbitMQ connection settings (log consumer + workflow broker)
+    RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq.internal")
+    RABBITMQ_PORT = int(os.getenv("RABBITMQ_PORT", "5672"))
+    RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST", "/")
+    RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME", "admin")
+    RABBITMQ_PASSWORD = os.getenv("RABBITMQ_PASSWORD", "admin123")
+
+    # MongoDB 配置（日志存储） / MongoDB settings for log storage
+    # 注意：默认使用 root 用户认证数据库 admin
+    # Note: default uses root user with authSource=admin
+    MONGODB_URI = os.getenv(
+        "MONGODB_URI",
+        "mongodb://admin:admin123@mongo.internal:27017/admin?authSource=admin",
+    )
+    MONGODB_DB = "infra_logging"
+    MONGODB_COLLECTION = "service_logs"
+
+    # Celery 调度配置（基于 RabbitMQ + Redis）
+    # Celery scheduling settings (broker via RabbitMQ, backend via Redis)
+    ENABLE_WORKFLOW_CELERY = True
+    CELERY_APP_NAME = "infrasrv_workflow"
+    CELERY_RESULT_BACKEND_DB = 5
+    CELERY_TIMEZONE = "Asia/Shanghai"
+    CELERY_ENABLE_UTC = True
+    CELERY_BEAT_SCHEDULE = {
+        "scan-workflow-triggers": {
+            "task": WorkflowTask.SCAN_SCHEDULED,
+            "schedule": 5.0,
+        }
+    }
+
+    # 日志配置 - 使用项目根目录
+    # - In Pod: /app/server_log_data/{}.log (volume 挂载到宿主机)
+    # - Out Pod: {project_root}/server_log_data/{}.log
+    LOGGING_ENABLE_FILE_HANDLER = True
+    LOGGING_FILE_ADDR_FORMAT = os.path.join(_PROJECT_ROOT, "server_log_data", "{}.log")
+
+
+class _ProductionConfig(_BaseConfig):
+    """生产环境配置"""
+    ENV_MODE = "prod"
+    DEBUG = False
+    LOGGING_LEVEL = logging.INFO
+
+
+service_conf_dict = {
+    'prod': _ProductionConfig,
+}

cancan_microstack/services/infrasrv/domain/health_check/__init__.py

@@ -0,0 +1,3 @@
+"""
+健康检查领域层模块
+"""