ls-algorithm-plugin-sdk 0.2.5__tar.gz → 0.3.1__tar.gz

ls_algorithm_plugin_sdk-0.3.1/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: ls-algorithm-plugin-sdk
+Version: 0.3.1
+Summary: Protocol-independent runtime SDK for dataset algorithms
+Author: Ling Robotics
+License: Proprietary
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: service
+Requires-Dist: fastapi<1,>=0.110; extra == "service"
+Requires-Dist: pydantic<3,>=2.0; extra == "service"
+Requires-Dist: uvicorn[standard]<1,>=0.29; extra == "service"
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == "dev"
+
+# Algorithm Plugin SDK
+
+## 算法侧最小接口
+
+```python
+from algorithm_plugin_sdk import Algorithm
+
+class MyAlgorithm(Algorithm):
+    @classmethod
+    def metadata(cls): ...
+    def execute(self, request, context): ...
+```
+
+算法包需要声明 `algorithm_plugin_sdk.algorithms` entry point。
+
+## 单次运行
+
+```bash
+algorithm-plugin run camera-space-mano \
+  --input '{"input_dataset":"/data/input1","output":"/data/output1"}' \
+  --input '{"input_dataset":"/data/input2","output":"/data/output2"}' \
+  --gpu-ids 0 \
+  --parameters '{"batch_size":16}'
+```
+
+`--gpu-ids` 和 `--parameters` 可省略，也可用 `--request-file` 读取完整请求 JSON。
+
+## 配置现有仓库
+
+目标算法仓库必须满足：
+
+- 仓库本身是 Git 工作树根目录。
+- 仓库包含自己的 `.venv`。
+- `<repository>/.venv/bin/algorithm-plugin` 存在且可执行。
+
+建议将生成的 `release-manifest.json` 和 `algorithm-plugin.json` 加入算法仓库的
+`.gitignore`，不要手动编辑或提交它们。
+
+通过目标仓库的虚拟环境执行：
+
+```bash
+sudo /srv/camera-space-mano/.venv/bin/algorithm-plugin configure \
+  --repository /srv/camera-space-mano \
+  --compute-url http://compute:5180 \
+  --api-key "$LDP_INTERNAL_API_KEY" \
+  --port 9030 \
+  --cluster production \
+  --gpu-ids 0,1
+```
+
+重复执行 configure 需要显式提供 `--force`。
+
+确认输出后执行：
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable camera-space-mano.service
+sudo systemctl restart camera-space-mano.service
+sudo systemctl status camera-space-mano.service --no-pager
+```
+
+## serve
+
+```bash
+/srv/camera-space-mano/.venv/bin/algorithm-plugin serve \
+  --config /srv/camera-space-mano/algorithm-plugin.json
+```
+
+## 安装与测试
+
+```bash
+python -m pip install .
+python -m unittest discover -s tests -v
+```

ls_algorithm_plugin_sdk-0.3.1/README.md

@@ -0,0 +1,74 @@
+# Algorithm Plugin SDK
+
+## 算法侧最小接口
+
+```python
+from algorithm_plugin_sdk import Algorithm
+
+class MyAlgorithm(Algorithm):
+    @classmethod
+    def metadata(cls): ...
+    def execute(self, request, context): ...
+```
+
+算法包需要声明 `algorithm_plugin_sdk.algorithms` entry point。
+
+## 单次运行
+
+```bash
+algorithm-plugin run camera-space-mano \
+  --input '{"input_dataset":"/data/input1","output":"/data/output1"}' \
+  --input '{"input_dataset":"/data/input2","output":"/data/output2"}' \
+  --gpu-ids 0 \
+  --parameters '{"batch_size":16}'
+```
+
+`--gpu-ids` 和 `--parameters` 可省略，也可用 `--request-file` 读取完整请求 JSON。
+
+## 配置现有仓库
+
+目标算法仓库必须满足：
+
+- 仓库本身是 Git 工作树根目录。
+- 仓库包含自己的 `.venv`。
+- `<repository>/.venv/bin/algorithm-plugin` 存在且可执行。
+
+建议将生成的 `release-manifest.json` 和 `algorithm-plugin.json` 加入算法仓库的
+`.gitignore`，不要手动编辑或提交它们。
+
+通过目标仓库的虚拟环境执行：
+
+```bash
+sudo /srv/camera-space-mano/.venv/bin/algorithm-plugin configure \
+  --repository /srv/camera-space-mano \
+  --compute-url http://compute:5180 \
+  --api-key "$LDP_INTERNAL_API_KEY" \
+  --port 9030 \
+  --cluster production \
+  --gpu-ids 0,1
+```
+
+重复执行 configure 需要显式提供 `--force`。
+
+确认输出后执行：
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable camera-space-mano.service
+sudo systemctl restart camera-space-mano.service
+sudo systemctl status camera-space-mano.service --no-pager
+```
+
+## serve
+
+```bash
+/srv/camera-space-mano/.venv/bin/algorithm-plugin serve \
+  --config /srv/camera-space-mano/algorithm-plugin.json
+```
+
+## 安装与测试
+
+```bash
+python -m pip install .
+python -m unittest discover -s tests -v
+```

{ls_algorithm_plugin_sdk-0.2.5 → ls_algorithm_plugin_sdk-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ls-algorithm-plugin-sdk"
-version = "0.2.5"
+version = "0.3.1"
 description = "Protocol-independent runtime SDK for dataset algorithms"
 readme = "README.md"
 requires-python = ">=3.10"

ls_algorithm_plugin_sdk-0.3.1/src/algorithm_plugin_sdk/cli.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+from .cli_impl import configure as configure_command
+from .cli_impl import run as run_command
+from .cli_impl import serve as serve_command
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="algorithm-plugin")
+    subs = parser.add_subparsers(dest="command", required=True)
+
+    run_command.configure_parser(
+        subs.add_parser("run", help="run datasets once")
+    )
+    serve_command.configure_parser(
+        subs.add_parser("serve", help="run the generated HTTP service")
+    )
+    configure_command.configure_parser(
+        subs.add_parser("configure", help="configure an existing Algorithm repository for systemd")
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s %(message)s",
+    )
+    args = build_parser().parse_args(argv)
+    try:
+        return args.handler(args)
+    except KeyboardInterrupt:
+        return 130
+    except Exception as exc:
+        print(f"ERROR: {type(exc).__name__}: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ls_algorithm_plugin_sdk-0.3.1/src/algorithm_plugin_sdk/cli_impl/__init__.py

@@ -0,0 +1 @@
+"""Implementation modules for the algorithm-plugin command line interface."""

ls_algorithm_plugin_sdk-0.3.1/src/algorithm_plugin_sdk/cli_impl/configure.py

@@ -0,0 +1,434 @@
+from __future__ import annotations
+
+import argparse
+import ipaddress
+import os
+import re
+import socket
+import subprocess
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+from ..deployment import (
+    DEFAULT_CONFIG_NAME,
+    DeploymentConfig,
+    repository_root,
+    stable_instance_key,
+)
+from .parsing import environment, gpu_ids
+
+
+SYSTEMD_UNIT_DIR = Path("/etc/systemd/system")
+INPUT_ROOT = Path("/mnt/ldp_uploads/ldp-uploads")
+PRIVATE_IPV4_NETWORKS = tuple(
+    ipaddress.ip_network(network)
+    for network in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")
+)
+
+SERVICE_NAME_PATTERN = re.compile(
+    r"^[A-Za-z0-9][A-Za-z0-9_.@-]*$"
+)
+
+
+def configure_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "algorithm",
+        nargs="?",
+        help="entry-point name or package.module:AlgorithmClass",
+    )
+    parser.add_argument(
+        "--repository",
+        type=Path,
+        default=Path.cwd(),
+        help="existing Algorithm repository containing its own .venv",
+    )
+    parser.add_argument(
+        "--config-output",
+        type=Path,
+        default=Path(DEFAULT_CONFIG_NAME),
+        help=f"config path relative to the repository (default: {DEFAULT_CONFIG_NAME})",
+    )
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=9000)
+    parser.add_argument(
+        "--webui",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+    )
+    parser.add_argument("--max-concurrency", type=int, default=1)
+    parser.add_argument("--scratch-dir")
+    parser.add_argument(
+        "--gpu-ids",
+        type=gpu_ids,
+        default=[],
+        help="GPU IDs injected into every service execution",
+    )
+    parser.add_argument("--service-token")
+    parser.add_argument(
+        "--compute-url",
+        default=os.getenv("LDP_COMPUTE_URL"),
+    )
+    parser.add_argument(
+        "--api-key",
+        default=os.getenv("LDP_INTERNAL_API_KEY"),
+    )
+    parser.add_argument(
+        "--instance-key",
+        default=os.getenv("LDP_PLUGIN_INSTANCE_KEY"),
+    )
+    parser.add_argument(
+        "--cluster",
+        default=os.getenv("LDP_CLUSTER"),
+    )
+    parser.add_argument(
+        "--local",
+        action="store_true",
+        help="configure a local service without compute registration",
+    )
+    parser.add_argument(
+        "--env",
+        action="append",
+        type=environment,
+        default=[],
+        metavar="NAME=VALUE",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="replace an existing systemd unit whose content differs",
+    )
+    parser.set_defaults(handler=execute)
+
+
+def resolve_service_name(
+    repository: Path,
+) -> str:
+    name = repository.name
+    if name.endswith(".service"):
+        raise ValueError(
+            "repository name must not include the .service suffix"
+        )
+    if not SERVICE_NAME_PATTERN.fullmatch(name):
+        raise ValueError(
+            "repository name may contain only letters, numbers, "
+            "underscore, dot, @, and hyphen"
+        )
+    return name
+
+
+def _escape_systemd_path(value: Path) -> str:
+    text = str(value)
+    if "\n" in text or "\r" in text or "\0" in text:
+        raise ValueError(
+            f"systemd path contains control characters: {value}"
+        )
+    return (
+        text.replace("\\", "\\\\")
+        .replace('"', '\\"')
+        .replace("%", "%%")
+    )
+
+
+def _quote_systemd_path(value: Path) -> str:
+    return f'"{_escape_systemd_path(value)}"'
+
+
+def render_systemd_unit(
+    *,
+    repository: Path,
+    config_path: Path,
+    service_name: str,
+) -> str:
+    root = repository.resolve()
+    launcher = root / ".venv" / "bin" / "algorithm-plugin"
+    config = config_path.resolve()
+    return "\n".join(
+        (
+            "[Unit]",
+            f"Description=Algorithm Plugin {service_name}",
+            "Wants=network-online.target",
+            "After=network-online.target",
+            "",
+            "[Service]",
+            "Type=simple",
+            "User=root",
+            f"WorkingDirectory={_escape_systemd_path(root)}",
+            (
+                f"ExecStart={_quote_systemd_path(launcher)} "
+                f"serve --config {_quote_systemd_path(config)}"
+            ),
+            "Environment=PYTHONUNBUFFERED=1",
+            "Restart=on-failure",
+            "RestartSec=5s",
+            "",
+            "[Install]",
+            "WantedBy=multi-user.target",
+            "",
+        )
+    )
+
+
+def require_root() -> None:
+    if os.geteuid() != 0:
+        raise PermissionError(
+            "configure must run as root to write systemd units"
+        )
+
+
+def inspect_systemd_unit(
+    *,
+    service_name: str,
+    content: str,
+    force: bool,
+    unit_dir: Path = SYSTEMD_UNIT_DIR,
+) -> tuple[Path, bool]:
+    require_root()
+    if not unit_dir.is_dir():
+        raise NotADirectoryError(
+            f"systemd unit directory is unavailable: {unit_dir}"
+        )
+
+    target = unit_dir / f"{service_name}.service"
+    if target.is_file():
+        existing = target.read_text(encoding="utf-8")
+        if existing == content:
+            return target, False
+        if not force:
+            raise FileExistsError(
+                f"systemd unit differs: {target}; "
+                "use --force to replace it"
+            )
+    elif target.exists():
+        raise FileExistsError(
+            f"systemd unit path is not a regular file: {target}"
+        )
+    return target, True
+
+
+def write_systemd_unit(
+    *,
+    service_name: str,
+    content: str,
+    force: bool,
+    unit_dir: Path = SYSTEMD_UNIT_DIR,
+) -> tuple[Path, bool]:
+    target, changed = inspect_systemd_unit(
+        service_name=service_name,
+        content=content,
+        force=force,
+        unit_dir=unit_dir,
+    )
+    if not changed:
+        return target, False
+
+    temporary = target.with_name(
+        f".{target.name}.{os.getpid()}.tmp"
+    )
+    try:
+        temporary.write_text(content, encoding="utf-8")
+        temporary.chmod(0o644)
+        temporary.replace(target)
+    finally:
+        temporary.unlink(missing_ok=True)
+    return target, True
+
+
+def validate_repository(value: Path) -> Path:
+    requested = value.expanduser().resolve()
+    if not requested.is_dir():
+        raise NotADirectoryError(
+            f"repository directory does not exist: {requested}"
+        )
+
+    repository = repository_root(requested)
+    launcher = repository / ".venv" / "bin" / "algorithm-plugin"
+    if not launcher.is_file():
+        raise FileNotFoundError(
+            f"repository virtualenv CLI is missing: {launcher}"
+        )
+    if not os.access(launcher, os.X_OK):
+        raise PermissionError(
+            f"repository virtualenv CLI is not executable: {launcher}"
+        )
+
+    environment = os.environ.copy()
+    environment.pop("PYTHONPATH", None)
+    for command in ("run", "serve"):
+        try:
+            completed = subprocess.run(
+                [str(launcher), command, "--help"],
+                cwd=repository,
+                env=environment,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+        except OSError as exc:
+            raise RuntimeError(
+                f"cannot execute repository virtualenv CLI {launcher}: {exc}"
+            ) from exc
+        if completed.returncode:
+            detail = (
+                completed.stderr.strip()
+                or completed.stdout.strip()
+                or "no output"
+            )
+            raise RuntimeError(
+                f"{launcher} {command} --help failed with exit code "
+                f"{completed.returncode}: {detail}"
+            )
+    return repository
+
+
+def validate_configure_runtime(repository: Path) -> Path:
+    expected = (repository / ".venv").resolve()
+    actual = Path(sys.prefix).resolve()
+    if actual != expected:
+        launcher = expected / "bin" / "algorithm-plugin"
+        raise RuntimeError(
+            "configure must run from the repository virtualenv; use: "
+            f"sudo {launcher} configure --repository {repository}"
+        )
+    return expected
+
+
+def resolve_config_path(
+    repository: Path,
+    output: Path,
+) -> Path:
+    path = output.expanduser()
+    if not path.is_absolute():
+        path = repository / path
+    return path.resolve()
+
+
+def resolve_private_ipv4(compute_url: str) -> str:
+    parsed = urlparse(compute_url)
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        raise ValueError("compute URL must be an http or https URL")
+    try:
+        port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    except ValueError as exc:
+        raise ValueError("compute URL has an invalid port") from exc
+
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
+            probe.connect((parsed.hostname, port))
+            value = probe.getsockname()[0]
+    except OSError as exc:
+        raise RuntimeError(
+            f"cannot resolve the local IP used to reach {parsed.hostname}: {exc}"
+        ) from exc
+
+    address = ipaddress.ip_address(value)
+    if not isinstance(address, ipaddress.IPv4Address) or not any(
+        address in network for network in PRIVATE_IPV4_NETWORKS
+    ):
+        raise RuntimeError(
+            f"the local IP used to reach {parsed.hostname} is not a private IPv4: "
+            f"{address}"
+        )
+    return str(address)
+
+
+def registration_from_args(
+    args: argparse.Namespace,
+    repository: Path,
+) -> dict[str, str] | None:
+    if args.local:
+        return None
+
+    supplied = {
+        "computeUrl": args.compute_url,
+        "apiKey": args.api_key,
+        "inputRoot": str(INPUT_ROOT),
+        "workspaceRoot": str(repository.resolve()),
+        "cluster": args.cluster,
+    }
+    missing = [
+        name
+        for name, value in supplied.items()
+        if not str(value or "").strip()
+    ]
+    if missing:
+        raise ValueError(
+            "configuration registration requires "
+            + ", ".join(missing)
+            + "; use --local to disable registration"
+        )
+
+    registration = {
+        name: str(value)
+        for name, value in supplied.items()
+    }
+    private_ipv4 = resolve_private_ipv4(registration["computeUrl"])
+    registration["publicUrl"] = f"http://{private_ipv4}:{args.port}"
+    registration["nodeName"] = private_ipv4
+    registration["instanceKey"] = stable_instance_key(
+        repository / ".algorithm-plugin" / "instance-key",
+        requested=args.instance_key,
+    )
+    return registration
+
+
+def generate_config(
+    args: argparse.Namespace,
+    repository: Path,
+) -> DeploymentConfig:
+    config = DeploymentConfig.generate(
+        output=args.config_output,
+        root=repository,
+        algorithm=args.algorithm,
+        host=args.host,
+        port=args.port,
+        webui=args.webui,
+        max_concurrency=args.max_concurrency,
+        scratch_dir=args.scratch_dir,
+        token=args.service_token,
+        registration=registration_from_args(args, repository),
+        gpu_ids=args.gpu_ids,
+        environment=dict(args.env),
+    )
+    config.save()
+    return config
+
+
+def execute(args: argparse.Namespace) -> int:
+    require_root()
+    repository = validate_repository(args.repository)
+    validate_configure_runtime(repository)
+    service_name = resolve_service_name(repository)
+    config_path = resolve_config_path(
+        repository,
+        args.config_output,
+    )
+    unit = render_systemd_unit(
+        repository=repository,
+        config_path=config_path,
+        service_name=service_name,
+    )
+
+    inspect_systemd_unit(
+        service_name=service_name,
+        content=unit,
+        force=args.force,
+    )
+    config = generate_config(args, repository)
+    unit_path, changed = write_systemd_unit(
+        service_name=service_name,
+        content=unit,
+        force=args.force,
+    )
+
+    unit_status = "updated" if changed else "unchanged"
+    print(f"Configured repository: {repository}")
+    print(f"Configuration: {config.path}")
+    print(f"Release manifest: {repository / 'release-manifest.json'}")
+    print(f"Systemd unit: {unit_path} ({unit_status})")
+    print("Next commands:")
+    print("  sudo systemctl daemon-reload")
+    print(f"  sudo systemctl enable {service_name}.service")
+    print(f"  sudo systemctl restart {service_name}.service")
+    print(f"  sudo systemctl status {service_name}.service --no-pager")
+    return 0

ls_algorithm_plugin_sdk-0.3.1/src/algorithm_plugin_sdk/cli_impl/parsing.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from typing import Any
+
+
+def env_bool(name: str, default: bool = False) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def json_object(value: str) -> dict[str, Any]:
+    try:
+        parsed = json.loads(value)
+    except json.JSONDecodeError as exc:
+        raise argparse.ArgumentTypeError(f"invalid JSON: {exc}") from exc
+    if not isinstance(parsed, dict):
+        raise argparse.ArgumentTypeError("value must be a JSON object")
+    return parsed
+
+
+def gpu_ids(value: str) -> list[int]:
+    if not value.strip():
+        return []
+    try:
+        return [int(item.strip()) for item in value.split(",")]
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError(
+            "GPU IDs must be comma-separated integers"
+        ) from exc
+
+
+def environment(value: str) -> tuple[str, str]:
+    name, separator, setting = value.partition("=")
+    if not separator or not name or not name.replace("_", "").isalnum():
+        raise argparse.ArgumentTypeError("environment must use NAME=VALUE")
+    if name.startswith("LDP_") or name.startswith("ALGORITHM_"):
+        raise argparse.ArgumentTypeError(
+            "SDK-managed LDP_/ALGORITHM_ settings must use dedicated options"
+        )
+    return name, setting