hutash-inference 0.1.0__tar.gz

hutash_inference-0.1.0/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: hutash-inference
+Version: 0.1.0
+Summary: Shared HTTP inference framework for Hutash models
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/Appsork/hutash
+Requires-Python: >=3.10
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: python-multipart

hutash_inference-0.1.0/__init__.py

@@ -0,0 +1,31 @@
+"""Aura Inference Library
+
+Shared framework for Aura model containers. Provides:
+- Inference base class that all model inference classes inherit from
+- @capability decorator for marking methods as HTTP endpoint handlers
+- FastAPI server boilerplate
+- I/O helpers (audio, image, file)
+- Validation (model.yaml / manifest.json <-> inference.py compatibility)
+
+See docs/strategy/model-ssot-architecture.md for architecture details.
+"""
+
+from hutash_inference.base import Inference, capability, resolve_local_weights_dir
+from hutash_inference.errors import (
+    HutashInferenceError,
+    ValidationError,
+    ModelLoadError,
+    GenerationError,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Inference",
+    "capability",
+    "resolve_local_weights_dir",
+    "HutashInferenceError",
+    "ValidationError",
+    "ModelLoadError",
+    "GenerationError",
+]

hutash_inference-0.1.0/base.py

@@ -0,0 +1,312 @@
+"""Inference base class and @capability decorator.
+
+Every model's inference.py defines a class that inherits from Inference
+and uses @capability to mark methods as capability handlers. The
+hutash_inference server reads manifest.json, finds the @capability methods,
+and wires them to HTTP endpoints.
+"""
+
+import io
+import logging
+import tempfile
+from typing import Callable, Optional
+
+
+def capability(capability_id: str) -> Callable:
+    """Decorator marking a method as implementing a capability.
+
+    The capability_id must match a key under 'capabilities' in the
+    model's model.yaml file.
+
+    Example:
+        @capability("voice-clone")
+        def clone_voice(self, text: str, reference_audio: bytes):
+            ...
+
+    Args:
+        capability_id: ID matching a key in model.yaml capabilities
+
+    Returns:
+        The original function with metadata attached
+    """
+    def decorator(func: Callable) -> Callable:
+        func._aura_capability = capability_id
+        return func
+    return decorator
+
+
+def resolve_local_weights_dir(model_id: str) -> str:
+    """Resolve the mounted weights snapshot dir for ``model_id``'s PINNED commit.
+
+    Weights-out: the model's weights are downloaded on the host and mounted at
+    ``HF_HUB_CACHE`` (``/weights``). This returns
+    ``<cache>/models--*/snapshots/<commit>/`` for the EXACT pinned commit so a
+    model can load from a local path (e.g. ``from_local(ckpt_dir=...)``) instead
+    of going through the HuggingFace cache / ``hf_hub_download`` (which needs a
+    blob/etag structure the staged cache does not have offline).
+
+    The pinned commit comes from ``HUTASH_HF_REVISION``, injected by
+    docker_manager from the catalogue's ``hf_revision``. This deliberately does
+    NOT fall back to "the first/only snapshot" — if the pinned commit's dir is
+    missing it raises, so a drifted/incomplete mount fails loudly instead of
+    silently loading the wrong weights.
+    """
+    import glob
+    import os
+
+    from hutash_inference.errors import ModelLoadError
+
+    cache = os.environ.get("HF_HUB_CACHE", "/weights")
+    revision = os.environ.get("HUTASH_HF_REVISION")
+    if not revision:
+        raise ModelLoadError(
+            f"resolve_local_weights_dir({model_id!r}): HUTASH_HF_REVISION is "
+            f"not set — cannot resolve the pinned weights snapshot. "
+            f"(docker_manager injects it from the catalogue hf_revision.)"
+        )
+    matches = [
+        m for m in glob.glob(os.path.join(cache, "models--*", "snapshots", revision))
+        if os.path.isdir(m)
+    ]
+    if not matches:
+        raise ModelLoadError(
+            f"resolve_local_weights_dir({model_id!r}): pinned snapshot "
+            f"{revision} not found under {cache} — the weights mount is "
+            f"missing or not staged at this revision."
+        )
+    if len(matches) > 1:
+        raise ModelLoadError(
+            f"resolve_local_weights_dir({model_id!r}): multiple repos contain "
+            f"snapshot {revision}: {matches}"
+        )
+    return matches[0]
+
+
+class Inference:
+    """Base class for all Aura model inference implementations.
+
+    Subclass this in your inference.py and override load() to initialize
+    your model. Use @capability decorator to mark methods as capability
+    handlers.
+
+    Example:
+        class MyModelInference(Inference):
+            def load(self):
+                from my_library import MyModel
+                self.model = MyModel.from_pretrained("...")
+
+            @capability("my-capability")
+            def do_something(self, text: str) -> dict:
+                result = self.model.run(text)
+                return {"output": result}
+    """
+
+    def __init__(self, config: Optional[dict] = None):
+        """Initialize inference with model.yaml config.
+
+        Args:
+            config: Parsed model.yaml contents, provided by server
+        """
+        self.config = config or {}
+        self.model_id = self.config.get("model_id", self.__class__.__name__)
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self._is_loaded = False
+
+    def load(self) -> None:
+        """Default Tier-1 weights-out loader for standard HuggingFace models.
+
+        Called once at container startup. Models whose weights load via the
+        standard HF convention (``SomeClass.from_pretrained(dir)``) need NO
+        custom load(): they declare an ``hf_load`` spec in their
+        catalogue/manifest and the framework loads them here — offline, from
+        the mounted pinned snapshot. Non-standard models (chatterbox,
+        kokoro) override load() instead (Tier-2).
+
+        ``hf_load`` spec::
+
+            {"library": "transformers",                  # default
+             "auto_class": "AutoModelForSpeechSeq2Seq",  # required
+             "processor_class": "AutoProcessor",         # optional
+             "tokenizer_class": "AutoTokenizer"}         # optional
+
+            {"library": "diffusers",
+             "pipeline_class": "DiffusionPipeline"}       # required
+
+        Everything loads with ``local_files_only=True`` from the mounted
+        snapshot — never the network, never a guessed class. Results land on
+        ``self.model`` (+ ``self.processor`` / ``self.tokenizer`` when
+        declared).
+        """
+        from hutash_inference.errors import ModelLoadError
+
+        spec = self.config.get("hf_load")
+        if not spec:
+            raise ModelLoadError(
+                f"{self.model_id!r} has no load() override and no 'hf_load' "
+                f"spec — a standard HuggingFace model needs an hf_load spec "
+                f"(Tier-1) in model.meta.yaml (flows into manifest.json); a "
+                f"non-standard model needs a custom load() override (Tier-2)."
+            )
+
+        weights_dir = resolve_local_weights_dir(self.model_id)
+        library = spec.get("library", "transformers")
+        self.logger.info(
+            "Tier-1 load: %s via %s from %s", self.model_id, library, weights_dir
+        )
+
+        if library == "transformers":
+            import transformers
+
+            auto_class = spec.get("auto_class")
+            if not auto_class:
+                raise ModelLoadError(
+                    f"{self.model_id!r} hf_load.library='transformers' requires "
+                    f"'auto_class' (e.g. AutoModelForSpeechSeq2Seq)."
+                )
+            self.model = self._hf_from_pretrained(transformers, auto_class, weights_dir)
+            if spec.get("processor_class"):
+                self.processor = self._hf_from_pretrained(
+                    transformers, spec["processor_class"], weights_dir
+                )
+            if spec.get("tokenizer_class"):
+                self.tokenizer = self._hf_from_pretrained(
+                    transformers, spec["tokenizer_class"], weights_dir
+                )
+        elif library == "diffusers":
+            import diffusers
+
+            pipeline_class = spec.get("pipeline_class")
+            if not pipeline_class:
+                raise ModelLoadError(
+                    f"{self.model_id!r} hf_load.library='diffusers' requires "
+                    f"'pipeline_class' (e.g. DiffusionPipeline)."
+                )
+            self.model = self._hf_from_pretrained(
+                diffusers, pipeline_class, weights_dir
+            )
+        else:
+            raise ModelLoadError(
+                f"{self.model_id!r} hf_load.library={library!r} is not supported "
+                f"(use 'transformers' or 'diffusers')."
+            )
+
+    def _hf_from_pretrained(self, module, class_name: str, weights_dir: str):
+        """``getattr(module, class_name).from_pretrained(dir, local_files_only=True)``.
+
+        Offline by construction; raises a clear error if the class name is
+        not found in the library (never guesses a default).
+        """
+        from hutash_inference.errors import ModelLoadError
+
+        cls = getattr(module, class_name, None)
+        if cls is None:
+            raise ModelLoadError(
+                f"{self.model_id!r}: {module.__name__}.{class_name} not found "
+                f"— check the hf_load spec's class name."
+            )
+        return cls.from_pretrained(weights_dir, local_files_only=True)
+
+    def unload(self) -> None:
+        """Called on graceful shutdown.
+
+        Override for cleanup. Base implementation does nothing.
+        """
+        pass
+
+    def health_check(self) -> dict:
+        """Return container health status.
+
+        Override for custom health logic. Default returns 'ok' if
+        load() has been called successfully.
+        """
+        return {
+            "status": "ok" if self._is_loaded else "not_ready",
+            "loaded": self._is_loaded,
+        }
+
+    def mark_loaded(self) -> None:
+        """Called by server after load() completes successfully."""
+        self._is_loaded = True
+
+    # ---- Helper methods for common I/O conversions ----
+
+    def _save_temp_audio(self, audio_bytes: bytes, suffix: str = ".wav") -> str:
+        """Save audio bytes to a temporary file, return path.
+
+        Useful for libraries that require file paths rather than bytes.
+        Caller is responsible for cleanup.
+        """
+        tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
+        tmp.write(audio_bytes)
+        tmp.close()
+        return tmp.name
+
+    def _audio_to_wav_bytes(self, audio, sample_rate: int, channels: int = 1) -> bytes:
+        """Convert audio array to WAV bytes.
+
+        Supports numpy arrays, torch tensors, lists.
+        """
+        import soundfile as sf
+        import numpy as np
+
+        try:
+            import torch
+            if isinstance(audio, torch.Tensor):
+                audio = audio.detach().cpu().numpy()
+        except ImportError:
+            pass
+
+        if not isinstance(audio, np.ndarray):
+            audio = np.array(audio)
+
+        if channels == 1 and audio.ndim > 1:
+            audio = audio.squeeze()
+
+        buffer = io.BytesIO()
+        sf.write(buffer, audio, sample_rate, format="WAV")
+        return buffer.getvalue()
+
+    def _image_to_png_bytes(self, image) -> bytes:
+        """Convert image to PNG bytes.
+
+        Supports PIL Images, numpy arrays, torch tensors.
+        """
+        from PIL import Image
+        import numpy as np
+
+        if isinstance(image, Image.Image):
+            pil_image = image
+        else:
+            try:
+                import torch
+                if isinstance(image, torch.Tensor):
+                    image = image.detach().cpu().numpy()
+            except ImportError:
+                pass
+
+            if image.dtype != np.uint8:
+                if image.max() <= 1.0:
+                    image = (image * 255).astype(np.uint8)
+                else:
+                    image = image.astype(np.uint8)
+
+            pil_image = Image.fromarray(image)
+
+        buffer = io.BytesIO()
+        pil_image.save(buffer, format="PNG")
+        return buffer.getvalue()
+
+
+def get_capabilities(instance: Inference) -> dict[str, Callable]:
+    """Discover @capability methods on an Inference instance.
+
+    Returns dict mapping capability_id to the bound method.
+    """
+    capabilities: dict[str, Callable] = {}
+    for attr_name in dir(instance):
+        if attr_name.startswith("_"):
+            continue
+        attr = getattr(instance, attr_name)
+        if callable(attr) and hasattr(attr, "_aura_capability"):
+            capabilities[attr._aura_capability] = attr
+    return capabilities

hutash_inference-0.1.0/errors.py

@@ -0,0 +1,26 @@
+"""Standardized error types for Aura Inference library."""
+
+
+class HutashInferenceError(Exception):
+    """Base class for all Aura inference errors."""
+    pass
+
+
+class ValidationError(HutashInferenceError):
+    """Raised when model.yaml and inference.py don't match."""
+    pass
+
+
+class ModelLoadError(HutashInferenceError):
+    """Raised when model fails to load at container startup."""
+    pass
+
+
+class GenerationError(HutashInferenceError):
+    """Raised when inference fails during a generation request."""
+    pass
+
+
+class CapabilityNotFoundError(HutashInferenceError):
+    """Raised when a requested capability isn't implemented."""
+    pass

hutash_inference-0.1.0/hutash_inference.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: hutash-inference
+Version: 0.1.0
+Summary: Shared HTTP inference framework for Hutash models
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/Appsork/hutash
+Requires-Python: >=3.10
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: python-multipart

hutash_inference-0.1.0/hutash_inference.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+__init__.py
+base.py
+errors.py
+io_handlers.py
+logging.py
+pyproject.toml
+server.py
+validation.py
+./__init__.py
+./base.py
+./errors.py
+./io_handlers.py
+./logging.py
+./server.py
+./validation.py
+hutash_inference.egg-info/PKG-INFO
+hutash_inference.egg-info/SOURCES.txt
+hutash_inference.egg-info/dependency_links.txt
+hutash_inference.egg-info/requires.txt
+hutash_inference.egg-info/top_level.txt

hutash_inference-0.1.0/hutash_inference.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hutash_inference-0.1.0/hutash_inference.egg-info/requires.txt

@@ -0,0 +1,3 @@
+fastapi
+uvicorn
+python-multipart

hutash_inference-0.1.0/hutash_inference.egg-info/top_level.txt

@@ -0,0 +1 @@
+hutash_inference

hutash_inference-0.1.0/io_handlers.py

@@ -0,0 +1,157 @@
+"""I/O handlers for converting between HTTP request data and Python values.
+
+The server uses these to parse incoming request parameters according to
+the types declared in manifest.json, and to serialize responses.
+"""
+
+import base64
+import json
+from typing import Any
+
+
+# Module-level registry — types whose form value carries File bytes
+# and that go through the sidecar's PCM-envelope decoder.
+#
+# Mirror of `api/services/models/schema_builder.py::INPUT_TYPE_MAP`'s
+# `carries_file: True` rows. Ships baked into every model image, so
+# this set is the in-container view; the sidecar reads its own SSOT.
+# Models that introduce a new composite primitive call
+# `register_carries_file_type()` at import time to extend the set
+# without modifying this file.
+_CARRIES_FILE_TYPES: set[str] = {
+    "audio_file",
+    "image_file",
+    "video_file",
+    "audio_with_transcript",
+    "text_file",
+}
+
+
+def register_carries_file_type(type_id: str) -> None:
+    """Register a primitive input type as carrying File bytes.
+
+    Called at import time by model packages that introduce a new
+    composite primitive the in-container PCM decoder must recognise.
+    """
+    _CARRIES_FILE_TYPES.add(type_id)
+
+
+def carries_file(type_id: str) -> bool:
+    """True if the type's form value carries File bytes (PCM envelope)."""
+    return type_id in _CARRIES_FILE_TYPES
+
+
+# Backwards-compatible public name. Existing callers do
+# `declared_type in CARRIES_FILE_TYPES`; `in` works identically on a
+# set, so the rename to a set semantics is a drop-in.
+CARRIES_FILE_TYPES = _CARRIES_FILE_TYPES
+
+
+def parse_input(value: Any, declared_type: str) -> Any:
+    """Parse input value according to declared type from manifest.
+
+    Args:
+        value: Raw value from HTTP request
+        declared_type: Type string from manifest (e.g., "string", "audio_file")
+
+    Returns:
+        Parsed value ready to pass to inference method
+    """
+    if declared_type in ("string", "text"):
+        return str(value)
+
+    if declared_type == "integer":
+        return int(value)
+
+    if declared_type == "number":
+        return float(value)
+
+    if declared_type == "boolean":
+        if isinstance(value, str):
+            return value.lower() in ("true", "1", "yes")
+        return bool(value)
+
+    if declared_type in CARRIES_FILE_TYPES:
+        # Sidecar-preprocessed PCM envelope. Aura's sidecar decodes
+        # audio_file inputs via FFmpeg and forwards raw float32 PCM
+        # under this shape so containers never need torchaudio /
+        # torchcodec / FFmpeg for decoding.
+        #
+        # Shape:
+        #   {
+        #     "__pcm__": True,
+        #     "pcm_b64": "<base64 raw float32 LE PCM>",
+        #     "sample_rate": <int>,
+        #     "channels": <int>,    # 1 or 2
+        #   }
+        #
+        # Returns 16-bit PCM WAV bytes (with header) — universal
+        # contract that works in every container (no torch import
+        # required) and matches the `file: bytes` signature every
+        # inference method already declares. Inference code that
+        # writes to a temp file and passes the path to its model
+        # keeps working unchanged.
+        if isinstance(value, dict) and value.get("__pcm__"):
+            import io
+            import wave
+            import numpy as np
+
+            pcm_bytes = base64.b64decode(value["pcm_b64"])
+            sample_rate = int(value.get("sample_rate", 16000))
+            channels = int(value.get("channels", 1)) or 1
+            # Convert float32 PCM → 16-bit WAV bytes
+            # Universal contract — works without torch, matches all inference.py signatures
+            f32 = np.frombuffer(pcm_bytes, dtype=np.float32)
+            s16 = (f32 * 32767.0).clip(-32768, 32767).astype(np.int16).tobytes()
+            buf = io.BytesIO()
+            with wave.open(buf, "wb") as w:
+                w.setnchannels(channels)
+                w.setsampwidth(2)
+                w.setframerate(sample_rate)
+                w.writeframes(s16)
+            return buf.getvalue()
+
+        # Legacy path: raw bytes or base64 string. Kept so any
+        # container that still receives undecoded bytes (e.g.
+        # a manifest input without target_sample_rate /
+        # target_channels declared) continues to work.
+        if isinstance(value, bytes):
+            return value
+        if isinstance(value, str):
+            try:
+                return base64.b64decode(value)
+            except Exception:
+                return value.encode("utf-8")
+        return value
+
+    if declared_type == "reference_asset":
+        if isinstance(value, dict):
+            return value
+        return json.loads(value) if isinstance(value, str) else value
+
+    # Unknown type — pass through
+    return value
+
+
+def serialize_output(value: Any, declared_type: str) -> Any:
+    """Serialize output value according to declared type for HTTP response.
+
+    Args:
+        value: Raw output from inference method
+        declared_type: Type string from manifest (e.g., "wav", "png")
+
+    Returns:
+        Serialized value for HTTP response (base64 for binary)
+    """
+    if declared_type in ("wav", "mp3", "flac", "png", "jpg", "webp", "mp4", "webm"):
+        if isinstance(value, bytes):
+            return base64.b64encode(value).decode("utf-8")
+        return value
+
+    if declared_type == "text":
+        return str(value)
+
+    if declared_type == "json":
+        return value
+
+    return value

hutash_inference-0.1.0/logging.py

@@ -0,0 +1,31 @@
+"""Logging configuration for Aura model containers.
+
+Ensures consistent log format across all containers so Aura's Core API
+can parse them uniformly.
+"""
+
+import logging
+import sys
+
+
+def configure_logging(model_id: str, level: str = "INFO") -> None:
+    """Set up logging for a model container.
+
+    Args:
+        model_id: The model's ID (appears in log lines)
+        level: Log level (DEBUG, INFO, WARNING, ERROR)
+    """
+    log_format = (
+        f"%(asctime)s [{model_id}] %(name)s %(levelname)s: %(message)s"
+    )
+
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        format=log_format,
+        datefmt="%Y-%m-%d %H:%M:%S",
+        stream=sys.stdout,
+    )
+
+    # Silence overly chatty third-party libraries
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("filelock").setLevel(logging.WARNING)

hutash_inference-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+# hutash-inference — the shared HTTP inference framework every Hutash model
+# imports. Making it a real, installable package (rather than a folder copied
+# next to each model or put on sys.path) is what lets a model's inference.py do
+#
+#     from hutash_inference import Inference, capability
+#
+# and have it resolve inside the model's own venv. hutashd installs this package
+# into a model venv first, then the model's own dependencies — the BentoML /
+# MLflow / TorchServe pattern: the framework is a package, models import it.
+#
+# The module ships as a flat layout (its .py files sit directly in this folder),
+# so package-dir maps the package name `hutash_inference` to this directory. That
+# keeps this pyproject.toml inside hutash_inference/ while still installing the
+# folder as the importable `hutash_inference` package.
+
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hutash-inference"
+version = "0.1.0"
+description = "Shared HTTP inference framework for Hutash models"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+dependencies = [
+    "fastapi",
+    "uvicorn",
+    # The server parses multipart form uploads (STT/audio models POST audio as
+    # multipart) via request.form(); FastAPI requires python-multipart for that
+    # path, so it is a hard framework dependency, not a per-model one.
+    "python-multipart",
+]
+
+[project.urls]
+Homepage = "https://github.com/Appsork/hutash"
+
+# Flat layout: the package `hutash_inference` lives in this directory (its .py
+# files are siblings of this file), not a nested hutash_inference/ subfolder.
+[tool.setuptools]
+packages = ["hutash_inference"]
+package-dir = { "hutash_inference" = "." }

hutash_inference-0.1.0/server.py

@@ -0,0 +1,400 @@
+"""FastAPI server for Aura model containers.
+
+Every model container runs this server. It:
+1. Reads /app/manifest.json to know what endpoints to expose
+2. Imports /app/inference.py to find the Inference subclass
+3. Instantiates and loads the model
+4. Validates inference.py matches manifest.json
+5. Wires @capability methods to HTTP endpoints from manifest
+6. Starts listening on the configured port
+
+Developers writing model inference code never need to touch this file.
+Their only Python file is inference.py.
+
+Container invocation (factory pattern — no module-level side effects):
+    uvicorn --factory hutash_inference.server:create_app \
+        --host 0.0.0.0 --port ${PORT}
+
+The module has no top-level app instantiation, so importing it on the
+host (tests, smoke checks) never triggers the container-only startup
+path. `create_app()` is only invoked when explicitly called.
+"""
+
+import importlib.util
+import inspect
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import JSONResponse, Response
+
+from hutash_inference.base import Inference, get_capabilities
+from hutash_inference.errors import (
+    HutashInferenceError,
+    GenerationError,
+    ModelLoadError,
+)
+from hutash_inference.io_handlers import parse_input, serialize_output
+from hutash_inference.logging import configure_logging
+from hutash_inference.validation import validate_inference_matches_manifest
+
+
+def model_dir() -> Path:
+    """Directory holding this model's manifest.json + inference.py.
+
+    In a container these live at ``/app`` (the image bakes them there). In a
+    native venv there is no ``/app``; hutashd points ``HUTASH_MODEL_DIR`` at the
+    model's package directory instead. Resolution order:
+    ``HUTASH_MODEL_DIR`` → ``HUTASH_APP_DIR`` → ``/app`` (the container default,
+    so existing images are unaffected).
+    """
+    d = os.environ.get("HUTASH_MODEL_DIR") or os.environ.get("HUTASH_APP_DIR")
+    return Path(d) if d else Path("/app")
+
+
+# Model file locations, resolved at call time so the same server code runs both
+# in a container (/app) and in a native venv (HUTASH_MODEL_DIR). Kept as
+# module-level accessors for readability at the call sites below.
+def _manifest_path() -> Path:
+    return model_dir() / "manifest.json"
+
+
+def _inference_module_path() -> Path:
+    return model_dir() / "inference.py"
+
+
+def _third_party_licenses_path() -> Path:
+    return model_dir() / "THIRD_PARTY_LICENSES.txt"
+
+
+# Composite primitive fanout — when an input's declared type is a key
+# in this map, the dispatcher also copies additional wire fields named
+# `<input_name><suffix>` from raw_data into the method kwargs. Mirrors
+# the frontend convention where a composite primitive (e.g. the React
+# `AudioWithTranscript` component) writes ONE manifest input but TWO
+# RHF form keys → splitFormValues sends both to the wire under
+# `<input_name>` and `<input_name><suffix>`.
+#
+# Sibling to `CARRIES_FILE_TYPES` in io_handlers.py and `carriesFile`
+# in src/ui/inputs/InputRegistry.ts. When adding a new composite
+# primitive that fans out to multiple wire fields, add it here.
+COMPOSITE_FANOUT: dict[str, tuple[str, ...]] = {
+    "audio_with_transcript": ("_transcript",),
+}
+
+
+def load_manifest() -> dict:
+    """Load and parse the model's manifest.json (see model_dir())."""
+    manifest_path = _manifest_path()
+    if not manifest_path.exists():
+        raise ModelLoadError(f"Manifest not found at {manifest_path}")
+    with open(manifest_path) as f:
+        return json.load(f)
+
+
+def load_inference_module():
+    """Dynamically import the model's inference.py (see model_dir())."""
+    inference_path = _inference_module_path()
+    if not inference_path.exists():
+        raise ModelLoadError(f"inference.py not found at {inference_path}")
+
+    spec = importlib.util.spec_from_file_location(
+        "inference", inference_path
+    )
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["inference"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def find_inference_class(module) -> type[Inference]:
+    """Find the Inference subclass in the given module."""
+    for name in dir(module):
+        obj = getattr(module, name)
+        if (
+            inspect.isclass(obj)
+            and issubclass(obj, Inference)
+            and obj is not Inference
+        ):
+            return obj
+    raise ModelLoadError(
+        "No Inference subclass found in inference.py. "
+        "Define a class inheriting from hutash_inference.Inference."
+    )
+
+
+def make_capability_handler(method, spec: dict):
+    """Create an async HTTP handler for a capability method.
+
+    Parses request data according to declared inputs + controls,
+    calls the method, serializes the result.
+
+    Controls with implementation_status in {"stub", "planned"} are
+    UI-only: the manifest declares them, but inference.py does not
+    accept them as kwargs. We inspect the method's signature once
+    at wiring time and drop any incoming kwargs the method doesn't
+    accept — lets Core API forward stub values without erroring.
+    """
+    inputs_spec = spec.get("inputs", {}) or {}
+    controls_spec = spec.get("controls", {}) or {}
+    outputs_spec = spec.get("outputs", {}) or {}
+
+    # If the capability declares exactly one output AND its type is a
+    # binary format, respond with raw bytes + the matching Content-Type
+    # instead of JSON-wrapping base64. Core API's _handle_audio_response
+    # does `response.content → write_bytes(...)` and has no decoding
+    # step, so JSON-wrapped binary lands on disk verbatim. Raw bytes
+    # restores pre-SSOT wire behavior for audio / image / video models
+    # with one binary output. Multi-field dicts (STT's text+segments+
+    # language+duration) and text/json outputs stay JSON.
+    _BINARY_TYPE_TO_MEDIA = {
+        "wav": "audio/wav",
+        "mp3": "audio/mpeg",
+        "flac": "audio/flac",
+        "png": "image/png",
+        "jpg": "image/jpeg",
+        "jpeg": "image/jpeg",
+        "webp": "image/webp",
+        "mp4": "video/mp4",
+        "webm": "video/webm",
+    }
+    single_binary_output = None
+    if len(outputs_spec) == 1:
+        only_key, only_spec = next(iter(outputs_spec.items()))
+        media_type = _BINARY_TYPE_TO_MEDIA.get(only_spec.get("type", ""))
+        if media_type is not None:
+            single_binary_output = (only_key, media_type)
+
+    sig = inspect.signature(method)
+    accepts_var_keyword = any(
+        p.kind is inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()
+    )
+    accepted_params = {
+        name for name in sig.parameters if name != "self"
+    }
+    logger = logging.getLogger("hutash_inference.handler")
+
+    async def handler(request: Request):
+        try:
+            if request.headers.get("content-type", "").startswith("multipart/"):
+                form = await request.form()
+                raw_data = {}
+                for key in form.keys():
+                    value = form[key]
+                    # FastAPI UploadFile → read bytes for file-typed inputs.
+                    # Detected via duck-typing on read() + filename so we
+                    # don't have to import starlette.datastructures here.
+                    if hasattr(value, "read") and hasattr(value, "filename"):
+                        raw_data[key] = await value.read()
+                    else:
+                        raw_data[key] = value
+            else:
+                raw_data = await request.json()
+
+            # Core API's build_json_payload wraps controls under a
+            # `parameters` key: {"prompt": "...", "parameters": {...}}.
+            # hutash_inference expects flat top-level keys matching the
+            # manifest's input + control names. Unwrap `parameters`
+            # here so both shapes work — this is a transition bridge;
+            # once Core API sends SSOT-native flat bodies, the unwrap
+            # is a no-op (no `parameters` key present).
+            if isinstance(raw_data, dict) and isinstance(raw_data.get("parameters"), dict):
+                nested = raw_data.pop("parameters")
+                for k, v in nested.items():
+                    raw_data.setdefault(k, v)
+
+            # Unwrap fields{} envelope (new wire format from Core API)
+            if isinstance(raw_data, dict) and isinstance(raw_data.get("fields"), dict):
+                nested = raw_data.pop("fields")
+                raw_data.update(nested)
+
+            parsed_args: dict = {}
+            for param_name, param_spec in inputs_spec.items():
+                if param_name in raw_data:
+                    parsed_args[param_name] = parse_input(
+                        raw_data[param_name], param_spec["type"],
+                    )
+                elif param_spec.get("required", False):
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Missing required input: {param_name}",
+                    )
+
+                # Composite primitive fanout — see COMPOSITE_FANOUT
+                # above. Pure passthrough (no parse_input — the suffix
+                # fields are sibling values, not file-shaped). The
+                # inference method declares them in its signature
+                # (e.g. `voice_ref_transcript: str | None = None`).
+                input_type = param_spec.get("type", "")
+                if input_type in COMPOSITE_FANOUT:
+                    for suffix in COMPOSITE_FANOUT[input_type]:
+                        fanout_key = f"{param_name}{suffix}"
+                        if fanout_key in raw_data:
+                            parsed_args[fanout_key] = raw_data[fanout_key]
+
+            for param_name, param_spec in controls_spec.items():
+                if param_name in raw_data:
+                    parsed_args[param_name] = parse_input(
+                        raw_data[param_name], param_spec["type"],
+                    )
+                # else: method uses its default
+
+            if not accepts_var_keyword:
+                dropped = [k for k in parsed_args if k not in accepted_params]
+                for k in dropped:
+                    logger.debug(
+                        "dropping non-accepted kwarg %r for capability method %s",
+                        k, method.__qualname__,
+                    )
+                    parsed_args.pop(k, None)
+
+            result = method(**parsed_args)
+
+            # Single-binary-output shortcut: return raw bytes with
+            # the matching Content-Type. Lets Core API's
+            # _handle_audio_response write response.content directly
+            # to disk as a valid WAV/PNG/MP4/etc.
+            if single_binary_output is not None and isinstance(result, dict):
+                key, media_type = single_binary_output
+                value = result.get(key)
+                if isinstance(value, (bytes, bytearray)):
+                    return Response(content=bytes(value), media_type=media_type)
+                # Fall through to JSON if the method returned something
+                # other than bytes under the declared key (shouldn't
+                # happen for a correctly-implemented capability).
+
+            if isinstance(result, dict):
+                serialized = {}
+                for key, value in result.items():
+                    if key in outputs_spec:
+                        serialized[key] = serialize_output(
+                            value, outputs_spec[key]["type"]
+                        )
+                    else:
+                        serialized[key] = value
+                return JSONResponse(content=serialized)
+            return JSONResponse(content={"result": result})
+
+        except GenerationError as e:
+            raise HTTPException(status_code=500, detail=str(e)) from e
+        except HutashInferenceError as e:
+            raise HTTPException(status_code=500, detail=str(e)) from e
+        except HTTPException:
+            raise
+        except Exception as e:  # noqa: BLE001 — boundary guard
+            raise HTTPException(
+                status_code=500,
+                detail=f"Unexpected error: {type(e).__name__}: {e}",
+            ) from e
+
+    return handler
+
+
+def create_app() -> FastAPI:
+    """Build the FastAPI app by reading manifest and wiring inference."""
+    manifest = load_manifest()
+    model_id = manifest.get("model_id", "unknown")
+    configure_logging(model_id)
+
+    module = load_inference_module()
+    InferenceClass = find_inference_class(module)
+
+    instance = InferenceClass(config=manifest)
+
+    try:
+        instance.load()
+        instance.mark_loaded()
+    except Exception as e:
+        raise ModelLoadError(f"Failed to load model: {e}") from e
+
+    validate_inference_matches_manifest(instance, manifest)
+
+    # Image in manifest is a fully-qualified registry string like
+    # "ghcr.io/appsork/aura-kokoro:v0.1.0" — the tag is after the last ':'.
+    image_ref = manifest.get("image", "")
+    image_tag = image_ref.rsplit(":", 1)[-1] if ":" in image_ref else "unknown"
+    app = FastAPI(
+        title=f"Aura Model: {model_id}",
+        version=image_tag,
+    )
+
+    @app.get("/health")
+    def health():
+        """Container readiness gate.
+
+        Returns 200 only when every data path this container exposes is
+        ready. Specifically:
+          (a) instance._is_loaded â€" the model finished load()
+          (b) THIRD_PARTY_LICENSES.txt readable on disk â€" backs
+              /third-party-licenses, which the host's install-time
+              cache populate fetches once and stores in SQLite KV
+          (c) manifest.json readable on disk â€" backs /manifest, which
+              the host's install-time manifest cache populate fetches
+
+        503 here causes the host backend's _check_health() probe in
+        docker_manager.py â€" and the Docker daemon's HEALTHCHECK â€" to
+        mark the container unhealthy. ensure_model_running() in the
+        host waits on exactly that 200 transition.
+
+        Contract: when /health returns 200, every other endpoint this
+        container serves is ready. A single-shot fetch from the host
+        backend will succeed.
+        """
+        if not instance._is_loaded:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "not_ready", "reason": "model_not_loaded"},
+            )
+        # THIRD_PARTY_LICENSES.txt is generated at container-build time and may
+        # be absent in a native venv install, so it does not gate readiness —
+        # /third-party-licenses simply 404s when it is missing. Model-loaded +
+        # manifest-present is the readiness contract that holds in both runtimes.
+        if not _manifest_path().exists():
+            return JSONResponse(
+                status_code=503,
+                content={"status": "not_ready", "reason": "manifest_missing"},
+            )
+        return {"status": "ok"}
+
+    @app.get("/manifest")
+    def get_manifest():
+        return manifest
+
+    @app.get("/third-party-licenses")
+    def get_third_party_licenses():
+        """Return the bundled THIRD_PARTY_LICENSES.txt as plain text.
+
+        Proxied by Core API at GET /api/v1/models/{id}/third-party-licenses
+        so the desktop app's Settings → About / Licenses tab can display
+        per-container Python-package attributions for the live fleet.
+        """
+        licenses_path = _third_party_licenses_path()
+        if not licenses_path.exists():
+            raise HTTPException(
+                status_code=404,
+                detail="THIRD_PARTY_LICENSES.txt not present in this install",
+            )
+        return Response(
+            content=licenses_path.read_text(encoding="utf-8"),
+            media_type="text/plain; charset=utf-8",
+        )
+
+    capabilities = get_capabilities(instance)
+    declared_capabilities = manifest.get("capabilities", {}) or {}
+
+    for cap_id, cap_spec in declared_capabilities.items():
+        endpoint = cap_spec.get("endpoint", f"/{cap_id}")
+        method = capabilities[cap_id]
+        handler = make_capability_handler(method, cap_spec)
+        app.add_api_route(endpoint, handler, methods=["POST"], name=cap_id)
+
+    return app
+
+
+# Factory pattern only — no module-level app instantiation.
+# Inside containers, uvicorn loads this via:
+#   uvicorn --factory hutash_inference.server:create_app --host 0.0.0.0 --port ${PORT}
+# On dev machines and in tests, create_app() is called explicitly when needed.

hutash_inference-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hutash_inference-0.1.0/validation.py

@@ -0,0 +1,92 @@
+"""Validate that inference.py matches model.yaml capabilities.
+
+Runs at container startup. If mismatch, raises ValidationError with
+clear message so operator knows what to fix.
+"""
+
+import inspect
+
+from hutash_inference.base import Inference, get_capabilities
+from hutash_inference.errors import ValidationError
+
+
+def validate_inference_matches_manifest(
+    instance: Inference,
+    manifest: dict,
+) -> None:
+    """Verify Inference instance implements all capabilities in manifest.
+
+    Checks:
+    - Every capability declared in manifest has a @capability method
+    - Method parameter names match declared inputs + controls
+    - Extra @capability methods without manifest declaration are tolerated
+      (developer may be iterating)
+
+    Args:
+        instance: Loaded Inference subclass instance
+        manifest: Parsed manifest.json contents
+
+    Raises:
+        ValidationError: If any capability mismatch detected
+    """
+    declared_capabilities = manifest.get("capabilities", {}) or {}
+    implemented_capabilities = get_capabilities(instance)
+
+    # Every declared capability must have an implementation
+    missing = [
+        cap_id for cap_id in declared_capabilities
+        if cap_id not in implemented_capabilities
+    ]
+    if missing:
+        raise ValidationError(
+            f"Manifest declares capabilities {missing} but inference.py "
+            f"has no @capability methods for them. "
+            f"Implemented: {list(implemented_capabilities.keys())}"
+        )
+
+    # Verify parameter names for each implemented capability
+    for cap_id, method in implemented_capabilities.items():
+        if cap_id not in declared_capabilities:
+            continue
+        validate_method_signature(cap_id, method, declared_capabilities[cap_id])
+
+
+def validate_method_signature(
+    capability_id: str,
+    method: callable,
+    spec: dict,
+) -> None:
+    """Verify method parameter names match declared inputs + wired controls.
+
+    Only controls whose implementation_status is "wired" (or unspecified)
+    must appear in the method signature. Stub and planned controls are
+    declarative-only — UI renders them, but inference.py need not accept
+    them as kwargs, and the server filters unexpected kwargs out before
+    calling the method.
+
+    Raises:
+        ValidationError: If required parameters are missing from the method
+    """
+    expected_params: set[str] = set()
+    expected_params.update((spec.get("inputs") or {}).keys())
+    for ctrl_id, ctrl_spec in (spec.get("controls") or {}).items():
+        status = (ctrl_spec or {}).get("implementation_status", "wired")
+        if status == "wired":
+            expected_params.add(ctrl_id)
+
+    sig = inspect.signature(method)
+    actual_params: set[str] = set()
+    for param_name in sig.parameters:
+        if param_name == "self":
+            continue
+        actual_params.add(param_name)
+
+    missing = expected_params - actual_params
+    if missing:
+        raise ValidationError(
+            f"Capability '{capability_id}' method missing parameters: "
+            f"{missing}. Manifest declares inputs + wired controls: "
+            f"{expected_params}. Method signature has: {actual_params}"
+        )
+
+    # Extra actual params are OK (could be internal defaults)