ai-inference 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ai-inference might be problematic. Click here for more details.
- ai_inference/__init__.py +57 -0
- ai_inference/config.py +95 -0
- ai_inference/contracts/__init__.py +1 -0
- ai_inference/contracts/data.py +63 -0
- ai_inference/contracts/processors.py +20 -0
- ai_inference/contracts/registry.py +26 -0
- ai_inference/contracts/validation.py +47 -0
- ai_inference/engines/__init__.py +8 -0
- ai_inference/engines/base.py +183 -0
- ai_inference/engines/om.py +79 -0
- ai_inference/engines/onnxruntime.py +153 -0
- ai_inference/engines/registry.py +5 -0
- ai_inference/inference/__init__.py +1 -0
- ai_inference/inference/batcher.py +115 -0
- ai_inference/inference/predictor.py +60 -0
- ai_inference/logging.py +64 -0
- ai_inference/model.py +250 -0
- ai_inference/processing/__init__.py +10 -0
- ai_inference/processing/common.py +61 -0
- ai_inference/processing/context.py +35 -0
- ai_inference/processing/decoders/__init__.py +1 -0
- ai_inference/processing/decoders/rfdetr.py +69 -0
- ai_inference/processing/decoders/rtdetr.py +63 -0
- ai_inference/processing/decoders/sface.py +80 -0
- ai_inference/processing/decoders/yolo.py +86 -0
- ai_inference/processing/decoders/yunet.py +91 -0
- ai_inference/processing/geometry.py +48 -0
- ai_inference/processing/pipeline.py +192 -0
- ai_inference/processing/steps/__init__.py +1 -0
- ai_inference/processing/steps/decode.py +168 -0
- ai_inference/processing/steps/postprocess.py +61 -0
- ai_inference/processing/steps/preprocess.py +344 -0
- ai_inference/py.typed +0 -0
- ai_inference/service/__init__.py +1 -0
- ai_inference/service/app.py +197 -0
- ai_inference/service/inputs.py +212 -0
- ai_inference/service/runtime.py +86 -0
- ai_inference/service/schemas.py +47 -0
- ai_inference/service/settings.py +32 -0
- ai_inference/visualization/__init__.py +1 -0
- ai_inference/visualization/plotter.py +221 -0
- ai_inference-0.2.1.dist-info/METADATA +185 -0
- ai_inference-0.2.1.dist-info/RECORD +46 -0
- ai_inference-0.2.1.dist-info/WHEEL +5 -0
- ai_inference-0.2.1.dist-info/licenses/LICENSE +21 -0
- ai_inference-0.2.1.dist-info/top_level.txt +1 -0
ai_inference/__init__.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Public SDK exports; implementations are grouped by lifecycle responsibility."""
|
|
2
|
+
|
|
3
|
+
from ai_inference.config import Config
|
|
4
|
+
from ai_inference.contracts.data import (
|
|
5
|
+
FaceInput,
|
|
6
|
+
FeaturePrediction,
|
|
7
|
+
ModelInput,
|
|
8
|
+
Prediction,
|
|
9
|
+
PreparedInput,
|
|
10
|
+
RawOutput,
|
|
11
|
+
Result,
|
|
12
|
+
)
|
|
13
|
+
from ai_inference.contracts.processors import Postprocessor, Preprocessor
|
|
14
|
+
from ai_inference.engines import BaseEngine, Engine, OMEngine, ONNXRuntimeEngine
|
|
15
|
+
from ai_inference.engines.registry import EngineRegistry
|
|
16
|
+
from ai_inference.inference.batcher import BatchChunk, Batcher
|
|
17
|
+
from ai_inference.inference.predictor import DefaultPredictor, Predictor
|
|
18
|
+
from ai_inference.logging import LOG_FORMAT, log_context, log_event
|
|
19
|
+
from ai_inference.model import Model
|
|
20
|
+
from ai_inference.processing import Pipeline, State, Step, StepRegistry
|
|
21
|
+
from ai_inference.processing.context import Context, Geometry, SampleContext
|
|
22
|
+
from ai_inference.visualization.plotter import Plotter
|
|
23
|
+
|
|
24
|
+
__version__ = "0.2.1"
|
|
25
|
+
__all__ = [
|
|
26
|
+
"Model",
|
|
27
|
+
"Config",
|
|
28
|
+
"PreparedInput",
|
|
29
|
+
"RawOutput",
|
|
30
|
+
"Prediction",
|
|
31
|
+
"FaceInput",
|
|
32
|
+
"FeaturePrediction",
|
|
33
|
+
"Result",
|
|
34
|
+
"ModelInput",
|
|
35
|
+
"EngineRegistry",
|
|
36
|
+
"Engine",
|
|
37
|
+
"BaseEngine",
|
|
38
|
+
"ONNXRuntimeEngine",
|
|
39
|
+
"OMEngine",
|
|
40
|
+
"Preprocessor",
|
|
41
|
+
"Postprocessor",
|
|
42
|
+
"Plotter",
|
|
43
|
+
"Predictor",
|
|
44
|
+
"DefaultPredictor",
|
|
45
|
+
"Batcher",
|
|
46
|
+
"BatchChunk",
|
|
47
|
+
"Context",
|
|
48
|
+
"Geometry",
|
|
49
|
+
"SampleContext",
|
|
50
|
+
"Pipeline",
|
|
51
|
+
"Step",
|
|
52
|
+
"StepRegistry",
|
|
53
|
+
"State",
|
|
54
|
+
"LOG_FORMAT",
|
|
55
|
+
"log_context",
|
|
56
|
+
"log_event",
|
|
57
|
+
]
|
ai_inference/config.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Validated YAML configuration. Tensor shapes always include the batch axis."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Literal
|
|
5
|
+
|
|
6
|
+
import yaml
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Section(BaseModel):
|
|
11
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ModelConfig(Section):
|
|
15
|
+
name: str = Field(min_length=1)
|
|
16
|
+
path: Path
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BatchConfig(Section):
|
|
20
|
+
mode: Literal["fixed", "dynamic"] = "fixed"
|
|
21
|
+
size: int = Field(default=4, gt=0, strict=True)
|
|
22
|
+
padding: Literal["repeat_last", "zeros"] = "repeat_last"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class RuntimeConfig(Section):
|
|
26
|
+
backend: str = "onnxruntime"
|
|
27
|
+
warmup: bool = Field(default=True, strict=True)
|
|
28
|
+
warmup_runs: int = Field(default=3, gt=0, strict=True)
|
|
29
|
+
batch: BatchConfig = Field(default_factory=BatchConfig)
|
|
30
|
+
# Each engine validates its own strict options schema.
|
|
31
|
+
options: dict[str, Any] = Field(default_factory=dict)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class TensorSpec(Section):
|
|
35
|
+
name: str = Field(min_length=1)
|
|
36
|
+
dtype: Literal["float32", "float16", "float64", "int64", "int32", "uint8", "bool"] | None = None
|
|
37
|
+
shape: tuple[int | None, ...] | None = None
|
|
38
|
+
|
|
39
|
+
@model_validator(mode="after")
|
|
40
|
+
def check_shape(self):
|
|
41
|
+
if self.shape is not None and (
|
|
42
|
+
not self.shape or any(x is not None and x <= 0 for x in self.shape)
|
|
43
|
+
):
|
|
44
|
+
raise ValueError("shape must have positive dimensions or null for dynamic dimensions")
|
|
45
|
+
return self
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Config(Section):
|
|
49
|
+
model: ModelConfig
|
|
50
|
+
runtime: RuntimeConfig = Field(default_factory=RuntimeConfig)
|
|
51
|
+
inputs: tuple[TensorSpec, ...] | None = Field(default=None, min_length=1)
|
|
52
|
+
outputs: tuple[TensorSpec, ...] | None = Field(default=None, min_length=1)
|
|
53
|
+
# Ordered Pipeline operations; factories validate their own strict options.
|
|
54
|
+
preprocess: dict[str, Any]
|
|
55
|
+
postprocess: dict[str, Any]
|
|
56
|
+
|
|
57
|
+
@model_validator(mode="after")
|
|
58
|
+
def check_contract(self):
|
|
59
|
+
if "steps" not in self.preprocess or "steps" not in self.postprocess:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
"preprocess and postprocess must both use ordered steps; flat parameters are unsupported"
|
|
62
|
+
)
|
|
63
|
+
batch = self.runtime.batch
|
|
64
|
+
for specs in (self.inputs, self.outputs):
|
|
65
|
+
if specs is None:
|
|
66
|
+
continue
|
|
67
|
+
if len({s.name for s in specs}) != len(specs):
|
|
68
|
+
raise ValueError("duplicate tensor names")
|
|
69
|
+
for spec in specs:
|
|
70
|
+
expected = batch.size if batch.mode == "fixed" else None
|
|
71
|
+
if spec.shape is not None and spec.shape[0] != expected:
|
|
72
|
+
raise ValueError(
|
|
73
|
+
"all tensor batch dimensions must match runtime.batch (null for dynamic)"
|
|
74
|
+
)
|
|
75
|
+
if self.runtime.backend not in ("onnxruntime", "om") and not self.has_tensor_contract:
|
|
76
|
+
raise ValueError("this backend requires explicit inputs/outputs with dtype and shape")
|
|
77
|
+
return self
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def has_tensor_contract(self) -> bool:
|
|
81
|
+
return all(
|
|
82
|
+
specs is not None and all(s.dtype is not None and s.shape is not None for s in specs)
|
|
83
|
+
for specs in (self.inputs, self.outputs)
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_yaml(cls, path: str | Path) -> "Config":
|
|
88
|
+
path = Path(path).expanduser().resolve()
|
|
89
|
+
config = cls.model_validate(yaml.safe_load(path.read_text(encoding="utf-8")))
|
|
90
|
+
model_path = config.model.path.expanduser()
|
|
91
|
+
if not model_path.is_absolute():
|
|
92
|
+
model_path = path.parent / model_path
|
|
93
|
+
return config.model_copy(
|
|
94
|
+
update={"model": config.model.model_copy(update={"path": model_path.resolve()})}
|
|
95
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""contracts package; public SDK exports live in ai_inference."""
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Transport-free intermediate and prediction types; no business metadata."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, Mapping
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class PreparedInput:
|
|
11
|
+
"""Unpadded tensors [N,...], one processing metadata entry per real sample."""
|
|
12
|
+
|
|
13
|
+
tensors: Mapping[str, np.ndarray]
|
|
14
|
+
metadata: tuple[Any, ...]
|
|
15
|
+
|
|
16
|
+
@property
|
|
17
|
+
def count(self) -> int:
|
|
18
|
+
return len(self.metadata)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class RawOutput:
|
|
23
|
+
"""Unpadded, ordered tensors [N,...]; metadata travels with its own request."""
|
|
24
|
+
|
|
25
|
+
tensors: Mapping[str, np.ndarray]
|
|
26
|
+
prepared: PreparedInput
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Prediction:
|
|
31
|
+
"""One image: xyxy pixels, float32 scores, int64 class IDs (decoder-defined order)."""
|
|
32
|
+
|
|
33
|
+
boxes: np.ndarray
|
|
34
|
+
scores: np.ndarray
|
|
35
|
+
class_ids: np.ndarray
|
|
36
|
+
original_shape: tuple[int, int]
|
|
37
|
+
|
|
38
|
+
# Optional instance data in original image coordinates; detection-only callers
|
|
39
|
+
# retain the same four constructor arguments and existing fields.
|
|
40
|
+
keypoints: np.ndarray | None = None # [instances, points, 2], xy pixels
|
|
41
|
+
masks: np.ndarray | None = None # [instances, H, W], original-resolution masks
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class FeaturePrediction:
|
|
46
|
+
"""One sample's finite float32 feature vector (L2 normalized by default)."""
|
|
47
|
+
|
|
48
|
+
embedding: np.ndarray
|
|
49
|
+
original_shape: tuple[int, int]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class FaceInput:
|
|
54
|
+
"""Full uint8 image and five original-coordinate points in YuNet order:
|
|
55
|
+
right eye, left eye, nose, right mouth, left mouth.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
image: np.ndarray
|
|
59
|
+
keypoints: np.ndarray
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
Result = Prediction | FeaturePrediction
|
|
63
|
+
ModelInput = np.ndarray | list[np.ndarray] | FaceInput | list[FaceInput]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Backend-independent processing contracts; implementations may use any library.
|
|
2
|
+
|
|
3
|
+
Custom processor pairs may define their own per-image metadata. When replacing
|
|
4
|
+
only one processor, preserve the metadata contract of its counterpart. Tensor
|
|
5
|
+
values currently remain CPU NumPy arrays as required by the engine/batching API.
|
|
6
|
+
Processing placement is separate from runtime provider selection; Model rejects
|
|
7
|
+
unimplemented processing devices and per-step plans before engine creation.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from typing import Protocol
|
|
11
|
+
|
|
12
|
+
from ai_inference.contracts.data import ModelInput, PreparedInput, RawOutput, Result
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Preprocessor(Protocol):
|
|
16
|
+
def __call__(self, images: ModelInput, /) -> PreparedInput: ...
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Postprocessor(Protocol):
|
|
20
|
+
def __call__(self, outputs: RawOutput, /) -> list[Result]: ...
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Explicit registries; register plugins before constructing Models."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Registry:
|
|
7
|
+
def __init__(self, kind: str):
|
|
8
|
+
self.kind = kind
|
|
9
|
+
self._factories: dict[str, Callable] = {}
|
|
10
|
+
|
|
11
|
+
def register(self, name: str, factory: Callable | None = None):
|
|
12
|
+
def install(value):
|
|
13
|
+
if not name or name in self._factories:
|
|
14
|
+
raise ValueError(f"Invalid or duplicate {self.kind}: {name!r}")
|
|
15
|
+
self._factories[name] = value
|
|
16
|
+
return value
|
|
17
|
+
|
|
18
|
+
return install if factory is None else install(factory)
|
|
19
|
+
|
|
20
|
+
def create(self, name: str, config: Any, **kwargs: Any):
|
|
21
|
+
if name not in self._factories:
|
|
22
|
+
raise ValueError(f"Unknown {self.kind} {name!r}; registered: {sorted(self._factories)}")
|
|
23
|
+
return self._factories[name](config, **kwargs)
|
|
24
|
+
|
|
25
|
+
def names(self) -> tuple[str, ...]:
|
|
26
|
+
return tuple(sorted(self._factories))
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Shared tensor contracts for logical requests and physical engine batches."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from ai_inference.config import Config, TensorSpec
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_tensors(
|
|
11
|
+
tensors: Mapping[str, np.ndarray], specs: tuple[TensorSpec, ...] | None, count: int
|
|
12
|
+
) -> None:
|
|
13
|
+
if not isinstance(tensors, Mapping):
|
|
14
|
+
raise ValueError("tensors must be a named tensor mapping")
|
|
15
|
+
if specs is None or any(s.dtype is None or s.shape is None for s in specs):
|
|
16
|
+
raise ValueError("tensor contract must be resolved before validation")
|
|
17
|
+
if set(tensors) != {s.name for s in specs}:
|
|
18
|
+
raise ValueError("tensor names differ from configured contract")
|
|
19
|
+
for spec in specs:
|
|
20
|
+
shape = spec.shape
|
|
21
|
+
dtype = spec.dtype
|
|
22
|
+
if shape is None or dtype is None:
|
|
23
|
+
raise ValueError("tensor contract must be resolved before validation")
|
|
24
|
+
tensor = tensors[spec.name]
|
|
25
|
+
if not isinstance(tensor, np.ndarray) or tensor.dtype != np.dtype(dtype):
|
|
26
|
+
raise ValueError(f"{spec.name}: expected ndarray dtype {dtype}")
|
|
27
|
+
if tensor.ndim != len(shape) or tensor.shape[0] != count:
|
|
28
|
+
raise ValueError(
|
|
29
|
+
f"{spec.name}: invalid rank or batch dimension {tensor.shape}; expected count={count}, rank={len(shape)}"
|
|
30
|
+
)
|
|
31
|
+
if any(want is not None and got != want for got, want in zip(tensor.shape[1:], shape[1:])):
|
|
32
|
+
raise ValueError(f"{spec.name}: unexpected shape {tensor.shape}; spec {spec.shape}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def validate_physical_inputs(tensors: Mapping[str, np.ndarray], config: Config) -> int:
|
|
36
|
+
"""Check physical batch size in addition to names, dtype and dimensions."""
|
|
37
|
+
if not isinstance(tensors, Mapping) or not tensors:
|
|
38
|
+
raise ValueError("inputs must be a nonempty tensor mapping")
|
|
39
|
+
first = next(iter(tensors.values()))
|
|
40
|
+
if not isinstance(first, np.ndarray) or first.ndim == 0:
|
|
41
|
+
raise ValueError("input tensors must be ndarrays with a batch dimension")
|
|
42
|
+
count = first.shape[0]
|
|
43
|
+
policy = config.runtime.batch
|
|
44
|
+
if count <= 0 or count > policy.size or (policy.mode == "fixed" and count != policy.size):
|
|
45
|
+
raise ValueError("invalid physical batch size")
|
|
46
|
+
validate_tensors(tensors, config.inputs, count)
|
|
47
|
+
return count
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Physical inference backends and their registration interface."""
|
|
2
|
+
|
|
3
|
+
from ai_inference.engines.base import BaseEngine, Engine
|
|
4
|
+
from ai_inference.engines.om import OMEngine
|
|
5
|
+
from ai_inference.engines.onnxruntime import ONNXRuntimeEngine
|
|
6
|
+
from ai_inference.engines.registry import EngineRegistry
|
|
7
|
+
|
|
8
|
+
__all__ = ["Engine", "BaseEngine", "ONNXRuntimeEngine", "OMEngine", "EngineRegistry"]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import TYPE_CHECKING, Mapping, Protocol
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from ai_inference.config import Config, TensorSpec
|
|
9
|
+
from ai_inference.contracts.validation import validate_physical_inputs, validate_tensors
|
|
10
|
+
from ai_inference.logging import log_context, log_event
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from loguru import Logger
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class TensorMetadata:
|
|
18
|
+
name: str
|
|
19
|
+
dtype: str
|
|
20
|
+
shape: tuple[int | str | None, ...]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Engine(Protocol):
|
|
24
|
+
"""Execute one physical batch, preserving names, dtype and leading batch axis."""
|
|
25
|
+
|
|
26
|
+
config: Config
|
|
27
|
+
|
|
28
|
+
def infer(self, tensors: Mapping[str, np.ndarray]) -> Mapping[str, np.ndarray]: ...
|
|
29
|
+
def warmup(self, tensors: Mapping[str, np.ndarray] | None = None) -> None: ...
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class BaseEngine:
|
|
33
|
+
"""Checked public inference entry; subclasses implement backend execution only."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, config: Config):
|
|
36
|
+
self.config = config
|
|
37
|
+
|
|
38
|
+
def read_tensor_metadata(self) -> dict[str, tuple[TensorMetadata, ...]]:
|
|
39
|
+
"""Return backend-neutral metadata; backends may inspect their session."""
|
|
40
|
+
return {
|
|
41
|
+
kind: tuple(
|
|
42
|
+
TensorMetadata(s.name, s.dtype, s.shape) for s in getattr(self.config, kind)
|
|
43
|
+
)
|
|
44
|
+
for kind in ("inputs", "outputs")
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
def _load_tensor_contract(self) -> None:
|
|
48
|
+
"""Resolve optional YAML constraints in the backend's actual tensor order."""
|
|
49
|
+
data = self.config.model_dump()
|
|
50
|
+
for kind, tensors in self.read_tensor_metadata().items():
|
|
51
|
+
data[kind] = self._resolve_metadata(tensors, getattr(self.config, kind), kind)
|
|
52
|
+
self.config = Config.model_validate(data)
|
|
53
|
+
|
|
54
|
+
def _resolve_metadata(self, actual, specs, kind):
|
|
55
|
+
nodes = {n.name: n for n in actual}
|
|
56
|
+
if not actual or len(nodes) != len(actual):
|
|
57
|
+
raise ValueError(
|
|
58
|
+
f"{self.config.runtime.backend} {kind} metadata must have unique tensor names"
|
|
59
|
+
)
|
|
60
|
+
if specs is not None and set(nodes) != {s.name for s in specs}:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"{self.config.runtime.backend} {kind} names do not match YAML contract"
|
|
63
|
+
)
|
|
64
|
+
expected = {s.name: s for s in specs or ()}
|
|
65
|
+
resolved = []
|
|
66
|
+
for node in actual:
|
|
67
|
+
dtype = node.dtype
|
|
68
|
+
shape = tuple(d if isinstance(d, int) else None for d in node.shape)
|
|
69
|
+
policy = self.config.runtime.batch
|
|
70
|
+
if shape:
|
|
71
|
+
model_batch = shape[0]
|
|
72
|
+
if model_batch is not None and (
|
|
73
|
+
policy.mode == "dynamic" or model_batch != policy.size
|
|
74
|
+
):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"{self.config.runtime.backend} {kind} tensor '{node.name}': "
|
|
77
|
+
f"model batch={model_batch} conflicts with "
|
|
78
|
+
f"runtime.batch.mode={policy.mode}, size={policy.size}; "
|
|
79
|
+
f"use mode=fixed, size={model_batch} for this static model"
|
|
80
|
+
)
|
|
81
|
+
if model_batch is None and policy.mode == "fixed":
|
|
82
|
+
shape = (policy.size, *shape[1:])
|
|
83
|
+
spec = expected.get(node.name)
|
|
84
|
+
if spec is not None:
|
|
85
|
+
if spec.dtype is not None and spec.dtype != dtype:
|
|
86
|
+
raise ValueError(
|
|
87
|
+
f"{self.config.runtime.backend} {kind} dtype mismatch: {node.name}"
|
|
88
|
+
)
|
|
89
|
+
if spec.shape is not None:
|
|
90
|
+
if len(shape) != len(spec.shape):
|
|
91
|
+
raise ValueError(
|
|
92
|
+
f"{self.config.runtime.backend} {kind} rank mismatch: {node.name}"
|
|
93
|
+
)
|
|
94
|
+
for actual_dim, configured in zip(shape, spec.shape):
|
|
95
|
+
if actual_dim is not None and actual_dim != configured:
|
|
96
|
+
raise ValueError(
|
|
97
|
+
f"{self.config.runtime.backend} {kind} shape mismatch: {node.name}"
|
|
98
|
+
)
|
|
99
|
+
# Concrete constraints on symbolic dimensions remain runtime checks.
|
|
100
|
+
shape = spec.shape
|
|
101
|
+
resolved.append(TensorSpec(name=node.name, dtype=dtype, shape=shape))
|
|
102
|
+
return resolved
|
|
103
|
+
|
|
104
|
+
def log_tensor_metadata(self, logger: Logger) -> None:
|
|
105
|
+
for kind, tensors in self.read_tensor_metadata().items():
|
|
106
|
+
resolved = {s.name: s for s in getattr(self.config, kind)}
|
|
107
|
+
for tensor in tensors:
|
|
108
|
+
resolved_shape = list(resolved[tensor.name].shape)
|
|
109
|
+
with log_context(logger, model=self.config.model.name):
|
|
110
|
+
direction = "Input" if kind == "inputs" else "Output"
|
|
111
|
+
log_event(
|
|
112
|
+
"init",
|
|
113
|
+
"engine.metadata",
|
|
114
|
+
f"{direction} '{tensor.name}' | {tensor.dtype} | shape={list(tensor.shape)} | resolved_shape={resolved_shape}",
|
|
115
|
+
kind=kind,
|
|
116
|
+
tensor_name=tensor.name,
|
|
117
|
+
dtype=tensor.dtype,
|
|
118
|
+
shape=list(tensor.shape),
|
|
119
|
+
resolved_shape=resolved_shape,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def create_warmup_inputs(self) -> Mapping[str, np.ndarray]:
|
|
123
|
+
"""Default zero inputs; override for model-specific values or shapes."""
|
|
124
|
+
batch = self.config.runtime.batch.size if self.config.runtime.batch.mode == "fixed" else 1
|
|
125
|
+
tensors = {}
|
|
126
|
+
specs = self.config.inputs
|
|
127
|
+
if specs is None:
|
|
128
|
+
raise ValueError("warmup requires a resolved input tensor contract")
|
|
129
|
+
for spec in specs:
|
|
130
|
+
shape, dtype = spec.shape, spec.dtype
|
|
131
|
+
if shape is None or dtype is None:
|
|
132
|
+
raise ValueError("warmup requires a resolved input tensor contract")
|
|
133
|
+
dimensions = []
|
|
134
|
+
for dim in shape[1:]:
|
|
135
|
+
if dim is None:
|
|
136
|
+
raise ValueError(
|
|
137
|
+
"warmup requires concrete non-batch input dimensions; provide tensors, specify sizes or set runtime.warmup=false"
|
|
138
|
+
)
|
|
139
|
+
dimensions.append(dim)
|
|
140
|
+
tensors[spec.name] = np.zeros((batch, *dimensions), dtype=dtype)
|
|
141
|
+
return tensors
|
|
142
|
+
|
|
143
|
+
def warmup(self, tensors: Mapping[str, np.ndarray] | None = None) -> None:
|
|
144
|
+
"""Create or accept valid physical inputs, then run the engine entry."""
|
|
145
|
+
completed = 0
|
|
146
|
+
try:
|
|
147
|
+
tensors = self.create_warmup_inputs() if tensors is None else tensors
|
|
148
|
+
batch = validate_physical_inputs(tensors, self.config)
|
|
149
|
+
for _ in range(self.config.runtime.warmup_runs):
|
|
150
|
+
self.infer(tensors)
|
|
151
|
+
completed += 1
|
|
152
|
+
except Exception as exc:
|
|
153
|
+
log_event(
|
|
154
|
+
"init",
|
|
155
|
+
"engine.warmup",
|
|
156
|
+
f"Failed | completed runs={completed} | error={type(exc).__name__}",
|
|
157
|
+
level="WARNING",
|
|
158
|
+
model=self.config.model.name,
|
|
159
|
+
error_type=type(exc).__name__,
|
|
160
|
+
completed_runs=completed,
|
|
161
|
+
status="failed",
|
|
162
|
+
)
|
|
163
|
+
raise
|
|
164
|
+
log_event(
|
|
165
|
+
"init",
|
|
166
|
+
"engine.warmup",
|
|
167
|
+
f"Completed | runs={self.config.runtime.warmup_runs} | batch={batch}",
|
|
168
|
+
model=self.config.model.name,
|
|
169
|
+
warmup_runs=self.config.runtime.warmup_runs,
|
|
170
|
+
physical_batch=batch,
|
|
171
|
+
status="completed",
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
def infer(self, tensors: Mapping[str, np.ndarray]) -> Mapping[str, np.ndarray]:
|
|
175
|
+
count = validate_physical_inputs(tensors, self.config)
|
|
176
|
+
outputs = self._infer(tensors)
|
|
177
|
+
if not isinstance(outputs, Mapping):
|
|
178
|
+
raise ValueError("outputs must be a tensor mapping")
|
|
179
|
+
validate_tensors(outputs, self.config.outputs, count)
|
|
180
|
+
return outputs
|
|
181
|
+
|
|
182
|
+
def _infer(self, tensors: Mapping[str, np.ndarray]) -> Mapping[str, np.ndarray]:
|
|
183
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Static Ascend OM execution through ais_bench, without image processing."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from ai_inference.config import Config, Section
|
|
9
|
+
from ai_inference.engines.base import BaseEngine, TensorMetadata
|
|
10
|
+
from ai_inference.engines.registry import EngineRegistry
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class OMOptions(Section):
|
|
14
|
+
device_id: int = Field(default=0, ge=0, strict=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@EngineRegistry.register("om")
|
|
18
|
+
class OMEngine(BaseEngine):
|
|
19
|
+
def __init__(self, config: Config):
|
|
20
|
+
super().__init__(config)
|
|
21
|
+
options = OMOptions.model_validate(config.runtime.options)
|
|
22
|
+
if config.runtime.batch.mode != "fixed":
|
|
23
|
+
raise ValueError("OMEngine currently supports static fixed-batch inference only")
|
|
24
|
+
self._load_session(options)
|
|
25
|
+
self._load_tensor_contract()
|
|
26
|
+
|
|
27
|
+
def _load_session(self, options: OMOptions) -> None:
|
|
28
|
+
try:
|
|
29
|
+
from ais_bench.infer.interface import InferSession
|
|
30
|
+
except ImportError as exc:
|
|
31
|
+
raise ImportError(
|
|
32
|
+
"OMEngine requires compatible ais_bench and ACLRuntime in an Ascend/CANN environment"
|
|
33
|
+
) from exc
|
|
34
|
+
self.session = InferSession(options.device_id, str(self.config.model.path))
|
|
35
|
+
|
|
36
|
+
def read_tensor_metadata(self) -> dict[str, tuple[TensorMetadata, ...]]:
|
|
37
|
+
"""Read actual OM names, dtypes and shapes in vendor execution order."""
|
|
38
|
+
dtypes = {
|
|
39
|
+
"float32": "float32",
|
|
40
|
+
"float16": "float16",
|
|
41
|
+
"double64": "float64",
|
|
42
|
+
"float64": "float64",
|
|
43
|
+
"int64": "int64",
|
|
44
|
+
"int32": "int32",
|
|
45
|
+
"uint8": "uint8",
|
|
46
|
+
"bool": "bool",
|
|
47
|
+
}
|
|
48
|
+
metadata = {}
|
|
49
|
+
for kind, nodes in (
|
|
50
|
+
("inputs", self.session.get_inputs()),
|
|
51
|
+
("outputs", self.session.get_outputs()),
|
|
52
|
+
):
|
|
53
|
+
tensors = []
|
|
54
|
+
for node in nodes:
|
|
55
|
+
# ACLRuntime dtype is a pybind enum, not a NumPy dtype or ACL type ID.
|
|
56
|
+
dtype = node.datatype.name
|
|
57
|
+
if dtype not in dtypes:
|
|
58
|
+
raise ValueError(f"unsupported OM {kind} dtype: {node.name}: {dtype}")
|
|
59
|
+
shape = tuple(node.shape)
|
|
60
|
+
if not shape or any(not isinstance(d, int) or d <= 0 for d in shape):
|
|
61
|
+
raise ValueError(f"OM requires static positive {kind} dimensions: {node.name}")
|
|
62
|
+
tensors.append(TensorMetadata(node.name, dtypes[dtype], shape))
|
|
63
|
+
metadata[kind] = tuple(tensors)
|
|
64
|
+
return metadata
|
|
65
|
+
|
|
66
|
+
def _infer(self, tensors: Mapping[str, np.ndarray]) -> dict[str, np.ndarray]:
|
|
67
|
+
inputs, outputs = self.config.inputs, self.config.outputs
|
|
68
|
+
if inputs is None or outputs is None:
|
|
69
|
+
raise ValueError("OM requires a resolved tensor contract")
|
|
70
|
+
feeds = [np.ascontiguousarray(tensors[spec.name]) for spec in inputs]
|
|
71
|
+
values = self.session.infer(feeds)
|
|
72
|
+
if len(values) != len(outputs):
|
|
73
|
+
raise ValueError("OM output count does not match configured outputs")
|
|
74
|
+
result = {}
|
|
75
|
+
for spec, value in zip(outputs, values):
|
|
76
|
+
if not isinstance(value, np.ndarray):
|
|
77
|
+
raise ValueError("OM outputs must be ndarrays")
|
|
78
|
+
result[spec.name] = value
|
|
79
|
+
return result
|