ai-inference 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ai-inference might be problematic. Click here for more details.

Files changed (46) hide show
  1. ai_inference/__init__.py +57 -0
  2. ai_inference/config.py +95 -0
  3. ai_inference/contracts/__init__.py +1 -0
  4. ai_inference/contracts/data.py +63 -0
  5. ai_inference/contracts/processors.py +20 -0
  6. ai_inference/contracts/registry.py +26 -0
  7. ai_inference/contracts/validation.py +47 -0
  8. ai_inference/engines/__init__.py +8 -0
  9. ai_inference/engines/base.py +183 -0
  10. ai_inference/engines/om.py +79 -0
  11. ai_inference/engines/onnxruntime.py +153 -0
  12. ai_inference/engines/registry.py +5 -0
  13. ai_inference/inference/__init__.py +1 -0
  14. ai_inference/inference/batcher.py +115 -0
  15. ai_inference/inference/predictor.py +60 -0
  16. ai_inference/logging.py +64 -0
  17. ai_inference/model.py +250 -0
  18. ai_inference/processing/__init__.py +10 -0
  19. ai_inference/processing/common.py +61 -0
  20. ai_inference/processing/context.py +35 -0
  21. ai_inference/processing/decoders/__init__.py +1 -0
  22. ai_inference/processing/decoders/rfdetr.py +69 -0
  23. ai_inference/processing/decoders/rtdetr.py +63 -0
  24. ai_inference/processing/decoders/sface.py +80 -0
  25. ai_inference/processing/decoders/yolo.py +86 -0
  26. ai_inference/processing/decoders/yunet.py +91 -0
  27. ai_inference/processing/geometry.py +48 -0
  28. ai_inference/processing/pipeline.py +192 -0
  29. ai_inference/processing/steps/__init__.py +1 -0
  30. ai_inference/processing/steps/decode.py +168 -0
  31. ai_inference/processing/steps/postprocess.py +61 -0
  32. ai_inference/processing/steps/preprocess.py +344 -0
  33. ai_inference/py.typed +0 -0
  34. ai_inference/service/__init__.py +1 -0
  35. ai_inference/service/app.py +197 -0
  36. ai_inference/service/inputs.py +212 -0
  37. ai_inference/service/runtime.py +86 -0
  38. ai_inference/service/schemas.py +47 -0
  39. ai_inference/service/settings.py +32 -0
  40. ai_inference/visualization/__init__.py +1 -0
  41. ai_inference/visualization/plotter.py +221 -0
  42. ai_inference-0.2.1.dist-info/METADATA +185 -0
  43. ai_inference-0.2.1.dist-info/RECORD +46 -0
  44. ai_inference-0.2.1.dist-info/WHEEL +5 -0
  45. ai_inference-0.2.1.dist-info/licenses/LICENSE +21 -0
  46. ai_inference-0.2.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,57 @@
1
+ """Public SDK exports; implementations are grouped by lifecycle responsibility."""
2
+
3
+ from ai_inference.config import Config
4
+ from ai_inference.contracts.data import (
5
+ FaceInput,
6
+ FeaturePrediction,
7
+ ModelInput,
8
+ Prediction,
9
+ PreparedInput,
10
+ RawOutput,
11
+ Result,
12
+ )
13
+ from ai_inference.contracts.processors import Postprocessor, Preprocessor
14
+ from ai_inference.engines import BaseEngine, Engine, OMEngine, ONNXRuntimeEngine
15
+ from ai_inference.engines.registry import EngineRegistry
16
+ from ai_inference.inference.batcher import BatchChunk, Batcher
17
+ from ai_inference.inference.predictor import DefaultPredictor, Predictor
18
+ from ai_inference.logging import LOG_FORMAT, log_context, log_event
19
+ from ai_inference.model import Model
20
+ from ai_inference.processing import Pipeline, State, Step, StepRegistry
21
+ from ai_inference.processing.context import Context, Geometry, SampleContext
22
+ from ai_inference.visualization.plotter import Plotter
23
+
24
+ __version__ = "0.2.1"
25
+ __all__ = [
26
+ "Model",
27
+ "Config",
28
+ "PreparedInput",
29
+ "RawOutput",
30
+ "Prediction",
31
+ "FaceInput",
32
+ "FeaturePrediction",
33
+ "Result",
34
+ "ModelInput",
35
+ "EngineRegistry",
36
+ "Engine",
37
+ "BaseEngine",
38
+ "ONNXRuntimeEngine",
39
+ "OMEngine",
40
+ "Preprocessor",
41
+ "Postprocessor",
42
+ "Plotter",
43
+ "Predictor",
44
+ "DefaultPredictor",
45
+ "Batcher",
46
+ "BatchChunk",
47
+ "Context",
48
+ "Geometry",
49
+ "SampleContext",
50
+ "Pipeline",
51
+ "Step",
52
+ "StepRegistry",
53
+ "State",
54
+ "LOG_FORMAT",
55
+ "log_context",
56
+ "log_event",
57
+ ]
ai_inference/config.py ADDED
@@ -0,0 +1,95 @@
1
+ """Validated YAML configuration. Tensor shapes always include the batch axis."""
2
+
3
+ from pathlib import Path
4
+ from typing import Any, Literal
5
+
6
+ import yaml
7
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
8
+
9
+
10
+ class Section(BaseModel):
11
+ model_config = ConfigDict(extra="forbid", frozen=True)
12
+
13
+
14
+ class ModelConfig(Section):
15
+ name: str = Field(min_length=1)
16
+ path: Path
17
+
18
+
19
+ class BatchConfig(Section):
20
+ mode: Literal["fixed", "dynamic"] = "fixed"
21
+ size: int = Field(default=4, gt=0, strict=True)
22
+ padding: Literal["repeat_last", "zeros"] = "repeat_last"
23
+
24
+
25
+ class RuntimeConfig(Section):
26
+ backend: str = "onnxruntime"
27
+ warmup: bool = Field(default=True, strict=True)
28
+ warmup_runs: int = Field(default=3, gt=0, strict=True)
29
+ batch: BatchConfig = Field(default_factory=BatchConfig)
30
+ # Each engine validates its own strict options schema.
31
+ options: dict[str, Any] = Field(default_factory=dict)
32
+
33
+
34
+ class TensorSpec(Section):
35
+ name: str = Field(min_length=1)
36
+ dtype: Literal["float32", "float16", "float64", "int64", "int32", "uint8", "bool"] | None = None
37
+ shape: tuple[int | None, ...] | None = None
38
+
39
+ @model_validator(mode="after")
40
+ def check_shape(self):
41
+ if self.shape is not None and (
42
+ not self.shape or any(x is not None and x <= 0 for x in self.shape)
43
+ ):
44
+ raise ValueError("shape must have positive dimensions or null for dynamic dimensions")
45
+ return self
46
+
47
+
48
+ class Config(Section):
49
+ model: ModelConfig
50
+ runtime: RuntimeConfig = Field(default_factory=RuntimeConfig)
51
+ inputs: tuple[TensorSpec, ...] | None = Field(default=None, min_length=1)
52
+ outputs: tuple[TensorSpec, ...] | None = Field(default=None, min_length=1)
53
+ # Ordered Pipeline operations; factories validate their own strict options.
54
+ preprocess: dict[str, Any]
55
+ postprocess: dict[str, Any]
56
+
57
+ @model_validator(mode="after")
58
+ def check_contract(self):
59
+ if "steps" not in self.preprocess or "steps" not in self.postprocess:
60
+ raise ValueError(
61
+ "preprocess and postprocess must both use ordered steps; flat parameters are unsupported"
62
+ )
63
+ batch = self.runtime.batch
64
+ for specs in (self.inputs, self.outputs):
65
+ if specs is None:
66
+ continue
67
+ if len({s.name for s in specs}) != len(specs):
68
+ raise ValueError("duplicate tensor names")
69
+ for spec in specs:
70
+ expected = batch.size if batch.mode == "fixed" else None
71
+ if spec.shape is not None and spec.shape[0] != expected:
72
+ raise ValueError(
73
+ "all tensor batch dimensions must match runtime.batch (null for dynamic)"
74
+ )
75
+ if self.runtime.backend not in ("onnxruntime", "om") and not self.has_tensor_contract:
76
+ raise ValueError("this backend requires explicit inputs/outputs with dtype and shape")
77
+ return self
78
+
79
+ @property
80
+ def has_tensor_contract(self) -> bool:
81
+ return all(
82
+ specs is not None and all(s.dtype is not None and s.shape is not None for s in specs)
83
+ for specs in (self.inputs, self.outputs)
84
+ )
85
+
86
+ @classmethod
87
+ def from_yaml(cls, path: str | Path) -> "Config":
88
+ path = Path(path).expanduser().resolve()
89
+ config = cls.model_validate(yaml.safe_load(path.read_text(encoding="utf-8")))
90
+ model_path = config.model.path.expanduser()
91
+ if not model_path.is_absolute():
92
+ model_path = path.parent / model_path
93
+ return config.model_copy(
94
+ update={"model": config.model.model_copy(update={"path": model_path.resolve()})}
95
+ )
@@ -0,0 +1 @@
1
+ """contracts package; public SDK exports live in ai_inference."""
@@ -0,0 +1,63 @@
1
+ """Transport-free intermediate and prediction types; no business metadata."""
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Any, Mapping
5
+
6
+ import numpy as np
7
+
8
+
9
+ @dataclass(frozen=True)
10
+ class PreparedInput:
11
+ """Unpadded tensors [N,...], one processing metadata entry per real sample."""
12
+
13
+ tensors: Mapping[str, np.ndarray]
14
+ metadata: tuple[Any, ...]
15
+
16
+ @property
17
+ def count(self) -> int:
18
+ return len(self.metadata)
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class RawOutput:
23
+ """Unpadded, ordered tensors [N,...]; metadata travels with its own request."""
24
+
25
+ tensors: Mapping[str, np.ndarray]
26
+ prepared: PreparedInput
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Prediction:
31
+ """One image: xyxy pixels, float32 scores, int64 class IDs (decoder-defined order)."""
32
+
33
+ boxes: np.ndarray
34
+ scores: np.ndarray
35
+ class_ids: np.ndarray
36
+ original_shape: tuple[int, int]
37
+
38
+ # Optional instance data in original image coordinates; detection-only callers
39
+ # retain the same four constructor arguments and existing fields.
40
+ keypoints: np.ndarray | None = None # [instances, points, 2], xy pixels
41
+ masks: np.ndarray | None = None # [instances, H, W], original-resolution masks
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class FeaturePrediction:
46
+ """One sample's finite float32 feature vector (L2 normalized by default)."""
47
+
48
+ embedding: np.ndarray
49
+ original_shape: tuple[int, int]
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class FaceInput:
54
+ """Full uint8 image and five original-coordinate points in YuNet order:
55
+ right eye, left eye, nose, right mouth, left mouth.
56
+ """
57
+
58
+ image: np.ndarray
59
+ keypoints: np.ndarray
60
+
61
+
62
+ Result = Prediction | FeaturePrediction
63
+ ModelInput = np.ndarray | list[np.ndarray] | FaceInput | list[FaceInput]
@@ -0,0 +1,20 @@
1
+ """Backend-independent processing contracts; implementations may use any library.
2
+
3
+ Custom processor pairs may define their own per-image metadata. When replacing
4
+ only one processor, preserve the metadata contract of its counterpart. Tensor
5
+ values currently remain CPU NumPy arrays as required by the engine/batching API.
6
+ Processing placement is separate from runtime provider selection; Model rejects
7
+ unimplemented processing devices and per-step plans before engine creation.
8
+ """
9
+
10
+ from typing import Protocol
11
+
12
+ from ai_inference.contracts.data import ModelInput, PreparedInput, RawOutput, Result
13
+
14
+
15
+ class Preprocessor(Protocol):
16
+ def __call__(self, images: ModelInput, /) -> PreparedInput: ...
17
+
18
+
19
+ class Postprocessor(Protocol):
20
+ def __call__(self, outputs: RawOutput, /) -> list[Result]: ...
@@ -0,0 +1,26 @@
1
+ """Explicit registries; register plugins before constructing Models."""
2
+
3
+ from typing import Any, Callable
4
+
5
+
6
+ class Registry:
7
+ def __init__(self, kind: str):
8
+ self.kind = kind
9
+ self._factories: dict[str, Callable] = {}
10
+
11
+ def register(self, name: str, factory: Callable | None = None):
12
+ def install(value):
13
+ if not name or name in self._factories:
14
+ raise ValueError(f"Invalid or duplicate {self.kind}: {name!r}")
15
+ self._factories[name] = value
16
+ return value
17
+
18
+ return install if factory is None else install(factory)
19
+
20
+ def create(self, name: str, config: Any, **kwargs: Any):
21
+ if name not in self._factories:
22
+ raise ValueError(f"Unknown {self.kind} {name!r}; registered: {sorted(self._factories)}")
23
+ return self._factories[name](config, **kwargs)
24
+
25
+ def names(self) -> tuple[str, ...]:
26
+ return tuple(sorted(self._factories))
@@ -0,0 +1,47 @@
1
+ """Shared tensor contracts for logical requests and physical engine batches."""
2
+
3
+ from collections.abc import Mapping
4
+
5
+ import numpy as np
6
+
7
+ from ai_inference.config import Config, TensorSpec
8
+
9
+
10
+ def validate_tensors(
11
+ tensors: Mapping[str, np.ndarray], specs: tuple[TensorSpec, ...] | None, count: int
12
+ ) -> None:
13
+ if not isinstance(tensors, Mapping):
14
+ raise ValueError("tensors must be a named tensor mapping")
15
+ if specs is None or any(s.dtype is None or s.shape is None for s in specs):
16
+ raise ValueError("tensor contract must be resolved before validation")
17
+ if set(tensors) != {s.name for s in specs}:
18
+ raise ValueError("tensor names differ from configured contract")
19
+ for spec in specs:
20
+ shape = spec.shape
21
+ dtype = spec.dtype
22
+ if shape is None or dtype is None:
23
+ raise ValueError("tensor contract must be resolved before validation")
24
+ tensor = tensors[spec.name]
25
+ if not isinstance(tensor, np.ndarray) or tensor.dtype != np.dtype(dtype):
26
+ raise ValueError(f"{spec.name}: expected ndarray dtype {dtype}")
27
+ if tensor.ndim != len(shape) or tensor.shape[0] != count:
28
+ raise ValueError(
29
+ f"{spec.name}: invalid rank or batch dimension {tensor.shape}; expected count={count}, rank={len(shape)}"
30
+ )
31
+ if any(want is not None and got != want for got, want in zip(tensor.shape[1:], shape[1:])):
32
+ raise ValueError(f"{spec.name}: unexpected shape {tensor.shape}; spec {spec.shape}")
33
+
34
+
35
+ def validate_physical_inputs(tensors: Mapping[str, np.ndarray], config: Config) -> int:
36
+ """Check physical batch size in addition to names, dtype and dimensions."""
37
+ if not isinstance(tensors, Mapping) or not tensors:
38
+ raise ValueError("inputs must be a nonempty tensor mapping")
39
+ first = next(iter(tensors.values()))
40
+ if not isinstance(first, np.ndarray) or first.ndim == 0:
41
+ raise ValueError("input tensors must be ndarrays with a batch dimension")
42
+ count = first.shape[0]
43
+ policy = config.runtime.batch
44
+ if count <= 0 or count > policy.size or (policy.mode == "fixed" and count != policy.size):
45
+ raise ValueError("invalid physical batch size")
46
+ validate_tensors(tensors, config.inputs, count)
47
+ return count
@@ -0,0 +1,8 @@
1
+ """Physical inference backends and their registration interface."""
2
+
3
+ from ai_inference.engines.base import BaseEngine, Engine
4
+ from ai_inference.engines.om import OMEngine
5
+ from ai_inference.engines.onnxruntime import ONNXRuntimeEngine
6
+ from ai_inference.engines.registry import EngineRegistry
7
+
8
+ __all__ = ["Engine", "BaseEngine", "ONNXRuntimeEngine", "OMEngine", "EngineRegistry"]
@@ -0,0 +1,183 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import TYPE_CHECKING, Mapping, Protocol
5
+
6
+ import numpy as np
7
+
8
+ from ai_inference.config import Config, TensorSpec
9
+ from ai_inference.contracts.validation import validate_physical_inputs, validate_tensors
10
+ from ai_inference.logging import log_context, log_event
11
+
12
+ if TYPE_CHECKING:
13
+ from loguru import Logger
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class TensorMetadata:
18
+ name: str
19
+ dtype: str
20
+ shape: tuple[int | str | None, ...]
21
+
22
+
23
+ class Engine(Protocol):
24
+ """Execute one physical batch, preserving names, dtype and leading batch axis."""
25
+
26
+ config: Config
27
+
28
+ def infer(self, tensors: Mapping[str, np.ndarray]) -> Mapping[str, np.ndarray]: ...
29
+ def warmup(self, tensors: Mapping[str, np.ndarray] | None = None) -> None: ...
30
+
31
+
32
+ class BaseEngine:
33
+ """Checked public inference entry; subclasses implement backend execution only."""
34
+
35
+ def __init__(self, config: Config):
36
+ self.config = config
37
+
38
+ def read_tensor_metadata(self) -> dict[str, tuple[TensorMetadata, ...]]:
39
+ """Return backend-neutral metadata; backends may inspect their session."""
40
+ return {
41
+ kind: tuple(
42
+ TensorMetadata(s.name, s.dtype, s.shape) for s in getattr(self.config, kind)
43
+ )
44
+ for kind in ("inputs", "outputs")
45
+ }
46
+
47
+ def _load_tensor_contract(self) -> None:
48
+ """Resolve optional YAML constraints in the backend's actual tensor order."""
49
+ data = self.config.model_dump()
50
+ for kind, tensors in self.read_tensor_metadata().items():
51
+ data[kind] = self._resolve_metadata(tensors, getattr(self.config, kind), kind)
52
+ self.config = Config.model_validate(data)
53
+
54
+ def _resolve_metadata(self, actual, specs, kind):
55
+ nodes = {n.name: n for n in actual}
56
+ if not actual or len(nodes) != len(actual):
57
+ raise ValueError(
58
+ f"{self.config.runtime.backend} {kind} metadata must have unique tensor names"
59
+ )
60
+ if specs is not None and set(nodes) != {s.name for s in specs}:
61
+ raise ValueError(
62
+ f"{self.config.runtime.backend} {kind} names do not match YAML contract"
63
+ )
64
+ expected = {s.name: s for s in specs or ()}
65
+ resolved = []
66
+ for node in actual:
67
+ dtype = node.dtype
68
+ shape = tuple(d if isinstance(d, int) else None for d in node.shape)
69
+ policy = self.config.runtime.batch
70
+ if shape:
71
+ model_batch = shape[0]
72
+ if model_batch is not None and (
73
+ policy.mode == "dynamic" or model_batch != policy.size
74
+ ):
75
+ raise ValueError(
76
+ f"{self.config.runtime.backend} {kind} tensor '{node.name}': "
77
+ f"model batch={model_batch} conflicts with "
78
+ f"runtime.batch.mode={policy.mode}, size={policy.size}; "
79
+ f"use mode=fixed, size={model_batch} for this static model"
80
+ )
81
+ if model_batch is None and policy.mode == "fixed":
82
+ shape = (policy.size, *shape[1:])
83
+ spec = expected.get(node.name)
84
+ if spec is not None:
85
+ if spec.dtype is not None and spec.dtype != dtype:
86
+ raise ValueError(
87
+ f"{self.config.runtime.backend} {kind} dtype mismatch: {node.name}"
88
+ )
89
+ if spec.shape is not None:
90
+ if len(shape) != len(spec.shape):
91
+ raise ValueError(
92
+ f"{self.config.runtime.backend} {kind} rank mismatch: {node.name}"
93
+ )
94
+ for actual_dim, configured in zip(shape, spec.shape):
95
+ if actual_dim is not None and actual_dim != configured:
96
+ raise ValueError(
97
+ f"{self.config.runtime.backend} {kind} shape mismatch: {node.name}"
98
+ )
99
+ # Concrete constraints on symbolic dimensions remain runtime checks.
100
+ shape = spec.shape
101
+ resolved.append(TensorSpec(name=node.name, dtype=dtype, shape=shape))
102
+ return resolved
103
+
104
+ def log_tensor_metadata(self, logger: Logger) -> None:
105
+ for kind, tensors in self.read_tensor_metadata().items():
106
+ resolved = {s.name: s for s in getattr(self.config, kind)}
107
+ for tensor in tensors:
108
+ resolved_shape = list(resolved[tensor.name].shape)
109
+ with log_context(logger, model=self.config.model.name):
110
+ direction = "Input" if kind == "inputs" else "Output"
111
+ log_event(
112
+ "init",
113
+ "engine.metadata",
114
+ f"{direction} '{tensor.name}' | {tensor.dtype} | shape={list(tensor.shape)} | resolved_shape={resolved_shape}",
115
+ kind=kind,
116
+ tensor_name=tensor.name,
117
+ dtype=tensor.dtype,
118
+ shape=list(tensor.shape),
119
+ resolved_shape=resolved_shape,
120
+ )
121
+
122
+ def create_warmup_inputs(self) -> Mapping[str, np.ndarray]:
123
+ """Default zero inputs; override for model-specific values or shapes."""
124
+ batch = self.config.runtime.batch.size if self.config.runtime.batch.mode == "fixed" else 1
125
+ tensors = {}
126
+ specs = self.config.inputs
127
+ if specs is None:
128
+ raise ValueError("warmup requires a resolved input tensor contract")
129
+ for spec in specs:
130
+ shape, dtype = spec.shape, spec.dtype
131
+ if shape is None or dtype is None:
132
+ raise ValueError("warmup requires a resolved input tensor contract")
133
+ dimensions = []
134
+ for dim in shape[1:]:
135
+ if dim is None:
136
+ raise ValueError(
137
+ "warmup requires concrete non-batch input dimensions; provide tensors, specify sizes or set runtime.warmup=false"
138
+ )
139
+ dimensions.append(dim)
140
+ tensors[spec.name] = np.zeros((batch, *dimensions), dtype=dtype)
141
+ return tensors
142
+
143
+ def warmup(self, tensors: Mapping[str, np.ndarray] | None = None) -> None:
144
+ """Create or accept valid physical inputs, then run the engine entry."""
145
+ completed = 0
146
+ try:
147
+ tensors = self.create_warmup_inputs() if tensors is None else tensors
148
+ batch = validate_physical_inputs(tensors, self.config)
149
+ for _ in range(self.config.runtime.warmup_runs):
150
+ self.infer(tensors)
151
+ completed += 1
152
+ except Exception as exc:
153
+ log_event(
154
+ "init",
155
+ "engine.warmup",
156
+ f"Failed | completed runs={completed} | error={type(exc).__name__}",
157
+ level="WARNING",
158
+ model=self.config.model.name,
159
+ error_type=type(exc).__name__,
160
+ completed_runs=completed,
161
+ status="failed",
162
+ )
163
+ raise
164
+ log_event(
165
+ "init",
166
+ "engine.warmup",
167
+ f"Completed | runs={self.config.runtime.warmup_runs} | batch={batch}",
168
+ model=self.config.model.name,
169
+ warmup_runs=self.config.runtime.warmup_runs,
170
+ physical_batch=batch,
171
+ status="completed",
172
+ )
173
+
174
+ def infer(self, tensors: Mapping[str, np.ndarray]) -> Mapping[str, np.ndarray]:
175
+ count = validate_physical_inputs(tensors, self.config)
176
+ outputs = self._infer(tensors)
177
+ if not isinstance(outputs, Mapping):
178
+ raise ValueError("outputs must be a tensor mapping")
179
+ validate_tensors(outputs, self.config.outputs, count)
180
+ return outputs
181
+
182
+ def _infer(self, tensors: Mapping[str, np.ndarray]) -> Mapping[str, np.ndarray]:
183
+ raise NotImplementedError
@@ -0,0 +1,79 @@
1
+ """Static Ascend OM execution through ais_bench, without image processing."""
2
+
3
+ from collections.abc import Mapping
4
+
5
+ import numpy as np
6
+ from pydantic import Field
7
+
8
+ from ai_inference.config import Config, Section
9
+ from ai_inference.engines.base import BaseEngine, TensorMetadata
10
+ from ai_inference.engines.registry import EngineRegistry
11
+
12
+
13
+ class OMOptions(Section):
14
+ device_id: int = Field(default=0, ge=0, strict=True)
15
+
16
+
17
+ @EngineRegistry.register("om")
18
+ class OMEngine(BaseEngine):
19
+ def __init__(self, config: Config):
20
+ super().__init__(config)
21
+ options = OMOptions.model_validate(config.runtime.options)
22
+ if config.runtime.batch.mode != "fixed":
23
+ raise ValueError("OMEngine currently supports static fixed-batch inference only")
24
+ self._load_session(options)
25
+ self._load_tensor_contract()
26
+
27
+ def _load_session(self, options: OMOptions) -> None:
28
+ try:
29
+ from ais_bench.infer.interface import InferSession
30
+ except ImportError as exc:
31
+ raise ImportError(
32
+ "OMEngine requires compatible ais_bench and ACLRuntime in an Ascend/CANN environment"
33
+ ) from exc
34
+ self.session = InferSession(options.device_id, str(self.config.model.path))
35
+
36
+ def read_tensor_metadata(self) -> dict[str, tuple[TensorMetadata, ...]]:
37
+ """Read actual OM names, dtypes and shapes in vendor execution order."""
38
+ dtypes = {
39
+ "float32": "float32",
40
+ "float16": "float16",
41
+ "double64": "float64",
42
+ "float64": "float64",
43
+ "int64": "int64",
44
+ "int32": "int32",
45
+ "uint8": "uint8",
46
+ "bool": "bool",
47
+ }
48
+ metadata = {}
49
+ for kind, nodes in (
50
+ ("inputs", self.session.get_inputs()),
51
+ ("outputs", self.session.get_outputs()),
52
+ ):
53
+ tensors = []
54
+ for node in nodes:
55
+ # ACLRuntime dtype is a pybind enum, not a NumPy dtype or ACL type ID.
56
+ dtype = node.datatype.name
57
+ if dtype not in dtypes:
58
+ raise ValueError(f"unsupported OM {kind} dtype: {node.name}: {dtype}")
59
+ shape = tuple(node.shape)
60
+ if not shape or any(not isinstance(d, int) or d <= 0 for d in shape):
61
+ raise ValueError(f"OM requires static positive {kind} dimensions: {node.name}")
62
+ tensors.append(TensorMetadata(node.name, dtypes[dtype], shape))
63
+ metadata[kind] = tuple(tensors)
64
+ return metadata
65
+
66
+ def _infer(self, tensors: Mapping[str, np.ndarray]) -> dict[str, np.ndarray]:
67
+ inputs, outputs = self.config.inputs, self.config.outputs
68
+ if inputs is None or outputs is None:
69
+ raise ValueError("OM requires a resolved tensor contract")
70
+ feeds = [np.ascontiguousarray(tensors[spec.name]) for spec in inputs]
71
+ values = self.session.infer(feeds)
72
+ if len(values) != len(outputs):
73
+ raise ValueError("OM output count does not match configured outputs")
74
+ result = {}
75
+ for spec, value in zip(outputs, values):
76
+ if not isinstance(value, np.ndarray):
77
+ raise ValueError("OM outputs must be ndarrays")
78
+ result[spec.name] = value
79
+ return result