inferlab-integration-specialized-engine 0.2.0__py3-none-any.whl

inferlab_integration_specialized_engine/__init__.py

@@ -0,0 +1,454 @@
+"""Planning and rendering for the shared token-only Specialized Engine contract."""
+
+from importlib.metadata import PackageNotFoundError, version
+from typing import Annotated
+
+from inferlab_adapter_sdk import (
+    AdapterErrorCode,
+    AdapterOperationError,
+    CaptureTargetRequirement,
+    CaptureWindowControlEndpoint,
+    CaptureWindowControlRequirement,
+    EndpointProtocol,
+    EndpointRequirement,
+    FrontendCoRendering,
+    FrontendGatewayComponent,
+    FrontendProcessRole,
+    GatewayFrontendBinding,
+    GatewayPlan,
+    GatewayTarget,
+    GatewayTargetEngine,
+    HttpActionSpec,
+    HttpMethod,
+    IntegrationIdentity,
+    Parallelism,
+    ParallelismAttention,
+    ParallelismExperts,
+    ParallelismOuter,
+    PlanServeInput,
+    PlanServeResult,
+    ProcessSpec,
+    ReadinessProbe,
+    ReadinessProbeHttp,
+    ReadinessProbeProcessAlive,
+    RenderedServeProcess,
+    RenderServeInput,
+    RenderServeResult,
+    RenderSource,
+    ServeProcessAllocationFrontend,
+    ServeProcessAllocationModelRank,
+    ServeReplicaRequirement,
+    ServeRoleKind,
+    ServeRoleLink,
+    ServeRoleLinkRequestRouting,
+    ServeRoleResult,
+    ServeTopology,
+    SettingValue,
+    effective_settings,
+    integration_identity,
+    rendered_frontend,
+    rendered_model_rank,
+    replica_id,
+    require_role,
+    split_serve_allocations,
+    validate_settings,
+)
+from pydantic import BaseModel, ConfigDict, Field
+
+_ADAPTER_DISTRIBUTION = "inferlab-integration-specialized-engine"
+_GATEWAY_BACKEND = "smg"
+_GATEWAY_IMPLEMENTATION = "tokenspeed-smg"
+_DEFERRED_WORKER_STARTUP_TIMEOUT_SECS = 2_147_483_647
+
+
+class EngineContractSettings(BaseModel):
+    """Settings shared by every implementation of the token Engine contract."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    default_max_output_tokens: Annotated[int, Field(ge=1)] = 16
+    max_num_batched_tokens: Annotated[int, Field(ge=1)] = 12_288
+
+
+def _identity() -> IntegrationIdentity:
+    return integration_identity(
+        adapter_id="inferlab-specialized-engine",
+        adapter_distribution=_ADAPTER_DISTRIBUTION,
+        framework="specialized-engine",
+        framework_distribution=_ADAPTER_DISTRIBUTION,
+        module_file=__file__,
+    )
+
+
+def _smg_version() -> str:
+    try:
+        return version("tokenspeed-smg")
+    except PackageNotFoundError:
+        return "unavailable"
+
+
+def _pure_tp_parallelism(
+    parallelism: Parallelism,
+    error_code: AdapterErrorCode = AdapterErrorCode.invalid_settings,
+) -> tuple[Parallelism, int]:
+    outer = parallelism.outer
+    tensor_parallel_size = (
+        outer.tensor_parallel_size
+        if outer is not None and outer.tensor_parallel_size is not None
+        else 1
+    )
+    non_tp_values = [
+        outer.pipeline_parallel_size if outer is not None else None,
+        (parallelism.attention.data_parallel_size if parallelism.attention is not None else None),
+        (
+            parallelism.attention.context_parallel_size
+            if parallelism.attention is not None
+            else None
+        ),
+        (parallelism.experts.data_parallel_size if parallelism.experts is not None else None),
+        (parallelism.experts.expert_parallel_size if parallelism.experts is not None else None),
+    ]
+    if any(value not in {None, 1} for value in non_tp_values):
+        raise AdapterOperationError(
+            error_code,
+            "the Specialized Engine contract supports only tensor parallelism",
+        )
+
+    component_tp_values = [
+        (parallelism.attention.tensor_parallel_size if parallelism.attention is not None else None),
+        (parallelism.experts.tensor_parallel_size if parallelism.experts is not None else None),
+        (
+            parallelism.experts.dense_tensor_parallel_size
+            if parallelism.experts is not None
+            else None
+        ),
+    ]
+    if any(value is not None and value != tensor_parallel_size for value in component_tp_values):
+        raise AdapterOperationError(
+            error_code,
+            "attention and expert tensor parallelism must match outer tensor parallelism",
+        )
+
+    effective = Parallelism(
+        outer=ParallelismOuter(
+            tensor_parallel_size=tensor_parallel_size,
+            pipeline_parallel_size=1,
+        ),
+        attention=ParallelismAttention(
+            tensor_parallel_size=tensor_parallel_size,
+            data_parallel_size=1,
+            context_parallel_size=1,
+        ),
+        experts=ParallelismExperts(
+            tensor_parallel_size=tensor_parallel_size,
+            data_parallel_size=1,
+            expert_parallel_size=1,
+            dense_tensor_parallel_size=tensor_parallel_size,
+        ),
+    )
+    return effective, tensor_parallel_size
+
+
+def _public_endpoint() -> EndpointRequirement:
+    return EndpointRequirement(
+        protocol=EndpointProtocol(),
+        completions_path="/v1/completions",
+        chat_completions_path="/v1/chat/completions",
+        prefix_cache_reset=HttpActionSpec(method=HttpMethod(), path="/flush_cache"),
+    )
+
+
+def plan_serve(input: PlanServeInput) -> PlanServeResult:
+    """Plan one token Engine behind one SMG Gateway."""
+    if input.topology != ServeTopology.single:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_settings,
+            "the Specialized Engine integration supports only single topology",
+        )
+    if input.gateway_backend != _GATEWAY_BACKEND:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_settings,
+            "the Specialized Engine integration requires Gateway backend smg",
+        )
+    if input.pd_router_backend is not None:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_settings,
+            "the Specialized Engine routed-single workflow must not select a P/D Router",
+        )
+    if input.kv_transfer is not None:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_settings,
+            "single topology does not use KV transfer",
+        )
+    role = require_role(input, ServeRoleKind.serve)
+    if role.replica_count != 1:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_settings,
+            "the Specialized Engine integration supports exactly one replica",
+        )
+    settings = validate_settings(EngineContractSettings, role.settings)
+    parallelism, tensor_parallel_size = _pure_tp_parallelism(role.parallelism)
+    role_result = ServeRoleResult(
+        id=role.id,
+        kind=role.kind,
+        declared_replica_count=role.replica_count,
+        effective_replica_count=role.replica_count,
+        effective_settings=effective_settings(settings),
+        effective_parallelism=parallelism,
+        public_endpoint=None,
+    )
+    gateway = GatewayPlan(
+        backend=_GATEWAY_BACKEND,
+        implementation=_GATEWAY_IMPLEMENTATION,
+        implementation_version=_smg_version(),
+        effective_settings={
+            "worker_protocol": SettingValue(root="tokenspeed_scheduler_v1"),
+            "policy": SettingValue(root="passthrough"),
+            "retries": SettingValue(root=False),
+            "circuit_breaker": SettingValue(root=False),
+        },
+        endpoint=_public_endpoint(),
+        readiness=ReadinessProbe(root=ReadinessProbeHttp(path="/readiness")),
+        ports=["prometheus"],
+        targets=[GatewayTarget(root=GatewayTargetEngine(role=role.id))],
+        render_inputs=[],
+        render_source=RenderSource.integration,
+        co_rendering=FrontendCoRendering(process_role=FrontendProcessRole()),
+    )
+    return PlanServeResult(
+        integration=_identity(),
+        roles=[role_result],
+        replicas=[
+            ServeReplicaRequirement(
+                id=replica_id(role, 0),
+                role_id=role.id,
+                replica_index=0,
+                device_count=tensor_parallel_size,
+                ports=[],
+                primary_ports=[],
+                primary_readiness=ReadinessProbe(root=ReadinessProbeProcessAlive()),
+                worker_readiness=ReadinessProbe(root=ReadinessProbeProcessAlive()),
+                capture_target=(
+                    CaptureTargetRequirement(
+                        window_control=CaptureWindowControlRequirement(
+                            endpoint=CaptureWindowControlEndpoint.gateway,
+                            start=HttpActionSpec(method=HttpMethod(), path="/start_profile"),
+                            stop=HttpActionSpec(method=HttpMethod(), path="/stop_profile"),
+                        )
+                    )
+                    if input.profiling
+                    else None
+                ),
+            )
+        ],
+        links=[
+            ServeRoleLink(root=ServeRoleLinkRequestRouting(source="gateway", targets=[role.id]))
+        ],
+        gateway=gateway,
+        pd_router=None,
+    )
+
+
+def _require_engine(
+    allocations: list[ServeProcessAllocationModelRank],
+) -> ServeProcessAllocationModelRank:
+    if len(allocations) != 1:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the Specialized Engine integration requires one model-rank allocation",
+        )
+    engine = allocations[0]
+    if (
+        engine.role_kind != ServeRoleKind.serve
+        or engine.replica != 0
+        or engine.rank != 0
+        or engine.rank_count != 1
+    ):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the Engine allocation must be serve replica 0 in one rank process",
+        )
+    effective_parallelism, tensor_parallel_size = _pure_tp_parallelism(
+        engine.effective_parallelism,
+        AdapterErrorCode.invalid_request,
+    )
+    if (
+        engine.effective_parallelism != effective_parallelism
+        or len(engine.devices) != tensor_parallel_size
+    ):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the Engine rank process must own one device per effective tensor-parallel rank",
+        )
+    if engine.endpoint is None:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the Engine allocation is missing its endpoint",
+        )
+    return engine
+
+
+def _require_gateway(allocation: object, engine_role: str) -> ServeProcessAllocationFrontend:
+    if not isinstance(allocation, ServeProcessAllocationFrontend):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the routed-single workflow requires one Gateway frontend allocation",
+        )
+    if not isinstance(allocation.components.root, GatewayFrontendBinding):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the routed-single frontend must bind only [gateway]",
+        )
+    if allocation.components.root.root != [FrontendGatewayComponent()]:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the routed-single frontend must bind only [gateway]",
+        )
+    gateway = allocation.gateway
+    if (
+        gateway.backend != _GATEWAY_BACKEND
+        or gateway.implementation != _GATEWAY_IMPLEMENTATION
+        or gateway.implementation_version != _smg_version()
+        or gateway.render_source != RenderSource.integration
+        or allocation.pd_router is not None
+        or allocation.process_role != gateway.co_rendering.process_role
+    ):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the frontend allocation does not preserve the planned SMG Gateway",
+        )
+    targets = gateway.targets
+    if (
+        len(targets) != 1
+        or not isinstance(targets[0].root, GatewayTargetEngine)
+        or targets[0].root.role != engine_role
+    ):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the SMG Gateway must target the sole Engine role",
+        )
+    return allocation
+
+
+def _render_engine(
+    input: RenderServeInput,
+    allocation: ServeProcessAllocationModelRank,
+) -> RenderedServeProcess:
+    endpoint = allocation.endpoint
+    if endpoint is None:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the Engine allocation is missing its endpoint",
+        )
+    settings = validate_settings(EngineContractSettings, allocation.effective_settings)
+    _, tensor_parallel_size = _pure_tp_parallelism(
+        allocation.effective_parallelism,
+        AdapterErrorCode.invalid_request,
+    )
+    return rendered_model_rank(
+        allocation,
+        ProcessSpec(
+            argv=[
+                "inferlab-token-engine",
+                "smg-worker",
+                "--listen",
+                f"{endpoint.host}:{endpoint.port}",
+                "--model",
+                allocation.model_locator,
+                "--served-model-name",
+                input.model.served_name,
+                "--tensor-parallel-size",
+                str(tensor_parallel_size),
+                "--default-max-output-tokens",
+                str(settings.default_max_output_tokens),
+                "--max-num-batched-tokens",
+                str(settings.max_num_batched_tokens),
+            ],
+            env={},
+        ),
+    )
+
+
+def _render_gateway(
+    allocation: ServeProcessAllocationFrontend,
+    engine: ServeProcessAllocationModelRank,
+) -> RenderedServeProcess:
+    engine_endpoint = engine.endpoint
+    if engine_endpoint is None:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the Engine allocation is missing its endpoint",
+        )
+    prometheus = allocation.ports.get("prometheus")
+    if prometheus is None:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the SMG Gateway allocation is missing its Prometheus port",
+        )
+    return rendered_frontend(
+        allocation,
+        ProcessSpec(
+            argv=[
+                "smg",
+                "launch",
+                "--host",
+                "0.0.0.0",
+                "--port",
+                str(allocation.endpoint.port),
+                "--prometheus-port",
+                str(prometheus.port),
+                "--worker-startup-timeout-secs",
+                str(_DEFERRED_WORKER_STARTUP_TIMEOUT_SECS),
+                "--worker-urls",
+                f"grpc://{engine_endpoint.host}:{engine_endpoint.port}",
+                "--model-path",
+                engine.model_locator,
+                "--tokenizer-path",
+                engine.model_locator,
+                "--policy",
+                "passthrough",
+                "--disable-retries",
+                "--disable-circuit-breaker",
+            ],
+            env={},
+        ),
+    )
+
+
+def render_serve(input: RenderServeInput) -> RenderServeResult:
+    if (
+        input.topology != ServeTopology.single
+        or input.gateway_backend != _GATEWAY_BACKEND
+        or input.pd_router_backend is not None
+        or input.kv_transfer is not None
+    ):
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "render input is not the planned routed-single SMG workflow",
+        )
+    allocations, model_allocations = split_serve_allocations(input.allocations)
+    if len(allocations) != 2:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the routed-single workflow requires one Engine and one Gateway allocation",
+        )
+    engine = _require_engine(model_allocations)
+    frontend_candidates = [
+        allocation
+        for allocation in allocations
+        if isinstance(allocation, ServeProcessAllocationFrontend)
+    ]
+    if len(frontend_candidates) != 1:
+        raise AdapterOperationError(
+            AdapterErrorCode.invalid_request,
+            "the routed-single workflow requires one Gateway allocation",
+        )
+    _require_gateway(frontend_candidates[0], engine.role)
+
+    processes: list[RenderedServeProcess] = []
+    for allocation in allocations:
+        if isinstance(allocation, ServeProcessAllocationModelRank):
+            processes.append(_render_engine(input, allocation))
+        elif isinstance(allocation, ServeProcessAllocationFrontend):
+            processes.append(_render_gateway(allocation, engine))
+    return RenderServeResult(integration=_identity(), processes=processes)

inferlab_integration_specialized_engine/__main__.py

@@ -0,0 +1,11 @@
+from inferlab_adapter_sdk import run_adapter
+
+from . import plan_serve, render_serve
+
+
+def main() -> None:
+    raise SystemExit(run_adapter(plan_serve, render_serve=render_serve))
+
+
+if __name__ == "__main__":
+    main()

inferlab_integration_specialized_engine-0.2.0.dist-info/METADATA

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: inferlab-integration-specialized-engine
+Version: 0.2.0
+Summary: Inferlab adapter integration for token-only Specialized Engines
+Project-URL: Repository, https://github.com/Infer-Lab/InferLab
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: inferlab-adapter-sdk==0.5.0
+Description-Content-Type: text/markdown
+
+# Inferlab Integration for Specialized Engines
+
+This package connects Inferlab to a hardware-by-model Specialized Engine that
+implements the canonical `inferlab-token-engine smg-worker` command. The Engine
+accepts prompt token IDs and returns generated token IDs over SMG's worker
+protocol. TokenSpeed SMG remains responsible for the public HTTP API,
+tokenization, chat templates, detokenization, and response formatting.
+
+The integration contains no model-, architecture-, hardware-, or
+Engine-implementation-specific lowering. A downstream workspace supplies the
+Rust Engine binary, SMG, model intent, source revision, locked environment, and
+private placement bindings. Consequently, a new conforming Engine does not
+need another Inferlab integration package.
+
+The supported shape is deliberately closed: one `single` Engine replica in one
+process behind one SMG Gateway. The process owns an arbitrary nonzero pure-TP
+device set; attention, expert, and dense-expert tensor parallelism all equal
+the outer TP width, while pipeline, data, context, and expert parallelism stay
+at one. The contract remains serial and has no P/D Router, KV-transfer,
+batching, or Engine-local profiling surface. InferLab can profile the Engine
+process tree while TokenSpeed SMG exposes the capture-window
+`POST /start_profile` and `POST /stop_profile` actions on the Gateway.

inferlab_integration_specialized_engine-0.2.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+inferlab_integration_specialized_engine/__init__.py,sha256=nIhQRyuKpHDC0JynfJ3hk2GIqDCsZ2Ttx2dgLQhzdT4,16347
+inferlab_integration_specialized_engine/__main__.py,sha256=e1uoS23_cXd5sMrRK_nSuUPgdz37Tx1xBhxqMlCYhBg,220
+inferlab_integration_specialized_engine-0.2.0.dist-info/METADATA,sha256=v5AnpmvUcWT73awqu2hSmTEDhHi0jPinY-fAOFZDKDA,1731
+inferlab_integration_specialized_engine-0.2.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+inferlab_integration_specialized_engine-0.2.0.dist-info/entry_points.txt,sha256=rXD9hNQ3ll_gLI7tVT5Dsu2WR9BApLuiOCiWHzALbZg,110
+inferlab_integration_specialized_engine-0.2.0.dist-info/licenses/LICENSE,sha256=_sFBKPJJjTLuE3lL3Ir9HDHTTrB9zsZtMBAOvBfBWA4,1065
+inferlab_integration_specialized_engine-0.2.0.dist-info/RECORD,,

inferlab_integration_specialized_engine-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

inferlab_integration_specialized_engine-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+inferlab-adapter-specialized-engine = inferlab_integration_specialized_engine.__main__:main

inferlab_integration_specialized_engine-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zihua Wu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.