hugpy-engine 0.3.5__tar.gz → 0.3.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hugpy_engine-0.3.7/PKG-INFO +204 -0
- hugpy_engine-0.3.7/README.md +185 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/pyproject.toml +1 -1
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/categories.py +26 -7
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/config/models/models_config.py +84 -28
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/dispatch/dispatch.py +187 -68
- hugpy_engine-0.3.7/src/hugpy_engine/imports/__init__.py +12 -0
- hugpy_engine-0.3.7/src/hugpy_engine/imports/classes.py +5 -0
- hugpy_engine-0.3.7/src/hugpy_engine/imports/constants.py +57 -0
- hugpy_engine-0.3.7/src/hugpy_engine/imports/functions.py +46 -0
- hugpy_engine-0.3.7/src/hugpy_engine/imports/init_imports.py +13 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/resolvers/__init__.py +2 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/resolvers/assure_model_key.py +26 -22
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/resolvers/model_resolver.py +118 -38
- hugpy_engine-0.3.7/src/hugpy_engine/resolvers/task_resolution.py +643 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/task_schemas.py +9 -0
- hugpy_engine-0.3.7/src/hugpy_engine/utils/__init__.py +1 -0
- hugpy_engine-0.3.7/src/hugpy_engine/utils/json_scavenge.py +111 -0
- hugpy_engine-0.3.7/src/hugpy_engine/utils/model_marker.py +35 -0
- hugpy_engine-0.3.7/src/hugpy_engine/utils/no_think.py +323 -0
- hugpy_engine-0.3.7/src/hugpy_engine.egg-info/PKG-INFO +204 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine.egg-info/SOURCES.txt +11 -1
- hugpy_engine-0.3.5/PKG-INFO +0 -22
- hugpy_engine-0.3.5/README.md +0 -3
- hugpy_engine-0.3.5/src/hugpy_engine.egg-info/PKG-INFO +0 -22
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/setup.cfg +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/config/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/config/main.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/config/models/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/config/models/models_default.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/dispatch/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/fit/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/fit/types.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/llama/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/llama/runners/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/llama/runners/fitevict_runner.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/name_match.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/placement.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/py.typed +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/resolvers/categories/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/resolvers/categories/builders.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/resolvers/categories/frameworks.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/__init__.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/chat_schemas.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/event_schemas.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/metadata_schemas.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/model_schemas.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/schemas/runner_schemas.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/task_deps.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/task_endpoints.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine/tasks.py +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine.egg-info/dependency_links.txt +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine.egg-info/requires.txt +0 -0
- {hugpy_engine-0.3.5 → hugpy_engine-0.3.7}/src/hugpy_engine.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hugpy-engine
|
|
3
|
+
Version: 0.3.7
|
|
4
|
+
Summary: Hugpy engine: model discovery and classification, allocation and eviction, native llama.cpp and GGUF runners, and the prompt-to-reply inference path with task and placement seams
|
|
5
|
+
Author-email: putkoff <support@hugpy.ai>
|
|
6
|
+
License-Expression: LicenseRef-Proprietary
|
|
7
|
+
Keywords: hugpy,llm,self-hosted,inference,llama.cpp,gguf,transformers,model-serving
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Requires-Dist: hugpy-platform
|
|
17
|
+
Requires-Dist: abstract-essentials
|
|
18
|
+
Requires-Dist: pydantic>=2
|
|
19
|
+
|
|
20
|
+
# hugpy-engine
|
|
21
|
+
|
|
22
|
+
Inference resolution for the Hugpy arms. It turns "a prompt and maybe a file"
|
|
23
|
+
into a model and a task, and hands the call to central's `/v1` relay. Central
|
|
24
|
+
picks the worker, and that worker's wrapper does evict-to-fit, load, serving
|
|
25
|
+
and the call log. The engine itself does not load models in fleet mode.
|
|
26
|
+
|
|
27
|
+
Distribution `hugpy-engine`, import `hugpy_engine`. Dependencies:
|
|
28
|
+
`hugpy-platform`, `abstract-essentials`, `pydantic>=2`.
|
|
29
|
+
|
|
30
|
+
**Status: LIVE.** It is a library only: no service, no console script.
|
|
31
|
+
|
|
32
|
+
- Installed in `/opt/hugpy-media/venv` (0.3.7) and `/opt/hugpy-video/venv`
|
|
33
|
+
(0.3.8).
|
|
34
|
+
- Imported by `hugpy_media`, `hugpy_video` and `hugpy_oracle`.
|
|
35
|
+
- Core's live basic mode does not use it. Only core's legacy
|
|
36
|
+
`config/prompt.py` does, and that code is not live.
|
|
37
|
+
|
|
38
|
+
On 2026-10-15 the engine's last ties to `hugpy-storage` and `hugpy-control`
|
|
39
|
+
are being removed, targeting 0.3.9 and later:
|
|
40
|
+
|
|
41
|
+
- `vl_gguf_tasks` is ported in from `hugpy_storage.hugpy_marker`.
|
|
42
|
+
- The `hugpy_storage.model_paths.route_destination` call is dropped; it was
|
|
43
|
+
dead in fleet mode.
|
|
44
|
+
- `hugpy_storage.console.downloader.model_status` is replaced by registry
|
|
45
|
+
availability from `/v1/models`.
|
|
46
|
+
- The guarded imports of the retired `hugpy_control.settings.settings_store`
|
|
47
|
+
are deleted.
|
|
48
|
+
|
|
49
|
+
Check `pyproject.toml` for the current version. See
|
|
50
|
+
[docs/packages.md](../../docs/packages.md#versions-in-flight).
|
|
51
|
+
|
|
52
|
+
## Public API
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from hugpy_engine import (
|
|
56
|
+
execute_prompt, execute_prompt_stream, execute_chat_stream,
|
|
57
|
+
resolve, resolve_model_key,
|
|
58
|
+
MODEL_REGISTRY, get_model_registry, refresh_registry,
|
|
59
|
+
normalize_prompt_kwargs, infer_arg_name, runner_for,
|
|
60
|
+
)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Positional arguments to `execute_prompt` are inferred by type:
|
|
64
|
+
|
|
65
|
+
| type | meaning |
|
|
66
|
+
|---|---|
|
|
67
|
+
| `str` | a model id or a prompt |
|
|
68
|
+
| `int` | `max_new_tokens` |
|
|
69
|
+
| `float` | `temperature`, then `top_p` |
|
|
70
|
+
| `bool` | `do_sample` |
|
|
71
|
+
| `list` | `messages` |
|
|
72
|
+
|
|
73
|
+
`execute_prompt_stream` is a single resolve, build and runner pass yielded as
|
|
74
|
+
events (`TokenEvent`, `DoneEvent`, `ErrorEvent`). It accepts `cancel_event`.
|
|
75
|
+
`execute_chat_stream` adds chat continuation on top of it.
|
|
76
|
+
|
|
77
|
+
Chat and vision run today. A task whose runner has no front-door endpoint yet
|
|
78
|
+
resolves to a clean, named error (fail-as-data).
|
|
79
|
+
|
|
80
|
+
## execute_prompt: infer, rank, assure
|
|
81
|
+
|
|
82
|
+
Full reference: [docs/execute_prompt.md](../../docs/execute_prompt.md).
|
|
83
|
+
|
|
84
|
+
`execute_prompt` is not plain task routing. Its defining contract
|
|
85
|
+
(operator): it "would infer programmatically what the user was looking to
|
|
86
|
+
prompt, via file inspection, prompt type, and anything that necessarily
|
|
87
|
+
indicated a model type — most assuredly to receive a response than most any
|
|
88
|
+
other query method." Task bindings are only ONE input. The mechanic has three
|
|
89
|
+
stages, all in the stdlib-only module `hugpy_engine.resolvers.task_resolution`,
|
|
90
|
+
so that central can later serve the same resolution.
|
|
91
|
+
|
|
92
|
+
1. **INFER.** `task_resolution.infer(args, kwargs)` works out WHAT to prompt
|
|
93
|
+
from several signals:
|
|
94
|
+
- positional args: a bare prompt, a messages list, an existing file, a
|
|
95
|
+
model-ish string, a `data:` URL or an http(s) URL
|
|
96
|
+
- file type: the extension maps to image, audio, video, document, code or
|
|
97
|
+
text
|
|
98
|
+
- message parts: `image_url`, `input_image`, audio, `data:` URLs
|
|
99
|
+
- explicit cues: `task=`, `model_format=`
|
|
100
|
+
|
|
101
|
+
It derives the task only when the caller gave no explicit `task` and no
|
|
102
|
+
explicit `model_key`:
|
|
103
|
+
|
|
104
|
+
| input | task |
|
|
105
|
+
|---|---|
|
|
106
|
+
| image | image-text-to-text |
|
|
107
|
+
| audio or video | asr |
|
|
108
|
+
| pdf or doc | extract + text |
|
|
109
|
+
| text | text-generation |
|
|
110
|
+
|
|
111
|
+
One-call use works: `execute_prompt("describe this", "/path/img.png")`
|
|
112
|
+
needs no task or model. Inference only ever *adds* a task. It never
|
|
113
|
+
overrides an explicit one, and a failure inside it never blocks the call.
|
|
114
|
+
|
|
115
|
+
2. **RANK.** `task_resolution.resolve(...)` returns an ORDERED candidate
|
|
116
|
+
list, strongest first, by source precedence:
|
|
117
|
+
|
|
118
|
+
explicit model_key → binding → configured default → capability match
|
|
119
|
+
|
|
120
|
+
Capability matches are ranked **resident > available on a healthy worker
|
|
121
|
+
> other**. A row whose `primary_task` matches the task is preferred, then
|
|
122
|
+
order is deterministic by id. "No model in the fleet lists task X" appears
|
|
123
|
+
ONLY when no registry model can serve X. It is never a dead-end "bind one
|
|
124
|
+
in Settings".
|
|
125
|
+
|
|
126
|
+
3. **ASSURE.** `execute_prompt` tries the candidates in order.
|
|
127
|
+
- On a MODEL-SIDE failure (worker down, load refusal, timeout, 5xx, empty
|
|
128
|
+
reply), it moves to the next candidate.
|
|
129
|
+
- A CALLER error (4xx validation, bad args) is surfaced immediately. The
|
|
130
|
+
same bad request would fail on every model, so it is never cascaded.
|
|
131
|
+
- The first real answer wins and carries `resolution` metadata:
|
|
132
|
+
`{task, source, model_key, attempts:[...]}`.
|
|
133
|
+
- When every candidate fails, it returns a fail-as-data result (`ok=False`)
|
|
134
|
+
with the full attempt list, never a bare exception that hides what was
|
|
135
|
+
tried.
|
|
136
|
+
|
|
137
|
+
### Registry and bindings source
|
|
138
|
+
|
|
139
|
+
**The registry** is built from basic core's `GET /v1/models` rows: `id`,
|
|
140
|
+
`hugpy_json.tasks`, `task_override`, `engine`, `resident`, `available`,
|
|
141
|
+
`workers` and `resident_on`. `derive_model_config_row` maps each row onto
|
|
142
|
+
`ModelConfig`. If that fails, it falls back to the legacy `GET /models`, and
|
|
143
|
+
then to a local `hugpy.json` disk scan of `HUGPY_MODELS_HOME` /
|
|
144
|
+
`LLAMA_CPP_MODEL_DIR`. The scan is fail-soft and fails fast on an EIO models
|
|
145
|
+
disk.
|
|
146
|
+
|
|
147
|
+
**Bindings** are read from `GET /v1/task-bindings` (`{"defaults": {...}}`),
|
|
148
|
+
then the legacy `GET /prompt/tasks`. A 404 means "no bindings". Live basic
|
|
149
|
+
core (0.4.40) serves neither route, so the binding stage is empty today and
|
|
150
|
+
resolution relies on explicit keys, configured defaults and capability
|
|
151
|
+
matches.
|
|
152
|
+
|
|
153
|
+
Both reads are TTL-cached and never raise on the resolve path.
|
|
154
|
+
|
|
155
|
+
### Configuration
|
|
156
|
+
|
|
157
|
+
`hugpy_engine.imports.constants` names every environment variable, timeout and
|
|
158
|
+
TTL the chain reads:
|
|
159
|
+
|
|
160
|
+
| variable | default | meaning |
|
|
161
|
+
|---|---|---|
|
|
162
|
+
| `HUGPY_CORE_URL` (legacy alias `HUGPY_FITEVICT_FRONT_DOOR`) | `http://127.0.0.1:7020` | central's base URL |
|
|
163
|
+
| `HUGPY_REGISTRY_TTL_S` | 30 | registry and bindings cache lifetime |
|
|
164
|
+
| `HUGPY_REGISTRY_TIMEOUT_S` | 10 | registry fetch timeout |
|
|
165
|
+
| `HUGPY_TASK_ENDPOINT_TIMEOUT_S` | 900 | task endpoint call timeout |
|
|
166
|
+
| `HUGPY_FITEVICT_TIMEOUT_S` | 900 | `/v1` call timeout |
|
|
167
|
+
| `HUGPY_MODELS_HOME`, `LLAMA_CPP_MODEL_DIR` | unset | roots for the local disk-scan fallback |
|
|
168
|
+
|
|
169
|
+
The same module holds the `TASK_DEFAULTS`, `MEDIA_DEFAULTS` and
|
|
170
|
+
`MEDIA_TYPE_TO_TASK` tables.
|
|
171
|
+
|
|
172
|
+
## The task plugin seam
|
|
173
|
+
|
|
174
|
+
`hugpy_engine.tasks` holds the task registry. Packages add tasks through the
|
|
175
|
+
`hugpy_engine.tasks` entry-point group (a zero-argument callable), which
|
|
176
|
+
`tasks.load_entry_points()` imports. Two packages register today:
|
|
177
|
+
|
|
178
|
+
- `hugpy-media`: `media = "hugpy_media.plugin:register"`
|
|
179
|
+
- `hugpy-video`: `video = "hugpy_video.plugin:register"`
|
|
180
|
+
|
|
181
|
+
`task_endpoints.TASK_ROUTES` maps tasks to the front-door endpoints.
|
|
182
|
+
|
|
183
|
+
## Layout
|
|
184
|
+
|
|
185
|
+
```
|
|
186
|
+
hugpy_engine/
|
|
187
|
+
dispatch/ execute_prompt / execute_prompt_stream / execute_chat_stream; per-(model, task) runner cache
|
|
188
|
+
resolvers/ task_resolution (infer / rank, stdlib only), model_resolver (resolve), assure_model_key,
|
|
189
|
+
categories/ (task + framework registries)
|
|
190
|
+
config/ models/ (MODEL_REGISTRY, models_config, defaults), main
|
|
191
|
+
llama/runners/ fitevict_runner: the runner that sends a call to central's /v1
|
|
192
|
+
schemas/ chat, event, metadata, model, runner, task schemas (pydantic)
|
|
193
|
+
fit/, placement.py placement types and null single-box defaults
|
|
194
|
+
tasks.py, task_endpoints.py, task_deps.py, categories.py, name_match.py
|
|
195
|
+
utils/ json_scavenge, no_think
|
|
196
|
+
imports/ constants (every env name), functions, classes, init_imports
|
|
197
|
+
(docs/packages.md#the-imports-layout-convention)
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
See also:
|
|
201
|
+
|
|
202
|
+
- [docs/execute_prompt.md](../../docs/execute_prompt.md)
|
|
203
|
+
- [docs/arms.md](../../docs/arms.md)
|
|
204
|
+
- [docs/packages.md](../../docs/packages.md)
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# hugpy-engine
|
|
2
|
+
|
|
3
|
+
Inference resolution for the Hugpy arms. It turns "a prompt and maybe a file"
|
|
4
|
+
into a model and a task, and hands the call to central's `/v1` relay. Central
|
|
5
|
+
picks the worker, and that worker's wrapper does evict-to-fit, load, serving
|
|
6
|
+
and the call log. The engine itself does not load models in fleet mode.
|
|
7
|
+
|
|
8
|
+
Distribution `hugpy-engine`, import `hugpy_engine`. Dependencies:
|
|
9
|
+
`hugpy-platform`, `abstract-essentials`, `pydantic>=2`.
|
|
10
|
+
|
|
11
|
+
**Status: LIVE.** It is a library only: no service, no console script.
|
|
12
|
+
|
|
13
|
+
- Installed in `/opt/hugpy-media/venv` (0.3.7) and `/opt/hugpy-video/venv`
|
|
14
|
+
(0.3.8).
|
|
15
|
+
- Imported by `hugpy_media`, `hugpy_video` and `hugpy_oracle`.
|
|
16
|
+
- Core's live basic mode does not use it. Only core's legacy
|
|
17
|
+
`config/prompt.py` does, and that code is not live.
|
|
18
|
+
|
|
19
|
+
On 2026-10-15 the engine's last ties to `hugpy-storage` and `hugpy-control`
|
|
20
|
+
are being removed, targeting 0.3.9 and later:
|
|
21
|
+
|
|
22
|
+
- `vl_gguf_tasks` is ported in from `hugpy_storage.hugpy_marker`.
|
|
23
|
+
- The `hugpy_storage.model_paths.route_destination` call is dropped; it was
|
|
24
|
+
dead in fleet mode.
|
|
25
|
+
- `hugpy_storage.console.downloader.model_status` is replaced by registry
|
|
26
|
+
availability from `/v1/models`.
|
|
27
|
+
- The guarded imports of the retired `hugpy_control.settings.settings_store`
|
|
28
|
+
are deleted.
|
|
29
|
+
|
|
30
|
+
Check `pyproject.toml` for the current version. See
|
|
31
|
+
[docs/packages.md](../../docs/packages.md#versions-in-flight).
|
|
32
|
+
|
|
33
|
+
## Public API
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from hugpy_engine import (
|
|
37
|
+
execute_prompt, execute_prompt_stream, execute_chat_stream,
|
|
38
|
+
resolve, resolve_model_key,
|
|
39
|
+
MODEL_REGISTRY, get_model_registry, refresh_registry,
|
|
40
|
+
normalize_prompt_kwargs, infer_arg_name, runner_for,
|
|
41
|
+
)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Positional arguments to `execute_prompt` are inferred by type:
|
|
45
|
+
|
|
46
|
+
| type | meaning |
|
|
47
|
+
|---|---|
|
|
48
|
+
| `str` | a model id or a prompt |
|
|
49
|
+
| `int` | `max_new_tokens` |
|
|
50
|
+
| `float` | `temperature`, then `top_p` |
|
|
51
|
+
| `bool` | `do_sample` |
|
|
52
|
+
| `list` | `messages` |
|
|
53
|
+
|
|
54
|
+
`execute_prompt_stream` is a single resolve, build and runner pass yielded as
|
|
55
|
+
events (`TokenEvent`, `DoneEvent`, `ErrorEvent`). It accepts `cancel_event`.
|
|
56
|
+
`execute_chat_stream` adds chat continuation on top of it.
|
|
57
|
+
|
|
58
|
+
Chat and vision run today. A task whose runner has no front-door endpoint yet
|
|
59
|
+
resolves to a clean, named error (fail-as-data).
|
|
60
|
+
|
|
61
|
+
## execute_prompt: infer, rank, assure
|
|
62
|
+
|
|
63
|
+
Full reference: [docs/execute_prompt.md](../../docs/execute_prompt.md).
|
|
64
|
+
|
|
65
|
+
`execute_prompt` is not plain task routing. Its defining contract
|
|
66
|
+
(operator): it "would infer programmatically what the user was looking to
|
|
67
|
+
prompt, via file inspection, prompt type, and anything that necessarily
|
|
68
|
+
indicated a model type — most assuredly to receive a response than most any
|
|
69
|
+
other query method." Task bindings are only ONE input. The mechanic has three
|
|
70
|
+
stages, all in the stdlib-only module `hugpy_engine.resolvers.task_resolution`,
|
|
71
|
+
so that central can later serve the same resolution.
|
|
72
|
+
|
|
73
|
+
1. **INFER.** `task_resolution.infer(args, kwargs)` works out WHAT to prompt
|
|
74
|
+
from several signals:
|
|
75
|
+
- positional args: a bare prompt, a messages list, an existing file, a
|
|
76
|
+
model-ish string, a `data:` URL or an http(s) URL
|
|
77
|
+
- file type: the extension maps to image, audio, video, document, code or
|
|
78
|
+
text
|
|
79
|
+
- message parts: `image_url`, `input_image`, audio, `data:` URLs
|
|
80
|
+
- explicit cues: `task=`, `model_format=`
|
|
81
|
+
|
|
82
|
+
It derives the task only when the caller gave no explicit `task` and no
|
|
83
|
+
explicit `model_key`:
|
|
84
|
+
|
|
85
|
+
| input | task |
|
|
86
|
+
|---|---|
|
|
87
|
+
| image | image-text-to-text |
|
|
88
|
+
| audio or video | asr |
|
|
89
|
+
| pdf or doc | extract + text |
|
|
90
|
+
| text | text-generation |
|
|
91
|
+
|
|
92
|
+
One-call use works: `execute_prompt("describe this", "/path/img.png")`
|
|
93
|
+
needs no task or model. Inference only ever *adds* a task. It never
|
|
94
|
+
overrides an explicit one, and a failure inside it never blocks the call.
|
|
95
|
+
|
|
96
|
+
2. **RANK.** `task_resolution.resolve(...)` returns an ORDERED candidate
|
|
97
|
+
list, strongest first, by source precedence:
|
|
98
|
+
|
|
99
|
+
explicit model_key → binding → configured default → capability match
|
|
100
|
+
|
|
101
|
+
Capability matches are ranked **resident > available on a healthy worker
|
|
102
|
+
> other**. A row whose `primary_task` matches the task is preferred, then
|
|
103
|
+
order is deterministic by id. "No model in the fleet lists task X" appears
|
|
104
|
+
ONLY when no registry model can serve X. It is never a dead-end "bind one
|
|
105
|
+
in Settings".
|
|
106
|
+
|
|
107
|
+
3. **ASSURE.** `execute_prompt` tries the candidates in order.
|
|
108
|
+
- On a MODEL-SIDE failure (worker down, load refusal, timeout, 5xx, empty
|
|
109
|
+
reply), it moves to the next candidate.
|
|
110
|
+
- A CALLER error (4xx validation, bad args) is surfaced immediately. The
|
|
111
|
+
same bad request would fail on every model, so it is never cascaded.
|
|
112
|
+
- The first real answer wins and carries `resolution` metadata:
|
|
113
|
+
`{task, source, model_key, attempts:[...]}`.
|
|
114
|
+
- When every candidate fails, it returns a fail-as-data result (`ok=False`)
|
|
115
|
+
with the full attempt list, never a bare exception that hides what was
|
|
116
|
+
tried.
|
|
117
|
+
|
|
118
|
+
### Registry and bindings source
|
|
119
|
+
|
|
120
|
+
**The registry** is built from basic core's `GET /v1/models` rows: `id`,
|
|
121
|
+
`hugpy_json.tasks`, `task_override`, `engine`, `resident`, `available`,
|
|
122
|
+
`workers` and `resident_on`. `derive_model_config_row` maps each row onto
|
|
123
|
+
`ModelConfig`. If that fails, it falls back to the legacy `GET /models`, and
|
|
124
|
+
then to a local `hugpy.json` disk scan of `HUGPY_MODELS_HOME` /
|
|
125
|
+
`LLAMA_CPP_MODEL_DIR`. The scan is fail-soft and fails fast on an EIO models
|
|
126
|
+
disk.
|
|
127
|
+
|
|
128
|
+
**Bindings** are read from `GET /v1/task-bindings` (`{"defaults": {...}}`),
|
|
129
|
+
then the legacy `GET /prompt/tasks`. A 404 means "no bindings". Live basic
|
|
130
|
+
core (0.4.40) serves neither route, so the binding stage is empty today and
|
|
131
|
+
resolution relies on explicit keys, configured defaults and capability
|
|
132
|
+
matches.
|
|
133
|
+
|
|
134
|
+
Both reads are TTL-cached and never raise on the resolve path.
|
|
135
|
+
|
|
136
|
+
### Configuration
|
|
137
|
+
|
|
138
|
+
`hugpy_engine.imports.constants` names every environment variable, timeout and
|
|
139
|
+
TTL the chain reads:
|
|
140
|
+
|
|
141
|
+
| variable | default | meaning |
|
|
142
|
+
|---|---|---|
|
|
143
|
+
| `HUGPY_CORE_URL` (legacy alias `HUGPY_FITEVICT_FRONT_DOOR`) | `http://127.0.0.1:7020` | central's base URL |
|
|
144
|
+
| `HUGPY_REGISTRY_TTL_S` | 30 | registry and bindings cache lifetime |
|
|
145
|
+
| `HUGPY_REGISTRY_TIMEOUT_S` | 10 | registry fetch timeout |
|
|
146
|
+
| `HUGPY_TASK_ENDPOINT_TIMEOUT_S` | 900 | task endpoint call timeout |
|
|
147
|
+
| `HUGPY_FITEVICT_TIMEOUT_S` | 900 | `/v1` call timeout |
|
|
148
|
+
| `HUGPY_MODELS_HOME`, `LLAMA_CPP_MODEL_DIR` | unset | roots for the local disk-scan fallback |
|
|
149
|
+
|
|
150
|
+
The same module holds the `TASK_DEFAULTS`, `MEDIA_DEFAULTS` and
|
|
151
|
+
`MEDIA_TYPE_TO_TASK` tables.
|
|
152
|
+
|
|
153
|
+
## The task plugin seam
|
|
154
|
+
|
|
155
|
+
`hugpy_engine.tasks` holds the task registry. Packages add tasks through the
|
|
156
|
+
`hugpy_engine.tasks` entry-point group (a zero-argument callable), which
|
|
157
|
+
`tasks.load_entry_points()` imports. Two packages register today:
|
|
158
|
+
|
|
159
|
+
- `hugpy-media`: `media = "hugpy_media.plugin:register"`
|
|
160
|
+
- `hugpy-video`: `video = "hugpy_video.plugin:register"`
|
|
161
|
+
|
|
162
|
+
`task_endpoints.TASK_ROUTES` maps tasks to the front-door endpoints.
|
|
163
|
+
|
|
164
|
+
## Layout
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
hugpy_engine/
|
|
168
|
+
dispatch/ execute_prompt / execute_prompt_stream / execute_chat_stream; per-(model, task) runner cache
|
|
169
|
+
resolvers/ task_resolution (infer / rank, stdlib only), model_resolver (resolve), assure_model_key,
|
|
170
|
+
categories/ (task + framework registries)
|
|
171
|
+
config/ models/ (MODEL_REGISTRY, models_config, defaults), main
|
|
172
|
+
llama/runners/ fitevict_runner: the runner that sends a call to central's /v1
|
|
173
|
+
schemas/ chat, event, metadata, model, runner, task schemas (pydantic)
|
|
174
|
+
fit/, placement.py placement types and null single-box defaults
|
|
175
|
+
tasks.py, task_endpoints.py, task_deps.py, categories.py, name_match.py
|
|
176
|
+
utils/ json_scavenge, no_think
|
|
177
|
+
imports/ constants (every env name), functions, classes, init_imports
|
|
178
|
+
(docs/packages.md#the-imports-layout-convention)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
See also:
|
|
182
|
+
|
|
183
|
+
- [docs/execute_prompt.md](../../docs/execute_prompt.md)
|
|
184
|
+
- [docs/arms.md](../../docs/arms.md)
|
|
185
|
+
- [docs/packages.md](../../docs/packages.md)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hugpy-engine"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.7"
|
|
8
8
|
description = "Hugpy engine: model discovery and classification, allocation and eviction, native llama.cpp and GGUF runners, and the prompt-to-reply inference path with task and placement seams"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -114,10 +114,29 @@ MEDIA_DEFAULTS: Dict[str, str] = {
|
|
|
114
114
|
"video": DEFAULT_WHISPER_MODEL,
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
#
|
|
118
|
-
#
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
#
|
|
122
|
-
#
|
|
123
|
-
|
|
117
|
+
# TASK_DEFAULTS — the configured task -> model_key seeds (ported back from the
|
|
118
|
+
# monolith, abstract_hugpy_dev .../imports/src/constants/categories.py:83). These
|
|
119
|
+
# are NO LONGER the sole authority (operator ruling 2026-10-04 made DB
|
|
120
|
+
# task_bindings the designated choice); they are ONE RANKED INPUT to
|
|
121
|
+
# resolvers.task_resolution.resolve(): a task resolves through
|
|
122
|
+
# explicit model_key -> DB binding -> THIS default -> capability match
|
|
123
|
+
# so a fresh box with no bindings still resolves every task to a real model
|
|
124
|
+
# instead of the dead-end "no model bound to task X". ``reconcile_defaults()``
|
|
125
|
+
# heals these seeds against the live registry in place; a seed the fleet does not
|
|
126
|
+
# serve is simply skipped (the default link falls through to capability match).
|
|
127
|
+
TASK_DEFAULTS: Dict[str, str] = {
|
|
128
|
+
"text-generation": DEFAULT_CHAT_MODEL,
|
|
129
|
+
"image-text-to-text": DEFAULT_VISION_MODEL,
|
|
130
|
+
"automatic-speech-recognition": DEFAULT_WHISPER_MODEL,
|
|
131
|
+
"text-summarization": DEFAULT_SUMMARIZE_MODEL,
|
|
132
|
+
"text2text-generation": DEFAULT_SUMMARIZE_MODEL,
|
|
133
|
+
"feature-extraction": DEFAULT_EMBED_MODEL,
|
|
134
|
+
"sentence-similarity": DEFAULT_EMBED_MODEL,
|
|
135
|
+
"text-to-image": DEFAULT_IMAGEGEN_MODEL,
|
|
136
|
+
"image-to-image": DEFAULT_IMAGEGEN_MODEL,
|
|
137
|
+
"keyword-extraction": DEFAULT_KEYWORDS_MODEL,
|
|
138
|
+
"depth-estimation": DEFAULT_DEPTH_MODEL,
|
|
139
|
+
"object-detection": DEFAULT_DETECT_MODEL,
|
|
140
|
+
"image-classification": DEFAULT_IMG_CLASSIFY_MODEL,
|
|
141
|
+
"image-segmentation": DEFAULT_SEGMENT_MODEL,
|
|
142
|
+
}
|
|
@@ -44,6 +44,12 @@ MODELS: Dict[str, dict] = {}
|
|
|
44
44
|
MODEL_REGISTRY: Dict[str, ModelConfig] = {}
|
|
45
45
|
MODEL_REGISTRY_DICT: Dict[str, dict] = {}
|
|
46
46
|
|
|
47
|
+
# The RAW catalog rows from the last successful fetch (the /v1/models shape).
|
|
48
|
+
# Kept alongside the ModelConfig registry because the ranked resolver
|
|
49
|
+
# (resolvers.task_resolution.resolve) ranks on the raw placement facts
|
|
50
|
+
# (resident/available/workers/task_override) — one fetch, two consumers.
|
|
51
|
+
_LAST_CATALOG: list = []
|
|
52
|
+
|
|
47
53
|
_CORE_URL = (os.environ.get("HUGPY_CORE_URL") or "http://127.0.0.1:7020").rstrip("/")
|
|
48
54
|
_TTL_S = float(os.environ.get("HUGPY_REGISTRY_TTL_S", "30"))
|
|
49
55
|
_TIMEOUT_S = float(os.environ.get("HUGPY_REGISTRY_TIMEOUT_S", "10"))
|
|
@@ -75,7 +81,10 @@ def derive_model_config_row(name, row) -> "Tuple[Optional[dict], Optional[str]]"
|
|
|
75
81
|
"""
|
|
76
82
|
hj = row.get("hugpy_json") if isinstance(row.get("hugpy_json"), dict) else {}
|
|
77
83
|
|
|
78
|
-
framework
|
|
84
|
+
# framework == the wrapper's engine. The basic core's /v1/models carries it
|
|
85
|
+
# as the top-level ``engine`` (hugpy_json.framework is authoritative when both
|
|
86
|
+
# are present); the legacy /models row used ``framework``. Read all three.
|
|
87
|
+
framework = hj.get("framework") or row.get("framework") or row.get("engine")
|
|
79
88
|
if isinstance(framework, (list, tuple)):
|
|
80
89
|
framework = framework[0] if framework else None
|
|
81
90
|
framework = str(framework or "").strip().lower() or None
|
|
@@ -85,6 +94,16 @@ def derive_model_config_row(name, row) -> "Tuple[Optional[dict], Optional[str]]"
|
|
|
85
94
|
tasks = hj.get("tasks") if isinstance(hj.get("tasks"), list) else None
|
|
86
95
|
if not tasks:
|
|
87
96
|
tasks = row.get("tasks") if isinstance(row.get("tasks"), list) else None
|
|
97
|
+
# task_override (operator override on the catalog row) is additive capability:
|
|
98
|
+
# a row that lists it CAN serve it. Fold it into the model's tasks so capability
|
|
99
|
+
# matching in resolvers.task_resolution sees it.
|
|
100
|
+
override = row.get("task_override")
|
|
101
|
+
if override is None:
|
|
102
|
+
override = hj.get("task_override")
|
|
103
|
+
if isinstance(override, (list, tuple)):
|
|
104
|
+
tasks = list(tasks or []) + [o for o in override if o]
|
|
105
|
+
elif isinstance(override, str) and override:
|
|
106
|
+
tasks = list(tasks or []) + [override]
|
|
88
107
|
primary = hj.get("primary_task") or row.get("primary_task") or row.get("task")
|
|
89
108
|
tasks = _dedup(tasks or ([primary] if primary else []))
|
|
90
109
|
if not tasks:
|
|
@@ -114,23 +133,54 @@ def derive_model_config_row(name, row) -> "Tuple[Optional[dict], Optional[str]]"
|
|
|
114
133
|
"filename": hj.get("filename") or row.get("file"),
|
|
115
134
|
"include": hj.get("include"),
|
|
116
135
|
"model_max_length": hj.get("model_max_length") or row.get("model_max_length"),
|
|
136
|
+
# PLACEMENT FACTS from the catalog row, carried verbatim into ModelConfig
|
|
137
|
+
# .extra (frozen, additive) so capability ranking in
|
|
138
|
+
# resolvers.task_resolution can prefer resident > available-on-a-healthy-
|
|
139
|
+
# worker > other without re-reading core. Never load-bearing for routing.
|
|
140
|
+
"resident": bool(row.get("resident")),
|
|
141
|
+
"available": bool(row.get("available")),
|
|
142
|
+
"workers": list(row.get("workers") or []),
|
|
143
|
+
"resident_on": list(row.get("resident_on") or []),
|
|
144
|
+
"dir": row.get("dir") or row.get("path"),
|
|
117
145
|
}, None
|
|
118
146
|
|
|
119
147
|
|
|
120
148
|
def _fetch_catalog() -> "Optional[list]":
|
|
121
|
-
"""
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
149
|
+
"""The live catalog as raw rows. None on total failure (fail-open).
|
|
150
|
+
|
|
151
|
+
Source order (resolvers.task_resolution.load_catalog, the dependency-light
|
|
152
|
+
fetcher hugpy_core will share): ``GET {core}/v1/models`` (the basic core's
|
|
153
|
+
real endpoint) -> legacy ``GET {core}/models`` -> a local hugpy.json disk
|
|
154
|
+
scan (HUGPY_MODELS_HOME / LLAMA_CPP_MODEL_DIR, fail-soft and fail-fast on the
|
|
155
|
+
EIO models disk). Stores the rows in ``_LAST_CATALOG`` so the ranked resolver
|
|
156
|
+
reads the same fetch."""
|
|
157
|
+
global _LAST_CATALOG
|
|
158
|
+
from hugpy_engine.resolvers import task_resolution as _tr
|
|
159
|
+
|
|
160
|
+
def _log(msg):
|
|
161
|
+
logger.warning("registry: %s; keeping the last known model set (%d models)",
|
|
162
|
+
msg, len(MODEL_REGISTRY))
|
|
163
|
+
|
|
164
|
+
rows = _tr.load_catalog(_CORE_URL, timeout=_TIMEOUT_S, on_log=_log)
|
|
165
|
+
if not rows:
|
|
166
|
+
# core gave us nothing usable — try the local disk marker scan before
|
|
167
|
+
# giving up (fail-open). The scan abandons an EIO root fast.
|
|
168
|
+
local = _tr.load_local_scan(on_log=lambda m: logger.warning("registry: %s", m))
|
|
169
|
+
if local:
|
|
170
|
+
logger.info("registry: core catalog empty/unreachable; local hugpy.json "
|
|
171
|
+
"scan found %d model(s)", len(local))
|
|
172
|
+
_LAST_CATALOG = local
|
|
173
|
+
return local
|
|
130
174
|
return None
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
175
|
+
_LAST_CATALOG = rows
|
|
176
|
+
return rows
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def catalog_rows() -> list:
|
|
180
|
+
"""The raw catalog rows from the last successful fetch (the /v1/models shape).
|
|
181
|
+
Read by the ranked resolver so it can prefer resident/available models. A copy,
|
|
182
|
+
so a caller can filter it without disturbing the cache."""
|
|
183
|
+
return list(_LAST_CATALOG)
|
|
134
184
|
|
|
135
185
|
|
|
136
186
|
def refresh_registry(force: bool = False) -> None:
|
|
@@ -205,25 +255,31 @@ _bindings_at = 0.0
|
|
|
205
255
|
|
|
206
256
|
|
|
207
257
|
def task_bindings(force: bool = False) -> dict:
|
|
208
|
-
"""The live {task: model_key} bindings
|
|
209
|
-
|
|
258
|
+
"""The live {task: model_key} bindings.
|
|
259
|
+
|
|
260
|
+
Source order (resolvers.task_resolution.load_bindings): ``GET
|
|
261
|
+
{core}/v1/task-bindings`` -> ``{"defaults": {...}}`` (the endpoint the core
|
|
262
|
+
owner is adding) -> legacy ``GET {core}/prompt/tasks`` -> {}. A 404 on both
|
|
263
|
+
means "no bindings" (returned as {}, the honest empty map that lets the
|
|
264
|
+
default/capability links of the chain pick instead). TTL-cached; a transient
|
|
265
|
+
unreachable read keeps the last-known map. Bindings are now ONE input to the
|
|
266
|
+
ranked resolver, not the sole authority."""
|
|
210
267
|
global _bindings, _bindings_at
|
|
211
268
|
now = time.time()
|
|
212
269
|
if not force and _bindings and (now - _bindings_at) < _TTL_S:
|
|
213
270
|
return dict(_bindings)
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
_bindings = {str(k): str(v) for k, v in defaults.items() if v}
|
|
271
|
+
from hugpy_engine.resolvers import task_resolution as _tr
|
|
272
|
+
_unreachable = []
|
|
273
|
+
fresh = _tr.load_bindings(_CORE_URL, timeout=_TIMEOUT_S,
|
|
274
|
+
on_log=lambda m: _unreachable.append(m))
|
|
275
|
+
if fresh:
|
|
276
|
+
_bindings = fresh
|
|
277
|
+
elif _unreachable and _bindings:
|
|
278
|
+
# a transient error (not a clean 404) — keep the last-known map.
|
|
279
|
+
logger.warning("registry: task bindings unreachable (%s); keeping the "
|
|
280
|
+
"last-known map (%d)", "; ".join(_unreachable), len(_bindings))
|
|
281
|
+
else:
|
|
282
|
+
_bindings = {} # clean "no bindings" answer
|
|
227
283
|
_bindings_at = now
|
|
228
284
|
return dict(_bindings)
|
|
229
285
|
|