meshapi 0.1.6__tar.gz → 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {meshapi-0.1.6 → meshapi-0.1.8}/CLAUDE.md +5 -5
- {meshapi-0.1.6 → meshapi-0.1.8}/PKG-INFO +79 -1
- {meshapi-0.1.6 → meshapi-0.1.8}/README.md +78 -0
- meshapi-0.1.8/livetests/.env.livetest.example +31 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/conftest.py +37 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_audio.py +3 -0
- meshapi-0.1.8/livetests/test_compare.py +73 -0
- meshapi-0.1.8/livetests/test_images_edit.py +51 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_models.py +28 -1
- meshapi-0.1.8/livetests/test_moderations.py +48 -0
- meshapi-0.1.8/livetests/test_responses.py +27 -0
- meshapi-0.1.8/livetests/test_router_select.py +45 -0
- meshapi-0.1.8/livetests/test_tool_calling.py +120 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_video.py +6 -1
- meshapi-0.1.8/livetests/test_web_search.py +40 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/__init__.py +67 -1
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/_http.py +1 -1
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/_types.py +365 -14
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/audio.py +43 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/batches.py +6 -4
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/images.py +15 -2
- meshapi-0.1.8/meshapi/resources/models.py +91 -0
- meshapi-0.1.8/meshapi/resources/moderations.py +24 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/rag.py +18 -5
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/responses.py +38 -3
- meshapi-0.1.8/meshapi/resources/router_select.py +31 -0
- meshapi-0.1.8/meshapi/resources/web_search.py +29 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/pyproject.toml +1 -1
- meshapi-0.1.6/livetests/test_compare.py +0 -38
- meshapi-0.1.6/livetests/tool_call.py +0 -101
- meshapi-0.1.6/meshapi/resources/models.py +0 -48
- {meshapi-0.1.6 → meshapi-0.1.8}/.gitignore +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/CHANGELOG.md +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/TESTING.md +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/compare.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/config.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/pytest.ini +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/requirements.txt +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/responses.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_chat.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_errors.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_feature_matrix.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_inference_resources.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_rag.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_realtime.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_stream.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_structured_output.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/livetests/test_templates.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/_errors.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/__init__.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/chat.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/compare.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/embeddings.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/realtime.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/templates.py +0 -0
- {meshapi-0.1.6 → meshapi-0.1.8}/meshapi/resources/videos.py +0 -0
|
@@ -20,13 +20,12 @@ python/
|
|
|
20
20
|
│ ├── chat.py # /v1/chat/completions
|
|
21
21
|
│ ├── responses.py # /v1/responses
|
|
22
22
|
│ ├── embeddings.py # /v1/embeddings
|
|
23
|
-
│ ├── compare.py # /v1/compare
|
|
24
|
-
│ ├──
|
|
25
|
-
│ ├── rag.py # /v1/files RAG endpoints (upload, embed, search)
|
|
23
|
+
│ ├── compare.py # /v1/chat/compare
|
|
24
|
+
│ ├── rag.py # /v1/files RAG endpoints (upload, list, get status, embed, search)
|
|
26
25
|
│ ├── batches.py # /v1/batches
|
|
27
26
|
│ ├── models.py # /v1/models
|
|
28
27
|
│ ├── templates.py # /v1/templates
|
|
29
|
-
│ └── images.py # /v1/images/generations
|
|
28
|
+
│ └── images.py # /v1/images/generations, /v1/images/edits
|
|
30
29
|
├── tests/
|
|
31
30
|
│ ├── unit/ # Fast, no-network tests
|
|
32
31
|
│ ├── contract/ # Pydantic model parsing against local fixtures
|
|
@@ -160,9 +159,10 @@ pytest test_rag.py::test_rag_upload_embed_search -v
|
|
|
160
159
|
| `test_feature_matrix.py` | Cross-model feature matrix |
|
|
161
160
|
| `test_rag.py` | RAG upload → embed → list → search |
|
|
162
161
|
| `test_realtime.py` | WebSocket connect/close, session lifecycle |
|
|
163
|
-
| `test_audio.py` | TTS synthesize, voice listing |
|
|
162
|
+
| `test_audio.py` | TTS synthesize, voice listing, audio_translate |
|
|
164
163
|
| `test_video.py` | Video list, generate → retrieve |
|
|
165
164
|
| `test_compare.py` | Non-streaming compare, streaming compare |
|
|
165
|
+
| `test_moderations.py` | Moderation classify: text and multimodal input |
|
|
166
166
|
|
|
167
167
|
---
|
|
168
168
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: meshapi
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.8
|
|
4
4
|
Summary: Official Python SDK for the MeshAPI AI model gateway
|
|
5
5
|
Project-URL: Homepage, https://meshapi.ai
|
|
6
6
|
Project-URL: Documentation, https://developers.meshapi.ai
|
|
@@ -82,6 +82,9 @@ Get a key at [meshapi.ai](https://meshapi.ai). Data-plane keys are prefixed `rsk
|
|
|
82
82
|
| **Multi-model compare** | Fire one prompt at N models in parallel and stream their replies side by side. |
|
|
83
83
|
| **Audio** | Text-to-speech, speech-to-text, transcription translation, and voice listing. |
|
|
84
84
|
| **Video** | Submit and poll async video generation tasks. |
|
|
85
|
+
| **Moderations** | Classify text/image content for policy violations via `moderations.create`. |
|
|
86
|
+
| **Web search** | Live web search with native + Tavily engines via `web.search`. |
|
|
87
|
+
| **Router select** | Ask the Auto Router which model it would pick, without running inference. |
|
|
85
88
|
| **RAG** | Upload files, embed them, and run vector search — all through the same client. |
|
|
86
89
|
| **Batches** | Async bulk inference jobs at discounted rates with inline request submission. |
|
|
87
90
|
| **Prompt templates** | Server-stored prompts with `{{variable}}` slots. Update prompts without redeploying. |
|
|
@@ -185,6 +188,10 @@ reply = client.responses.create(
|
|
|
185
188
|
max_output_tokens=512,
|
|
186
189
|
)
|
|
187
190
|
)
|
|
191
|
+
|
|
192
|
+
# List background response jobs, or fetch one by id
|
|
193
|
+
jobs = client.responses.list(limit=20)
|
|
194
|
+
job = client.responses.get("resp_abc123")
|
|
188
195
|
```
|
|
189
196
|
|
|
190
197
|
## Embeddings
|
|
@@ -301,6 +308,25 @@ result = client.images.generate(
|
|
|
301
308
|
print(result.data[0].url)
|
|
302
309
|
```
|
|
303
310
|
|
|
311
|
+
### Editing an image
|
|
312
|
+
|
|
313
|
+
`image` (and optional `mask` / `reference_images`) take a base64 or
|
|
314
|
+
`data:` URL — remote http(s) URLs are rejected by this endpoint.
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
from meshapi import ImageEditParams
|
|
318
|
+
|
|
319
|
+
edited = client.images.edit(
|
|
320
|
+
ImageEditParams(
|
|
321
|
+
model="openai/gpt-image-1",
|
|
322
|
+
image="data:image/png;base64,<...>",
|
|
323
|
+
prompt="Replace the background with a beach at sunset",
|
|
324
|
+
operation="edit", # or inpaint / outpaint / mix / reframe / upscale / remove_background
|
|
325
|
+
)
|
|
326
|
+
)
|
|
327
|
+
print(edited.data[0].url or edited.data[0].b64_json[:32])
|
|
328
|
+
```
|
|
329
|
+
|
|
304
330
|
## Compare (multi-model fanout)
|
|
305
331
|
|
|
306
332
|
```python
|
|
@@ -432,9 +458,61 @@ async with AsyncMeshAPI(base_url="...", token="rsk_...") as client:
|
|
|
432
458
|
## Models
|
|
433
459
|
|
|
434
460
|
```python
|
|
461
|
+
from meshapi import ModelSearchParams
|
|
462
|
+
|
|
435
463
|
all_models = client.models.list()
|
|
436
464
|
free = client.models.free()
|
|
437
465
|
paid = client.models.paid()
|
|
466
|
+
|
|
467
|
+
# Paginated catalog search (DB-only, no model cost)
|
|
468
|
+
page = client.models.search(ModelSearchParams(q="gpt", free=False, sort="name", limit=10))
|
|
469
|
+
print(page.total, page.brands)
|
|
470
|
+
|
|
471
|
+
# Fetch one model's detail
|
|
472
|
+
gpt4o = client.models.get("openai/gpt-4o")
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
## Moderations
|
|
476
|
+
|
|
477
|
+
```python
|
|
478
|
+
from meshapi import ModerationParams
|
|
479
|
+
|
|
480
|
+
result = client.moderations.create(ModerationParams(input="text to classify"))
|
|
481
|
+
if result.results[0].flagged:
|
|
482
|
+
print("flagged:", result.results[0].categories)
|
|
483
|
+
|
|
484
|
+
# Batch several inputs in one call
|
|
485
|
+
client.moderations.create(ModerationParams(input=["first text", "second text"]))
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
## Web search
|
|
489
|
+
|
|
490
|
+
Gated server-side by `WEB_SEARCH_ENABLED`. Native-first with Tavily fallback;
|
|
491
|
+
inspect `response.provider` to see which engine served the request.
|
|
492
|
+
|
|
493
|
+
```python
|
|
494
|
+
from meshapi import WebSearchParams
|
|
495
|
+
|
|
496
|
+
res = client.web.search(
|
|
497
|
+
WebSearchParams(query="latest news on Mars rovers", max_results=5, include_answer=True)
|
|
498
|
+
)
|
|
499
|
+
print(res.provider, res.answer)
|
|
500
|
+
for hit in res.results:
|
|
501
|
+
print(hit.title, hit.url)
|
|
502
|
+
```
|
|
503
|
+
|
|
504
|
+
## Router select
|
|
505
|
+
|
|
506
|
+
Gated server-side by `AUTO_ROUTER_ENABLED`. Returns the model the Auto Router
|
|
507
|
+
*would* pick — without running inference — so you can run it on your own path.
|
|
508
|
+
|
|
509
|
+
```python
|
|
510
|
+
from meshapi import RouterSelectParams, ChatMessage
|
|
511
|
+
|
|
512
|
+
sel = client.router.select(
|
|
513
|
+
RouterSelectParams(messages=[ChatMessage(role="user", content="Prove that 2+2=4.")])
|
|
514
|
+
)
|
|
515
|
+
print(sel.model, sel.auto_router.fallback_used)
|
|
438
516
|
```
|
|
439
517
|
|
|
440
518
|
## Prompt templates
|
|
@@ -45,6 +45,9 @@ Get a key at [meshapi.ai](https://meshapi.ai). Data-plane keys are prefixed `rsk
|
|
|
45
45
|
| **Multi-model compare** | Fire one prompt at N models in parallel and stream their replies side by side. |
|
|
46
46
|
| **Audio** | Text-to-speech, speech-to-text, transcription translation, and voice listing. |
|
|
47
47
|
| **Video** | Submit and poll async video generation tasks. |
|
|
48
|
+
| **Moderations** | Classify text/image content for policy violations via `moderations.create`. |
|
|
49
|
+
| **Web search** | Live web search with native + Tavily engines via `web.search`. |
|
|
50
|
+
| **Router select** | Ask the Auto Router which model it would pick, without running inference. |
|
|
48
51
|
| **RAG** | Upload files, embed them, and run vector search — all through the same client. |
|
|
49
52
|
| **Batches** | Async bulk inference jobs at discounted rates with inline request submission. |
|
|
50
53
|
| **Prompt templates** | Server-stored prompts with `{{variable}}` slots. Update prompts without redeploying. |
|
|
@@ -148,6 +151,10 @@ reply = client.responses.create(
|
|
|
148
151
|
max_output_tokens=512,
|
|
149
152
|
)
|
|
150
153
|
)
|
|
154
|
+
|
|
155
|
+
# List background response jobs, or fetch one by id
|
|
156
|
+
jobs = client.responses.list(limit=20)
|
|
157
|
+
job = client.responses.get("resp_abc123")
|
|
151
158
|
```
|
|
152
159
|
|
|
153
160
|
## Embeddings
|
|
@@ -264,6 +271,25 @@ result = client.images.generate(
|
|
|
264
271
|
print(result.data[0].url)
|
|
265
272
|
```
|
|
266
273
|
|
|
274
|
+
### Editing an image
|
|
275
|
+
|
|
276
|
+
`image` (and optional `mask` / `reference_images`) take a base64 or
|
|
277
|
+
`data:` URL — remote http(s) URLs are rejected by this endpoint.
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
from meshapi import ImageEditParams
|
|
281
|
+
|
|
282
|
+
edited = client.images.edit(
|
|
283
|
+
ImageEditParams(
|
|
284
|
+
model="openai/gpt-image-1",
|
|
285
|
+
image="data:image/png;base64,<...>",
|
|
286
|
+
prompt="Replace the background with a beach at sunset",
|
|
287
|
+
operation="edit", # or inpaint / outpaint / mix / reframe / upscale / remove_background
|
|
288
|
+
)
|
|
289
|
+
)
|
|
290
|
+
print(edited.data[0].url or edited.data[0].b64_json[:32])
|
|
291
|
+
```
|
|
292
|
+
|
|
267
293
|
## Compare (multi-model fanout)
|
|
268
294
|
|
|
269
295
|
```python
|
|
@@ -395,9 +421,61 @@ async with AsyncMeshAPI(base_url="...", token="rsk_...") as client:
|
|
|
395
421
|
## Models
|
|
396
422
|
|
|
397
423
|
```python
|
|
424
|
+
from meshapi import ModelSearchParams
|
|
425
|
+
|
|
398
426
|
all_models = client.models.list()
|
|
399
427
|
free = client.models.free()
|
|
400
428
|
paid = client.models.paid()
|
|
429
|
+
|
|
430
|
+
# Paginated catalog search (DB-only, no model cost)
|
|
431
|
+
page = client.models.search(ModelSearchParams(q="gpt", free=False, sort="name", limit=10))
|
|
432
|
+
print(page.total, page.brands)
|
|
433
|
+
|
|
434
|
+
# Fetch one model's detail
|
|
435
|
+
gpt4o = client.models.get("openai/gpt-4o")
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
## Moderations
|
|
439
|
+
|
|
440
|
+
```python
|
|
441
|
+
from meshapi import ModerationParams
|
|
442
|
+
|
|
443
|
+
result = client.moderations.create(ModerationParams(input="text to classify"))
|
|
444
|
+
if result.results[0].flagged:
|
|
445
|
+
print("flagged:", result.results[0].categories)
|
|
446
|
+
|
|
447
|
+
# Batch several inputs in one call
|
|
448
|
+
client.moderations.create(ModerationParams(input=["first text", "second text"]))
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
## Web search
|
|
452
|
+
|
|
453
|
+
Gated server-side by `WEB_SEARCH_ENABLED`. Native-first with Tavily fallback;
|
|
454
|
+
inspect `response.provider` to see which engine served the request.
|
|
455
|
+
|
|
456
|
+
```python
|
|
457
|
+
from meshapi import WebSearchParams
|
|
458
|
+
|
|
459
|
+
res = client.web.search(
|
|
460
|
+
WebSearchParams(query="latest news on Mars rovers", max_results=5, include_answer=True)
|
|
461
|
+
)
|
|
462
|
+
print(res.provider, res.answer)
|
|
463
|
+
for hit in res.results:
|
|
464
|
+
print(hit.title, hit.url)
|
|
465
|
+
```
|
|
466
|
+
|
|
467
|
+
## Router select
|
|
468
|
+
|
|
469
|
+
Gated server-side by `AUTO_ROUTER_ENABLED`. Returns the model the Auto Router
|
|
470
|
+
*would* pick — without running inference — so you can run it on your own path.
|
|
471
|
+
|
|
472
|
+
```python
|
|
473
|
+
from meshapi import RouterSelectParams, ChatMessage
|
|
474
|
+
|
|
475
|
+
sel = client.router.select(
|
|
476
|
+
RouterSelectParams(messages=[ChatMessage(role="user", content="Prove that 2+2=4.")])
|
|
477
|
+
)
|
|
478
|
+
print(sel.model, sel.auto_router.fallback_used)
|
|
401
479
|
```
|
|
402
480
|
|
|
403
481
|
## Prompt templates
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# MeshAPI Python SDK — live-test configuration.
|
|
2
|
+
# Copy to `python/.env.livetest` (read automatically by the harness) and fill in.
|
|
3
|
+
|
|
4
|
+
# ── Required ───────────────────────────────────────────────────────────────
|
|
5
|
+
MESHAPI_BASE_URL=https://api-dev.meshapi.ai
|
|
6
|
+
MESHAPI_TOKEN=rsk_your_key_here
|
|
7
|
+
|
|
8
|
+
# ── Core models (have sensible defaults) ─────────────────────────────────────
|
|
9
|
+
MESHAPI_MODEL=openai/gpt-4o-mini
|
|
10
|
+
MESHAPI_SECOND_MODEL=anthropic/claude-haiku-4.5
|
|
11
|
+
MESHAPI_EMBEDDINGS_MODEL=openai/text-embedding-3-small
|
|
12
|
+
MESHAPI_REALTIME_MODEL=openai/gpt-realtime-mini
|
|
13
|
+
|
|
14
|
+
# ── Optional-feature models/inputs ───────────────────────────────────────────
|
|
15
|
+
# When these are UNSET the corresponding tests skip silently (a green run then
|
|
16
|
+
# proves little). Set them all and MESHAPI_STRICT_LIVETESTS=1 for the
|
|
17
|
+
# pre-hackathon gate so every feature test actually runs.
|
|
18
|
+
MESHAPI_IMAGE_GEN_MODEL=openai/gpt-image-1
|
|
19
|
+
MESHAPI_IMAGE_URL=https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg
|
|
20
|
+
MESHAPI_INPUT_AUDIO_B64= # base64 of a short wav/mp3 clip
|
|
21
|
+
MESHAPI_INPUT_AUDIO_FORMAT=wav
|
|
22
|
+
MESHAPI_AUDIO_OUT_MODEL=openai/gpt-4o-mini-audio-preview
|
|
23
|
+
MESHAPI_VIDEO_GEN_MODEL=byteplus/seedance-1-5-pro-251215
|
|
24
|
+
# Image editing is opt-in (costly, niche) — not part of the strict gate.
|
|
25
|
+
# MESHAPI_IMAGE_EDIT_MODEL=openai/gpt-image-1
|
|
26
|
+
# MESHAPI_IMAGE_EDIT_INPUT= # optional base64/data-URL source image
|
|
27
|
+
|
|
28
|
+
# ── Pre-hackathon gate ───────────────────────────────────────────────────────
|
|
29
|
+
# When set, the suite fails fast unless every optional-feature var above is set,
|
|
30
|
+
# so skip-by-default tests can't hide behind a green run.
|
|
31
|
+
# MESHAPI_STRICT_LIVETESTS=1
|
|
@@ -30,6 +30,43 @@ from config import BASE_URL, TOKEN, MODEL, get_env # noqa: E402
|
|
|
30
30
|
from meshapi import MeshAPI # noqa: E402
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# Strict-mode preflight (pre-hackathon gate)
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
#
|
|
37
|
+
# Several feature tests skip-by-default when an env var is unset (image gen,
|
|
38
|
+
# vision, audio in/out, video). A skipped test reads as "passed" — so a green
|
|
39
|
+
# run can mean almost nothing ran. Set MESHAPI_STRICT_LIVETESTS=1 in the
|
|
40
|
+
# pre-hackathon gate: the run fails fast unless every optional-feature env var
|
|
41
|
+
# is present, forcing those tests to actually execute. See .env.livetest.example.
|
|
42
|
+
|
|
43
|
+
# Env vars whose absence turns a real feature test into a silent skip.
|
|
44
|
+
STRICT_REQUIRED_ENV = [
|
|
45
|
+
"MESHAPI_IMAGE_GEN_MODEL",
|
|
46
|
+
"MESHAPI_IMAGE_URL",
|
|
47
|
+
"MESHAPI_INPUT_AUDIO_B64",
|
|
48
|
+
"MESHAPI_AUDIO_OUT_MODEL",
|
|
49
|
+
"MESHAPI_VIDEO_GEN_MODEL",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _strict_mode() -> bool:
|
|
54
|
+
return (os.getenv("MESHAPI_STRICT_LIVETESTS") or "").lower() in ("1", "true", "yes")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def pytest_configure(config: "pytest.Config") -> None:
|
|
58
|
+
if not _strict_mode():
|
|
59
|
+
return
|
|
60
|
+
missing = [name for name in STRICT_REQUIRED_ENV if not get_env(name)]
|
|
61
|
+
if missing:
|
|
62
|
+
pytest.exit(
|
|
63
|
+
"MESHAPI_STRICT_LIVETESTS is set but these env vars are unset, so their "
|
|
64
|
+
"feature tests would silently skip:\n - " + "\n - ".join(missing) + "\n"
|
|
65
|
+
"Set them (see .env.livetest.example) or unset MESHAPI_STRICT_LIVETESTS.",
|
|
66
|
+
returncode=1,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
33
70
|
# ---------------------------------------------------------------------------
|
|
34
71
|
# Session-scoped shared client fixture
|
|
35
72
|
# ---------------------------------------------------------------------------
|
|
@@ -19,6 +19,9 @@ def test_audio_synthesize(client: MeshAPI) -> None:
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
def test_audio_stt_from_tts(client: MeshAPI) -> None:
|
|
22
|
+
# NB: not forcing response_format — the TTS model (sarvam) rejects an
|
|
23
|
+
# explicit "wav" with a 422, and its default codec already transcribes
|
|
24
|
+
# end-to-end, so the round-trip is valid as-is.
|
|
22
25
|
audio_bytes = client.audio.synthesize(
|
|
23
26
|
SpeechParams(input="Hello from MeshAPI audio test.", model=TTS_MODEL)
|
|
24
27
|
)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Live tests: Model Compare API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from meshapi import MeshAPI, CompareParams, ChatMessage, ModelOverride
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_compare_nonstreaming(client: MeshAPI, model: str, second_model: str) -> None:
|
|
10
|
+
result = client.compare.create(
|
|
11
|
+
CompareParams(
|
|
12
|
+
models=[model, second_model],
|
|
13
|
+
messages=[ChatMessage(role="user", content="What is 2+2? Reply in one word.")],
|
|
14
|
+
skip_comparison=True,
|
|
15
|
+
max_tokens=20,
|
|
16
|
+
)
|
|
17
|
+
)
|
|
18
|
+
assert result.comparison_id, "expected comparison_id"
|
|
19
|
+
assert len(result.results) == 2, f"expected 2 results, got {len(result.results)}"
|
|
20
|
+
found_models = {r.model for r in result.results}
|
|
21
|
+
assert model in found_models, f"expected {model} in results"
|
|
22
|
+
assert second_model in found_models, f"expected {second_model} in results"
|
|
23
|
+
for r in result.results:
|
|
24
|
+
assert r.content or r.error, f"result for {r.model} has neither content nor error"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_compare_streaming(client: MeshAPI, model: str, second_model: str) -> None:
|
|
28
|
+
events = list(
|
|
29
|
+
client.compare.stream(
|
|
30
|
+
CompareParams(
|
|
31
|
+
models=[model, second_model],
|
|
32
|
+
messages=[ChatMessage(role="user", content="Tell me a joke.")],
|
|
33
|
+
skip_comparison=True,
|
|
34
|
+
max_tokens=50,
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
)
|
|
38
|
+
assert len(events) > 0, "expected at least one streaming event"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_compare_with_synthesis(client: MeshAPI, model: str, second_model: str) -> None:
|
|
42
|
+
"""Exercise the synthesis path (skip_comparison=False) — previously never tested."""
|
|
43
|
+
result = client.compare.create(
|
|
44
|
+
CompareParams(
|
|
45
|
+
models=[model, second_model],
|
|
46
|
+
messages=[ChatMessage(role="user", content="In one sentence, what is TCP?")],
|
|
47
|
+
comparison_instructions="Briefly state which answer is clearer.",
|
|
48
|
+
skip_comparison=False,
|
|
49
|
+
max_tokens=60,
|
|
50
|
+
)
|
|
51
|
+
)
|
|
52
|
+
assert result.comparison_id
|
|
53
|
+
assert len(result.results) == 2
|
|
54
|
+
# When at least one per-model answer succeeded and the comparison model did
|
|
55
|
+
# not fall back, a synthesized comparison must be present with usage.
|
|
56
|
+
if any(r.content for r in result.results) and not result.comparison_fallback_used:
|
|
57
|
+
assert result.comparison, "expected a synthesized comparison when skip_comparison=False"
|
|
58
|
+
assert result.comparison_model, "expected comparison_model to be reported"
|
|
59
|
+
assert result.comparison_usage is not None, "expected comparison_usage to be populated"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_compare_model_overrides(client: MeshAPI, model: str, second_model: str) -> None:
|
|
63
|
+
"""Per-model overrides (temperature/max_tokens) — previously untested."""
|
|
64
|
+
result = client.compare.create(
|
|
65
|
+
CompareParams(
|
|
66
|
+
models=[model, second_model],
|
|
67
|
+
messages=[ChatMessage(role="user", content="Say hi in one word.")],
|
|
68
|
+
model_overrides=[ModelOverride(model=model, temperature=0.0, max_tokens=10)],
|
|
69
|
+
skip_comparison=True,
|
|
70
|
+
max_tokens=20,
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
assert len(result.results) == 2, "overrides must not drop any model from the fan-out"
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Live tests: image editing — POST /v1/images/edits.
|
|
2
|
+
|
|
3
|
+
Requires an edit-capable model. Set MESHAPI_IMAGE_EDIT_MODEL (and optionally
|
|
4
|
+
MESHAPI_IMAGE_EDIT_INPUT, a base64/data-URL source image) to run; otherwise
|
|
5
|
+
skipped. Image editing costs real inference, so this is opt-in.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
from config import get_env
|
|
12
|
+
from meshapi import ImageEditParams, MeshAPI, MeshAPIError
|
|
13
|
+
|
|
14
|
+
# 1x1 transparent PNG, used when MESHAPI_IMAGE_EDIT_INPUT is not provided.
|
|
15
|
+
_PIXEL_PNG = (
|
|
16
|
+
"data:image/png;base64,"
|
|
17
|
+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_image_edit_basic(client: MeshAPI) -> None:
|
|
22
|
+
model = get_env("MESHAPI_IMAGE_EDIT_MODEL")
|
|
23
|
+
if not model:
|
|
24
|
+
pytest.skip("set MESHAPI_IMAGE_EDIT_MODEL to run the image-edit live test")
|
|
25
|
+
source = get_env("MESHAPI_IMAGE_EDIT_INPUT") or _PIXEL_PNG
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
resp = client.images.edit(
|
|
29
|
+
ImageEditParams(
|
|
30
|
+
model=model,
|
|
31
|
+
image=source,
|
|
32
|
+
prompt="Make the background a solid blue.",
|
|
33
|
+
operation="edit",
|
|
34
|
+
)
|
|
35
|
+
)
|
|
36
|
+
except MeshAPIError as exc:
|
|
37
|
+
if exc.status == 400 and exc.error_code == "invalid_request":
|
|
38
|
+
# Upstream (provider) content/safety rejection of the synthetic test
|
|
39
|
+
# image — the request reached the provider, so the SDK path is
|
|
40
|
+
# validated. Skip rather than fail.
|
|
41
|
+
pytest.skip(f"provider rejected the test image: {exc}")
|
|
42
|
+
if exc.status in (400, 501) and exc.error_code in (
|
|
43
|
+
"model_capability_not_supported",
|
|
44
|
+
"not_implemented",
|
|
45
|
+
):
|
|
46
|
+
pytest.skip(f"model does not support image edits: {exc.error_code}")
|
|
47
|
+
raise
|
|
48
|
+
|
|
49
|
+
assert resp.data, "expected at least one edited image"
|
|
50
|
+
first = resp.data[0]
|
|
51
|
+
assert first.url or first.b64_json, "edited image should have a url or b64_json"
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from meshapi import MeshAPI
|
|
5
|
+
from meshapi import MeshAPI, ModelSearchParams
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def test_models_list(client: MeshAPI) -> None:
|
|
@@ -36,3 +36,30 @@ def test_models_list_filter_free(client: MeshAPI) -> None:
|
|
|
36
36
|
def test_models_list_filter_paid(client: MeshAPI) -> None:
|
|
37
37
|
filtered = client.models.list(free=False)
|
|
38
38
|
assert all(not m.is_free for m in filtered), "list(free=False) returned free models"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_models_search_paginated(client: MeshAPI) -> None:
|
|
42
|
+
page = client.models.search(ModelSearchParams(limit=5))
|
|
43
|
+
assert page.total >= 0, "expected a non-negative total"
|
|
44
|
+
assert page.limit == 5, "page should echo the requested limit"
|
|
45
|
+
assert len(page.items) <= 5, "page must not exceed the limit"
|
|
46
|
+
assert isinstance(page.brands, list), "expected a brands facet list"
|
|
47
|
+
for m in page.items:
|
|
48
|
+
assert m.id and m.name
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_models_search_query_filter(client: MeshAPI) -> None:
|
|
52
|
+
page = client.models.search(ModelSearchParams(q="gpt", limit=10))
|
|
53
|
+
# Fuzzy match over id/name/brand — every returned model should relate to the query.
|
|
54
|
+
for m in page.items:
|
|
55
|
+
haystack = f"{m.id} {m.name}".lower()
|
|
56
|
+
assert "gpt" in haystack, f"unexpected model {m.id!r} for q='gpt'"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_models_get_by_id(client: MeshAPI) -> None:
|
|
60
|
+
listed = client.models.list()
|
|
61
|
+
assert listed, "need at least one model to fetch by id"
|
|
62
|
+
target = listed[0].id
|
|
63
|
+
model = client.models.get(target)
|
|
64
|
+
assert model.id == target, f"get({target!r}) returned {model.id!r}"
|
|
65
|
+
assert model.name
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Live tests: moderations — POST /v1/moderations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from meshapi import MeshAPI, MeshAPIError, ModerationParams
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_moderation_flags_harmful_text(client: MeshAPI) -> None:
|
|
10
|
+
try:
|
|
11
|
+
resp = client.moderations.create(
|
|
12
|
+
ModerationParams(input="I want to hurt and kill someone right now.")
|
|
13
|
+
)
|
|
14
|
+
except MeshAPIError as exc:
|
|
15
|
+
if exc.status in (403, 404, 501, 503):
|
|
16
|
+
pytest.skip(f"moderations unavailable on this deployment: {exc.error_code}")
|
|
17
|
+
raise
|
|
18
|
+
|
|
19
|
+
assert resp.results, "expected at least one moderation result"
|
|
20
|
+
result = resp.results[0]
|
|
21
|
+
assert result.flagged is True, "expected harmful text to be flagged"
|
|
22
|
+
assert result.categories, "expected category booleans"
|
|
23
|
+
assert result.category_scores, "expected category scores"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_moderation_passes_benign_text(client: MeshAPI) -> None:
|
|
27
|
+
try:
|
|
28
|
+
resp = client.moderations.create(ModerationParams(input="I love sunny days at the park."))
|
|
29
|
+
except MeshAPIError as exc:
|
|
30
|
+
if exc.status in (403, 404, 501, 503):
|
|
31
|
+
pytest.skip(f"moderations unavailable on this deployment: {exc.error_code}")
|
|
32
|
+
raise
|
|
33
|
+
|
|
34
|
+
assert resp.results
|
|
35
|
+
assert resp.results[0].flagged is False, "expected benign text not to be flagged"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_moderation_batch_input(client: MeshAPI) -> None:
|
|
39
|
+
try:
|
|
40
|
+
resp = client.moderations.create(
|
|
41
|
+
ModerationParams(input=["hello friend", "have a nice day"])
|
|
42
|
+
)
|
|
43
|
+
except MeshAPIError as exc:
|
|
44
|
+
if exc.status in (403, 404, 501, 503):
|
|
45
|
+
pytest.skip(f"moderations unavailable on this deployment: {exc.error_code}")
|
|
46
|
+
raise
|
|
47
|
+
|
|
48
|
+
assert len(resp.results) == 2, "expected one result per input"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Live tests: responses list/get — GET /v1/responses, GET /v1/responses/{id}.
|
|
2
|
+
|
|
3
|
+
(POST /v1/responses create/stream is covered in test_inference_resources.py.)
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
from meshapi import MeshAPI, MeshAPIError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_responses_list_shape(client: MeshAPI) -> None:
|
|
13
|
+
page = client.responses.list(limit=5)
|
|
14
|
+
# OpenAI list envelope; data may be empty if the account has no background jobs.
|
|
15
|
+
if page.object is not None:
|
|
16
|
+
assert page.object == "list"
|
|
17
|
+
assert isinstance(page.data, list)
|
|
18
|
+
assert len(page.data) <= 5
|
|
19
|
+
for item in page.data:
|
|
20
|
+
assert item.id, "each job must have an id"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_responses_get_unknown_id_404(client: MeshAPI) -> None:
|
|
24
|
+
"""get() on a non-existent id should raise a structured 404 (exercises the path)."""
|
|
25
|
+
with pytest.raises(MeshAPIError) as excinfo:
|
|
26
|
+
client.responses.get("resp_does_not_exist_000000000000")
|
|
27
|
+
assert excinfo.value.status in (400, 404), f"unexpected status {excinfo.value.status}"
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Live tests: router select — POST /v1/router/select.
|
|
2
|
+
|
|
3
|
+
Gated server-side by AUTO_ROUTER_ENABLED. When disabled the endpoint returns
|
|
4
|
+
403/404, in which case these tests skip (deployment config), not fail.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
from meshapi import ChatMessage, MeshAPI, MeshAPIError, RouterSelectParams
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _select(client: MeshAPI, params: RouterSelectParams):
|
|
14
|
+
try:
|
|
15
|
+
return client.router.select(params)
|
|
16
|
+
except MeshAPIError as exc:
|
|
17
|
+
if exc.status in (403, 404, 501):
|
|
18
|
+
pytest.skip(f"auto router disabled on this deployment (AUTO_ROUTER_ENABLED): {exc.error_code}")
|
|
19
|
+
raise
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_router_select_returns_a_model(client: MeshAPI) -> None:
|
|
23
|
+
resp = _select(
|
|
24
|
+
client,
|
|
25
|
+
RouterSelectParams(
|
|
26
|
+
messages=[ChatMessage(role="user", content="Write a Python function to reverse a string.")]
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
assert resp.model, "router must always return a model (fail-soft)"
|
|
30
|
+
assert resp.auto_router is not None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_router_select_honors_exclusions(client: MeshAPI) -> None:
|
|
34
|
+
excluded = "openai/gpt-4o-mini"
|
|
35
|
+
resp = _select(
|
|
36
|
+
client,
|
|
37
|
+
RouterSelectParams(
|
|
38
|
+
messages=[ChatMessage(role="user", content="Explain the theory of relativity simply.")],
|
|
39
|
+
exclude_models=[excluded],
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
assert resp.model, "router must return a model even with exclusions"
|
|
43
|
+
# Unless it fell back to the configured default, the excluded model must not be picked.
|
|
44
|
+
if not resp.auto_router.fallback_used:
|
|
45
|
+
assert resp.model != excluded, "excluded model should not be selected"
|