metergraphrelay 0.7.3__tar.gz → 0.7.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {metergraphrelay-0.7.3/src/metergraphrelay.egg-info → metergraphrelay-0.7.6}/PKG-INFO +25 -1
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/README.md +24 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/pyproject.toml +1 -1
- metergraphrelay-0.7.6/src/metergraphrelay/__init__.py +1 -0
- metergraphrelay-0.7.6/src/metergraphrelay/providers/cache_shape.py +21 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/langfuse.py +2 -3
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/portkey.py +74 -4
- metergraphrelay-0.7.6/src/metergraphrelay/providers/portkey_batch.py +127 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6/src/metergraphrelay.egg-info}/PKG-INFO +25 -1
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/SOURCES.txt +2 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_portkey_sync.py +23 -0
- metergraphrelay-0.7.3/src/metergraphrelay/__init__.py +0 -1
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/LICENSE +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/setup.cfg +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/billing_evidence.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/capture_contract.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/cli.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/config.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/demo.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/finish_reason.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/http_limits.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/import_identity.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/metergraph_sync.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/portkey_sync.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/provider_sync.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/__init__.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/braintrust.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/langsmith.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/openai.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/phoenix.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/portkey_export.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/push.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/py.typed +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/sync_core.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/window.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/dependency_links.txt +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/entry_points.txt +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/requires.txt +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/top_level.txt +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_billing_evidence.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_capture_contract.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_cli.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_config.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_demo.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_finish_reason.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_metergraph_sync.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_provider_sync.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_push.py +0 -0
- {metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/tests/test_window.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: metergraphrelay
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.6
|
|
4
4
|
Summary: Move LLM trace data between systems: pull from providers, push to metergraph.
|
|
5
5
|
Author: Vasiliy Radostev
|
|
6
6
|
License-Expression: MIT
|
|
@@ -151,6 +151,30 @@ so `status` is always `"success"`. `error`/`error_type` flag a *partial*
|
|
|
151
151
|
record: `--include-content` was requested but the follow-up message fetch
|
|
152
152
|
failed, so token counts are still real while the content is missing.
|
|
153
153
|
|
|
154
|
+
### Token counts follow the publisher
|
|
155
|
+
|
|
156
|
+
A record carries token counts the way the model's publisher bills them, and
|
|
157
|
+
the metergraph catalog applies that publisher's arithmetic. Providers do not
|
|
158
|
+
agree on what `input_tokens` contains:
|
|
159
|
+
|
|
160
|
+
| Publisher | `input_tokens` contains cached tokens? | Cache buckets |
|
|
161
|
+
| --- | --- | --- |
|
|
162
|
+
| OpenAI, Google, DeepSeek, xAI | Yes, as a total | `cache_read_tokens` is a subset of input |
|
|
163
|
+
| Anthropic, Bedrock | No | `cache_read_tokens` and `cache_write_tokens` are separate buckets, added to input |
|
|
164
|
+
|
|
165
|
+
Gateways and tracing tools reshape these. Portkey's `req_units` is a total
|
|
166
|
+
for every provider; Vercel's gateway reports totals; Langfuse subtracts the
|
|
167
|
+
buckets for every provider. A relay adapter converts whatever shape it
|
|
168
|
+
receives back into the publisher's shape, using the provider's own usage
|
|
169
|
+
block where the source passes it through. Keeping a gateway's total for an
|
|
170
|
+
Anthropic call bills every cached token twice, once in input and once at the
|
|
171
|
+
cache rate; folding Langfuse's buckets into an OpenAI total does the same.
|
|
172
|
+
The gateway's own amount, where it reports one, is kept separately as
|
|
173
|
+
`reported_cost_usd`, so nothing the gateway said is lost.
|
|
174
|
+
|
|
175
|
+
`providers/cache_shape.py` names the publishers that count without the cache
|
|
176
|
+
buckets; add a provider there, not in an adapter.
|
|
177
|
+
|
|
154
178
|
## Pull from Langfuse
|
|
155
179
|
|
|
156
180
|
Import Langfuse **GENERATION** observations (the LLM call records
|
|
@@ -128,6 +128,30 @@ so `status` is always `"success"`. `error`/`error_type` flag a *partial*
|
|
|
128
128
|
record: `--include-content` was requested but the follow-up message fetch
|
|
129
129
|
failed, so token counts are still real while the content is missing.
|
|
130
130
|
|
|
131
|
+
### Token counts follow the publisher
|
|
132
|
+
|
|
133
|
+
A record carries token counts the way the model's publisher bills them, and
|
|
134
|
+
the metergraph catalog applies that publisher's arithmetic. Providers do not
|
|
135
|
+
agree on what `input_tokens` contains:
|
|
136
|
+
|
|
137
|
+
| Publisher | `input_tokens` contains cached tokens? | Cache buckets |
|
|
138
|
+
| --- | --- | --- |
|
|
139
|
+
| OpenAI, Google, DeepSeek, xAI | Yes, as a total | `cache_read_tokens` is a subset of input |
|
|
140
|
+
| Anthropic, Bedrock | No | `cache_read_tokens` and `cache_write_tokens` are separate buckets, added to input |
|
|
141
|
+
|
|
142
|
+
Gateways and tracing tools reshape these. Portkey's `req_units` is a total
|
|
143
|
+
for every provider; Vercel's gateway reports totals; Langfuse subtracts the
|
|
144
|
+
buckets for every provider. A relay adapter converts whatever shape it
|
|
145
|
+
receives back into the publisher's shape, using the provider's own usage
|
|
146
|
+
block where the source passes it through. Keeping a gateway's total for an
|
|
147
|
+
Anthropic call bills every cached token twice, once in input and once at the
|
|
148
|
+
cache rate; folding Langfuse's buckets into an OpenAI total does the same.
|
|
149
|
+
The gateway's own amount, where it reports one, is kept separately as
|
|
150
|
+
`reported_cost_usd`, so nothing the gateway said is lost.
|
|
151
|
+
|
|
152
|
+
`providers/cache_shape.py` names the publishers that count without the cache
|
|
153
|
+
buckets; add a provider there, not in an adapter.
|
|
154
|
+
|
|
131
155
|
## Pull from Langfuse
|
|
132
156
|
|
|
133
157
|
Import Langfuse **GENERATION** observations (the LLM call records
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "metergraphrelay"
|
|
7
|
-
version = "0.7.
|
|
7
|
+
version = "0.7.6"
|
|
8
8
|
description = "Move LLM trace data between systems: pull from providers, push to metergraph."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.7.6"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Which providers report input tokens without the cached ones.
|
|
2
|
+
|
|
3
|
+
MeterGraph prices cache handling per publisher, so a row has to arrive in the
|
|
4
|
+
shape that publisher's API uses. Anthropic (direct or on Bedrock) reports
|
|
5
|
+
input excluding the cache buckets it lists separately; OpenAI and Google
|
|
6
|
+
report totals that include them.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
CACHE_EXCLUSIVE_INPUT_PROVIDERS = frozenset(
|
|
12
|
+
{"anthropic", "bedrock", "aws-bedrock", "aws", "amazon-bedrock"}
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def reports_input_without_cache(provider: object, model: object = None) -> bool:
|
|
17
|
+
"""Whether this call's publisher counts input tokens without the cache
|
|
18
|
+
buckets. A Claude model served elsewhere (Vertex, a gateway) keeps
|
|
19
|
+
Anthropic's accounting."""
|
|
20
|
+
name = str(provider or "").strip().lower()
|
|
21
|
+
return name in CACHE_EXCLUSIVE_INPUT_PROVIDERS or str(model or "").strip().lower().startswith("claude")
|
|
@@ -11,6 +11,7 @@ import urllib.request
|
|
|
11
11
|
from typing import Any, Callable
|
|
12
12
|
|
|
13
13
|
from .. import __version__
|
|
14
|
+
from .cache_shape import CACHE_EXCLUSIVE_INPUT_PROVIDERS
|
|
14
15
|
from ..http_limits import ResponseTooLarge, read_bounded
|
|
15
16
|
from ..billing_evidence import reported_cost
|
|
16
17
|
from ..capture_contract import capture_response_text, capture_row
|
|
@@ -313,9 +314,7 @@ _NON_BUCKET_MARKERS = ("priority",)
|
|
|
313
314
|
#
|
|
314
315
|
# For OpenAI and Google the opposite is true: their own totals include the
|
|
315
316
|
# cached tokens, so the buckets Langfuse subtracted do have to be added back.
|
|
316
|
-
_CACHE_EXCLUSIVE_INPUT_PROVIDERS =
|
|
317
|
-
{"anthropic", "bedrock", "aws-bedrock", "aws", "amazon-bedrock"}
|
|
318
|
-
)
|
|
317
|
+
_CACHE_EXCLUSIVE_INPUT_PROVIDERS = CACHE_EXCLUSIVE_INPUT_PROVIDERS
|
|
319
318
|
|
|
320
319
|
|
|
321
320
|
def _is_int(value: Any) -> bool:
|
|
@@ -13,6 +13,7 @@ from .. import __version__
|
|
|
13
13
|
from ..billing_evidence import reported_cost
|
|
14
14
|
from ..capture_contract import capture_row, capture_text, capture_tool_calls
|
|
15
15
|
from ..finish_reason import extract_stop_reason
|
|
16
|
+
from .cache_shape import reports_input_without_cache
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
# Shared with the other sync providers; re-exported here for existing imports.
|
|
@@ -365,6 +366,32 @@ def _usage_detail(response: dict) -> dict[str, Any]:
|
|
|
365
366
|
return detail
|
|
366
367
|
|
|
367
368
|
|
|
369
|
+
def _exclude_cache_from_input(result: dict, response: dict) -> None:
|
|
370
|
+
"""Report Anthropic input the way Anthropic bills it.
|
|
371
|
+
|
|
372
|
+
Portkey's `req_units` is the whole prompt, cache reads and writes included,
|
|
373
|
+
for every provider. Anthropic's own usage lists input without them, and
|
|
374
|
+
the catalog prices Anthropic that way, so a row that keeps `req_units`
|
|
375
|
+
bills every cached token twice: once at the input rate and once at the
|
|
376
|
+
cache rate. The provider's own `input_tokens` is used where the usage
|
|
377
|
+
block carries it; otherwise the cache buckets come off the total.
|
|
378
|
+
"""
|
|
379
|
+
if not reports_input_without_cache(result.get("provider"), result.get("model")):
|
|
380
|
+
return
|
|
381
|
+
cache_read = result.get("cache_read_tokens") or 0
|
|
382
|
+
cache_write = result.get("cache_write_tokens") or 0
|
|
383
|
+
if not (cache_read or cache_write):
|
|
384
|
+
return
|
|
385
|
+
usage = response.get("usage")
|
|
386
|
+
native = usage.get("input_tokens") if isinstance(usage, dict) else None
|
|
387
|
+
if _is_int(native):
|
|
388
|
+
result["input_tokens"] = native
|
|
389
|
+
return
|
|
390
|
+
total = result.get("input_tokens")
|
|
391
|
+
if _is_int(total):
|
|
392
|
+
result["input_tokens"] = max(0, total - cache_read - cache_write)
|
|
393
|
+
|
|
394
|
+
|
|
368
395
|
def _web_search_calls(response: dict) -> int | None:
|
|
369
396
|
"""How many searches the model ran, for providers that bill per search.
|
|
370
397
|
|
|
@@ -409,6 +436,12 @@ def _endpoint(response: dict) -> str | None:
|
|
|
409
436
|
return "responses"
|
|
410
437
|
if isinstance(response.get("choices"), list):
|
|
411
438
|
return "chat.completions"
|
|
439
|
+
# Anthropic's Messages API answers with a top-level `type: message` and a
|
|
440
|
+
# `content` list rather than `choices`. Without this the call has no
|
|
441
|
+
# endpoint, the billing evidence never qualifies, and Portkey's cost for it
|
|
442
|
+
# is recorded but not used.
|
|
443
|
+
if response.get("type") == "message":
|
|
444
|
+
return "messages"
|
|
412
445
|
return None
|
|
413
446
|
|
|
414
447
|
|
|
@@ -507,6 +540,7 @@ def normalize_portkey_row(
|
|
|
507
540
|
if stop_reason is not None:
|
|
508
541
|
result["stop_reason"] = stop_reason
|
|
509
542
|
result.update(_usage_detail(response))
|
|
543
|
+
_exclude_cache_from_input(result, response)
|
|
510
544
|
if import_context is not None:
|
|
511
545
|
result["import_source"] = import_context.source
|
|
512
546
|
result["import_source_scope"] = import_context.source_scope
|
|
@@ -514,6 +548,41 @@ def normalize_portkey_row(
|
|
|
514
548
|
return capture_row(result)
|
|
515
549
|
|
|
516
550
|
|
|
551
|
+
def normalize_portkey_rows(
|
|
552
|
+
row: dict, *, import_context: ImportContext | None = None,
|
|
553
|
+
batch_requests: dict[str, dict] | None = None,
|
|
554
|
+
) -> list[dict]:
|
|
555
|
+
"""Normalize one export row to zero or more individual inference outcomes.
|
|
556
|
+
|
|
557
|
+
Ordinary rows keep the singular converter's contract. Batch management is
|
|
558
|
+
excluded; result messages use native usage and stable per-message identity.
|
|
559
|
+
The optional request map must belong to this batch, keyed by custom_id.
|
|
560
|
+
"""
|
|
561
|
+
from .portkey_batch import BatchShapeError, expand_batch
|
|
562
|
+
|
|
563
|
+
try:
|
|
564
|
+
children = expand_batch(row, batch_requests=batch_requests)
|
|
565
|
+
except BatchShapeError as exc:
|
|
566
|
+
raise PortkeyConversionError(str(exc)) from None
|
|
567
|
+
if children is None:
|
|
568
|
+
return [normalize_portkey_row(row, import_context=import_context)]
|
|
569
|
+
results = []
|
|
570
|
+
for child in children:
|
|
571
|
+
result = normalize_portkey_row(child, import_context=import_context)
|
|
572
|
+
if child.get('request') is None:
|
|
573
|
+
result.pop('request_json', None)
|
|
574
|
+
result['batch'] = True
|
|
575
|
+
result['endpoint'] = 'messages'
|
|
576
|
+
result['batch_custom_id'] = child['batch_custom_id']
|
|
577
|
+
result['batch_outcome'] = child['batch_outcome']
|
|
578
|
+
result['portkey_export_id'] = row.get('id')
|
|
579
|
+
# These describe a whole download, not an individual model completion.
|
|
580
|
+
for key in ('cost_usd', 'reported_cost_usd', 'reported_cost_source', 'latency_ms'):
|
|
581
|
+
result.pop(key, None)
|
|
582
|
+
results.append(result)
|
|
583
|
+
return results
|
|
584
|
+
|
|
585
|
+
|
|
517
586
|
def convert_portkey_export(
|
|
518
587
|
input_path: str,
|
|
519
588
|
output_path: str,
|
|
@@ -548,8 +617,8 @@ def convert_portkey_export(
|
|
|
548
617
|
)
|
|
549
618
|
continue
|
|
550
619
|
try:
|
|
551
|
-
|
|
552
|
-
|
|
620
|
+
normalized_rows = normalize_portkey_rows(row, import_context=import_context)
|
|
621
|
+
serialized_rows = [json.dumps(normalized) for normalized in normalized_rows]
|
|
553
622
|
except (KeyError, TypeError, AttributeError) as exc:
|
|
554
623
|
skipped += 1
|
|
555
624
|
row_id = (
|
|
@@ -560,6 +629,7 @@ def convert_portkey_export(
|
|
|
560
629
|
file=sys.stderr,
|
|
561
630
|
)
|
|
562
631
|
continue
|
|
563
|
-
|
|
564
|
-
|
|
632
|
+
for serialized in serialized_rows:
|
|
633
|
+
dst.write(serialized + "\n")
|
|
634
|
+
converted += 1
|
|
565
635
|
return converted, skipped
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Identify Anthropic batch operations and expand their individual outcomes.
|
|
2
|
+
|
|
3
|
+
The download is transport, not inference: its cost, latency and token counters
|
|
4
|
+
cannot be assigned to any child message. Successful message IDs are the stable
|
|
5
|
+
identity even when an application downloads the same results more than once.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BatchShapeError(ValueError):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
_CONTROL = re.compile(r"(?:anthropic/)?(?:batches|msgbatch_[A-Za-z0-9_-]+)(?:/cancel)?\Z")
|
|
19
|
+
_RESULTS = re.compile(r"(?:anthropic/)?(?:results|msgbatch_[A-Za-z0-9_-]+/results)\Z")
|
|
20
|
+
_RESULT_BATCH = re.compile(r"(?:anthropic/)?(?P<batch>msgbatch_[A-Za-z0-9_-]+)/results\Z")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _validate_cache(usage: dict) -> None:
|
|
24
|
+
for key in ('cache_read_input_tokens', 'cache_creation_input_tokens'):
|
|
25
|
+
if key in usage and (type(usage[key]) is not int or usage[key] < 0):
|
|
26
|
+
raise BatchShapeError('invalid batch message cache usage')
|
|
27
|
+
if 'cache_creation' in usage:
|
|
28
|
+
creation = usage['cache_creation']
|
|
29
|
+
if not isinstance(creation, dict):
|
|
30
|
+
raise BatchShapeError('invalid batch message cache usage')
|
|
31
|
+
for key in ('ephemeral_5m_input_tokens', 'ephemeral_1h_input_tokens'):
|
|
32
|
+
if key in creation and (type(creation[key]) is not int or creation[key] < 0):
|
|
33
|
+
raise BatchShapeError('invalid batch message cache usage')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _envelopes(response: Any) -> list[dict]:
|
|
37
|
+
if isinstance(response, str):
|
|
38
|
+
try:
|
|
39
|
+
response = [json.loads(line) for line in response.splitlines() if line.strip()]
|
|
40
|
+
except (ValueError, RecursionError):
|
|
41
|
+
raise BatchShapeError('invalid Anthropic batch results JSONL') from None
|
|
42
|
+
if len(response) == 1 and isinstance(response[0], list):
|
|
43
|
+
response = response[0]
|
|
44
|
+
if isinstance(response, dict):
|
|
45
|
+
response = [response]
|
|
46
|
+
if not isinstance(response, list) or not response:
|
|
47
|
+
raise BatchShapeError('Anthropic batch results must contain outcomes')
|
|
48
|
+
if not all(isinstance(item, dict) for item in response):
|
|
49
|
+
raise BatchShapeError('invalid Anthropic batch result envelope')
|
|
50
|
+
return response
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def expand_batch(row: dict, *, batch_requests: dict | None = None) -> list[dict] | None:
|
|
54
|
+
"""Return ordinary-shaped Portkey child rows, [] for control, None otherwise.
|
|
55
|
+
|
|
56
|
+
Request correlation is supplied by a caller that knows the batch identity.
|
|
57
|
+
Joining custom_id across an arbitrary export would mix unrelated batches.
|
|
58
|
+
"""
|
|
59
|
+
if str(row.get('ai_org', '')).strip().lower() != 'anthropic':
|
|
60
|
+
return None
|
|
61
|
+
response = row.get('response')
|
|
62
|
+
model = str(row.get('ai_model') or '')
|
|
63
|
+
if (isinstance(response, dict) and response.get('type') == 'message_batch') or _CONTROL.fullmatch(model):
|
|
64
|
+
return []
|
|
65
|
+
envelope = isinstance(response, dict) and 'custom_id' in response and 'result' in response
|
|
66
|
+
if not envelope and not _RESULTS.fullmatch(model):
|
|
67
|
+
return None
|
|
68
|
+
status = row.get('response_status_code')
|
|
69
|
+
if type(status) is not int or not 200 <= status < 300:
|
|
70
|
+
# A failed/not-ready fetch is batch transport, just like status polls.
|
|
71
|
+
# Only a successful response can claim to contain inference results.
|
|
72
|
+
return []
|
|
73
|
+
|
|
74
|
+
expanded = []
|
|
75
|
+
seen = set()
|
|
76
|
+
for item in _envelopes(response):
|
|
77
|
+
custom_id = item.get('custom_id')
|
|
78
|
+
outcome = item.get('result')
|
|
79
|
+
if not isinstance(custom_id, str) or not custom_id or not isinstance(outcome, dict):
|
|
80
|
+
raise BatchShapeError('invalid Anthropic batch result envelope')
|
|
81
|
+
if custom_id in seen:
|
|
82
|
+
raise BatchShapeError('duplicate custom_id in Anthropic batch results')
|
|
83
|
+
seen.add(custom_id)
|
|
84
|
+
request = (batch_requests or {}).get(custom_id)
|
|
85
|
+
if request is not None and not isinstance(request, dict):
|
|
86
|
+
raise BatchShapeError('invalid correlated Anthropic batch request')
|
|
87
|
+
outcome_type = outcome.get('type')
|
|
88
|
+
if outcome_type == 'succeeded':
|
|
89
|
+
message = outcome.get('message')
|
|
90
|
+
if not isinstance(message, dict) or message.get('type') != 'message':
|
|
91
|
+
raise BatchShapeError('successful batch outcome is missing a message')
|
|
92
|
+
message_id, actual_model, usage = message.get('id'), message.get('model'), message.get('usage')
|
|
93
|
+
if not isinstance(message_id, str) or not message_id.strip():
|
|
94
|
+
raise BatchShapeError('successful batch message is missing its identity')
|
|
95
|
+
if not isinstance(actual_model, str) or not actual_model.strip() or not isinstance(usage, dict):
|
|
96
|
+
raise BatchShapeError('successful batch message is missing model or usage')
|
|
97
|
+
for field in ('input_tokens', 'output_tokens'):
|
|
98
|
+
if type(usage.get(field)) is not int or usage[field] < 0:
|
|
99
|
+
raise BatchShapeError('invalid batch message token usage')
|
|
100
|
+
_validate_cache(usage)
|
|
101
|
+
identity = 'anthropic-batch:' + message_id.strip()
|
|
102
|
+
status = 200
|
|
103
|
+
elif outcome_type in ('errored', 'canceled', 'expired'):
|
|
104
|
+
actual_model = request.get('model') if request else None
|
|
105
|
+
usage = {}
|
|
106
|
+
batch_id = row.get('batch_id')
|
|
107
|
+
endpoint_batch = _RESULT_BATCH.fullmatch(model)
|
|
108
|
+
if not batch_id and endpoint_batch:
|
|
109
|
+
batch_id = endpoint_batch['batch']
|
|
110
|
+
# Without a provider batch ID, the export ID is the only available
|
|
111
|
+
# stable boundary. Never deduplicate unrelated batches by custom_id.
|
|
112
|
+
boundary = batch_id if isinstance(batch_id, str) and batch_id else row.get('id')
|
|
113
|
+
if not isinstance(boundary, str) or not boundary.strip():
|
|
114
|
+
raise BatchShapeError('failed batch outcome is missing its identity')
|
|
115
|
+
identity = 'anthropic-batch:' + boundary.strip() + ':' + custom_id
|
|
116
|
+
message = {'error': outcome.get('error') or {'type': outcome_type, 'message': outcome_type}}
|
|
117
|
+
status = 400
|
|
118
|
+
else:
|
|
119
|
+
raise BatchShapeError('unknown Anthropic batch outcome type')
|
|
120
|
+
expanded.append({
|
|
121
|
+
**row, 'id': identity, 'ai_model': actual_model, 'request': request,
|
|
122
|
+
'response': message, 'response_status_code': status,
|
|
123
|
+
'req_units': usage.get('input_tokens'), 'res_units': usage.get('output_tokens'),
|
|
124
|
+
'cost': None, 'response_time': None, 'batch_custom_id': custom_id,
|
|
125
|
+
'batch_outcome': outcome_type,
|
|
126
|
+
})
|
|
127
|
+
return expanded
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: metergraphrelay
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.6
|
|
4
4
|
Summary: Move LLM trace data between systems: pull from providers, push to metergraph.
|
|
5
5
|
Author: Vasiliy Radostev
|
|
6
6
|
License-Expression: MIT
|
|
@@ -151,6 +151,30 @@ so `status` is always `"success"`. `error`/`error_type` flag a *partial*
|
|
|
151
151
|
record: `--include-content` was requested but the follow-up message fetch
|
|
152
152
|
failed, so token counts are still real while the content is missing.
|
|
153
153
|
|
|
154
|
+
### Token counts follow the publisher
|
|
155
|
+
|
|
156
|
+
A record carries token counts the way the model's publisher bills them, and
|
|
157
|
+
the metergraph catalog applies that publisher's arithmetic. Providers do not
|
|
158
|
+
agree on what `input_tokens` contains:
|
|
159
|
+
|
|
160
|
+
| Publisher | `input_tokens` contains cached tokens? | Cache buckets |
|
|
161
|
+
| --- | --- | --- |
|
|
162
|
+
| OpenAI, Google, DeepSeek, xAI | Yes, as a total | `cache_read_tokens` is a subset of input |
|
|
163
|
+
| Anthropic, Bedrock | No | `cache_read_tokens` and `cache_write_tokens` are separate buckets, added to input |
|
|
164
|
+
|
|
165
|
+
Gateways and tracing tools reshape these. Portkey's `req_units` is a total
|
|
166
|
+
for every provider; Vercel's gateway reports totals; Langfuse subtracts the
|
|
167
|
+
buckets for every provider. A relay adapter converts whatever shape it
|
|
168
|
+
receives back into the publisher's shape, using the provider's own usage
|
|
169
|
+
block where the source passes it through. Keeping a gateway's total for an
|
|
170
|
+
Anthropic call bills every cached token twice, once in input and once at the
|
|
171
|
+
cache rate; folding Langfuse's buckets into an OpenAI total does the same.
|
|
172
|
+
The gateway's own amount, where it reports one, is kept separately as
|
|
173
|
+
`reported_cost_usd`, so nothing the gateway said is lost.
|
|
174
|
+
|
|
175
|
+
`providers/cache_shape.py` names the publishers that count without the cache
|
|
176
|
+
buckets; add a provider there, not in an adapter.
|
|
177
|
+
|
|
154
178
|
## Pull from Langfuse
|
|
155
179
|
|
|
156
180
|
Import Langfuse **GENERATION** observations (the LLM call records
|
|
@@ -25,11 +25,13 @@ src/metergraphrelay.egg-info/requires.txt
|
|
|
25
25
|
src/metergraphrelay.egg-info/top_level.txt
|
|
26
26
|
src/metergraphrelay/providers/__init__.py
|
|
27
27
|
src/metergraphrelay/providers/braintrust.py
|
|
28
|
+
src/metergraphrelay/providers/cache_shape.py
|
|
28
29
|
src/metergraphrelay/providers/langfuse.py
|
|
29
30
|
src/metergraphrelay/providers/langsmith.py
|
|
30
31
|
src/metergraphrelay/providers/openai.py
|
|
31
32
|
src/metergraphrelay/providers/phoenix.py
|
|
32
33
|
src/metergraphrelay/providers/portkey.py
|
|
34
|
+
src/metergraphrelay/providers/portkey_batch.py
|
|
33
35
|
src/metergraphrelay/providers/portkey_export.py
|
|
34
36
|
tests/test_billing_evidence.py
|
|
35
37
|
tests/test_capture_contract.py
|
|
@@ -184,6 +184,29 @@ def _run(mg, pk, pushes, **overrides):
|
|
|
184
184
|
# -- _LeaseRenewer: time-based renewal policy ------------------------------
|
|
185
185
|
|
|
186
186
|
|
|
187
|
+
@pytest.mark.parametrize('status', [None, 404, 429, 500])
|
|
188
|
+
def test_failed_batch_results_fetch_does_not_block_checkpoint(status):
|
|
189
|
+
mg, pushes = FakeMeterGraph(_acquired()), []
|
|
190
|
+
pending = {**_portkey_row('not-ready'), 'ai_org': 'anthropic',
|
|
191
|
+
'ai_model': 'anthropic/results', 'response_status_code': status,
|
|
192
|
+
'response': {'error': {'type': 'not_found_error'}}}
|
|
193
|
+
pk = FakePortkey({(WINDOW_START, WINDOW_END): [pending, _portkey_row('real-call')]})
|
|
194
|
+
outcome = _run(mg, pk, pushes)
|
|
195
|
+
assert outcome.status == 'completed'
|
|
196
|
+
assert mg.completed == ['lease-1'] and mg.abandoned == []
|
|
197
|
+
assert [row['request_id'] for push in pushes for row in push['rows']] == ['real-call']
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_corrupt_successful_batch_results_still_blocks_checkpoint():
|
|
201
|
+
mg, pushes = FakeMeterGraph(_acquired()), []
|
|
202
|
+
corrupt = {**_portkey_row('corrupt-download'), 'ai_org': 'anthropic',
|
|
203
|
+
'ai_model': 'anthropic/results', 'response': 'not-json'}
|
|
204
|
+
pk = FakePortkey({(WINDOW_START, WINDOW_END): [corrupt]})
|
|
205
|
+
outcome = _run(mg, pk, pushes)
|
|
206
|
+
assert outcome.status == 'failed'
|
|
207
|
+
assert mg.completed == [] and mg.abandoned == ['lease-1']
|
|
208
|
+
|
|
209
|
+
|
|
187
210
|
def test_lease_renewer_does_not_renew_before_interval_elapses():
|
|
188
211
|
clock = FakeClock()
|
|
189
212
|
renews = []
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.7.3"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay/providers/portkey_export.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{metergraphrelay-0.7.3 → metergraphrelay-0.7.6}/src/metergraphrelay.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|