typevet 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- typevet/__init__.py +67 -0
- typevet/_version.py +43 -0
- typevet/adapters/__init__.py +16 -0
- typevet/adapters/diagnostics/__init__.py +59 -0
- typevet/adapters/diagnostics/fields.py +108 -0
- typevet/adapters/diagnostics/generation_events.py +75 -0
- typevet/adapters/diagnostics/http_events.py +91 -0
- typevet/adapters/diagnostics/logs.py +116 -0
- typevet/adapters/diagnostics/redaction.py +94 -0
- typevet/adapters/diagnostics/settings.py +77 -0
- typevet/adapters/inbound/__init__.py +65 -0
- typevet/adapters/inbound/api.py +61 -0
- typevet/adapters/inbound/backend_settings.py +608 -0
- typevet/adapters/inbound/cord_semantic_acceptance_cli.py +135 -0
- typevet/adapters/inbound/helpers.py +67 -0
- typevet/adapters/inbound/settings.py +154 -0
- typevet/adapters/outbound/__init__.py +62 -0
- typevet/adapters/outbound/async_fake.py +123 -0
- typevet/adapters/outbound/async_llama_cpp.py +168 -0
- typevet/adapters/outbound/chat_completion.py +148 -0
- typevet/adapters/outbound/fake.py +119 -0
- typevet/adapters/outbound/gemma/__init__.py +84 -0
- typevet/adapters/outbound/gemma/answer_binding.py +290 -0
- typevet/adapters/outbound/gemma/scoring_prefix.py +102 -0
- typevet/adapters/outbound/gemma/served_template.py +131 -0
- typevet/adapters/outbound/gemma_native_vision_factory.py +286 -0
- typevet/adapters/outbound/generation_finite.py +50 -0
- typevet/adapters/outbound/http_errors.py +38 -0
- typevet/adapters/outbound/judgment_scoring.py +471 -0
- typevet/adapters/outbound/llama_cpp.py +187 -0
- typevet/adapters/outbound/llama_cpp_http.py +103 -0
- typevet/adapters/outbound/llama_cpp_multimodal.py +145 -0
- typevet/adapters/outbound/llama_cpp_scoring.py +333 -0
- typevet/adapters/outbound/vllm_content.py +63 -0
- typevet/adapters/outbound/vllm_generation.py +163 -0
- typevet/adapters/outbound/vllm_generation_async.py +257 -0
- typevet/adapters/outbound/vllm_http.py +121 -0
- typevet/adapters/outbound/vllm_judgment_factory.py +210 -0
- typevet/adapters/outbound/vllm_scoring.py +372 -0
- typevet/domain/__init__.py +193 -0
- typevet/domain/candidate_scoring_request.py +134 -0
- typevet/domain/candidate_scoring_response.py +96 -0
- typevet/domain/candidate_scoring_validate.py +197 -0
- typevet/domain/decision_compile.py +368 -0
- typevet/domain/decision_execute.py +229 -0
- typevet/domain/decisions.py +129 -0
- typevet/domain/errors.py +235 -0
- typevet/domain/field_instructions.py +134 -0
- typevet/domain/judgment_answers.py +198 -0
- typevet/domain/judgment_normalize.py +243 -0
- typevet/domain/judgment_questions.py +187 -0
- typevet/domain/judgment_response.py +102 -0
- typevet/domain/media.py +101 -0
- typevet/domain/models.py +115 -0
- typevet/domain/question_schema.py +254 -0
- typevet/domain/scoring_stage.py +34 -0
- typevet/evaluation/__init__.py +19 -0
- typevet/evaluation/consumer_http_accounting.py +135 -0
- typevet/evaluation/cord_expense_call_accounting.py +108 -0
- typevet/evaluation/cord_expense_live_harness.py +75 -0
- typevet/evaluation/cord_expense_receipt_requirement.py +212 -0
- typevet/evaluation/cord_expense_smoke.py +328 -0
- typevet/evaluation/cord_semantic_acceptance.py +463 -0
- typevet/evaluation/cord_semantic_acceptance_report.py +90 -0
- typevet/evaluation/cord_semantic_metrics.py +57 -0
- typevet/evaluation/datasets/__init__.py +30 -0
- typevet/evaluation/datasets/banking77.py +270 -0
- typevet/evaluation/datasets/boolq.py +550 -0
- typevet/evaluation/datasets/boolq_download.py +132 -0
- typevet/evaluation/datasets/civil_comments.py +450 -0
- typevet/evaluation/datasets/clinc.py +125 -0
- typevet/evaluation/datasets/clinc_domains.json +172 -0
- typevet/evaluation/datasets/clinc_download.py +103 -0
- typevet/evaluation/datasets/clinc_plus_intent_names.json +153 -0
- typevet/evaluation/datasets/clinc_rows.py +306 -0
- typevet/evaluation/datasets/clinc_shard.py +156 -0
- typevet/evaluation/datasets/cord_expense.py +381 -0
- typevet/evaluation/datasets/difraud.py +256 -0
- typevet/evaluation/datasets/go_emotions.py +368 -0
- typevet/evaluation/datasets/go_emotions_download.py +154 -0
- typevet/evaluation/datasets/hyperpartisan.py +562 -0
- typevet/evaluation/datasets/partner_guard.py +170 -0
- typevet/evaluation/datasets/psai.py +415 -0
- typevet/evaluation/datasets/psai_download.py +167 -0
- typevet/evaluation/datasets/psai_evidence_pilot.py +570 -0
- typevet/evaluation/datasets/psai_schema.py +157 -0
- typevet/evaluation/datasets/psai_stream.py +166 -0
- typevet/evaluation/datasets/psai_vision.py +534 -0
- typevet/evaluation/datasets/psai_vision_controls.py +502 -0
- typevet/evaluation/datasets/pubmedqa.py +420 -0
- typevet/evaluation/experiment_identity.py +677 -0
- typevet/evaluation/psai_vision_consumer_accounting.py +190 -0
- typevet/evaluation/psai_vision_consumer_dispatch.py +216 -0
- typevet/evaluation/psai_vision_consumer_harness.py +342 -0
- typevet/evaluation/psai_vision_consumer_live.py +332 -0
- typevet/evaluation/psai_vision_consumer_live_identity.py +156 -0
- typevet/evaluation/psai_vision_consumer_live_receipt.py +132 -0
- typevet/evaluation/psai_vision_consumer_live_router.py +162 -0
- typevet/evaluation/psai_vision_consumer_offline.py +418 -0
- typevet/evaluation/psai_vision_consumer_outcomes.py +224 -0
- typevet/evaluation/psai_vision_consumer_protocol.py +69 -0
- typevet/evaluation/psai_vision_consumer_receipt.py +161 -0
- typevet/evaluation/psai_vision_consumer_receipt_structure.py +263 -0
- typevet/evaluation/psai_vision_probability_evidence.py +336 -0
- typevet/evaluation/runner/__init__.py +56 -0
- typevet/evaluation/runner/core.py +104 -0
- typevet/evaluation/runner/datasets.py +186 -0
- typevet/evaluation/runner/live_gate.py +149 -0
- typevet/evaluation/runner/report.py +116 -0
- typevet/evaluation/tpjep/__init__.py +81 -0
- typevet/evaluation/tpjep/loader.py +237 -0
- typevet/evaluation/tpjep/outcome.py +101 -0
- typevet/evaluation/tpjep/records.py +428 -0
- typevet/evaluation/tpjep/runner.py +325 -0
- typevet/ports/__init__.py +39 -0
- typevet/ports/async_generation.py +57 -0
- typevet/ports/framing.py +59 -0
- typevet/ports/generation.py +58 -0
- typevet/ports/judgment.py +71 -0
- typevet/ports/scoring.py +64 -0
- typevet/py.typed +0 -0
- typevet/runtime/__init__.py +50 -0
- typevet/runtime/categorical.py +196 -0
- typevet/runtime/judgment.py +17 -0
- typevet/runtime/llama_cpp_gemma_vision.py +28 -0
- typevet/runtime/scoring_prefix.py +14 -0
- typevet/runtime/vllm_judgment.py +28 -0
- typevet/testing/__init__.py +22 -0
- typevet/testing/fakes.py +152 -0
- typevet-0.1.0.dev1.dist-info/METADATA +125 -0
- typevet-0.1.0.dev1.dist-info/RECORD +133 -0
- typevet-0.1.0.dev1.dist-info/WHEEL +4 -0
- typevet-0.1.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Environment-backed log settings for the composition root.
|
|
2
|
+
|
|
3
|
+
Diagnostics are operator signals on stderr, not telemetry and not remote export.
|
|
4
|
+
Importing this module does not configure structlog.
|
|
5
|
+
|
|
6
|
+
Examples:
|
|
7
|
+
```python
|
|
8
|
+
from typevet.adapters.diagnostics.settings import load_log_settings
|
|
9
|
+
|
|
10
|
+
settings = load_log_settings({"TYPEVET_LOG__LEVEL": "debug"})
|
|
11
|
+
assert settings.level == "debug"
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
See Also:
|
|
15
|
+
- [typevet.adapters.diagnostics.logs][]: Applies these settings to structlog
|
|
16
|
+
- [typevet.adapters.diagnostics.redaction][]: Honors ``log_prompts`` when masking
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
from collections.abc import Mapping
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
|
|
25
|
+
_VALID_FORMATS = frozenset({"auto", "json", "console"})
|
|
26
|
+
_VALID_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
|
|
27
|
+
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class LogSettings:
|
|
32
|
+
"""How diagnostic lines are rendered and which severities are kept.
|
|
33
|
+
|
|
34
|
+
Attributes:
|
|
35
|
+
format (str): ``auto``, ``json`` or ``console``. Default ``auto``.
|
|
36
|
+
level (str): ``debug`` through ``critical``. Default ``info``.
|
|
37
|
+
log_prompts (bool): When false, prompt-like fields are stripped from output.
|
|
38
|
+
|
|
39
|
+
Examples:
|
|
40
|
+
```python
|
|
41
|
+
from typevet.adapters.diagnostics.settings import LogSettings
|
|
42
|
+
|
|
43
|
+
LogSettings(format="json", level="debug", log_prompts=True)
|
|
44
|
+
```
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
format: str = "auto"
|
|
48
|
+
level: str = "info"
|
|
49
|
+
log_prompts: bool = False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def load_log_settings(
|
|
53
|
+
environ: Mapping[str, str] | None = None,
|
|
54
|
+
) -> LogSettings:
|
|
55
|
+
"""Read ``TYPEVET_LOG__*`` variables for the composition root.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
environ: Mapping to read. Defaults to ``os.environ``.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
Frozen settings with defaults for missing keys.
|
|
62
|
+
|
|
63
|
+
Raises:
|
|
64
|
+
ValueError: When format or level is not an allowed value.
|
|
65
|
+
"""
|
|
66
|
+
source = os.environ if environ is None else environ
|
|
67
|
+
fmt = source.get("TYPEVET_LOG__FORMAT", "auto").strip().lower()
|
|
68
|
+
level = source.get("TYPEVET_LOG__LEVEL", "info").strip().lower()
|
|
69
|
+
log_prompts_raw = source.get("TYPEVET_LOG__LOG_PROMPTS", "").strip().lower()
|
|
70
|
+
if fmt not in _VALID_FORMATS:
|
|
71
|
+
msg = f"TYPEVET_LOG__FORMAT must be one of {sorted(_VALID_FORMATS)}"
|
|
72
|
+
raise ValueError(msg)
|
|
73
|
+
if level not in _VALID_LEVELS:
|
|
74
|
+
msg = f"TYPEVET_LOG__LEVEL must be one of {sorted(_VALID_LEVELS)}"
|
|
75
|
+
raise ValueError(msg)
|
|
76
|
+
log_prompts = log_prompts_raw in _TRUTHY
|
|
77
|
+
return LogSettings(format=fmt, level=level, log_prompts=log_prompts)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Inbound adapters (library entry points).
|
|
2
|
+
|
|
3
|
+
Examples:
|
|
4
|
+
```python
|
|
5
|
+
from typevet.adapters.inbound import generate
|
|
6
|
+
from typevet.testing import StaticGenerationFake
|
|
7
|
+
|
|
8
|
+
result = generate(
|
|
9
|
+
StaticGenerationFake({"ok": True}),
|
|
10
|
+
prompt="hi",
|
|
11
|
+
schema={"type": "object", "additionalProperties": False},
|
|
12
|
+
model="fake",
|
|
13
|
+
)
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
See Also:
|
|
17
|
+
- [typevet.adapters.inbound.api][]: ``generate`` helper
|
|
18
|
+
- [typevet.adapters.inbound.helpers][]: ``run_sync`` helper
|
|
19
|
+
- [typevet.adapters.inbound.settings][]: ``TYPEVET_LLAMA__*`` composition root
|
|
20
|
+
- [typevet.adapters.inbound.backend_settings][]: ``TYPEVET_BACKEND`` selection
|
|
21
|
+
- [typevet.ports.generation][]: GenerationPort
|
|
22
|
+
|
|
23
|
+
Attributes:
|
|
24
|
+
generate (function): Build a request and invoke a generation port.
|
|
25
|
+
run_sync (function): Run an async generation coroutine from sync code.
|
|
26
|
+
LlamaSettings (type): llama.cpp connection settings for composition roots.
|
|
27
|
+
load_llama_settings (function): Read ``TYPEVET_LLAMA__*`` from the environment.
|
|
28
|
+
llama_cpp_adapter (function): Build ``LlamaCppGenerationAdapter`` from settings.
|
|
29
|
+
VllmSettings (type): vLLM connection settings; ``api_key`` is not in ``repr``.
|
|
30
|
+
load_backend (function): Read ``TYPEVET_BACKEND``.
|
|
31
|
+
load_vllm_settings (function): Read ``TYPEVET_VLLM__*`` from the environment.
|
|
32
|
+
vllm_http_client (function): Build the shared vLLM ``httpx.Client``.
|
|
33
|
+
generation_adapter (function): Build the adapter ``TYPEVET_BACKEND`` selects.
|
|
34
|
+
async_vllm_generation_adapter (function): Build the async vLLM adapter.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from typevet.adapters.inbound.api import generate
|
|
38
|
+
from typevet.adapters.inbound.backend_settings import (
|
|
39
|
+
VllmSettings,
|
|
40
|
+
async_vllm_generation_adapter,
|
|
41
|
+
generation_adapter,
|
|
42
|
+
load_backend,
|
|
43
|
+
load_vllm_settings,
|
|
44
|
+
vllm_http_client,
|
|
45
|
+
)
|
|
46
|
+
from typevet.adapters.inbound.helpers import run_sync
|
|
47
|
+
from typevet.adapters.inbound.settings import (
|
|
48
|
+
LlamaSettings,
|
|
49
|
+
llama_cpp_adapter,
|
|
50
|
+
load_llama_settings,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"LlamaSettings",
|
|
55
|
+
"VllmSettings",
|
|
56
|
+
"async_vllm_generation_adapter",
|
|
57
|
+
"generate",
|
|
58
|
+
"generation_adapter",
|
|
59
|
+
"llama_cpp_adapter",
|
|
60
|
+
"load_backend",
|
|
61
|
+
"load_llama_settings",
|
|
62
|
+
"load_vllm_settings",
|
|
63
|
+
"run_sync",
|
|
64
|
+
"vllm_http_client",
|
|
65
|
+
]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Library entry: call a generation port with a typed request.
|
|
2
|
+
|
|
3
|
+
Examples:
|
|
4
|
+
```python
|
|
5
|
+
from typevet.adapters.inbound.api import generate
|
|
6
|
+
from typevet.testing import StaticGenerationFake
|
|
7
|
+
|
|
8
|
+
generate(
|
|
9
|
+
StaticGenerationFake({"n": 1}),
|
|
10
|
+
prompt="n",
|
|
11
|
+
schema={
|
|
12
|
+
"type": "object",
|
|
13
|
+
"properties": {"n": {"type": "integer"}},
|
|
14
|
+
"required": ["n"],
|
|
15
|
+
"additionalProperties": False,
|
|
16
|
+
},
|
|
17
|
+
model="fake",
|
|
18
|
+
)
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
See Also:
|
|
22
|
+
- [typevet.domain.media][]: ImageInput and the media marker
|
|
23
|
+
- [typevet.domain.models][]: GenerationRequest
|
|
24
|
+
- [typevet.ports.generation][]: GenerationPort
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from collections.abc import Mapping
|
|
30
|
+
from typing import TYPE_CHECKING, Any
|
|
31
|
+
|
|
32
|
+
from typevet.domain.models import GenerationRequest, GenerationResult
|
|
33
|
+
from typevet.ports.generation import GenerationPort
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from typevet.domain.media import ImageInput
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def generate(
|
|
40
|
+
port: GenerationPort,
|
|
41
|
+
*,
|
|
42
|
+
prompt: str,
|
|
43
|
+
schema: Mapping[str, Any],
|
|
44
|
+
model: str,
|
|
45
|
+
media: tuple[ImageInput, ...] = (),
|
|
46
|
+
) -> GenerationResult:
|
|
47
|
+
"""Build a request and invoke the generation port.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
port: Outbound adapter that implements ``GenerationPort``.
|
|
51
|
+
prompt: Natural-language instruction.
|
|
52
|
+
schema: JSON Schema object as a mapping.
|
|
53
|
+
model: Backend model id or alias.
|
|
54
|
+
media: Images the prompt marks, one ``MEDIA_MARKER`` each, in
|
|
55
|
+
marker order. Empty for a text ask.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
Validated generation result from the port.
|
|
59
|
+
"""
|
|
60
|
+
request = GenerationRequest(prompt=prompt, schema=schema, model=model, media=media)
|
|
61
|
+
return port.generate(request)
|