typevet 0.1.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. typevet/__init__.py +67 -0
  2. typevet/_version.py +43 -0
  3. typevet/adapters/__init__.py +16 -0
  4. typevet/adapters/diagnostics/__init__.py +59 -0
  5. typevet/adapters/diagnostics/fields.py +108 -0
  6. typevet/adapters/diagnostics/generation_events.py +75 -0
  7. typevet/adapters/diagnostics/http_events.py +91 -0
  8. typevet/adapters/diagnostics/logs.py +116 -0
  9. typevet/adapters/diagnostics/redaction.py +94 -0
  10. typevet/adapters/diagnostics/settings.py +77 -0
  11. typevet/adapters/inbound/__init__.py +65 -0
  12. typevet/adapters/inbound/api.py +61 -0
  13. typevet/adapters/inbound/backend_settings.py +608 -0
  14. typevet/adapters/inbound/cord_semantic_acceptance_cli.py +135 -0
  15. typevet/adapters/inbound/helpers.py +67 -0
  16. typevet/adapters/inbound/settings.py +154 -0
  17. typevet/adapters/outbound/__init__.py +62 -0
  18. typevet/adapters/outbound/async_fake.py +123 -0
  19. typevet/adapters/outbound/async_llama_cpp.py +168 -0
  20. typevet/adapters/outbound/chat_completion.py +148 -0
  21. typevet/adapters/outbound/fake.py +119 -0
  22. typevet/adapters/outbound/gemma/__init__.py +84 -0
  23. typevet/adapters/outbound/gemma/answer_binding.py +290 -0
  24. typevet/adapters/outbound/gemma/scoring_prefix.py +102 -0
  25. typevet/adapters/outbound/gemma/served_template.py +131 -0
  26. typevet/adapters/outbound/gemma_native_vision_factory.py +286 -0
  27. typevet/adapters/outbound/generation_finite.py +50 -0
  28. typevet/adapters/outbound/http_errors.py +38 -0
  29. typevet/adapters/outbound/judgment_scoring.py +471 -0
  30. typevet/adapters/outbound/llama_cpp.py +187 -0
  31. typevet/adapters/outbound/llama_cpp_http.py +103 -0
  32. typevet/adapters/outbound/llama_cpp_multimodal.py +145 -0
  33. typevet/adapters/outbound/llama_cpp_scoring.py +333 -0
  34. typevet/adapters/outbound/vllm_content.py +63 -0
  35. typevet/adapters/outbound/vllm_generation.py +163 -0
  36. typevet/adapters/outbound/vllm_generation_async.py +257 -0
  37. typevet/adapters/outbound/vllm_http.py +121 -0
  38. typevet/adapters/outbound/vllm_judgment_factory.py +210 -0
  39. typevet/adapters/outbound/vllm_scoring.py +372 -0
  40. typevet/domain/__init__.py +193 -0
  41. typevet/domain/candidate_scoring_request.py +134 -0
  42. typevet/domain/candidate_scoring_response.py +96 -0
  43. typevet/domain/candidate_scoring_validate.py +197 -0
  44. typevet/domain/decision_compile.py +368 -0
  45. typevet/domain/decision_execute.py +229 -0
  46. typevet/domain/decisions.py +129 -0
  47. typevet/domain/errors.py +235 -0
  48. typevet/domain/field_instructions.py +134 -0
  49. typevet/domain/judgment_answers.py +198 -0
  50. typevet/domain/judgment_normalize.py +243 -0
  51. typevet/domain/judgment_questions.py +187 -0
  52. typevet/domain/judgment_response.py +102 -0
  53. typevet/domain/media.py +101 -0
  54. typevet/domain/models.py +115 -0
  55. typevet/domain/question_schema.py +254 -0
  56. typevet/domain/scoring_stage.py +34 -0
  57. typevet/evaluation/__init__.py +19 -0
  58. typevet/evaluation/consumer_http_accounting.py +135 -0
  59. typevet/evaluation/cord_expense_call_accounting.py +108 -0
  60. typevet/evaluation/cord_expense_live_harness.py +75 -0
  61. typevet/evaluation/cord_expense_receipt_requirement.py +212 -0
  62. typevet/evaluation/cord_expense_smoke.py +328 -0
  63. typevet/evaluation/cord_semantic_acceptance.py +463 -0
  64. typevet/evaluation/cord_semantic_acceptance_report.py +90 -0
  65. typevet/evaluation/cord_semantic_metrics.py +57 -0
  66. typevet/evaluation/datasets/__init__.py +30 -0
  67. typevet/evaluation/datasets/banking77.py +270 -0
  68. typevet/evaluation/datasets/boolq.py +550 -0
  69. typevet/evaluation/datasets/boolq_download.py +132 -0
  70. typevet/evaluation/datasets/civil_comments.py +450 -0
  71. typevet/evaluation/datasets/clinc.py +125 -0
  72. typevet/evaluation/datasets/clinc_domains.json +172 -0
  73. typevet/evaluation/datasets/clinc_download.py +103 -0
  74. typevet/evaluation/datasets/clinc_plus_intent_names.json +153 -0
  75. typevet/evaluation/datasets/clinc_rows.py +306 -0
  76. typevet/evaluation/datasets/clinc_shard.py +156 -0
  77. typevet/evaluation/datasets/cord_expense.py +381 -0
  78. typevet/evaluation/datasets/difraud.py +256 -0
  79. typevet/evaluation/datasets/go_emotions.py +368 -0
  80. typevet/evaluation/datasets/go_emotions_download.py +154 -0
  81. typevet/evaluation/datasets/hyperpartisan.py +562 -0
  82. typevet/evaluation/datasets/partner_guard.py +170 -0
  83. typevet/evaluation/datasets/psai.py +415 -0
  84. typevet/evaluation/datasets/psai_download.py +167 -0
  85. typevet/evaluation/datasets/psai_evidence_pilot.py +570 -0
  86. typevet/evaluation/datasets/psai_schema.py +157 -0
  87. typevet/evaluation/datasets/psai_stream.py +166 -0
  88. typevet/evaluation/datasets/psai_vision.py +534 -0
  89. typevet/evaluation/datasets/psai_vision_controls.py +502 -0
  90. typevet/evaluation/datasets/pubmedqa.py +420 -0
  91. typevet/evaluation/experiment_identity.py +677 -0
  92. typevet/evaluation/psai_vision_consumer_accounting.py +190 -0
  93. typevet/evaluation/psai_vision_consumer_dispatch.py +216 -0
  94. typevet/evaluation/psai_vision_consumer_harness.py +342 -0
  95. typevet/evaluation/psai_vision_consumer_live.py +332 -0
  96. typevet/evaluation/psai_vision_consumer_live_identity.py +156 -0
  97. typevet/evaluation/psai_vision_consumer_live_receipt.py +132 -0
  98. typevet/evaluation/psai_vision_consumer_live_router.py +162 -0
  99. typevet/evaluation/psai_vision_consumer_offline.py +418 -0
  100. typevet/evaluation/psai_vision_consumer_outcomes.py +224 -0
  101. typevet/evaluation/psai_vision_consumer_protocol.py +69 -0
  102. typevet/evaluation/psai_vision_consumer_receipt.py +161 -0
  103. typevet/evaluation/psai_vision_consumer_receipt_structure.py +263 -0
  104. typevet/evaluation/psai_vision_probability_evidence.py +336 -0
  105. typevet/evaluation/runner/__init__.py +56 -0
  106. typevet/evaluation/runner/core.py +104 -0
  107. typevet/evaluation/runner/datasets.py +186 -0
  108. typevet/evaluation/runner/live_gate.py +149 -0
  109. typevet/evaluation/runner/report.py +116 -0
  110. typevet/evaluation/tpjep/__init__.py +81 -0
  111. typevet/evaluation/tpjep/loader.py +237 -0
  112. typevet/evaluation/tpjep/outcome.py +101 -0
  113. typevet/evaluation/tpjep/records.py +428 -0
  114. typevet/evaluation/tpjep/runner.py +325 -0
  115. typevet/ports/__init__.py +39 -0
  116. typevet/ports/async_generation.py +57 -0
  117. typevet/ports/framing.py +59 -0
  118. typevet/ports/generation.py +58 -0
  119. typevet/ports/judgment.py +71 -0
  120. typevet/ports/scoring.py +64 -0
  121. typevet/py.typed +0 -0
  122. typevet/runtime/__init__.py +50 -0
  123. typevet/runtime/categorical.py +196 -0
  124. typevet/runtime/judgment.py +17 -0
  125. typevet/runtime/llama_cpp_gemma_vision.py +28 -0
  126. typevet/runtime/scoring_prefix.py +14 -0
  127. typevet/runtime/vllm_judgment.py +28 -0
  128. typevet/testing/__init__.py +22 -0
  129. typevet/testing/fakes.py +152 -0
  130. typevet-0.1.0.dev1.dist-info/METADATA +125 -0
  131. typevet-0.1.0.dev1.dist-info/RECORD +133 -0
  132. typevet-0.1.0.dev1.dist-info/WHEEL +4 -0
  133. typevet-0.1.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,77 @@
1
+ """Environment-backed log settings for the composition root.
2
+
3
+ Diagnostics are operator signals on stderr, not telemetry and not remote export.
4
+ Importing this module does not configure structlog.
5
+
6
+ Examples:
7
+ ```python
8
+ from typevet.adapters.diagnostics.settings import load_log_settings
9
+
10
+ settings = load_log_settings({"TYPEVET_LOG__LEVEL": "debug"})
11
+ assert settings.level == "debug"
12
+ ```
13
+
14
+ See Also:
15
+ - [typevet.adapters.diagnostics.logs][]: Applies these settings to structlog
16
+ - [typevet.adapters.diagnostics.redaction][]: Honors ``log_prompts`` when masking
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import os
22
+ from collections.abc import Mapping
23
+ from dataclasses import dataclass
24
+
25
+ _VALID_FORMATS = frozenset({"auto", "json", "console"})
26
+ _VALID_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
27
+ _TRUTHY = frozenset({"1", "true", "yes", "on"})
28
+
29
+
30
+ @dataclass(frozen=True, slots=True)
31
+ class LogSettings:
32
+ """How diagnostic lines are rendered and which severities are kept.
33
+
34
+ Attributes:
35
+ format (str): ``auto``, ``json`` or ``console``. Default ``auto``.
36
+ level (str): ``debug`` through ``critical``. Default ``info``.
37
+ log_prompts (bool): When false, prompt-like fields are stripped from output.
38
+
39
+ Examples:
40
+ ```python
41
+ from typevet.adapters.diagnostics.settings import LogSettings
42
+
43
+ LogSettings(format="json", level="debug", log_prompts=True)
44
+ ```
45
+ """
46
+
47
+ format: str = "auto"
48
+ level: str = "info"
49
+ log_prompts: bool = False
50
+
51
+
52
+ def load_log_settings(
53
+ environ: Mapping[str, str] | None = None,
54
+ ) -> LogSettings:
55
+ """Read ``TYPEVET_LOG__*`` variables for the composition root.
56
+
57
+ Args:
58
+ environ: Mapping to read. Defaults to ``os.environ``.
59
+
60
+ Returns:
61
+ Frozen settings with defaults for missing keys.
62
+
63
+ Raises:
64
+ ValueError: When format or level is not an allowed value.
65
+ """
66
+ source = os.environ if environ is None else environ
67
+ fmt = source.get("TYPEVET_LOG__FORMAT", "auto").strip().lower()
68
+ level = source.get("TYPEVET_LOG__LEVEL", "info").strip().lower()
69
+ log_prompts_raw = source.get("TYPEVET_LOG__LOG_PROMPTS", "").strip().lower()
70
+ if fmt not in _VALID_FORMATS:
71
+ msg = f"TYPEVET_LOG__FORMAT must be one of {sorted(_VALID_FORMATS)}"
72
+ raise ValueError(msg)
73
+ if level not in _VALID_LEVELS:
74
+ msg = f"TYPEVET_LOG__LEVEL must be one of {sorted(_VALID_LEVELS)}"
75
+ raise ValueError(msg)
76
+ log_prompts = log_prompts_raw in _TRUTHY
77
+ return LogSettings(format=fmt, level=level, log_prompts=log_prompts)
@@ -0,0 +1,65 @@
1
+ """Inbound adapters (library entry points).
2
+
3
+ Examples:
4
+ ```python
5
+ from typevet.adapters.inbound import generate
6
+ from typevet.testing import StaticGenerationFake
7
+
8
+ result = generate(
9
+ StaticGenerationFake({"ok": True}),
10
+ prompt="hi",
11
+ schema={"type": "object", "additionalProperties": False},
12
+ model="fake",
13
+ )
14
+ ```
15
+
16
+ See Also:
17
+ - [typevet.adapters.inbound.api][]: ``generate`` helper
18
+ - [typevet.adapters.inbound.helpers][]: ``run_sync`` helper
19
+ - [typevet.adapters.inbound.settings][]: ``TYPEVET_LLAMA__*`` composition root
20
+ - [typevet.adapters.inbound.backend_settings][]: ``TYPEVET_BACKEND`` selection
21
+ - [typevet.ports.generation][]: GenerationPort
22
+
23
+ Attributes:
24
+ generate (function): Build a request and invoke a generation port.
25
+ run_sync (function): Run an async generation coroutine from sync code.
26
+ LlamaSettings (type): llama.cpp connection settings for composition roots.
27
+ load_llama_settings (function): Read ``TYPEVET_LLAMA__*`` from the environment.
28
+ llama_cpp_adapter (function): Build ``LlamaCppGenerationAdapter`` from settings.
29
+ VllmSettings (type): vLLM connection settings; ``api_key`` is not in ``repr``.
30
+ load_backend (function): Read ``TYPEVET_BACKEND``.
31
+ load_vllm_settings (function): Read ``TYPEVET_VLLM__*`` from the environment.
32
+ vllm_http_client (function): Build the shared vLLM ``httpx.Client``.
33
+ generation_adapter (function): Build the adapter ``TYPEVET_BACKEND`` selects.
34
+ async_vllm_generation_adapter (function): Build the async vLLM adapter.
35
+ """
36
+
37
+ from typevet.adapters.inbound.api import generate
38
+ from typevet.adapters.inbound.backend_settings import (
39
+ VllmSettings,
40
+ async_vllm_generation_adapter,
41
+ generation_adapter,
42
+ load_backend,
43
+ load_vllm_settings,
44
+ vllm_http_client,
45
+ )
46
+ from typevet.adapters.inbound.helpers import run_sync
47
+ from typevet.adapters.inbound.settings import (
48
+ LlamaSettings,
49
+ llama_cpp_adapter,
50
+ load_llama_settings,
51
+ )
52
+
53
+ __all__ = [
54
+ "LlamaSettings",
55
+ "VllmSettings",
56
+ "async_vllm_generation_adapter",
57
+ "generate",
58
+ "generation_adapter",
59
+ "llama_cpp_adapter",
60
+ "load_backend",
61
+ "load_llama_settings",
62
+ "load_vllm_settings",
63
+ "run_sync",
64
+ "vllm_http_client",
65
+ ]
@@ -0,0 +1,61 @@
1
+ """Library entry: call a generation port with a typed request.
2
+
3
+ Examples:
4
+ ```python
5
+ from typevet.adapters.inbound.api import generate
6
+ from typevet.testing import StaticGenerationFake
7
+
8
+ generate(
9
+ StaticGenerationFake({"n": 1}),
10
+ prompt="n",
11
+ schema={
12
+ "type": "object",
13
+ "properties": {"n": {"type": "integer"}},
14
+ "required": ["n"],
15
+ "additionalProperties": False,
16
+ },
17
+ model="fake",
18
+ )
19
+ ```
20
+
21
+ See Also:
22
+ - [typevet.domain.media][]: ImageInput and the media marker
23
+ - [typevet.domain.models][]: GenerationRequest
24
+ - [typevet.ports.generation][]: GenerationPort
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ from collections.abc import Mapping
30
+ from typing import TYPE_CHECKING, Any
31
+
32
+ from typevet.domain.models import GenerationRequest, GenerationResult
33
+ from typevet.ports.generation import GenerationPort
34
+
35
+ if TYPE_CHECKING:
36
+ from typevet.domain.media import ImageInput
37
+
38
+
39
+ def generate(
40
+ port: GenerationPort,
41
+ *,
42
+ prompt: str,
43
+ schema: Mapping[str, Any],
44
+ model: str,
45
+ media: tuple[ImageInput, ...] = (),
46
+ ) -> GenerationResult:
47
+ """Build a request and invoke the generation port.
48
+
49
+ Args:
50
+ port: Outbound adapter that implements ``GenerationPort``.
51
+ prompt: Natural-language instruction.
52
+ schema: JSON Schema object as a mapping.
53
+ model: Backend model id or alias.
54
+ media: Images the prompt marks, one ``MEDIA_MARKER`` each, in
55
+ marker order. Empty for a text ask.
56
+
57
+ Returns:
58
+ Validated generation result from the port.
59
+ """
60
+ request = GenerationRequest(prompt=prompt, schema=schema, model=model, media=media)
61
+ return port.generate(request)