docent-python 0.1.20a0__py3-none-any.whl → 0.1.21a0__py3-none-any.whl

docent/judges/util/parse_output.py

@@ -0,0 +1,95 @@
+import json
+from typing import Any, cast
+
+import jsonschema
+
+from docent._llm_util.data_models.exceptions import ValidationFailedException
+from docent._llm_util.data_models.llm_output import LLMOutput
+from docent._log_util import get_logger
+from docent.data_models.agent_run import AgentRun
+from docent.data_models.remove_invalid_citation_ranges import remove_invalid_citation_ranges
+from docent.judges.types import traverse_schema_and_transform
+from docent.judges.util.forgiving_json import forgiving_json_loads
+
+logger = get_logger(__name__)
+
+
+def _validate_rubric_output(
+    output: dict[str, Any], output_schema: dict[str, Any], agent_run: AgentRun
+) -> dict[str, Any]:
+    """Validate and filter citation text ranges in rubric results.
+    Also check that the output conforms to the output schema.
+
+    Args:
+        output: Raw results from LLM judge
+        agent_run: Agent run containing transcript data for validation
+
+    Returns:
+        Validated result dict with invalid citations removed
+
+    Raises:
+        ValidationFailedException: If validation fails
+    """
+
+    def _validate_citation_string(text: str) -> str:
+        validated_text = remove_invalid_citation_ranges(text, agent_run)
+        if validated_text != text:
+            logger.warning(
+                f"Citation validation removed invalid text range from citation in judge result. "
+                f"Agent run ID: {agent_run.id}, "
+                f"Original text: {text}, "
+                f"Validated text: {validated_text}, "
+            )
+        return validated_text
+
+    try:
+        jsonschema.validate(output, output_schema)
+    except jsonschema.ValidationError as e:
+        raise ValidationFailedException(f"Schema validation failed: {e}", failed_output=str(output))
+
+    try:
+        return traverse_schema_and_transform(output, output_schema, _validate_citation_string)
+    except Exception as e:
+        raise ValidationFailedException(
+            f"Citation validation failed: {e}", failed_output=str(output)
+        )
+
+
+def parse_and_validate_llm_output(
+    llm_output: LLMOutput,
+    output_schema: dict[str, Any],
+    agent_run: AgentRun,
+) -> dict[str, Any]:
+    """Parse and validate LLM output for rubric evaluation.
+
+    Args:
+        llm_output: The LLM output to parse
+        output_schema: The schema to validate against
+        agent_run: Agent run for citation validation
+
+    Returns:
+        Validated output dict
+
+    Raises:
+        ValidationFailedException: If parsing or validation fails
+    """
+    if llm_output.first_text is None:
+        raise ValidationFailedException("LLM output has no text", failed_output=None)
+
+    try:
+        output = forgiving_json_loads(llm_output.first_text)
+    except json.JSONDecodeError as e:
+        raise ValidationFailedException(
+            f"Failed to parse JSON: {e}. Raw text: `{llm_output.first_text}`",
+            failed_output=llm_output.first_text,
+        )
+
+    if not isinstance(output, dict):
+        logger.error(f"Expected dict output, got {type(output)}")
+        logger.error(f"LLM output: {llm_output.first_text}")
+        raise ValidationFailedException(
+            f"Expected dict output, got {type(output)}. Raw text: {llm_output.first_text}",
+            failed_output=llm_output.first_text,
+        )
+
+    return _validate_rubric_output(cast(dict[str, Any], output), output_schema, agent_run)

docent/judges/util/voting.py

@@ -0,0 +1,84 @@
+from collections import Counter
+from typing import Any, cast
+
+
+def get_agreement_keys(schema: dict[str, Any]) -> list[str]:
+    """Get list of top-level keys in schema that we want to measure agreement on.
+
+    This includes enum, bool, and int fields. We skip float and strings.
+
+    Args:
+        schema: JSON schema dict
+
+    Returns:
+        List of field names (keys) that should be used for measuring agreement
+    """
+    agreement_keys: list[str] = []
+
+    properties = schema.get("properties", {})
+    assert isinstance(properties, dict)
+    properties = cast(dict[str, Any], properties)
+
+    for key, field_schema in properties.items():
+        assert isinstance(field_schema, dict)
+        field_schema = cast(dict[str, Any], field_schema)
+
+        field_type = field_schema.get("type")
+        assert isinstance(field_type, str)
+
+        # Include boolean fields
+        if field_type == "boolean":
+            agreement_keys.append(key)
+        # Include integer fields
+        elif field_type == "integer":
+            agreement_keys.append(key)
+        # Include enum fields (even strings)
+        elif "enum" in field_schema:
+            agreement_keys.append(key)
+
+    return agreement_keys
+
+
+def find_modal_result(indep_results: list[dict[str, Any]], agreement_keys: list[str]):
+    """Find the result that best matches modal values across agreement keys.
+
+    Args:
+        indep_results: List of independent results to analyze
+        agreement_keys: Keys to measure agreement on
+
+    Returns:
+        Tuple of (max_idx, agt_key_modes_and_counts) where:
+        - max_idx is the index of the result that best matches modal values
+        - agt_key_modes_and_counts maps each key to (modal_value, count) or None if no values exist for that key
+
+    Raises:
+        ValueError: If no results are provided
+    """
+    if not indep_results:
+        raise ValueError("No results to score")
+
+    # For each agreement key, compute the mode and count (or None, if no values exist for that key)
+    agt_key_modes_and_counts: dict[str, tuple[str | bool | int, int] | None] = {}
+    for key in agreement_keys:
+        key_modes = Counter(v for r in indep_results if (v := r.get(key)) is not None)
+        if most_common_one := key_modes.most_common(1):
+            agt_key_modes_and_counts[key] = most_common_one[0]
+        else:
+            agt_key_modes_and_counts[key] = None
+
+    # Score each rollout based on how many agreement keys they match
+    # If there is no mode for a key, or if a certain result doesn't have that key, it doesn't count.
+    # TODO(mengk): This may bias towards results that have more keys.
+    indep_result_scores: list[int] = []
+    for r in indep_results:
+        score = 0
+        for key in agreement_keys:
+            mode_and_count = agt_key_modes_and_counts[key]
+            if mode_and_count and r.get(key) == mode_and_count[0]:
+                score += 1
+        indep_result_scores.append(score)
+
+    # Argmax
+    max_idx = indep_result_scores.index(max(indep_result_scores))
+
+    return max_idx, agt_key_modes_and_counts

docent/trace.py

@@ -226,7 +226,7 @@ class DocentTracer:
         try:
 
             # Check for OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT environment variable
-            default_attribute_limit = 1024
+            default_attribute_limit = 1024 * 16
             env_value = os.environ.get("OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", "0")
             env_limit = int(env_value) if env_value.isdigit() else 0
             attribute_limit = max(env_limit, default_attribute_limit)

{docent_python-0.1.20a0.dist-info → docent_python-0.1.21a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docent-python
-Version: 0.1.20a0
+Version: 0.1.21a0
 Summary: Docent SDK
 Project-URL: Homepage, https://github.com/TransluceAI/docent
 Project-URL: Issues, https://github.com/TransluceAI/docent/issues
@@ -9,8 +9,12 @@ Author-email: Transluce <info@transluce.org>
 License-Expression: Apache-2.0
 License-File: LICENSE.md
 Requires-Python: >=3.11
+Requires-Dist: anthropic>=0.47.0
 Requires-Dist: backoff>=2.2.1
+Requires-Dist: google-genai>=1.16.1
 Requires-Dist: inspect-ai>=0.3.132
+Requires-Dist: jsonschema>=4.24.0
+Requires-Dist: openai>=1.68.0
 Requires-Dist: opentelemetry-api>=1.34.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.34.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.34.1
@@ -20,6 +24,7 @@ Requires-Dist: opentelemetry-instrumentation-langchain>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-openai>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-threading>=0.55b1
 Requires-Dist: opentelemetry-sdk>=1.34.1
+Requires-Dist: orjson>=3.11.3
 Requires-Dist: pydantic>=2.11.7
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: tiktoken>=0.7.0

docent_python-0.1.21a0.dist-info/RECORD

@@ -0,0 +1,58 @@
+docent/__init__.py,sha256=fuhETwJPcesiB76Zxa64HBJxeaaTyRalIH-fs77TWsU,112
+docent/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docent/trace.py,sha256=_JvDmtWVFARPYvXsNx8-RKRdev4mMxNK6iq9AARzoJE,66362
+docent/trace_2.py,sha256=-OxzXF2kOFkhto1UGXHWVM797EN_BT_uwDSbzgMme8o,67145
+docent/trace_temp.py,sha256=Z0lAPwVzXjFvxpiU-CuvfWIslq9Q4alNkZMoQ77Xudk,40711
+docent/_llm_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docent/_llm_util/llm_cache.py,sha256=p8pJ-B1vyJQlNn48ce1Pqv2gLocBVD6fZLPsd2VC5LA,6386
+docent/_llm_util/model_registry.py,sha256=8Y4VwrA2f2EX78cG1VBIBHVvT_p4qqBTdu9a9zJpfTo,3382
+docent/_llm_util/prod_llms.py,sha256=HuGOg5Bhnpk_TijC3mOH8CTRIBy2C8w0_SebiEouNoE,16859
+docent/_llm_util/data_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docent/_llm_util/data_models/exceptions.py,sha256=IW4BVMVp8r5TufNXyrhy3acgwJiQQQPQjB9VA4RVXw8,1489
+docent/_llm_util/data_models/llm_output.py,sha256=fuYPJ-SwxZjB4XGATA6XpLyc42Ix-kXHgBqFr_jPhK8,10123
+docent/_llm_util/data_models/simple_svc.py,sha256=0twuXP6aEU-jYY0obDSEgjT2lDSJCuZG_NgiqEzZIPM,2881
+docent/_llm_util/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docent/_llm_util/providers/anthropic.py,sha256=-1oPd5FB4aFwKSmNvXzG8PVewjhgsogLRX1SCpnCxoA,18720
+docent/_llm_util/providers/common.py,sha256=dgcTuU4XkCKoAaM48UW8zMgRYUzj7TDBhvWqtnxBO7g,1166
+docent/_llm_util/providers/google.py,sha256=2D9mDgenZW0pt0_V7koX-aoZzpl8jo8xE5EWOLK7I0k,20314
+docent/_llm_util/providers/openai.py,sha256=4niQV9CNaJ-iiEwYG0BSFxCwcsCAWZz0JuUs4wBKu9M,25904
+docent/_llm_util/providers/openrouter.py,sha256=sT2onpeQ1gAwJLjkQbzD2RodJWTm013Q-siTXezca10,11958
+docent/_llm_util/providers/preference_types.py,sha256=z-TOxj_es1_cs5DzknZaganGyjMkmh5NgtiDYKMRI1I,3751
+docent/_llm_util/providers/provider_registry.py,sha256=EPYGQlegYPtg4ogEusCftm_5PZP-_XVKH1qg3xjPFTU,6337
+docent/_log_util/__init__.py,sha256=3HXXrxrSm8PxwG4llotrCnSnp7GuroK1FNHsdg6f7aE,73
+docent/_log_util/logger.py,sha256=kwM0yRW1IJd6-XTorjWn48B4l8qvD2ZM6VDjY5eskQI,4422
+docent/data_models/__init__.py,sha256=bE_Wy4Ql-9-0ZPcolMCPHhYvaE_Ug6h-jV7wOJ_DAi0,399
+docent/data_models/_tiktoken_util.py,sha256=hC0EDDWItv5-0cONBnHWgZtQOflDU7ZNEhXPFo4DvPc,3057
+docent/data_models/agent_run.py,sha256=7_37I9aS9rhDTkAvMPwoJGssQldvvKte8qVb93EnAiY,19329
+docent/data_models/citation.py,sha256=2_M1-_olVOJtjCGGFx1GIwGYWl0ILHxRsW8-EFDS9j0,7844
+docent/data_models/judge.py,sha256=zPbTqztn-yWu6tgD3R5JTyGnNiDhY6cWQ-gz3e_eM5k,340
+docent/data_models/metadata_util.py,sha256=E-EClAP5vVm9xbfTlPSz0tUyCalOfN9Jujd6JGoRnBg,487
+docent/data_models/regex.py,sha256=0ciIerkrNwb91bY5mTcyO5nDWH67xx2tZYObV52fmBo,1684
+docent/data_models/remove_invalid_citation_ranges.py,sha256=3RSMsOzFO2cSjkxI549TAo12qdvD-AGHd05Jxu0amvs,6282
+docent/data_models/shared_types.py,sha256=jjm-Dh5S6v7UKInW7SEqoziOsx6Z7Uu4e3VzgCbTWvc,225
+docent/data_models/transcript.py,sha256=Hkj9-rQfRk6ywICpwM4P1vgpTJ_7T3jxjFtodfFcwPw,20087
+docent/data_models/util.py,sha256=dK0dviDkDe8PiNCZisryctT-dzScWenLXQi6DOk9Ts4,7194
+docent/data_models/chat/__init__.py,sha256=ws77P3raDiOv6XesAMycUwu-uT75D5f9aNgjFeJbUH8,631
+docent/data_models/chat/content.py,sha256=Co-jO8frQa_DSP11wJuhPX0s-GpJk8yqtKqPeiAIZ_U,1672
+docent/data_models/chat/message.py,sha256=_72xeTdgv8ogQd4WLl1P3yXfIDkIEQrHlWgdvObeQxY,4291
+docent/data_models/chat/tool.py,sha256=MMglNHzkwHqUoK0xDWqs2FtelPsgHqwVpGpI1F8KZyw,3049
+docent/judges/__init__.py,sha256=Sob1uxJRgmr2S2sz4J6skHP8iqcVoiUq7Jlh8S5Sj9Y,462
+docent/judges/impl.py,sha256=qiItNKWPvB0KlB5b0rQoIfT-7m1xzyI028WtgvgvRhU,8864
+docent/judges/types.py,sha256=NlLv42iLDORbPAHppCz-YWZ6ksR4QYDWAweGw75izJ0,8439
+docent/judges/util/forgiving_json.py,sha256=zSh0LF3UVHdSjuMNvEiqUmSxpxPaqK1rSLiI6KCNihg,3549
+docent/judges/util/meta_schema.json,sha256=g3MUa_6e38I3GqZryy8b1w_Y9Krx2xSiWIuaG8Zpszc,2055
+docent/judges/util/meta_schema.py,sha256=6IrIRHERJ6tkRcUtUShJ84I68yUJgkwfFeBjgt42qEA,930
+docent/judges/util/parse_output.py,sha256=qvqt7TEnrAqvzYHqip48boMQSUcoGa-1PA1gIGn-w4s,3381
+docent/judges/util/voting.py,sha256=cAty9b4w7M1OWeW-j8t6vxpZn7VXyE3aBL9Ex2ERKcU,3071
+docent/loaders/load_inspect.py,sha256=VLrtpvcVZ44n2DIPMwUivXqbvOWjaooGw6moY8UQ0VE,6789
+docent/samples/__init__.py,sha256=roDFnU6515l9Q8v17Es_SpWyY9jbm5d6X9lV01V0MZo,143
+docent/samples/load.py,sha256=ZGE07r83GBNO4A0QBh5aQ18WAu3mTWA1vxUoHd90nrM,207
+docent/samples/log.eval,sha256=orrW__9WBfANq7NwKsPSq9oTsQRcG6KohG5tMr_X_XY,397708
+docent/samples/tb_airline.json,sha256=eR2jFFRtOw06xqbEglh6-dPewjifOk-cuxJq67Dtu5I,47028
+docent/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docent/sdk/agent_run_writer.py,sha256=0AWdxejoqZyuj9JSA39WlEwGcMSYTWNqnzIuluySY-M,11043
+docent/sdk/client.py,sha256=K1NVkj_CFj0q-2mSFvWfh8NTqXqosED--dv5aLD7yOE,18239
+docent_python-0.1.21a0.dist-info/METADATA,sha256=H0iEQ39cv90MW0lRZ94XhER6C_znvNU3DBPc6M72i9g,1277
+docent_python-0.1.21a0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+docent_python-0.1.21a0.dist-info/licenses/LICENSE.md,sha256=QIMv2UiT6MppRasso4ymaA0w7ltkqmlL0HCt8CLD7Rc,580
+docent_python-0.1.21a0.dist-info/RECORD,,

docent_python-0.1.20a0.dist-info/RECORD

@@ -1,34 +0,0 @@
-docent/__init__.py,sha256=fuhETwJPcesiB76Zxa64HBJxeaaTyRalIH-fs77TWsU,112
-docent/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docent/trace.py,sha256=CEDT7StBE6DaEffWEiW-Lntx5OxNmv9vyXFGI9UlW28,66357
-docent/trace_2.py,sha256=-OxzXF2kOFkhto1UGXHWVM797EN_BT_uwDSbzgMme8o,67145
-docent/trace_temp.py,sha256=Z0lAPwVzXjFvxpiU-CuvfWIslq9Q4alNkZMoQ77Xudk,40711
-docent/_log_util/__init__.py,sha256=3HXXrxrSm8PxwG4llotrCnSnp7GuroK1FNHsdg6f7aE,73
-docent/_log_util/logger.py,sha256=kwM0yRW1IJd6-XTorjWn48B4l8qvD2ZM6VDjY5eskQI,4422
-docent/data_models/__init__.py,sha256=bE_Wy4Ql-9-0ZPcolMCPHhYvaE_Ug6h-jV7wOJ_DAi0,399
-docent/data_models/_tiktoken_util.py,sha256=hC0EDDWItv5-0cONBnHWgZtQOflDU7ZNEhXPFo4DvPc,3057
-docent/data_models/agent_run.py,sha256=7_37I9aS9rhDTkAvMPwoJGssQldvvKte8qVb93EnAiY,19329
-docent/data_models/citation.py,sha256=2_M1-_olVOJtjCGGFx1GIwGYWl0ILHxRsW8-EFDS9j0,7844
-docent/data_models/judge.py,sha256=zPbTqztn-yWu6tgD3R5JTyGnNiDhY6cWQ-gz3e_eM5k,340
-docent/data_models/metadata_util.py,sha256=E-EClAP5vVm9xbfTlPSz0tUyCalOfN9Jujd6JGoRnBg,487
-docent/data_models/regex.py,sha256=0ciIerkrNwb91bY5mTcyO5nDWH67xx2tZYObV52fmBo,1684
-docent/data_models/remove_invalid_citation_ranges.py,sha256=3RSMsOzFO2cSjkxI549TAo12qdvD-AGHd05Jxu0amvs,6282
-docent/data_models/shared_types.py,sha256=jjm-Dh5S6v7UKInW7SEqoziOsx6Z7Uu4e3VzgCbTWvc,225
-docent/data_models/transcript.py,sha256=7cdj2KAO_e2k3rj7OPzJzmzrkxPHIW7fbHygKTr7EZg,19940
-docent/data_models/util.py,sha256=dK0dviDkDe8PiNCZisryctT-dzScWenLXQi6DOk9Ts4,7194
-docent/data_models/chat/__init__.py,sha256=ws77P3raDiOv6XesAMycUwu-uT75D5f9aNgjFeJbUH8,631
-docent/data_models/chat/content.py,sha256=Co-jO8frQa_DSP11wJuhPX0s-GpJk8yqtKqPeiAIZ_U,1672
-docent/data_models/chat/message.py,sha256=_72xeTdgv8ogQd4WLl1P3yXfIDkIEQrHlWgdvObeQxY,4291
-docent/data_models/chat/tool.py,sha256=MMglNHzkwHqUoK0xDWqs2FtelPsgHqwVpGpI1F8KZyw,3049
-docent/loaders/load_inspect.py,sha256=VLrtpvcVZ44n2DIPMwUivXqbvOWjaooGw6moY8UQ0VE,6789
-docent/samples/__init__.py,sha256=roDFnU6515l9Q8v17Es_SpWyY9jbm5d6X9lV01V0MZo,143
-docent/samples/load.py,sha256=ZGE07r83GBNO4A0QBh5aQ18WAu3mTWA1vxUoHd90nrM,207
-docent/samples/log.eval,sha256=orrW__9WBfANq7NwKsPSq9oTsQRcG6KohG5tMr_X_XY,397708
-docent/samples/tb_airline.json,sha256=eR2jFFRtOw06xqbEglh6-dPewjifOk-cuxJq67Dtu5I,47028
-docent/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docent/sdk/agent_run_writer.py,sha256=0AWdxejoqZyuj9JSA39WlEwGcMSYTWNqnzIuluySY-M,11043
-docent/sdk/client.py,sha256=K1NVkj_CFj0q-2mSFvWfh8NTqXqosED--dv5aLD7yOE,18239
-docent_python-0.1.20a0.dist-info/METADATA,sha256=1XprRqUJ22jFi1WF_4X06nI3rL_fvQLQz7cjGqRXz-s,1114
-docent_python-0.1.20a0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-docent_python-0.1.20a0.dist-info/licenses/LICENSE.md,sha256=QIMv2UiT6MppRasso4ymaA0w7ltkqmlL0HCt8CLD7Rc,580
-docent_python-0.1.20a0.dist-info/RECORD,,