docent-python 0.1.21a0__py3-none-any.whl → 0.1.22a0__py3-none-any.whl

docent/judges/impl.py

@@ -8,7 +8,11 @@ from docent._log_util import get_logger
 from docent.data_models.agent_run import AgentRun
 from docent.judges.types import JudgeResult, ResultType, Rubric
 from docent.judges.util.parse_output import parse_and_validate_llm_output
-from docent.judges.util.voting import find_modal_result, get_agreement_keys
+from docent.judges.util.voting import (
+    compute_output_distribution,
+    find_modal_result,
+    get_agreement_keys,
+)
 
 logger = get_logger(__name__)
 
@@ -74,6 +78,11 @@ class MajorityVotingJudge(BaseJudge):
         )
         final_output = indep_results[final_max_idx]
 
+        # Compute the distribution of the output across the agreement keys
+        final_output_distribution = compute_output_distribution(
+            indep_results, self.cfg.output_schema, agreement_keys
+        )
+
         return JudgeResult(
             agent_run_id=agent_run.id,
             rubric_id=self.cfg.id,
@@ -85,6 +94,7 @@ class MajorityVotingJudge(BaseJudge):
                 "final_results": indep_results,
                 "final_agt_key_modes_and_counts": final_agt_key_modes_and_counts,
                 "final_max_idx": final_max_idx,
+                "final_output_distribution": final_output_distribution,
             },
             result_type=ResultType.DIRECT_RESULT,
         )

docent/judges/util/voting.py

@@ -5,7 +5,7 @@ from typing import Any, cast
 def get_agreement_keys(schema: dict[str, Any]) -> list[str]:
     """Get list of top-level keys in schema that we want to measure agreement on.
 
-    This includes enum, bool, and int fields. We skip float and strings.
+    This includes enum and bool fields.
 
     Args:
         schema: JSON schema dict
@@ -29,10 +29,7 @@ def get_agreement_keys(schema: dict[str, Any]) -> list[str]:
         # Include boolean fields
         if field_type == "boolean":
             agreement_keys.append(key)
-        # Include integer fields
-        elif field_type == "integer":
-            agreement_keys.append(key)
-        # Include enum fields (even strings)
+        # Include enum fields (strings and numbers must be in this category)
         elif "enum" in field_schema:
             agreement_keys.append(key)
 
@@ -82,3 +79,36 @@ def find_modal_result(indep_results: list[dict[str, Any]], agreement_keys: list[
     max_idx = indep_result_scores.index(max(indep_result_scores))
 
     return max_idx, agt_key_modes_and_counts
+
+
+def compute_output_distribution(
+    indep_results: list[dict[str, Any]], output_schema: dict[str, Any], agreement_keys: list[str]
+):
+    def _get_possible_values(key: str) -> list[str | bool | int | float]:
+        if "enum" in output_schema.get("properties", {}).get(key, {}):
+            return output_schema.get("properties", {}).get(key, {}).get("enum", [])
+        elif output_schema.get("properties", {}).get(key, {}).get("type") == "boolean":
+            return [True, False]
+        else:
+            return []
+
+    distributions: dict[str, dict[str | bool | int | float, float]] = {
+        key: {value: 0.0 for value in _get_possible_values(key)} for key in agreement_keys
+    }
+    # Collect counts for each possible value
+    for result in indep_results:
+        for key in agreement_keys:
+            if (value := result.get(key)) is not None:  # Could be none if the key is optional
+                assert (
+                    value in distributions[key]
+                ), "this should never happen; the value must be in possible values, since judge results have been validated against the schema"
+                distributions[key][value] += 1
+    # Normalize
+    for key in distributions:
+        total = sum(distributions[key].values())
+        if total == 0:
+            continue
+        for value in distributions[key]:
+            distributions[key][value] /= total
+
+    return distributions

docent/trace.py

@@ -43,6 +43,7 @@ class Instruments(Enum):
     ANTHROPIC = "anthropic"
     BEDROCK = "bedrock"
     LANGCHAIN = "langchain"
+    GOOGLE_GENERATIVEAI = "google_generativeai"
 
 
 class DocentTracer:
@@ -392,6 +393,23 @@ class DocentTracer:
                 except Exception as e:
                     logger.warning(f"Failed to instrument LangChain: {e}")
 
+            # Instrument Google Generative AI with our isolated tracer provider
+            if Instruments.GOOGLE_GENERATIVEAI in enabled_instruments:
+                try:
+                    if is_package_installed("google-generativeai") or is_package_installed(
+                        "google-genai"
+                    ):
+                        from opentelemetry.instrumentation.google_generativeai import (
+                            GoogleGenerativeAiInstrumentor,
+                        )
+
+                        GoogleGenerativeAiInstrumentor().instrument(
+                            tracer_provider=self._tracer_provider
+                        )
+                        logger.info("Instrumented Google Generative AI")
+                except Exception as e:
+                    logger.warning(f"Failed to instrument Google Generative AI: {e}")
+
             # Register cleanup handlers
             self._register_cleanup()
 

{docent_python-0.1.21a0.dist-info → docent_python-0.1.22a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docent-python
-Version: 0.1.21a0
+Version: 0.1.22a0
 Summary: Docent SDK
 Project-URL: Homepage, https://github.com/TransluceAI/docent
 Project-URL: Issues, https://github.com/TransluceAI/docent/issues
@@ -20,6 +20,7 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.34.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.34.1
 Requires-Dist: opentelemetry-instrumentation-anthropic>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-bedrock>=0.40.14
+Requires-Dist: opentelemetry-instrumentation-google-generativeai>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-langchain>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-openai>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-threading>=0.55b1

{docent_python-0.1.21a0.dist-info → docent_python-0.1.22a0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 docent/__init__.py,sha256=fuhETwJPcesiB76Zxa64HBJxeaaTyRalIH-fs77TWsU,112
 docent/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docent/trace.py,sha256=_JvDmtWVFARPYvXsNx8-RKRdev4mMxNK6iq9AARzoJE,66362
+docent/trace.py,sha256=u_1M_B1ncCR5a0Yy5ppQY_0k0AFYe4UhSIGeTuZCH0w,67271
 docent/trace_2.py,sha256=-OxzXF2kOFkhto1UGXHWVM797EN_BT_uwDSbzgMme8o,67145
 docent/trace_temp.py,sha256=Z0lAPwVzXjFvxpiU-CuvfWIslq9Q4alNkZMoQ77Xudk,40711
 docent/_llm_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -37,13 +37,13 @@ docent/data_models/chat/content.py,sha256=Co-jO8frQa_DSP11wJuhPX0s-GpJk8yqtKqPei
 docent/data_models/chat/message.py,sha256=_72xeTdgv8ogQd4WLl1P3yXfIDkIEQrHlWgdvObeQxY,4291
 docent/data_models/chat/tool.py,sha256=MMglNHzkwHqUoK0xDWqs2FtelPsgHqwVpGpI1F8KZyw,3049
 docent/judges/__init__.py,sha256=Sob1uxJRgmr2S2sz4J6skHP8iqcVoiUq7Jlh8S5Sj9Y,462
-docent/judges/impl.py,sha256=qiItNKWPvB0KlB5b0rQoIfT-7m1xzyI028WtgvgvRhU,8864
+docent/judges/impl.py,sha256=Cqu-qJDfCnca7CZ_TgDoMPt6DijNv-SjK2RwExRo-9I,9199
 docent/judges/types.py,sha256=NlLv42iLDORbPAHppCz-YWZ6ksR4QYDWAweGw75izJ0,8439
 docent/judges/util/forgiving_json.py,sha256=zSh0LF3UVHdSjuMNvEiqUmSxpxPaqK1rSLiI6KCNihg,3549
 docent/judges/util/meta_schema.json,sha256=g3MUa_6e38I3GqZryy8b1w_Y9Krx2xSiWIuaG8Zpszc,2055
 docent/judges/util/meta_schema.py,sha256=6IrIRHERJ6tkRcUtUShJ84I68yUJgkwfFeBjgt42qEA,930
 docent/judges/util/parse_output.py,sha256=qvqt7TEnrAqvzYHqip48boMQSUcoGa-1PA1gIGn-w4s,3381
-docent/judges/util/voting.py,sha256=cAty9b4w7M1OWeW-j8t6vxpZn7VXyE3aBL9Ex2ERKcU,3071
+docent/judges/util/voting.py,sha256=I0Ti0eP7DirdmEQlz-5plrvURUZGmJw0o9yGFgIzE9k,4377
 docent/loaders/load_inspect.py,sha256=VLrtpvcVZ44n2DIPMwUivXqbvOWjaooGw6moY8UQ0VE,6789
 docent/samples/__init__.py,sha256=roDFnU6515l9Q8v17Es_SpWyY9jbm5d6X9lV01V0MZo,143
 docent/samples/load.py,sha256=ZGE07r83GBNO4A0QBh5aQ18WAu3mTWA1vxUoHd90nrM,207
@@ -52,7 +52,7 @@ docent/samples/tb_airline.json,sha256=eR2jFFRtOw06xqbEglh6-dPewjifOk-cuxJq67Dtu5
 docent/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docent/sdk/agent_run_writer.py,sha256=0AWdxejoqZyuj9JSA39WlEwGcMSYTWNqnzIuluySY-M,11043
 docent/sdk/client.py,sha256=K1NVkj_CFj0q-2mSFvWfh8NTqXqosED--dv5aLD7yOE,18239
-docent_python-0.1.21a0.dist-info/METADATA,sha256=H0iEQ39cv90MW0lRZ94XhER6C_znvNU3DBPc6M72i9g,1277
-docent_python-0.1.21a0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-docent_python-0.1.21a0.dist-info/licenses/LICENSE.md,sha256=QIMv2UiT6MppRasso4ymaA0w7ltkqmlL0HCt8CLD7Rc,580
-docent_python-0.1.21a0.dist-info/RECORD,,
+docent_python-0.1.22a0.dist-info/METADATA,sha256=4JAnMdDEsuaTCe48gr3wfNqU1SmdxrdAr1mfBQ3WtGQ,1351
+docent_python-0.1.22a0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+docent_python-0.1.22a0.dist-info/licenses/LICENSE.md,sha256=QIMv2UiT6MppRasso4ymaA0w7ltkqmlL0HCt8CLD7Rc,580
+docent_python-0.1.22a0.dist-info/RECORD,,