docent-python 0.1.21a0__tar.gz → 0.1.22a0__tar.gz

{docent_python-0.1.21a0 → docent_python-0.1.22a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docent-python
-Version: 0.1.21a0
+Version: 0.1.22a0
 Summary: Docent SDK
 Project-URL: Homepage, https://github.com/TransluceAI/docent
 Project-URL: Issues, https://github.com/TransluceAI/docent/issues
@@ -20,6 +20,7 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.34.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.34.1
 Requires-Dist: opentelemetry-instrumentation-anthropic>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-bedrock>=0.40.14
+Requires-Dist: opentelemetry-instrumentation-google-generativeai>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-langchain>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-openai>=0.40.14
 Requires-Dist: opentelemetry-instrumentation-threading>=0.55b1

{docent_python-0.1.21a0 → docent_python-0.1.22a0}/docent/judges/impl.py

@@ -8,7 +8,11 @@ from docent._log_util import get_logger
 from docent.data_models.agent_run import AgentRun
 from docent.judges.types import JudgeResult, ResultType, Rubric
 from docent.judges.util.parse_output import parse_and_validate_llm_output
-from docent.judges.util.voting import find_modal_result, get_agreement_keys
+from docent.judges.util.voting import (
+    compute_output_distribution,
+    find_modal_result,
+    get_agreement_keys,
+)
 
 logger = get_logger(__name__)
 
@@ -74,6 +78,11 @@ class MajorityVotingJudge(BaseJudge):
         )
         final_output = indep_results[final_max_idx]
 
+        # Compute the distribution of the output across the agreement keys
+        final_output_distribution = compute_output_distribution(
+            indep_results, self.cfg.output_schema, agreement_keys
+        )
+
         return JudgeResult(
             agent_run_id=agent_run.id,
             rubric_id=self.cfg.id,
@@ -85,6 +94,7 @@ class MajorityVotingJudge(BaseJudge):
                 "final_results": indep_results,
                 "final_agt_key_modes_and_counts": final_agt_key_modes_and_counts,
                 "final_max_idx": final_max_idx,
+                "final_output_distribution": final_output_distribution,
             },
             result_type=ResultType.DIRECT_RESULT,
         )

{docent_python-0.1.21a0 → docent_python-0.1.22a0}/docent/judges/util/voting.py

@@ -5,7 +5,7 @@ from typing import Any, cast
 def get_agreement_keys(schema: dict[str, Any]) -> list[str]:
     """Get list of top-level keys in schema that we want to measure agreement on.
 
-    This includes enum, bool, and int fields. We skip float and strings.
+    This includes enum and bool fields.
 
     Args:
         schema: JSON schema dict
@@ -29,10 +29,7 @@ def get_agreement_keys(schema: dict[str, Any]) -> list[str]:
         # Include boolean fields
         if field_type == "boolean":
             agreement_keys.append(key)
-        # Include integer fields
-        elif field_type == "integer":
-            agreement_keys.append(key)
-        # Include enum fields (even strings)
+        # Include enum fields (strings and numbers must be in this category)
         elif "enum" in field_schema:
             agreement_keys.append(key)
 
@@ -82,3 +79,36 @@ def find_modal_result(indep_results: list[dict[str, Any]], agreement_keys: list[
     max_idx = indep_result_scores.index(max(indep_result_scores))
 
     return max_idx, agt_key_modes_and_counts
+
+
+def compute_output_distribution(
+    indep_results: list[dict[str, Any]], output_schema: dict[str, Any], agreement_keys: list[str]
+):
+    def _get_possible_values(key: str) -> list[str | bool | int | float]:
+        if "enum" in output_schema.get("properties", {}).get(key, {}):
+            return output_schema.get("properties", {}).get(key, {}).get("enum", [])
+        elif output_schema.get("properties", {}).get(key, {}).get("type") == "boolean":
+            return [True, False]
+        else:
+            return []
+
+    distributions: dict[str, dict[str | bool | int | float, float]] = {
+        key: {value: 0.0 for value in _get_possible_values(key)} for key in agreement_keys
+    }
+    # Collect counts for each possible value
+    for result in indep_results:
+        for key in agreement_keys:
+            if (value := result.get(key)) is not None:  # Could be none if the key is optional
+                assert (
+                    value in distributions[key]
+                ), "this should never happen; the value must be in possible values, since judge results have been validated against the schema"
+                distributions[key][value] += 1
+    # Normalize
+    for key in distributions:
+        total = sum(distributions[key].values())
+        if total == 0:
+            continue
+        for value in distributions[key]:
+            distributions[key][value] /= total
+
+    return distributions

{docent_python-0.1.21a0 → docent_python-0.1.22a0}/docent/trace.py

@@ -43,6 +43,7 @@ class Instruments(Enum):
     ANTHROPIC = "anthropic"
     BEDROCK = "bedrock"
     LANGCHAIN = "langchain"
+    GOOGLE_GENERATIVEAI = "google_generativeai"
 
 
 class DocentTracer:
@@ -392,6 +393,23 @@ class DocentTracer:
                 except Exception as e:
                     logger.warning(f"Failed to instrument LangChain: {e}")
 
+            # Instrument Google Generative AI with our isolated tracer provider
+            if Instruments.GOOGLE_GENERATIVEAI in enabled_instruments:
+                try:
+                    if is_package_installed("google-generativeai") or is_package_installed(
+                        "google-genai"
+                    ):
+                        from opentelemetry.instrumentation.google_generativeai import (
+                            GoogleGenerativeAiInstrumentor,
+                        )
+
+                        GoogleGenerativeAiInstrumentor().instrument(
+                            tracer_provider=self._tracer_provider
+                        )
+                        logger.info("Instrumented Google Generative AI")
+                except Exception as e:
+                    logger.warning(f"Failed to instrument Google Generative AI: {e}")
+
             # Register cleanup handlers
             self._register_cleanup()
 

{docent_python-0.1.21a0 → docent_python-0.1.22a0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "docent-python"
 description = "Docent SDK"
-version = "0.1.21-alpha"
+version = "0.1.22-alpha"
 authors = [
   { name="Transluce", email="info@transluce.org" },
 ]
@@ -28,11 +28,12 @@ dependencies = [
     "opentelemetry-exporter-otlp-proto-grpc>=1.34.1",
     "opentelemetry-exporter-otlp-proto-http>=1.34.1",
     # Instrumentation
+    "opentelemetry-instrumentation-threading>=0.55b1",
     "opentelemetry-instrumentation-anthropic>=0.40.14",
     "opentelemetry-instrumentation-bedrock>=0.40.14",
     "opentelemetry-instrumentation-langchain>=0.40.14",
     "opentelemetry-instrumentation-openai>=0.40.14",
-    "opentelemetry-instrumentation-threading>=0.55b1",
+    "opentelemetry-instrumentation-google-generativeai>=0.40.14",
 ]
 
 [build-system]