gatewaysdk 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gatewaysdk/__init__.py +540 -0
- gatewaysdk/adapter/__init__.py +15 -0
- gatewaysdk/adapter/base.py +94 -0
- gatewaysdk/adapter/messages.py +270 -0
- gatewaysdk/adapter/triplet.py +1026 -0
- gatewaysdk/algorithm/__init__.py +39 -0
- gatewaysdk/algorithm/apo/__init__.py +5 -0
- gatewaysdk/algorithm/apo/apo.py +898 -0
- gatewaysdk/algorithm/apo/prompts/apply_edit_variant01.poml +22 -0
- gatewaysdk/algorithm/apo/prompts/apply_edit_variant02.poml +18 -0
- gatewaysdk/algorithm/apo/prompts/text_gradient_variant01.poml +18 -0
- gatewaysdk/algorithm/apo/prompts/text_gradient_variant02.poml +16 -0
- gatewaysdk/algorithm/apo/prompts/text_gradient_variant03.poml +107 -0
- gatewaysdk/algorithm/base.py +262 -0
- gatewaysdk/algorithm/decorator.py +264 -0
- gatewaysdk/algorithm/evals/__init__.py +7 -0
- gatewaysdk/algorithm/evals/evals.py +217 -0
- gatewaysdk/algorithm/fast.py +250 -0
- gatewaysdk/algorithm/gepa/__init__.py +61 -0
- gatewaysdk/algorithm/gepa/adapter.py +495 -0
- gatewaysdk/algorithm/gepa/gepa.py +570 -0
- gatewaysdk/algorithm/gepa/lib/__init__.py +18 -0
- gatewaysdk/algorithm/gepa/lib/adapters/README.md +12 -0
- gatewaysdk/algorithm/gepa/lib/adapters/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/README.md +341 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/__init__.py +1 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/anymaths_adapter.py +174 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/requirements.txt +1 -0
- gatewaysdk/algorithm/gepa/lib/adapters/default_adapter/README.md +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/default_adapter/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/default_adapter/default_adapter.py +209 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_adapter/README.md +7 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_adapter/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_adapter/dspy_adapter.py +307 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_full_program_adapter/README.md +99 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_full_program_adapter/dspy_program_proposal_signature.py +137 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_full_program_adapter/full_program_adapter.py +268 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/GEPA_RAG.md +621 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/__init__.py +56 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/evaluation_metrics.py +226 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/generic_rag_adapter.py +496 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/rag_pipeline.py +238 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_store_interface.py +212 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/__init__.py +2 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/chroma_store.py +196 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/lancedb_store.py +422 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/milvus_store.py +409 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/qdrant_store.py +368 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/weaviate_store.py +418 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/README.md +552 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/__init__.py +37 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/mcp_adapter.py +699 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/mcp_client.py +364 -0
- gatewaysdk/algorithm/gepa/lib/adapters/terminal_bench_adapter/README.md +9 -0
- gatewaysdk/algorithm/gepa/lib/adapters/terminal_bench_adapter/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/terminal_bench_adapter/terminal_bench_adapter.py +217 -0
- gatewaysdk/algorithm/gepa/lib/api.py +382 -0
- gatewaysdk/algorithm/gepa/lib/core/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/core/adapter.py +180 -0
- gatewaysdk/algorithm/gepa/lib/core/data_loader.py +74 -0
- gatewaysdk/algorithm/gepa/lib/core/engine.py +379 -0
- gatewaysdk/algorithm/gepa/lib/core/result.py +233 -0
- gatewaysdk/algorithm/gepa/lib/core/state.py +636 -0
- gatewaysdk/algorithm/gepa/lib/examples/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/examples/aime.py +24 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/eval_default.py +111 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/prompt-templates/instruction_prompt.txt +9 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/prompt-templates/optimal_prompt.txt +24 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/train_anymaths.py +177 -0
- gatewaysdk/algorithm/gepa/lib/examples/dspy_full_program_evolution/arc_agi.ipynb +25705 -0
- gatewaysdk/algorithm/gepa/lib/examples/dspy_full_program_evolution/example.ipynb +348 -0
- gatewaysdk/algorithm/gepa/lib/examples/mcp_adapter/__init__.py +4 -0
- gatewaysdk/algorithm/gepa/lib/examples/mcp_adapter/mcp_optimization_example.py +456 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/RAG_GUIDE.md +613 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/__init__.py +9 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/rag_optimization.py +820 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/requirements-rag.txt +29 -0
- gatewaysdk/algorithm/gepa/lib/examples/terminal-bench/prompt-templates/instruction_prompt.txt +16 -0
- gatewaysdk/algorithm/gepa/lib/examples/terminal-bench/prompt-templates/terminus.txt +9 -0
- gatewaysdk/algorithm/gepa/lib/examples/terminal-bench/train_terminus.py +161 -0
- gatewaysdk/algorithm/gepa/lib/gepa_utils.py +117 -0
- gatewaysdk/algorithm/gepa/lib/logging/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/logging/experiment_tracker.py +187 -0
- gatewaysdk/algorithm/gepa/lib/logging/logger.py +75 -0
- gatewaysdk/algorithm/gepa/lib/logging/utils.py +103 -0
- gatewaysdk/algorithm/gepa/lib/proposer/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/proposer/base.py +31 -0
- gatewaysdk/algorithm/gepa/lib/proposer/merge.py +357 -0
- gatewaysdk/algorithm/gepa/lib/proposer/reflective_mutation/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/proposer/reflective_mutation/base.py +49 -0
- gatewaysdk/algorithm/gepa/lib/proposer/reflective_mutation/reflective_mutation.py +176 -0
- gatewaysdk/algorithm/gepa/lib/py.typed +0 -0
- gatewaysdk/algorithm/gepa/lib/strategies/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/strategies/batch_sampler.py +77 -0
- gatewaysdk/algorithm/gepa/lib/strategies/candidate_selector.py +50 -0
- gatewaysdk/algorithm/gepa/lib/strategies/component_selector.py +36 -0
- gatewaysdk/algorithm/gepa/lib/strategies/eval_policy.py +64 -0
- gatewaysdk/algorithm/gepa/lib/strategies/instruction_proposal.py +126 -0
- gatewaysdk/algorithm/gepa/lib/utils/__init__.py +10 -0
- gatewaysdk/algorithm/gepa/lib/utils/stop_condition.py +196 -0
- gatewaysdk/algorithm/gepa/tracing.py +105 -0
- gatewaysdk/algorithm/utils.py +177 -0
- gatewaysdk/algorithm/verl/__init__.py +5 -0
- gatewaysdk/algorithm/verl/interface.py +202 -0
- gatewaysdk/automations.py +111 -0
- gatewaysdk/benchmark_hub/README.md +88 -0
- gatewaysdk/benchmark_hub/__init__.py +119 -0
- gatewaysdk/benchmark_hub/_archive.py +189 -0
- gatewaysdk/benchmark_hub/_tar.py +6 -0
- gatewaysdk/benchmark_hub/client.py +992 -0
- gatewaysdk/benchmark_hub/dispatch_shard.py +34 -0
- gatewaysdk/benchmark_hub/eval_config.py +20 -0
- gatewaysdk/benchmark_hub/evals.py +351 -0
- gatewaysdk/benchmark_hub/harbor_adapter.py +673 -0
- gatewaysdk/benchmark_hub/path_utils.py +81 -0
- gatewaysdk/benchmark_hub/save_utils.py +101 -0
- gatewaysdk/benchmark_hub/verifiers_adapter.py +327 -0
- gatewaysdk/benchmark_hub/versioning.py +72 -0
- gatewaysdk/build.py +515 -0
- gatewaysdk/cli/__init__.py +58 -0
- gatewaysdk/cli/agent_runner.py +132 -0
- gatewaysdk/cli/http_client.py +115 -0
- gatewaysdk/cli/platform.py +4086 -0
- gatewaysdk/cli/prometheus.py +115 -0
- gatewaysdk/cli/release_gate.py +215 -0
- gatewaysdk/cli/store.py +131 -0
- gatewaysdk/cli/vllm.py +29 -0
- gatewaysdk/client.py +406 -0
- gatewaysdk/config.py +348 -0
- gatewaysdk/connectors/__init__.py +25 -0
- gatewaysdk/connectors/client.py +203 -0
- gatewaysdk/connectors/skill.py +25 -0
- gatewaysdk/connectors/template.py +106 -0
- gatewaysdk/context.py +606 -0
- gatewaysdk/emitter/__init__.py +43 -0
- gatewaysdk/emitter/annotation.py +370 -0
- gatewaysdk/emitter/exception.py +54 -0
- gatewaysdk/emitter/message.py +61 -0
- gatewaysdk/emitter/object.py +117 -0
- gatewaysdk/emitter/reward.py +320 -0
- gatewaysdk/env_var.py +156 -0
- gatewaysdk/environment/__init__.py +108 -0
- gatewaysdk/environment/_bundle.py +281 -0
- gatewaysdk/environment/_harbor.py +276 -0
- gatewaysdk/environment/_materialize.py +97 -0
- gatewaysdk/environment/_tar.py +33 -0
- gatewaysdk/environment/_world.py +346 -0
- gatewaysdk/environment/core.py +527 -0
- gatewaysdk/environment/errors.py +58 -0
- gatewaysdk/environment/runtime.py +150 -0
- gatewaysdk/environment/schema/__init__.py +35 -0
- gatewaysdk/environment/schema/__main__.py +225 -0
- gatewaysdk/environment/schema/_slack_fidelity.py +116 -0
- gatewaysdk/environment/schema/_slack_scenario.py +183 -0
- gatewaysdk/environment/schema/api.py +1628 -0
- gatewaysdk/environment/schema/batch.py +755 -0
- gatewaysdk/environment/schema/compiler.py +615 -0
- gatewaysdk/environment/schema/conform.py +428 -0
- gatewaysdk/environment/schema/connector.py +777 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/handlers.py +17 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/parity.json +28 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/provenance.json +233 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/scope.toml +112 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/world.json +1380 -0
- gatewaysdk/environment/schema/connectors/base.json +28 -0
- gatewaysdk/environment/schema/connectors/custom/handlers.py +18 -0
- gatewaysdk/environment/schema/connectors/custom/world.json +123 -0
- gatewaysdk/environment/schema/connectors/github/handlers.py +81 -0
- gatewaysdk/environment/schema/connectors/github/parity.json +37 -0
- gatewaysdk/environment/schema/connectors/github/provenance.json +134 -0
- gatewaysdk/environment/schema/connectors/github/scope.toml +158 -0
- gatewaysdk/environment/schema/connectors/github/world.json +5869 -0
- gatewaysdk/environment/schema/connectors/google-calendar/handlers.py +23 -0
- gatewaysdk/environment/schema/connectors/google-calendar/world.json +209 -0
- gatewaysdk/environment/schema/connectors/google-drive/handlers.py +18 -0
- gatewaysdk/environment/schema/connectors/google-drive/world.json +286 -0
- gatewaysdk/environment/schema/connectors/jamf/handlers.py +34 -0
- gatewaysdk/environment/schema/connectors/jamf/parity.json +49 -0
- gatewaysdk/environment/schema/connectors/jamf/provenance.json +181 -0
- gatewaysdk/environment/schema/connectors/jamf/scope.toml +128 -0
- gatewaysdk/environment/schema/connectors/jamf/world.json +2218 -0
- gatewaysdk/environment/schema/connectors/jira/handlers.py +28 -0
- gatewaysdk/environment/schema/connectors/jira/world.json +389 -0
- gatewaysdk/environment/schema/connectors/kandji/handlers.py +38 -0
- gatewaysdk/environment/schema/connectors/kandji/parity.json +29 -0
- gatewaysdk/environment/schema/connectors/kandji/provenance.json +152 -0
- gatewaysdk/environment/schema/connectors/kandji/scope.toml +120 -0
- gatewaysdk/environment/schema/connectors/kandji/world.json +765 -0
- gatewaysdk/environment/schema/connectors/linear/handlers.py +20 -0
- gatewaysdk/environment/schema/connectors/linear/world.json +363 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/fidelity.json +45 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/handlers.py +1 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/provenance.json +851 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/scope.md +231 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/teams-types.json +1710 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/world.json +858 -0
- gatewaysdk/environment/schema/connectors/netsuite/handlers.py +32 -0
- gatewaysdk/environment/schema/connectors/netsuite/parity.json +50 -0
- gatewaysdk/environment/schema/connectors/netsuite/provenance.json +137 -0
- gatewaysdk/environment/schema/connectors/netsuite/scope.toml +78 -0
- gatewaysdk/environment/schema/connectors/netsuite/world.json +16243 -0
- gatewaysdk/environment/schema/connectors/salesforce/handlers.py +806 -0
- gatewaysdk/environment/schema/connectors/salesforce/parity.json +46 -0
- gatewaysdk/environment/schema/connectors/salesforce/provenance.json +143 -0
- gatewaysdk/environment/schema/connectors/salesforce/scope.toml +174 -0
- gatewaysdk/environment/schema/connectors/salesforce/world.json +4695 -0
- gatewaysdk/environment/schema/connectors/slack/MODEL.md +236 -0
- gatewaysdk/environment/schema/connectors/slack/capabilities.json +1039 -0
- gatewaysdk/environment/schema/connectors/slack/handlers.py +762 -0
- gatewaysdk/environment/schema/connectors/slack/parity.json +65 -0
- gatewaysdk/environment/schema/connectors/slack/provenance.json +294 -0
- gatewaysdk/environment/schema/connectors/slack/scope.toml +44 -0
- gatewaysdk/environment/schema/connectors/slack/world.json +4993 -0
- gatewaysdk/environment/schema/connectors/workday/handlers.py +75 -0
- gatewaysdk/environment/schema/connectors/workday/parity.json +36 -0
- gatewaysdk/environment/schema/connectors/workday/provenance.json +179 -0
- gatewaysdk/environment/schema/connectors/workday/scope.toml +180 -0
- gatewaysdk/environment/schema/connectors/workday/world.json +806 -0
- gatewaysdk/environment/schema/data.py +593 -0
- gatewaysdk/environment/schema/extract.py +263 -0
- gatewaysdk/environment/schema/host.py +2437 -0
- gatewaysdk/environment/schema/host_surfaces.py +275 -0
- gatewaysdk/environment/schema/platform.py +1272 -0
- gatewaysdk/environment/schema/scaffold.py +250 -0
- gatewaysdk/environment/schema/skill.py +118 -0
- gatewaysdk/environment/schema/skills/agents/connector-template-author.md +202 -0
- gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- gatewaysdk/environment/schema/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +313 -0
- gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +223 -0
- gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +293 -0
- gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +390 -0
- gatewaysdk/environment/schema/slack.py +757 -0
- gatewaysdk/environment/schema/snapshot.py +479 -0
- gatewaysdk/environment/schema/store.py +904 -0
- gatewaysdk/environment/schema/tasks.py +525 -0
- gatewaysdk/environment/schema/tools_world.py +322 -0
- gatewaysdk/environment/schema/validation.py +577 -0
- gatewaysdk/environment/schema/worker.py +78 -0
- gatewaysdk/environment/schema/workers.py +500 -0
- gatewaysdk/environment/schema/world_tests.py +336 -0
- gatewaysdk/environments/__init__.py +24 -0
- gatewaysdk/environments/client.py +486 -0
- gatewaysdk/environments/types.py +318 -0
- gatewaysdk/execution/__init__.py +15 -0
- gatewaysdk/execution/base.py +64 -0
- gatewaysdk/execution/client_server.py +443 -0
- gatewaysdk/execution/events.py +69 -0
- gatewaysdk/execution/inter_process.py +16 -0
- gatewaysdk/execution/shared_memory.py +282 -0
- gatewaysdk/experiments/__init__.py +90 -0
- gatewaysdk/experiments/assignment.py +177 -0
- gatewaysdk/experiments/client.py +877 -0
- gatewaysdk/experiments/exposure.py +222 -0
- gatewaysdk/experiments/types.py +81 -0
- gatewaysdk/importers/__init__.py +49 -0
- gatewaysdk/importers/_normalize.py +91 -0
- gatewaysdk/importers/builder.py +173 -0
- gatewaysdk/importers/client.py +95 -0
- gatewaysdk/importers/langsmith.py +196 -0
- gatewaysdk/importers/recipes.py +249 -0
- gatewaysdk/instrumentation/__init__.py +300 -0
- gatewaysdk/instrumentation/agentops.py +314 -0
- gatewaysdk/instrumentation/agentops_langchain.py +45 -0
- gatewaysdk/instrumentation/base.py +119 -0
- gatewaysdk/instrumentation/litellm.py +83 -0
- gatewaysdk/instrumentation/registry.py +273 -0
- gatewaysdk/instrumentation/vllm.py +81 -0
- gatewaysdk/instrumentation/weave.py +500 -0
- gatewaysdk/integrations/__init__.py +15 -0
- gatewaysdk/integrations/gateway/__init__.py +11 -0
- gatewaysdk/integrations/gateway/client.py +171 -0
- gatewaysdk/integrations/tool_access.py +549 -0
- gatewaysdk/litagent/__init__.py +11 -0
- gatewaysdk/litagent/decorator.py +536 -0
- gatewaysdk/litagent/litagent.py +252 -0
- gatewaysdk/llm_proxy.py +1742 -0
- gatewaysdk/logging.py +370 -0
- gatewaysdk/memory.py +278 -0
- gatewaysdk/personas.py +186 -0
- gatewaysdk/platform/__init__.py +17 -0
- gatewaysdk/platform/builder.py +221 -0
- gatewaysdk/platform/compatibility.py +67 -0
- gatewaysdk/platform/manifest.py +185 -0
- gatewaysdk/platform/orchestrator.py +901 -0
- gatewaysdk/platform/registry.py +122 -0
- gatewaysdk/platform/worker.py +864 -0
- gatewaysdk/replay/__init__.py +1032 -0
- gatewaysdk/replay/pytest.py +56 -0
- gatewaysdk/reward.py +7 -0
- gatewaysdk/run.py +1781 -0
- gatewaysdk/runner/__init__.py +11 -0
- gatewaysdk/runner/agent.py +878 -0
- gatewaysdk/runner/base.py +182 -0
- gatewaysdk/runner/legacy.py +309 -0
- gatewaysdk/security.py +700 -0
- gatewaysdk/semconv.py +170 -0
- gatewaysdk/server.py +399 -0
- gatewaysdk/sessions.py +282 -0
- gatewaysdk/store/__init__.py +45 -0
- gatewaysdk/store/base.py +908 -0
- gatewaysdk/store/client_server.py +2093 -0
- gatewaysdk/store/collection/__init__.py +30 -0
- gatewaysdk/store/collection/base.py +587 -0
- gatewaysdk/store/collection/memory.py +970 -0
- gatewaysdk/store/collection/mongo.py +1412 -0
- gatewaysdk/store/collection_based.py +1823 -0
- gatewaysdk/store/gateway.py +983 -0
- gatewaysdk/store/gateway_listener.py +465 -0
- gatewaysdk/store/listener.py +58 -0
- gatewaysdk/store/memory.py +396 -0
- gatewaysdk/store/mongo.py +165 -0
- gatewaysdk/store/redis_stream.py +517 -0
- gatewaysdk/store/sqlite.py +3 -0
- gatewaysdk/store/threading.py +370 -0
- gatewaysdk/store/utils.py +142 -0
- gatewaysdk/tracer/__init__.py +14 -0
- gatewaysdk/tracer/base.py +286 -0
- gatewaysdk/tracer/dummy.py +106 -0
- gatewaysdk/tracer/otel.py +559 -0
- gatewaysdk/tracing/__init__.py +110 -0
- gatewaysdk/tracing/api.py +808 -0
- gatewaysdk/tracing/attributes.py +9 -0
- gatewaysdk/tracing/context.py +272 -0
- gatewaysdk/tracing/exporters/__init__.py +10 -0
- gatewaysdk/tracing/exporters/gateway.py +228 -0
- gatewaysdk/tracing/identity.py +288 -0
- gatewaysdk/tracing/init.py +620 -0
- gatewaysdk/tracing/instrumentors/__init__.py +15 -0
- gatewaysdk/tracing/instrumentors/claude_agent_sdk.py +766 -0
- gatewaysdk/tracing/instrumentors/instrumentation_principles.md +294 -0
- gatewaysdk/tracing/instrumentors/registry.py +352 -0
- gatewaysdk/tracing/mapping.py +729 -0
- gatewaysdk/tracing/processors.py +393 -0
- gatewaysdk/tracing/push.py +617 -0
- gatewaysdk/tracing/push_models.py +247 -0
- gatewaysdk/tracing/semconv.py +294 -0
- gatewaysdk/tracing/span_builder.py +356 -0
- gatewaysdk/trainer/__init__.py +6 -0
- gatewaysdk/trainer/init_utils.py +263 -0
- gatewaysdk/trainer/legacy.py +359 -0
- gatewaysdk/trainer/registry.py +12 -0
- gatewaysdk/trainer/trainer.py +638 -0
- gatewaysdk/types/__init__.py +63 -0
- gatewaysdk/types/core.py +556 -0
- gatewaysdk/types/resources.py +204 -0
- gatewaysdk/types/tracer.py +515 -0
- gatewaysdk/types/tracing.py +162 -0
- gatewaysdk/users.py +251 -0
- gatewaysdk/utils/__init__.py +1 -0
- gatewaysdk/utils/id.py +18 -0
- gatewaysdk/utils/metrics.py +1025 -0
- gatewaysdk/utils/otel.py +550 -0
- gatewaysdk/utils/otlp.py +556 -0
- gatewaysdk/utils/redact.py +22 -0
- gatewaysdk/utils/server_launcher.py +1045 -0
- gatewaysdk/utils/system_snapshot.py +90 -0
- gatewaysdk/verl/__init__.py +8 -0
- gatewaysdk/verl/__main__.py +6 -0
- gatewaysdk/verl/async_server.py +46 -0
- gatewaysdk/verl/config.yaml +27 -0
- gatewaysdk/verl/daemon.py +1154 -0
- gatewaysdk/verl/dataset.py +44 -0
- gatewaysdk/verl/entrypoint.py +248 -0
- gatewaysdk/verl/trainer.py +549 -0
- gatewaysdk/world_browser.py +748 -0
- gatewaysdk/world_data.py +453 -0
- gatewaysdk/world_sessions.py +978 -0
- gatewaysdk/world_tasks.py +285 -0
- gatewaysdk/world_tools.py +115 -0
- gatewaysdk/worlds.py +639 -0
- gatewaysdk-0.3.2.dist-info/METADATA +236 -0
- gatewaysdk-0.3.2.dist-info/RECORD +376 -0
- gatewaysdk-0.3.2.dist-info/WHEEL +4 -0
- gatewaysdk-0.3.2.dist-info/entry_points.txt +5 -0
- gatewaysdk-0.3.2.dist-info/licenses/LICENSE +19 -0
|
@@ -0,0 +1,877 @@
|
|
|
1
|
+
# Copyright (c) Gateway. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""Main client API for A/B experiments.
|
|
4
|
+
|
|
5
|
+
This module provides the ExperimentClient class that users interact with
|
|
6
|
+
to register experiments and get variant assignments. It also manages the
|
|
7
|
+
context variable that the span processor reads from.
|
|
8
|
+
|
|
9
|
+
The client supports two modes:
|
|
10
|
+
1. **Local-only**: Experiments registered in code stay in-memory
|
|
11
|
+
2. **Backend-synced**: Experiments are fetched from and synced to the Gateway backend
|
|
12
|
+
|
|
13
|
+
Usage::
|
|
14
|
+
|
|
15
|
+
import gatewaysdk as msk
|
|
16
|
+
|
|
17
|
+
# Initialize with backend sync (recommended)
|
|
18
|
+
msk.init_experiments(
|
|
19
|
+
api_endpoint="https://gateway.example.com/api/public/experiments/exposures/batch",
|
|
20
|
+
public_key="pk-lf-...",
|
|
21
|
+
secret_key="sk-lf-...",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# Option A: Register experiment in code (syncs to backend in background)
|
|
25
|
+
msk.experiments.register(
|
|
26
|
+
key="prompt-test-v1",
|
|
27
|
+
variants=[
|
|
28
|
+
{"key": "control", "weight": 50, "config": {"prompt": "..."}},
|
|
29
|
+
{"key": "treatment", "weight": 50, "config": {"prompt": "..."}},
|
|
30
|
+
],
|
|
31
|
+
traffic_percent=100,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# Option B: Fetch experiment from backend (defined in UI)
|
|
35
|
+
msk.experiments.fetch("prompt-test-v1")
|
|
36
|
+
|
|
37
|
+
# Get variant for a user (~0ms, purely in-memory)
|
|
38
|
+
variant = msk.experiments.get_variant("prompt-test-v1", user_id="user-123")
|
|
39
|
+
if variant:
|
|
40
|
+
print(f"Assigned to {variant.key}")
|
|
41
|
+
system_prompt = variant.config["prompt"]
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import logging
|
|
47
|
+
import threading
|
|
48
|
+
from contextvars import ContextVar
|
|
49
|
+
from datetime import datetime, timezone
|
|
50
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
51
|
+
|
|
52
|
+
import requests
|
|
53
|
+
|
|
54
|
+
from .assignment import assign_variant
|
|
55
|
+
from .types import ExperimentConfig, Exposure, Variant
|
|
56
|
+
|
|
57
|
+
if TYPE_CHECKING:
|
|
58
|
+
from .exposure import ExposureLogger
|
|
59
|
+
|
|
60
|
+
logger = logging.getLogger(__name__)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# =============================================================================
|
|
64
|
+
# Exceptions
|
|
65
|
+
# =============================================================================
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class BalancedAssignmentError(Exception):
|
|
69
|
+
"""Raised when server-side balanced assignment fails due to network/server issues.
|
|
70
|
+
|
|
71
|
+
This exception is raised by get_variant_balanced() when the server cannot be reached
|
|
72
|
+
or returns an error. The caller (get_variant) catches this and falls back to
|
|
73
|
+
local hash-based assignment for graceful degradation.
|
|
74
|
+
|
|
75
|
+
This is NOT raised when the user is legitimately not in traffic (server returns
|
|
76
|
+
variantKey: null) — in that case, None is returned normally.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
pass
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# =============================================================================
|
|
83
|
+
# Context Variable for Experiment Assignment
|
|
84
|
+
# =============================================================================
|
|
85
|
+
|
|
86
|
+
# Stores the current experiment context for span injection.
|
|
87
|
+
# The TracingContextSpanProcessor reads this and injects attributes into spans.
|
|
88
|
+
_current_experiment: ContextVar[Optional[Dict[str, Any]]] = ContextVar("msk_current_experiment", default=None)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def get_current_experiment() -> Optional[Dict[str, Any]]:
|
|
92
|
+
"""Get the current experiment assignment context.
|
|
93
|
+
|
|
94
|
+
Returns a dict with keys:
|
|
95
|
+
- experiment_id: The experiment key
|
|
96
|
+
- variant_key: The assigned variant key
|
|
97
|
+
- user_id: The user identifier
|
|
98
|
+
|
|
99
|
+
Returns None if no experiment is active.
|
|
100
|
+
|
|
101
|
+
This is called by TracingContextSpanProcessor to inject experiment
|
|
102
|
+
attributes into spans.
|
|
103
|
+
"""
|
|
104
|
+
return _current_experiment.get()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _set_current_experiment(ctx: Optional[Dict[str, Any]]) -> None:
|
|
108
|
+
"""Set the current experiment context.
|
|
109
|
+
|
|
110
|
+
Internal function - called by ExperimentClient.get_variant().
|
|
111
|
+
"""
|
|
112
|
+
_current_experiment.set(ctx)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def clear_current_experiment() -> None:
|
|
116
|
+
"""Clear the current experiment context.
|
|
117
|
+
|
|
118
|
+
Call this when you want to stop attributing traces to an experiment,
|
|
119
|
+
for example when switching users or ending an experiment session.
|
|
120
|
+
"""
|
|
121
|
+
_current_experiment.set(None)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# =============================================================================
|
|
125
|
+
# Experiment Client
|
|
126
|
+
# =============================================================================
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class ExperimentClient:
|
|
130
|
+
"""Client for managing A/B experiments.
|
|
131
|
+
|
|
132
|
+
This class provides the main API for:
|
|
133
|
+
- Registering experiments with variants (locally and to backend)
|
|
134
|
+
- Fetching experiments from the backend
|
|
135
|
+
- Getting variant assignments for users (~0ms, in-memory)
|
|
136
|
+
- Managing exposure logging (async, background thread)
|
|
137
|
+
- Background config polling (keeps in-memory cache fresh)
|
|
138
|
+
|
|
139
|
+
The client is typically accessed via the gatewaysdk module:
|
|
140
|
+
msk.experiments.register(...)
|
|
141
|
+
msk.experiments.fetch(...)
|
|
142
|
+
msk.experiments.get_variant(...)
|
|
143
|
+
|
|
144
|
+
Attributes:
|
|
145
|
+
exposure_logger: The background logger for exposure events
|
|
146
|
+
api_base_url: Base URL for the experiments API
|
|
147
|
+
public_key: Gateway public key for authentication
|
|
148
|
+
secret_key: Gateway secret key for authentication
|
|
149
|
+
poll_interval: Seconds between background config refreshes (default 30)
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
def __init__(
|
|
153
|
+
self,
|
|
154
|
+
exposure_logger: Optional["ExposureLogger"] = None,
|
|
155
|
+
api_base_url: Optional[str] = None,
|
|
156
|
+
public_key: Optional[str] = None,
|
|
157
|
+
secret_key: Optional[str] = None,
|
|
158
|
+
poll_interval: float = 30.0,
|
|
159
|
+
):
|
|
160
|
+
"""Initialize the experiment client.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
exposure_logger: Optional ExposureLogger for sending exposure events.
|
|
164
|
+
If not provided, exposures will be logged but not sent.
|
|
165
|
+
api_base_url: Base URL for the experiments API (e.g., "https://gateway.example.com").
|
|
166
|
+
If not provided, backend sync is disabled.
|
|
167
|
+
public_key: Gateway public key for API authentication.
|
|
168
|
+
secret_key: Gateway secret key for API authentication.
|
|
169
|
+
poll_interval: Seconds between background polls for config updates.
|
|
170
|
+
Default is 30 seconds. Set to 0 to disable polling.
|
|
171
|
+
"""
|
|
172
|
+
# PREVIOUSLY: plain dict, written from main thread only.
|
|
173
|
+
# NOW: protected by a lock because the background refresh thread also
|
|
174
|
+
# writes to it while get_variant() reads from the main thread.
|
|
175
|
+
self._experiments: Dict[str, ExperimentConfig] = {}
|
|
176
|
+
self._experiments_lock = threading.Lock()
|
|
177
|
+
|
|
178
|
+
self._exposure_logger = exposure_logger
|
|
179
|
+
self._api_base_url = api_base_url
|
|
180
|
+
self._public_key = public_key
|
|
181
|
+
self._secret_key = secret_key
|
|
182
|
+
self._poll_interval = poll_interval
|
|
183
|
+
|
|
184
|
+
# Background polling state
|
|
185
|
+
self._poll_thread: Optional[threading.Thread] = None
|
|
186
|
+
self._poll_shutdown = threading.Event()
|
|
187
|
+
self._poll_started = False
|
|
188
|
+
|
|
189
|
+
def set_exposure_logger(self, logger: "ExposureLogger") -> None:
|
|
190
|
+
"""Set the exposure logger after initialization.
|
|
191
|
+
|
|
192
|
+
This is useful when the client is created before the API configuration
|
|
193
|
+
is known (e.g., during module initialization).
|
|
194
|
+
"""
|
|
195
|
+
self._exposure_logger = logger
|
|
196
|
+
|
|
197
|
+
def configure_backend(
|
|
198
|
+
self,
|
|
199
|
+
api_base_url: str,
|
|
200
|
+
public_key: str,
|
|
201
|
+
secret_key: str,
|
|
202
|
+
) -> None:
|
|
203
|
+
"""Configure backend connectivity for fetching/syncing experiments.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
api_base_url: Base URL for the experiments API.
|
|
207
|
+
public_key: Gateway public key.
|
|
208
|
+
secret_key: Gateway secret key.
|
|
209
|
+
"""
|
|
210
|
+
self._api_base_url = api_base_url
|
|
211
|
+
self._public_key = public_key
|
|
212
|
+
self._secret_key = secret_key
|
|
213
|
+
logger.debug(f"Configured experiments backend: {api_base_url}")
|
|
214
|
+
|
|
215
|
+
def _get_auth_header(self) -> Dict[str, str]:
|
|
216
|
+
"""Get authorization header for API requests."""
|
|
217
|
+
if not self._public_key or not self._secret_key:
|
|
218
|
+
return {}
|
|
219
|
+
import base64
|
|
220
|
+
|
|
221
|
+
credentials = base64.b64encode(f"{self._public_key}:{self._secret_key}".encode()).decode()
|
|
222
|
+
return {"Authorization": f"Basic {credentials}"}
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def _parse_experiment_response(data: Dict[str, Any]) -> ExperimentConfig:
|
|
226
|
+
"""Parse an experiment JSON response into an ExperimentConfig."""
|
|
227
|
+
return ExperimentConfig(
|
|
228
|
+
key=data["key"],
|
|
229
|
+
variants=[
|
|
230
|
+
Variant(
|
|
231
|
+
key=v["key"],
|
|
232
|
+
weight=v["weight"],
|
|
233
|
+
config=v.get("config", {}),
|
|
234
|
+
)
|
|
235
|
+
for v in data["variants"]
|
|
236
|
+
],
|
|
237
|
+
traffic_percent=data["trafficPercent"],
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
# -------------------------------------------------------------------------
|
|
241
|
+
# Background Config Polling
|
|
242
|
+
# -------------------------------------------------------------------------
|
|
243
|
+
|
|
244
|
+
def start_background_refresh(self) -> None:
|
|
245
|
+
"""Start a background daemon thread that polls the backend for config updates.
|
|
246
|
+
|
|
247
|
+
WHY: Previously, config was only loaded when the caller explicitly called
|
|
248
|
+
fetch() or fetch_all() — a blocking HTTP call. Now, a daemon thread runs
|
|
249
|
+
silently in the background and atomically swaps in fresh configs every
|
|
250
|
+
poll_interval seconds. get_variant() always reads from warm memory (~0ms).
|
|
251
|
+
|
|
252
|
+
Safe to call multiple times — only starts once. No-op if poll_interval=0
|
|
253
|
+
or backend is not configured.
|
|
254
|
+
"""
|
|
255
|
+
if self._poll_started or not self._api_base_url or self._poll_interval <= 0:
|
|
256
|
+
return
|
|
257
|
+
|
|
258
|
+
self._poll_shutdown.clear()
|
|
259
|
+
self._poll_thread = threading.Thread(
|
|
260
|
+
target=self._poll_loop,
|
|
261
|
+
daemon=True,
|
|
262
|
+
name="msk-experiment-refresh",
|
|
263
|
+
)
|
|
264
|
+
self._poll_thread.start()
|
|
265
|
+
self._poll_started = True
|
|
266
|
+
logger.debug(f"Background experiment refresh started (interval={self._poll_interval}s)")
|
|
267
|
+
|
|
268
|
+
def stop_background_refresh(self) -> None:
|
|
269
|
+
"""Stop the background refresh thread. Blocks until it exits (up to 5s)."""
|
|
270
|
+
if not self._poll_started:
|
|
271
|
+
return
|
|
272
|
+
self._poll_shutdown.set()
|
|
273
|
+
if self._poll_thread:
|
|
274
|
+
self._poll_thread.join(timeout=5.0)
|
|
275
|
+
self._poll_started = False
|
|
276
|
+
logger.debug("Background experiment refresh stopped")
|
|
277
|
+
|
|
278
|
+
def _poll_loop(self) -> None:
|
|
279
|
+
"""Background loop: refresh all known experiment keys every poll_interval."""
|
|
280
|
+
# Run immediately on first iteration, then wait between subsequent ones.
|
|
281
|
+
while not self._poll_shutdown.is_set():
|
|
282
|
+
self._refresh_all_from_backend()
|
|
283
|
+
# wait() returns True if shutdown was requested, False on timeout.
|
|
284
|
+
# Using wait() instead of sleep() so shutdown is instant.
|
|
285
|
+
self._poll_shutdown.wait(timeout=self._poll_interval)
|
|
286
|
+
|
|
287
|
+
def _refresh_all_from_backend(self) -> None:
|
|
288
|
+
"""Silently re-fetch all registered experiment keys from the backend.
|
|
289
|
+
|
|
290
|
+
Called by the background poll loop. Updates _experiments atomically
|
|
291
|
+
under a lock so get_variant() never sees a partial state.
|
|
292
|
+
"""
|
|
293
|
+
if not self._api_base_url:
|
|
294
|
+
return
|
|
295
|
+
|
|
296
|
+
with self._experiments_lock:
|
|
297
|
+
keys = list(self._experiments.keys())
|
|
298
|
+
|
|
299
|
+
if not keys:
|
|
300
|
+
return
|
|
301
|
+
|
|
302
|
+
headers = self._get_auth_header()
|
|
303
|
+
updated = 0
|
|
304
|
+
|
|
305
|
+
for key in keys:
|
|
306
|
+
url = f"{self._api_base_url}/api/public/experiments/{key}"
|
|
307
|
+
try:
|
|
308
|
+
response = requests.get(url, headers=headers, timeout=5)
|
|
309
|
+
if response.status_code == 200:
|
|
310
|
+
config = self._parse_experiment_response(response.json())
|
|
311
|
+
with self._experiments_lock:
|
|
312
|
+
self._experiments[key] = config
|
|
313
|
+
updated += 1
|
|
314
|
+
elif response.status_code == 404:
|
|
315
|
+
logger.debug(f"Experiment '{key}' no longer found in backend")
|
|
316
|
+
else:
|
|
317
|
+
logger.debug(f"Background refresh for '{key}' got {response.status_code}")
|
|
318
|
+
except Exception as e:
|
|
319
|
+
# Never crash the background thread — just log and move on.
|
|
320
|
+
logger.debug(f"Background refresh failed for '{key}': {e}")
|
|
321
|
+
|
|
322
|
+
if updated:
|
|
323
|
+
logger.debug(f"Background refresh updated {updated}/{len(keys)} experiments")
|
|
324
|
+
|
|
325
|
+
# -------------------------------------------------------------------------
|
|
326
|
+
# Register / Sync
|
|
327
|
+
# -------------------------------------------------------------------------
|
|
328
|
+
|
|
329
|
+
def register(
|
|
330
|
+
self,
|
|
331
|
+
key: str,
|
|
332
|
+
variants: List[Dict[str, Any]],
|
|
333
|
+
traffic_percent: int = 100,
|
|
334
|
+
sync_to_backend: bool = True,
|
|
335
|
+
) -> None:
|
|
336
|
+
"""Register an experiment configuration.
|
|
337
|
+
|
|
338
|
+
Experiments are defined in code and can optionally be synced to the
|
|
339
|
+
backend database. This makes them visible in the UI and allows for
|
|
340
|
+
evaluator configuration.
|
|
341
|
+
|
|
342
|
+
PREVIOUSLY: if sync_to_backend=True, this blocked the caller for the
|
|
343
|
+
full HTTP round-trip (~100–500ms) before returning.
|
|
344
|
+
NOW: the backend sync is fire-and-forget (daemon thread). register()
|
|
345
|
+
returns immediately after writing to in-memory state.
|
|
346
|
+
|
|
347
|
+
Args:
|
|
348
|
+
key: Unique experiment identifier (e.g., "prompt-test-v1").
|
|
349
|
+
Should be descriptive and versioned.
|
|
350
|
+
variants: List of variant dicts, each with:
|
|
351
|
+
- key: Variant identifier (e.g., "control", "treatment")
|
|
352
|
+
- weight: Relative weight for traffic (e.g., 50 = 50%)
|
|
353
|
+
- config: Arbitrary config dict passed to your code
|
|
354
|
+
traffic_percent: Percentage of users to include (0-100).
|
|
355
|
+
Use < 100 to gradually roll out experiments.
|
|
356
|
+
sync_to_backend: If True and backend is configured, syncs the
|
|
357
|
+
experiment to the Gateway database. Default True.
|
|
358
|
+
|
|
359
|
+
Example::
|
|
360
|
+
|
|
361
|
+
msk.experiments.register(
|
|
362
|
+
key="prompt-test-v1",
|
|
363
|
+
variants=[
|
|
364
|
+
{"key": "control", "weight": 50, "config": {"prompt": "You are a helpful assistant."}},
|
|
365
|
+
{"key": "treatment", "weight": 50, "config": {"prompt": "You are a concise assistant."}},
|
|
366
|
+
],
|
|
367
|
+
traffic_percent=100,
|
|
368
|
+
)
|
|
369
|
+
"""
|
|
370
|
+
if traffic_percent < 0 or traffic_percent > 100:
|
|
371
|
+
raise ValueError(f"traffic_percent must be 0-100, got {traffic_percent}")
|
|
372
|
+
|
|
373
|
+
if len(variants) < 2:
|
|
374
|
+
raise ValueError("Experiments require at least 2 variants")
|
|
375
|
+
|
|
376
|
+
config = ExperimentConfig(
|
|
377
|
+
key=key,
|
|
378
|
+
variants=[
|
|
379
|
+
Variant(
|
|
380
|
+
key=v["key"],
|
|
381
|
+
weight=v.get("weight", 1),
|
|
382
|
+
config=v.get("config", {}),
|
|
383
|
+
)
|
|
384
|
+
for v in variants
|
|
385
|
+
],
|
|
386
|
+
traffic_percent=traffic_percent,
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
with self._experiments_lock:
|
|
390
|
+
self._experiments[key] = config
|
|
391
|
+
|
|
392
|
+
logger.debug(f"Registered experiment '{key}' with {len(variants)} variants, " f"{traffic_percent}% traffic")
|
|
393
|
+
|
|
394
|
+
# PREVIOUSLY: blocking requests.post() here — caller waited for it.
|
|
395
|
+
# NOW: spawns a daemon thread and returns immediately.
|
|
396
|
+
if sync_to_backend and self._api_base_url:
|
|
397
|
+
self._sync_experiment_to_backend_async(key, variants, traffic_percent)
|
|
398
|
+
|
|
399
|
+
def _sync_experiment_to_backend_async(
|
|
400
|
+
self,
|
|
401
|
+
key: str,
|
|
402
|
+
variants: List[Dict[str, Any]],
|
|
403
|
+
traffic_percent: int,
|
|
404
|
+
) -> None:
|
|
405
|
+
"""Fire-and-forget backend sync — runs in a daemon thread.
|
|
406
|
+
|
|
407
|
+
PREVIOUSLY: this was _sync_experiment_to_backend() and was called
|
|
408
|
+
directly (blocking). Now it spawns a thread and returns in ~0ms.
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
def _do_sync() -> None:
|
|
412
|
+
url = f"{self._api_base_url}/api/public/experiments"
|
|
413
|
+
headers = {
|
|
414
|
+
"Content-Type": "application/json",
|
|
415
|
+
**self._get_auth_header(),
|
|
416
|
+
}
|
|
417
|
+
payload = {
|
|
418
|
+
"key": key,
|
|
419
|
+
"trafficPercent": traffic_percent,
|
|
420
|
+
"variants": [
|
|
421
|
+
{
|
|
422
|
+
"key": v["key"],
|
|
423
|
+
"weight": v.get("weight", 1),
|
|
424
|
+
"config": v.get("config", {}),
|
|
425
|
+
}
|
|
426
|
+
for v in variants
|
|
427
|
+
],
|
|
428
|
+
}
|
|
429
|
+
try:
|
|
430
|
+
response = requests.post(url, json=payload, headers=headers, timeout=5)
|
|
431
|
+
if response.status_code in (200, 201):
|
|
432
|
+
data = response.json()
|
|
433
|
+
action = "created" if data.get("created") else "updated"
|
|
434
|
+
logger.info(f"Experiment '{key}' {action} in backend (id={data.get('id')})")
|
|
435
|
+
else:
|
|
436
|
+
logger.warning(
|
|
437
|
+
f"Failed to sync experiment '{key}' to backend: " f"{response.status_code} {response.text}"
|
|
438
|
+
)
|
|
439
|
+
except Exception as e:
|
|
440
|
+
logger.warning(f"Failed to sync experiment '{key}' to backend: {e}")
|
|
441
|
+
|
|
442
|
+
thread = threading.Thread(
|
|
443
|
+
target=_do_sync,
|
|
444
|
+
daemon=True,
|
|
445
|
+
name=f"msk-sync-{key}",
|
|
446
|
+
)
|
|
447
|
+
thread.start()
|
|
448
|
+
|
|
449
|
+
# -------------------------------------------------------------------------
|
|
450
|
+
# Fetch (explicit blocking fetch — for startup patterns)
|
|
451
|
+
# -------------------------------------------------------------------------
|
|
452
|
+
|
|
453
|
+
def fetch(
|
|
454
|
+
self,
|
|
455
|
+
experiment_key: str,
|
|
456
|
+
auto_register: bool = True,
|
|
457
|
+
) -> Optional[ExperimentConfig]:
|
|
458
|
+
"""Fetch an experiment configuration from the backend.
|
|
459
|
+
|
|
460
|
+
This is intentionally synchronous — use it at startup when you need
|
|
461
|
+
the config to be available before proceeding. For runtime updates,
|
|
462
|
+
rely on start_background_refresh() instead.
|
|
463
|
+
|
|
464
|
+
Args:
|
|
465
|
+
experiment_key: The experiment identifier to fetch.
|
|
466
|
+
auto_register: If True, automatically register the fetched
|
|
467
|
+
experiment locally so get_variant() works.
|
|
468
|
+
|
|
469
|
+
Returns:
|
|
470
|
+
The ExperimentConfig if found, None otherwise.
|
|
471
|
+
|
|
472
|
+
Example::
|
|
473
|
+
|
|
474
|
+
# Fetch experiment defined in the UI
|
|
475
|
+
config = msk.experiments.fetch("prompt-test-v1")
|
|
476
|
+
if config:
|
|
477
|
+
variant = msk.experiments.get_variant("prompt-test-v1", user_id="user-123")
|
|
478
|
+
"""
|
|
479
|
+
if not self._api_base_url:
|
|
480
|
+
logger.warning("Backend not configured, cannot fetch experiment")
|
|
481
|
+
return None
|
|
482
|
+
|
|
483
|
+
url = f"{self._api_base_url}/api/public/experiments/{experiment_key}"
|
|
484
|
+
headers = self._get_auth_header()
|
|
485
|
+
|
|
486
|
+
try:
|
|
487
|
+
response = requests.get(url, headers=headers, timeout=5)
|
|
488
|
+
if response.status_code == 200:
|
|
489
|
+
config = self._parse_experiment_response(response.json())
|
|
490
|
+
|
|
491
|
+
if auto_register:
|
|
492
|
+
with self._experiments_lock:
|
|
493
|
+
self._experiments[experiment_key] = config
|
|
494
|
+
logger.info(
|
|
495
|
+
f"Fetched and registered experiment '{experiment_key}' "
|
|
496
|
+
f"from backend ({len(config.variants)} variants)"
|
|
497
|
+
)
|
|
498
|
+
else:
|
|
499
|
+
logger.info(f"Fetched experiment '{experiment_key}' from backend")
|
|
500
|
+
|
|
501
|
+
return config
|
|
502
|
+
elif response.status_code == 404:
|
|
503
|
+
logger.warning(f"Experiment '{experiment_key}' not found in backend")
|
|
504
|
+
return None
|
|
505
|
+
else:
|
|
506
|
+
logger.warning(
|
|
507
|
+
f"Failed to fetch experiment '{experiment_key}': " f"{response.status_code} {response.text}"
|
|
508
|
+
)
|
|
509
|
+
return None
|
|
510
|
+
except Exception as e:
|
|
511
|
+
logger.warning(f"Failed to fetch experiment '{experiment_key}': {e}")
|
|
512
|
+
return None
|
|
513
|
+
|
|
514
|
+
def fetch_all(self, auto_register: bool = True) -> List[ExperimentConfig]:
|
|
515
|
+
"""Fetch all active experiments from the backend.
|
|
516
|
+
|
|
517
|
+
The listing endpoint returns summaries only (no variant details), so
|
|
518
|
+
each experiment must be fetched individually via ``fetch()``.
|
|
519
|
+
|
|
520
|
+
Useful at startup to pre-populate the in-memory cache. After calling
|
|
521
|
+
this, ``start_background_refresh()`` will keep configs fresh.
|
|
522
|
+
|
|
523
|
+
Args:
|
|
524
|
+
auto_register: If True, automatically register fetched experiments.
|
|
525
|
+
|
|
526
|
+
Returns:
|
|
527
|
+
List of ExperimentConfig objects that were successfully fetched.
|
|
528
|
+
"""
|
|
529
|
+
if not self._api_base_url:
|
|
530
|
+
logger.warning("Backend not configured, cannot fetch experiments")
|
|
531
|
+
return []
|
|
532
|
+
|
|
533
|
+
url = f"{self._api_base_url}/api/public/experiments"
|
|
534
|
+
headers = self._get_auth_header()
|
|
535
|
+
|
|
536
|
+
try:
|
|
537
|
+
response = requests.get(url, headers=headers, timeout=10)
|
|
538
|
+
if response.status_code != 200:
|
|
539
|
+
logger.warning(f"Failed to fetch experiments: {response.status_code} {response.text}")
|
|
540
|
+
return []
|
|
541
|
+
|
|
542
|
+
experiment_keys = [exp["key"] for exp in response.json().get("experiments", [])]
|
|
543
|
+
except Exception as e:
|
|
544
|
+
logger.warning(f"Failed to fetch experiments: {e}")
|
|
545
|
+
return []
|
|
546
|
+
|
|
547
|
+
configs: List[ExperimentConfig] = []
|
|
548
|
+
for key in experiment_keys:
|
|
549
|
+
config = self.fetch(key, auto_register=auto_register)
|
|
550
|
+
if config:
|
|
551
|
+
configs.append(config)
|
|
552
|
+
|
|
553
|
+
logger.info(f"Fetched {len(configs)}/{len(experiment_keys)} experiments from backend")
|
|
554
|
+
return configs
|
|
555
|
+
|
|
556
|
+
# -------------------------------------------------------------------------
|
|
557
|
+
# Get Variant (~0ms hot path)
|
|
558
|
+
# -------------------------------------------------------------------------
|
|
559
|
+
|
|
560
|
+
def get_variant(
|
|
561
|
+
self,
|
|
562
|
+
experiment_key: str,
|
|
563
|
+
user_id: str,
|
|
564
|
+
) -> Optional[Variant]:
|
|
565
|
+
"""Get the variant for a user in an experiment.
|
|
566
|
+
|
|
567
|
+
This is the main entry point for A/B testing. It:
|
|
568
|
+
1. Deterministically assigns the user to a variant (or None if not in traffic)
|
|
569
|
+
2. Sets the context variable for span injection
|
|
570
|
+
3. Logs an exposure event (sent to backend asynchronously)
|
|
571
|
+
|
|
572
|
+
When the backend is configured, uses server-side balanced assignment
|
|
573
|
+
(Redis counter) for exact proportional allocation at any sample size.
|
|
574
|
+
Falls back to local hash-based assignment when the backend is unavailable.
|
|
575
|
+
|
|
576
|
+
Args:
|
|
577
|
+
experiment_key: The experiment identifier (must be registered first)
|
|
578
|
+
user_id: The user/subject identifier. This should be a stable ID
|
|
579
|
+
that uniquely identifies the user across sessions.
|
|
580
|
+
|
|
581
|
+
Returns:
|
|
582
|
+
The assigned Variant with its config, or None if:
|
|
583
|
+
- The experiment is not registered
|
|
584
|
+
- The user is not in the traffic allocation
|
|
585
|
+
|
|
586
|
+
Example::
|
|
587
|
+
|
|
588
|
+
variant = msk.experiments.get_variant("prompt-test-v1", user_id="user-alice-789")
|
|
589
|
+
if variant:
|
|
590
|
+
print(f"User assigned to {variant.key}")
|
|
591
|
+
system_prompt = variant.config["prompt"]
|
|
592
|
+
# Run your agent with this prompt
|
|
593
|
+
else:
|
|
594
|
+
# User not in experiment, use default behavior
|
|
595
|
+
system_prompt = DEFAULT_PROMPT
|
|
596
|
+
"""
|
|
597
|
+
if self._api_base_url:
|
|
598
|
+
try:
|
|
599
|
+
result = self.get_variant_balanced(experiment_key, user_id)
|
|
600
|
+
# result is None only when user is legitimately not in traffic
|
|
601
|
+
# (server returned variantKey: null) — do NOT fall back
|
|
602
|
+
return result
|
|
603
|
+
except BalancedAssignmentError:
|
|
604
|
+
# Network error, timeout, server error — fall back to hash-based
|
|
605
|
+
logger.warning(
|
|
606
|
+
f"Balanced assignment failed for '{experiment_key}', " f"falling back to hash-based assignment"
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
return self._get_variant_hash(experiment_key, user_id)
|
|
610
|
+
|
|
611
|
+
def _get_variant_hash(
|
|
612
|
+
self,
|
|
613
|
+
experiment_key: str,
|
|
614
|
+
user_id: str,
|
|
615
|
+
) -> Optional[Variant]:
|
|
616
|
+
"""Hash-based local assignment (fallback when backend is unavailable)."""
|
|
617
|
+
with self._experiments_lock:
|
|
618
|
+
config = self._experiments.get(experiment_key)
|
|
619
|
+
|
|
620
|
+
if config is None:
|
|
621
|
+
logger.warning(
|
|
622
|
+
f"Experiment '{experiment_key}' not registered. "
|
|
623
|
+
f"Call msk.experiments.register() or msk.experiments.fetch() first."
|
|
624
|
+
)
|
|
625
|
+
return None
|
|
626
|
+
|
|
627
|
+
variant = assign_variant(config, user_id)
|
|
628
|
+
if variant is None:
|
|
629
|
+
logger.debug(f"User '{user_id}' not in traffic for '{experiment_key}'")
|
|
630
|
+
return None
|
|
631
|
+
|
|
632
|
+
_set_current_experiment(
|
|
633
|
+
{
|
|
634
|
+
"experiment_id": experiment_key,
|
|
635
|
+
"variant_key": variant.key,
|
|
636
|
+
"user_id": user_id,
|
|
637
|
+
}
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
if self._exposure_logger:
|
|
641
|
+
self._exposure_logger.log(
|
|
642
|
+
Exposure(
|
|
643
|
+
user_id=user_id,
|
|
644
|
+
experiment_id=experiment_key,
|
|
645
|
+
variant_key=variant.key,
|
|
646
|
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
|
647
|
+
)
|
|
648
|
+
)
|
|
649
|
+
else:
|
|
650
|
+
logger.debug(
|
|
651
|
+
f"No exposure logger configured, exposure not sent "
|
|
652
|
+
f"(experiment={experiment_key}, variant={variant.key}, user={user_id})"
|
|
653
|
+
)
|
|
654
|
+
|
|
655
|
+
logger.debug(f"Assigned user '{user_id}' to variant '{variant.key}' " f"in experiment '{experiment_key}'")
|
|
656
|
+
|
|
657
|
+
return variant
|
|
658
|
+
|
|
659
|
+
def assign_with_override(
|
|
660
|
+
self,
|
|
661
|
+
experiment_key: str,
|
|
662
|
+
user_id: str,
|
|
663
|
+
variant_key: str,
|
|
664
|
+
) -> Optional[Variant]:
|
|
665
|
+
"""Assign a user to a specific variant, bypassing hash-based selection.
|
|
666
|
+
|
|
667
|
+
Use this when you have pre-computed balanced assignments (e.g. via
|
|
668
|
+
``compute_balanced_assignments``) and want to force a specific variant
|
|
669
|
+
while still setting the tracing context and logging the exposure.
|
|
670
|
+
|
|
671
|
+
Args:
|
|
672
|
+
experiment_key: The experiment identifier (must be registered first).
|
|
673
|
+
user_id: The user/subject identifier.
|
|
674
|
+
variant_key: The variant key to force (e.g. "control", "treatment").
|
|
675
|
+
|
|
676
|
+
Returns:
|
|
677
|
+
The matching Variant with its config, or None if the experiment is
|
|
678
|
+
not registered or variant_key is not found.
|
|
679
|
+
|
|
680
|
+
Example::
|
|
681
|
+
|
|
682
|
+
assignments = compute_balanced_assignments(config, all_users)
|
|
683
|
+
for user_id, variant in assignments.items():
|
|
684
|
+
if variant:
|
|
685
|
+
msk.experiments.assign_with_override(
|
|
686
|
+
"my-exp", user_id, variant.key
|
|
687
|
+
)
|
|
688
|
+
"""
|
|
689
|
+
with self._experiments_lock:
|
|
690
|
+
config = self._experiments.get(experiment_key)
|
|
691
|
+
|
|
692
|
+
if config is None:
|
|
693
|
+
logger.warning(
|
|
694
|
+
f"Experiment '{experiment_key}' not registered. "
|
|
695
|
+
f"Call msk.experiments.register() or msk.experiments.fetch() first."
|
|
696
|
+
)
|
|
697
|
+
return None
|
|
698
|
+
|
|
699
|
+
variant = next((v for v in config.variants if v.key == variant_key), None)
|
|
700
|
+
if variant is None:
|
|
701
|
+
logger.warning(f"Variant '{variant_key}' not found in experiment '{experiment_key}'")
|
|
702
|
+
return None
|
|
703
|
+
|
|
704
|
+
_set_current_experiment(
|
|
705
|
+
{
|
|
706
|
+
"experiment_id": experiment_key,
|
|
707
|
+
"variant_key": variant.key,
|
|
708
|
+
"user_id": user_id,
|
|
709
|
+
}
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
if self._exposure_logger:
|
|
713
|
+
self._exposure_logger.log(
|
|
714
|
+
Exposure(
|
|
715
|
+
user_id=user_id,
|
|
716
|
+
experiment_id=experiment_key,
|
|
717
|
+
variant_key=variant.key,
|
|
718
|
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
|
719
|
+
)
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
logger.debug(f"Force-assigned user '{user_id}' to variant '{variant.key}' " f"in experiment '{experiment_key}'")
|
|
723
|
+
|
|
724
|
+
return variant
|
|
725
|
+
|
|
726
|
+
def get_variant_balanced(
|
|
727
|
+
self,
|
|
728
|
+
experiment_key: str,
|
|
729
|
+
user_id: str,
|
|
730
|
+
) -> Optional[Variant]:
|
|
731
|
+
"""Get variant using server-side balanced assignment (guaranteed 50/50).
|
|
732
|
+
|
|
733
|
+
Unlike ``get_variant`` which uses local hash-based assignment (can produce
|
|
734
|
+
unequal splits at small sample sizes), this method calls the server which
|
|
735
|
+
uses a Redis atomic counter to guarantee exact proportional allocation.
|
|
736
|
+
|
|
737
|
+
Trade-off: requires a network call (~10-50ms) vs hash-based (~0ms local).
|
|
738
|
+
Use this when you have small sample sizes and need exact balance.
|
|
739
|
+
|
|
740
|
+
The assignment is:
|
|
741
|
+
1. Deterministic per user — same user always gets same variant
|
|
742
|
+
2. Perfectly balanced — guaranteed 50/50 (or proportional to weights)
|
|
743
|
+
3. Persistent — stored in Redis, survives restarts
|
|
744
|
+
|
|
745
|
+
Args:
|
|
746
|
+
experiment_key: The experiment identifier (must exist in backend).
|
|
747
|
+
user_id: The user/subject identifier.
|
|
748
|
+
|
|
749
|
+
Returns:
|
|
750
|
+
The assigned Variant with its config, or None if:
|
|
751
|
+
- User is legitimately not in traffic (server returned variantKey: null)
|
|
752
|
+
|
|
753
|
+
Raises:
|
|
754
|
+
BalancedAssignmentError: If network error, timeout, or server error occurs.
|
|
755
|
+
Caller can catch this to fall back to hash-based assignment.
|
|
756
|
+
|
|
757
|
+
Example::
|
|
758
|
+
|
|
759
|
+
# Use balanced assignment for small-sample experiments
|
|
760
|
+
variant = msk.experiments.get_variant_balanced(
|
|
761
|
+
"prompt-test-v1", user_id="user-alice-789"
|
|
762
|
+
)
|
|
763
|
+
if variant:
|
|
764
|
+
print(f"User assigned to {variant.key} (server-balanced)")
|
|
765
|
+
"""
|
|
766
|
+
if not self._api_base_url:
|
|
767
|
+
logger.warning(
|
|
768
|
+
"Backend not configured — cannot use balanced assignment. "
|
|
769
|
+
"Falling back to local hash-based assignment."
|
|
770
|
+
)
|
|
771
|
+
return self.get_variant(experiment_key, user_id)
|
|
772
|
+
|
|
773
|
+
url = f"{self._api_base_url}/api/public/experiments/assign"
|
|
774
|
+
headers = self._get_auth_header()
|
|
775
|
+
payload = {
|
|
776
|
+
"experimentKey": experiment_key,
|
|
777
|
+
"userId": user_id,
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
try:
|
|
781
|
+
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
|
782
|
+
|
|
783
|
+
if response.status_code != 200:
|
|
784
|
+
error_msg = response.text[:200] if response.text else "Unknown error"
|
|
785
|
+
logger.error(f"Server-balanced assignment failed: {response.status_code} — {error_msg}")
|
|
786
|
+
raise BalancedAssignmentError(f"Server returned {response.status_code}: {error_msg}")
|
|
787
|
+
|
|
788
|
+
data = response.json()
|
|
789
|
+
|
|
790
|
+
if data.get("variantKey") is None:
|
|
791
|
+
logger.debug(f"User '{user_id}' not in traffic for '{experiment_key}' (server response)")
|
|
792
|
+
return None
|
|
793
|
+
|
|
794
|
+
variant = Variant(
|
|
795
|
+
key=data["variantKey"],
|
|
796
|
+
weight=0, # Weight not returned by assignment endpoint
|
|
797
|
+
config=data.get("config", {}),
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
# Set context for span processor
|
|
801
|
+
_set_current_experiment(
|
|
802
|
+
{
|
|
803
|
+
"experiment_id": experiment_key,
|
|
804
|
+
"variant_key": variant.key,
|
|
805
|
+
"user_id": user_id,
|
|
806
|
+
}
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
# Log exposure
|
|
810
|
+
if self._exposure_logger:
|
|
811
|
+
self._exposure_logger.log(
|
|
812
|
+
Exposure(
|
|
813
|
+
user_id=user_id,
|
|
814
|
+
experiment_id=experiment_key,
|
|
815
|
+
variant_key=variant.key,
|
|
816
|
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
|
817
|
+
)
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
is_new = data.get("isNewAssignment", True)
|
|
821
|
+
logger.debug(
|
|
822
|
+
f"Server-balanced assignment: user '{user_id}' → '{variant.key}' "
|
|
823
|
+
f"in experiment '{experiment_key}' (new={is_new})"
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
return variant
|
|
827
|
+
|
|
828
|
+
except requests.exceptions.Timeout:
|
|
829
|
+
logger.error(f"Server-balanced assignment timed out for '{experiment_key}'")
|
|
830
|
+
raise BalancedAssignmentError(f"Timeout for '{experiment_key}'")
|
|
831
|
+
except requests.exceptions.RequestException as e:
|
|
832
|
+
logger.error(f"Server-balanced assignment failed for '{experiment_key}': {e}")
|
|
833
|
+
raise BalancedAssignmentError(f"Network error for '{experiment_key}': {e}")
|
|
834
|
+
except BalancedAssignmentError:
|
|
835
|
+
raise # Re-raise our custom exception
|
|
836
|
+
except Exception as e:
|
|
837
|
+
logger.error(f"Unexpected error in server-balanced assignment: {e}")
|
|
838
|
+
raise BalancedAssignmentError(f"Unexpected error for '{experiment_key}': {e}")
|
|
839
|
+
|
|
840
|
+
# -------------------------------------------------------------------------
|
|
841
|
+
# Introspection
|
|
842
|
+
# -------------------------------------------------------------------------
|
|
843
|
+
|
|
844
|
+
def is_registered(self, experiment_key: str) -> bool:
|
|
845
|
+
"""Check if an experiment is registered."""
|
|
846
|
+
with self._experiments_lock:
|
|
847
|
+
return experiment_key in self._experiments
|
|
848
|
+
|
|
849
|
+
def list_experiments(self) -> List[str]:
|
|
850
|
+
"""List all registered experiment keys."""
|
|
851
|
+
with self._experiments_lock:
|
|
852
|
+
return list(self._experiments.keys())
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
# =============================================================================
|
|
856
|
+
# Module-level instance (populated by gatewaysdk.init())
|
|
857
|
+
# =============================================================================
|
|
858
|
+
|
|
859
|
+
# This will be set by the main gatewaysdk module during init()
|
|
860
|
+
_experiments_client: Optional[ExperimentClient] = None
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def get_experiments_client() -> Optional[ExperimentClient]:
|
|
864
|
+
"""Get the global experiments client instance.
|
|
865
|
+
|
|
866
|
+
Returns None if gatewaysdk has not been initialized.
|
|
867
|
+
"""
|
|
868
|
+
return _experiments_client
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def _set_experiments_client(client: Optional[ExperimentClient]) -> None:
|
|
872
|
+
"""Set the global experiments client instance.
|
|
873
|
+
|
|
874
|
+
Internal function - called by gatewaysdk.init().
|
|
875
|
+
"""
|
|
876
|
+
global _experiments_client
|
|
877
|
+
_experiments_client = client
|