gatewaysdk 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gatewaysdk/__init__.py +540 -0
- gatewaysdk/adapter/__init__.py +15 -0
- gatewaysdk/adapter/base.py +94 -0
- gatewaysdk/adapter/messages.py +270 -0
- gatewaysdk/adapter/triplet.py +1026 -0
- gatewaysdk/algorithm/__init__.py +39 -0
- gatewaysdk/algorithm/apo/__init__.py +5 -0
- gatewaysdk/algorithm/apo/apo.py +898 -0
- gatewaysdk/algorithm/apo/prompts/apply_edit_variant01.poml +22 -0
- gatewaysdk/algorithm/apo/prompts/apply_edit_variant02.poml +18 -0
- gatewaysdk/algorithm/apo/prompts/text_gradient_variant01.poml +18 -0
- gatewaysdk/algorithm/apo/prompts/text_gradient_variant02.poml +16 -0
- gatewaysdk/algorithm/apo/prompts/text_gradient_variant03.poml +107 -0
- gatewaysdk/algorithm/base.py +262 -0
- gatewaysdk/algorithm/decorator.py +264 -0
- gatewaysdk/algorithm/evals/__init__.py +7 -0
- gatewaysdk/algorithm/evals/evals.py +217 -0
- gatewaysdk/algorithm/fast.py +250 -0
- gatewaysdk/algorithm/gepa/__init__.py +61 -0
- gatewaysdk/algorithm/gepa/adapter.py +495 -0
- gatewaysdk/algorithm/gepa/gepa.py +570 -0
- gatewaysdk/algorithm/gepa/lib/__init__.py +18 -0
- gatewaysdk/algorithm/gepa/lib/adapters/README.md +12 -0
- gatewaysdk/algorithm/gepa/lib/adapters/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/README.md +341 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/__init__.py +1 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/anymaths_adapter.py +174 -0
- gatewaysdk/algorithm/gepa/lib/adapters/anymaths_adapter/requirements.txt +1 -0
- gatewaysdk/algorithm/gepa/lib/adapters/default_adapter/README.md +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/default_adapter/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/default_adapter/default_adapter.py +209 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_adapter/README.md +7 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_adapter/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_adapter/dspy_adapter.py +307 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_full_program_adapter/README.md +99 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_full_program_adapter/dspy_program_proposal_signature.py +137 -0
- gatewaysdk/algorithm/gepa/lib/adapters/dspy_full_program_adapter/full_program_adapter.py +268 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/GEPA_RAG.md +621 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/__init__.py +56 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/evaluation_metrics.py +226 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/generic_rag_adapter.py +496 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/rag_pipeline.py +238 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_store_interface.py +212 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/__init__.py +2 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/chroma_store.py +196 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/lancedb_store.py +422 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/milvus_store.py +409 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/qdrant_store.py +368 -0
- gatewaysdk/algorithm/gepa/lib/adapters/generic_rag_adapter/vector_stores/weaviate_store.py +418 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/README.md +552 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/__init__.py +37 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/mcp_adapter.py +699 -0
- gatewaysdk/algorithm/gepa/lib/adapters/mcp_adapter/mcp_client.py +364 -0
- gatewaysdk/algorithm/gepa/lib/adapters/terminal_bench_adapter/README.md +9 -0
- gatewaysdk/algorithm/gepa/lib/adapters/terminal_bench_adapter/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/adapters/terminal_bench_adapter/terminal_bench_adapter.py +217 -0
- gatewaysdk/algorithm/gepa/lib/api.py +382 -0
- gatewaysdk/algorithm/gepa/lib/core/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/core/adapter.py +180 -0
- gatewaysdk/algorithm/gepa/lib/core/data_loader.py +74 -0
- gatewaysdk/algorithm/gepa/lib/core/engine.py +379 -0
- gatewaysdk/algorithm/gepa/lib/core/result.py +233 -0
- gatewaysdk/algorithm/gepa/lib/core/state.py +636 -0
- gatewaysdk/algorithm/gepa/lib/examples/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/examples/aime.py +24 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/eval_default.py +111 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/prompt-templates/instruction_prompt.txt +9 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/prompt-templates/optimal_prompt.txt +24 -0
- gatewaysdk/algorithm/gepa/lib/examples/anymaths-bench/train_anymaths.py +177 -0
- gatewaysdk/algorithm/gepa/lib/examples/dspy_full_program_evolution/arc_agi.ipynb +25705 -0
- gatewaysdk/algorithm/gepa/lib/examples/dspy_full_program_evolution/example.ipynb +348 -0
- gatewaysdk/algorithm/gepa/lib/examples/mcp_adapter/__init__.py +4 -0
- gatewaysdk/algorithm/gepa/lib/examples/mcp_adapter/mcp_optimization_example.py +456 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/RAG_GUIDE.md +613 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/__init__.py +9 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/rag_optimization.py +820 -0
- gatewaysdk/algorithm/gepa/lib/examples/rag_adapter/requirements-rag.txt +29 -0
- gatewaysdk/algorithm/gepa/lib/examples/terminal-bench/prompt-templates/instruction_prompt.txt +16 -0
- gatewaysdk/algorithm/gepa/lib/examples/terminal-bench/prompt-templates/terminus.txt +9 -0
- gatewaysdk/algorithm/gepa/lib/examples/terminal-bench/train_terminus.py +161 -0
- gatewaysdk/algorithm/gepa/lib/gepa_utils.py +117 -0
- gatewaysdk/algorithm/gepa/lib/logging/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/logging/experiment_tracker.py +187 -0
- gatewaysdk/algorithm/gepa/lib/logging/logger.py +75 -0
- gatewaysdk/algorithm/gepa/lib/logging/utils.py +103 -0
- gatewaysdk/algorithm/gepa/lib/proposer/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/proposer/base.py +31 -0
- gatewaysdk/algorithm/gepa/lib/proposer/merge.py +357 -0
- gatewaysdk/algorithm/gepa/lib/proposer/reflective_mutation/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/proposer/reflective_mutation/base.py +49 -0
- gatewaysdk/algorithm/gepa/lib/proposer/reflective_mutation/reflective_mutation.py +176 -0
- gatewaysdk/algorithm/gepa/lib/py.typed +0 -0
- gatewaysdk/algorithm/gepa/lib/strategies/__init__.py +0 -0
- gatewaysdk/algorithm/gepa/lib/strategies/batch_sampler.py +77 -0
- gatewaysdk/algorithm/gepa/lib/strategies/candidate_selector.py +50 -0
- gatewaysdk/algorithm/gepa/lib/strategies/component_selector.py +36 -0
- gatewaysdk/algorithm/gepa/lib/strategies/eval_policy.py +64 -0
- gatewaysdk/algorithm/gepa/lib/strategies/instruction_proposal.py +126 -0
- gatewaysdk/algorithm/gepa/lib/utils/__init__.py +10 -0
- gatewaysdk/algorithm/gepa/lib/utils/stop_condition.py +196 -0
- gatewaysdk/algorithm/gepa/tracing.py +105 -0
- gatewaysdk/algorithm/utils.py +177 -0
- gatewaysdk/algorithm/verl/__init__.py +5 -0
- gatewaysdk/algorithm/verl/interface.py +202 -0
- gatewaysdk/automations.py +111 -0
- gatewaysdk/benchmark_hub/README.md +88 -0
- gatewaysdk/benchmark_hub/__init__.py +119 -0
- gatewaysdk/benchmark_hub/_archive.py +189 -0
- gatewaysdk/benchmark_hub/_tar.py +6 -0
- gatewaysdk/benchmark_hub/client.py +992 -0
- gatewaysdk/benchmark_hub/dispatch_shard.py +34 -0
- gatewaysdk/benchmark_hub/eval_config.py +20 -0
- gatewaysdk/benchmark_hub/evals.py +351 -0
- gatewaysdk/benchmark_hub/harbor_adapter.py +673 -0
- gatewaysdk/benchmark_hub/path_utils.py +81 -0
- gatewaysdk/benchmark_hub/save_utils.py +101 -0
- gatewaysdk/benchmark_hub/verifiers_adapter.py +327 -0
- gatewaysdk/benchmark_hub/versioning.py +72 -0
- gatewaysdk/build.py +515 -0
- gatewaysdk/cli/__init__.py +58 -0
- gatewaysdk/cli/agent_runner.py +132 -0
- gatewaysdk/cli/http_client.py +115 -0
- gatewaysdk/cli/platform.py +4086 -0
- gatewaysdk/cli/prometheus.py +115 -0
- gatewaysdk/cli/release_gate.py +215 -0
- gatewaysdk/cli/store.py +131 -0
- gatewaysdk/cli/vllm.py +29 -0
- gatewaysdk/client.py +406 -0
- gatewaysdk/config.py +348 -0
- gatewaysdk/connectors/__init__.py +25 -0
- gatewaysdk/connectors/client.py +203 -0
- gatewaysdk/connectors/skill.py +25 -0
- gatewaysdk/connectors/template.py +106 -0
- gatewaysdk/context.py +606 -0
- gatewaysdk/emitter/__init__.py +43 -0
- gatewaysdk/emitter/annotation.py +370 -0
- gatewaysdk/emitter/exception.py +54 -0
- gatewaysdk/emitter/message.py +61 -0
- gatewaysdk/emitter/object.py +117 -0
- gatewaysdk/emitter/reward.py +320 -0
- gatewaysdk/env_var.py +156 -0
- gatewaysdk/environment/__init__.py +108 -0
- gatewaysdk/environment/_bundle.py +281 -0
- gatewaysdk/environment/_harbor.py +276 -0
- gatewaysdk/environment/_materialize.py +97 -0
- gatewaysdk/environment/_tar.py +33 -0
- gatewaysdk/environment/_world.py +346 -0
- gatewaysdk/environment/core.py +527 -0
- gatewaysdk/environment/errors.py +58 -0
- gatewaysdk/environment/runtime.py +150 -0
- gatewaysdk/environment/schema/__init__.py +35 -0
- gatewaysdk/environment/schema/__main__.py +225 -0
- gatewaysdk/environment/schema/_slack_fidelity.py +116 -0
- gatewaysdk/environment/schema/_slack_scenario.py +183 -0
- gatewaysdk/environment/schema/api.py +1628 -0
- gatewaysdk/environment/schema/batch.py +755 -0
- gatewaysdk/environment/schema/compiler.py +615 -0
- gatewaysdk/environment/schema/conform.py +428 -0
- gatewaysdk/environment/schema/connector.py +777 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/handlers.py +17 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/parity.json +28 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/provenance.json +233 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/scope.toml +112 -0
- gatewaysdk/environment/schema/connectors/apple-business-manager/world.json +1380 -0
- gatewaysdk/environment/schema/connectors/base.json +28 -0
- gatewaysdk/environment/schema/connectors/custom/handlers.py +18 -0
- gatewaysdk/environment/schema/connectors/custom/world.json +123 -0
- gatewaysdk/environment/schema/connectors/github/handlers.py +81 -0
- gatewaysdk/environment/schema/connectors/github/parity.json +37 -0
- gatewaysdk/environment/schema/connectors/github/provenance.json +134 -0
- gatewaysdk/environment/schema/connectors/github/scope.toml +158 -0
- gatewaysdk/environment/schema/connectors/github/world.json +5869 -0
- gatewaysdk/environment/schema/connectors/google-calendar/handlers.py +23 -0
- gatewaysdk/environment/schema/connectors/google-calendar/world.json +209 -0
- gatewaysdk/environment/schema/connectors/google-drive/handlers.py +18 -0
- gatewaysdk/environment/schema/connectors/google-drive/world.json +286 -0
- gatewaysdk/environment/schema/connectors/jamf/handlers.py +34 -0
- gatewaysdk/environment/schema/connectors/jamf/parity.json +49 -0
- gatewaysdk/environment/schema/connectors/jamf/provenance.json +181 -0
- gatewaysdk/environment/schema/connectors/jamf/scope.toml +128 -0
- gatewaysdk/environment/schema/connectors/jamf/world.json +2218 -0
- gatewaysdk/environment/schema/connectors/jira/handlers.py +28 -0
- gatewaysdk/environment/schema/connectors/jira/world.json +389 -0
- gatewaysdk/environment/schema/connectors/kandji/handlers.py +38 -0
- gatewaysdk/environment/schema/connectors/kandji/parity.json +29 -0
- gatewaysdk/environment/schema/connectors/kandji/provenance.json +152 -0
- gatewaysdk/environment/schema/connectors/kandji/scope.toml +120 -0
- gatewaysdk/environment/schema/connectors/kandji/world.json +765 -0
- gatewaysdk/environment/schema/connectors/linear/handlers.py +20 -0
- gatewaysdk/environment/schema/connectors/linear/world.json +363 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/fidelity.json +45 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/handlers.py +1 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/provenance.json +851 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/scope.md +231 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/teams-types.json +1710 -0
- gatewaysdk/environment/schema/connectors/microsoft-teams/world.json +858 -0
- gatewaysdk/environment/schema/connectors/netsuite/handlers.py +32 -0
- gatewaysdk/environment/schema/connectors/netsuite/parity.json +50 -0
- gatewaysdk/environment/schema/connectors/netsuite/provenance.json +137 -0
- gatewaysdk/environment/schema/connectors/netsuite/scope.toml +78 -0
- gatewaysdk/environment/schema/connectors/netsuite/world.json +16243 -0
- gatewaysdk/environment/schema/connectors/salesforce/handlers.py +806 -0
- gatewaysdk/environment/schema/connectors/salesforce/parity.json +46 -0
- gatewaysdk/environment/schema/connectors/salesforce/provenance.json +143 -0
- gatewaysdk/environment/schema/connectors/salesforce/scope.toml +174 -0
- gatewaysdk/environment/schema/connectors/salesforce/world.json +4695 -0
- gatewaysdk/environment/schema/connectors/slack/MODEL.md +236 -0
- gatewaysdk/environment/schema/connectors/slack/capabilities.json +1039 -0
- gatewaysdk/environment/schema/connectors/slack/handlers.py +762 -0
- gatewaysdk/environment/schema/connectors/slack/parity.json +65 -0
- gatewaysdk/environment/schema/connectors/slack/provenance.json +294 -0
- gatewaysdk/environment/schema/connectors/slack/scope.toml +44 -0
- gatewaysdk/environment/schema/connectors/slack/world.json +4993 -0
- gatewaysdk/environment/schema/connectors/workday/handlers.py +75 -0
- gatewaysdk/environment/schema/connectors/workday/parity.json +36 -0
- gatewaysdk/environment/schema/connectors/workday/provenance.json +179 -0
- gatewaysdk/environment/schema/connectors/workday/scope.toml +180 -0
- gatewaysdk/environment/schema/connectors/workday/world.json +806 -0
- gatewaysdk/environment/schema/data.py +593 -0
- gatewaysdk/environment/schema/extract.py +263 -0
- gatewaysdk/environment/schema/host.py +2437 -0
- gatewaysdk/environment/schema/host_surfaces.py +275 -0
- gatewaysdk/environment/schema/platform.py +1272 -0
- gatewaysdk/environment/schema/scaffold.py +250 -0
- gatewaysdk/environment/schema/skill.py +118 -0
- gatewaysdk/environment/schema/skills/agents/connector-template-author.md +202 -0
- gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- gatewaysdk/environment/schema/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +313 -0
- gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +223 -0
- gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +293 -0
- gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +390 -0
- gatewaysdk/environment/schema/slack.py +757 -0
- gatewaysdk/environment/schema/snapshot.py +479 -0
- gatewaysdk/environment/schema/store.py +904 -0
- gatewaysdk/environment/schema/tasks.py +525 -0
- gatewaysdk/environment/schema/tools_world.py +322 -0
- gatewaysdk/environment/schema/validation.py +577 -0
- gatewaysdk/environment/schema/worker.py +78 -0
- gatewaysdk/environment/schema/workers.py +500 -0
- gatewaysdk/environment/schema/world_tests.py +336 -0
- gatewaysdk/environments/__init__.py +24 -0
- gatewaysdk/environments/client.py +486 -0
- gatewaysdk/environments/types.py +318 -0
- gatewaysdk/execution/__init__.py +15 -0
- gatewaysdk/execution/base.py +64 -0
- gatewaysdk/execution/client_server.py +443 -0
- gatewaysdk/execution/events.py +69 -0
- gatewaysdk/execution/inter_process.py +16 -0
- gatewaysdk/execution/shared_memory.py +282 -0
- gatewaysdk/experiments/__init__.py +90 -0
- gatewaysdk/experiments/assignment.py +177 -0
- gatewaysdk/experiments/client.py +877 -0
- gatewaysdk/experiments/exposure.py +222 -0
- gatewaysdk/experiments/types.py +81 -0
- gatewaysdk/importers/__init__.py +49 -0
- gatewaysdk/importers/_normalize.py +91 -0
- gatewaysdk/importers/builder.py +173 -0
- gatewaysdk/importers/client.py +95 -0
- gatewaysdk/importers/langsmith.py +196 -0
- gatewaysdk/importers/recipes.py +249 -0
- gatewaysdk/instrumentation/__init__.py +300 -0
- gatewaysdk/instrumentation/agentops.py +314 -0
- gatewaysdk/instrumentation/agentops_langchain.py +45 -0
- gatewaysdk/instrumentation/base.py +119 -0
- gatewaysdk/instrumentation/litellm.py +83 -0
- gatewaysdk/instrumentation/registry.py +273 -0
- gatewaysdk/instrumentation/vllm.py +81 -0
- gatewaysdk/instrumentation/weave.py +500 -0
- gatewaysdk/integrations/__init__.py +15 -0
- gatewaysdk/integrations/gateway/__init__.py +11 -0
- gatewaysdk/integrations/gateway/client.py +171 -0
- gatewaysdk/integrations/tool_access.py +549 -0
- gatewaysdk/litagent/__init__.py +11 -0
- gatewaysdk/litagent/decorator.py +536 -0
- gatewaysdk/litagent/litagent.py +252 -0
- gatewaysdk/llm_proxy.py +1742 -0
- gatewaysdk/logging.py +370 -0
- gatewaysdk/memory.py +278 -0
- gatewaysdk/personas.py +186 -0
- gatewaysdk/platform/__init__.py +17 -0
- gatewaysdk/platform/builder.py +221 -0
- gatewaysdk/platform/compatibility.py +67 -0
- gatewaysdk/platform/manifest.py +185 -0
- gatewaysdk/platform/orchestrator.py +901 -0
- gatewaysdk/platform/registry.py +122 -0
- gatewaysdk/platform/worker.py +864 -0
- gatewaysdk/replay/__init__.py +1032 -0
- gatewaysdk/replay/pytest.py +56 -0
- gatewaysdk/reward.py +7 -0
- gatewaysdk/run.py +1781 -0
- gatewaysdk/runner/__init__.py +11 -0
- gatewaysdk/runner/agent.py +878 -0
- gatewaysdk/runner/base.py +182 -0
- gatewaysdk/runner/legacy.py +309 -0
- gatewaysdk/security.py +700 -0
- gatewaysdk/semconv.py +170 -0
- gatewaysdk/server.py +399 -0
- gatewaysdk/sessions.py +282 -0
- gatewaysdk/store/__init__.py +45 -0
- gatewaysdk/store/base.py +908 -0
- gatewaysdk/store/client_server.py +2093 -0
- gatewaysdk/store/collection/__init__.py +30 -0
- gatewaysdk/store/collection/base.py +587 -0
- gatewaysdk/store/collection/memory.py +970 -0
- gatewaysdk/store/collection/mongo.py +1412 -0
- gatewaysdk/store/collection_based.py +1823 -0
- gatewaysdk/store/gateway.py +983 -0
- gatewaysdk/store/gateway_listener.py +465 -0
- gatewaysdk/store/listener.py +58 -0
- gatewaysdk/store/memory.py +396 -0
- gatewaysdk/store/mongo.py +165 -0
- gatewaysdk/store/redis_stream.py +517 -0
- gatewaysdk/store/sqlite.py +3 -0
- gatewaysdk/store/threading.py +370 -0
- gatewaysdk/store/utils.py +142 -0
- gatewaysdk/tracer/__init__.py +14 -0
- gatewaysdk/tracer/base.py +286 -0
- gatewaysdk/tracer/dummy.py +106 -0
- gatewaysdk/tracer/otel.py +559 -0
- gatewaysdk/tracing/__init__.py +110 -0
- gatewaysdk/tracing/api.py +808 -0
- gatewaysdk/tracing/attributes.py +9 -0
- gatewaysdk/tracing/context.py +272 -0
- gatewaysdk/tracing/exporters/__init__.py +10 -0
- gatewaysdk/tracing/exporters/gateway.py +228 -0
- gatewaysdk/tracing/identity.py +288 -0
- gatewaysdk/tracing/init.py +620 -0
- gatewaysdk/tracing/instrumentors/__init__.py +15 -0
- gatewaysdk/tracing/instrumentors/claude_agent_sdk.py +766 -0
- gatewaysdk/tracing/instrumentors/instrumentation_principles.md +294 -0
- gatewaysdk/tracing/instrumentors/registry.py +352 -0
- gatewaysdk/tracing/mapping.py +729 -0
- gatewaysdk/tracing/processors.py +393 -0
- gatewaysdk/tracing/push.py +617 -0
- gatewaysdk/tracing/push_models.py +247 -0
- gatewaysdk/tracing/semconv.py +294 -0
- gatewaysdk/tracing/span_builder.py +356 -0
- gatewaysdk/trainer/__init__.py +6 -0
- gatewaysdk/trainer/init_utils.py +263 -0
- gatewaysdk/trainer/legacy.py +359 -0
- gatewaysdk/trainer/registry.py +12 -0
- gatewaysdk/trainer/trainer.py +638 -0
- gatewaysdk/types/__init__.py +63 -0
- gatewaysdk/types/core.py +556 -0
- gatewaysdk/types/resources.py +204 -0
- gatewaysdk/types/tracer.py +515 -0
- gatewaysdk/types/tracing.py +162 -0
- gatewaysdk/users.py +251 -0
- gatewaysdk/utils/__init__.py +1 -0
- gatewaysdk/utils/id.py +18 -0
- gatewaysdk/utils/metrics.py +1025 -0
- gatewaysdk/utils/otel.py +550 -0
- gatewaysdk/utils/otlp.py +556 -0
- gatewaysdk/utils/redact.py +22 -0
- gatewaysdk/utils/server_launcher.py +1045 -0
- gatewaysdk/utils/system_snapshot.py +90 -0
- gatewaysdk/verl/__init__.py +8 -0
- gatewaysdk/verl/__main__.py +6 -0
- gatewaysdk/verl/async_server.py +46 -0
- gatewaysdk/verl/config.yaml +27 -0
- gatewaysdk/verl/daemon.py +1154 -0
- gatewaysdk/verl/dataset.py +44 -0
- gatewaysdk/verl/entrypoint.py +248 -0
- gatewaysdk/verl/trainer.py +549 -0
- gatewaysdk/world_browser.py +748 -0
- gatewaysdk/world_data.py +453 -0
- gatewaysdk/world_sessions.py +978 -0
- gatewaysdk/world_tasks.py +285 -0
- gatewaysdk/world_tools.py +115 -0
- gatewaysdk/worlds.py +639 -0
- gatewaysdk-0.3.2.dist-info/METADATA +236 -0
- gatewaysdk-0.3.2.dist-info/RECORD +376 -0
- gatewaysdk-0.3.2.dist-info/WHEEL +4 -0
- gatewaysdk-0.3.2.dist-info/entry_points.txt +5 -0
- gatewaysdk-0.3.2.dist-info/licenses/LICENSE +19 -0
|
@@ -0,0 +1,4086 @@
|
|
|
1
|
+
# Copyright (c) Gateway. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""Gateway platform CLI — register agents and manage algorithms.
|
|
4
|
+
|
|
5
|
+
Entry point: `gateway` (registered in pyproject.toml).
|
|
6
|
+
|
|
7
|
+
Usage::
|
|
8
|
+
|
|
9
|
+
gateway deploy <image> [--manifest-dir ./gateway_manifest] [--skip-build]
|
|
10
|
+
gateway algorithm list
|
|
11
|
+
gateway algorithm new <name>
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import sys
|
|
18
|
+
from dataclasses import asdict
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
import typer
|
|
24
|
+
except ImportError:
|
|
25
|
+
print(
|
|
26
|
+
"typer is required for the gateway CLI. Install with: pip install typer",
|
|
27
|
+
file=sys.stderr,
|
|
28
|
+
)
|
|
29
|
+
sys.exit(1)
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
import httpx
|
|
33
|
+
except ImportError:
|
|
34
|
+
print(
|
|
35
|
+
"httpx is required for the gateway CLI. Install with: pip install httpx",
|
|
36
|
+
file=sys.stderr,
|
|
37
|
+
)
|
|
38
|
+
sys.exit(1)
|
|
39
|
+
|
|
40
|
+
app = typer.Typer(
|
|
41
|
+
name="gateway",
|
|
42
|
+
help="Gateway platform CLI — register agents and manage algorithms.",
|
|
43
|
+
no_args_is_help=True,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# =============================================================================
|
|
47
|
+
# deploy command
|
|
48
|
+
# =============================================================================
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command()
|
|
52
|
+
def deploy(
|
|
53
|
+
image: Optional[str] = typer.Argument(
|
|
54
|
+
None,
|
|
55
|
+
help="Docker image to register. If omitted, builds from project dir.",
|
|
56
|
+
),
|
|
57
|
+
project_dir: str = typer.Option(
|
|
58
|
+
".",
|
|
59
|
+
"--project-dir",
|
|
60
|
+
help="Agent project directory (for building).",
|
|
61
|
+
),
|
|
62
|
+
manifest_dir: str = typer.Option(
|
|
63
|
+
"./gateway_manifest",
|
|
64
|
+
help="Path to the gateway_manifest/ directory.",
|
|
65
|
+
),
|
|
66
|
+
skip_build: bool = typer.Option(
|
|
67
|
+
False,
|
|
68
|
+
"--skip-build",
|
|
69
|
+
help="Skip Docker build, register image only.",
|
|
70
|
+
),
|
|
71
|
+
dockerfile: Optional[str] = typer.Option(
|
|
72
|
+
None,
|
|
73
|
+
"--dockerfile",
|
|
74
|
+
help="Use specific Dockerfile (e.g., Dockerfile.gpu).",
|
|
75
|
+
),
|
|
76
|
+
tag: Optional[str] = typer.Option(
|
|
77
|
+
None,
|
|
78
|
+
"--tag",
|
|
79
|
+
help="Override image tag.",
|
|
80
|
+
),
|
|
81
|
+
dry_run: bool = typer.Option(
|
|
82
|
+
False,
|
|
83
|
+
"--dry-run",
|
|
84
|
+
help="Validate manifest only, don't build or register.",
|
|
85
|
+
),
|
|
86
|
+
host: Optional[str] = typer.Option(
|
|
87
|
+
None,
|
|
88
|
+
envvar="GATEWAY_HOST",
|
|
89
|
+
help="Gateway server URL (e.g., http://localhost:3000).",
|
|
90
|
+
),
|
|
91
|
+
api_key: Optional[str] = typer.Option(
|
|
92
|
+
None,
|
|
93
|
+
envvar="GATEWAY_SECRET_KEY",
|
|
94
|
+
help="Gateway project API key (sk-lf-...).",
|
|
95
|
+
),
|
|
96
|
+
) -> None:
|
|
97
|
+
"""Build, push, and register an agent with the Gateway platform.
|
|
98
|
+
|
|
99
|
+
Examples::
|
|
100
|
+
|
|
101
|
+
# Full pipeline: build + push + register
|
|
102
|
+
gateway deploy --tag myagent:v1
|
|
103
|
+
|
|
104
|
+
# Register pre-built image (skip build)
|
|
105
|
+
gateway deploy myregistry/myagent:v1 --skip-build
|
|
106
|
+
|
|
107
|
+
# Use specific Dockerfile
|
|
108
|
+
gateway deploy --tag myagent:v1 --dockerfile Dockerfile.gpu
|
|
109
|
+
|
|
110
|
+
# Validate manifest only
|
|
111
|
+
gateway deploy --dry-run
|
|
112
|
+
"""
|
|
113
|
+
from gatewaysdk.platform.manifest import parse_manifest, parse_secrets, parse_task_schema
|
|
114
|
+
|
|
115
|
+
manifest_path = Path(manifest_dir)
|
|
116
|
+
if not manifest_path.exists():
|
|
117
|
+
typer.echo(f"Error: Manifest directory not found: {manifest_path}", err=True)
|
|
118
|
+
raise typer.Exit(1)
|
|
119
|
+
|
|
120
|
+
# Parse manifest
|
|
121
|
+
try:
|
|
122
|
+
manifest = parse_manifest(manifest_path)
|
|
123
|
+
except (FileNotFoundError, ValueError) as e:
|
|
124
|
+
typer.echo(f"Error parsing manifest: {e}", err=True)
|
|
125
|
+
raise typer.Exit(1)
|
|
126
|
+
|
|
127
|
+
# Parse optional task schema
|
|
128
|
+
task_schema = parse_task_schema(manifest_path)
|
|
129
|
+
|
|
130
|
+
typer.echo(f"Validating manifest...")
|
|
131
|
+
typer.echo(f" Name: {manifest.name}")
|
|
132
|
+
typer.echo(f" Version: {manifest.version}")
|
|
133
|
+
typer.echo(f" Capabilities: {', '.join(manifest.capabilities) or 'none'}")
|
|
134
|
+
if task_schema:
|
|
135
|
+
typer.echo(f" Task schema: found")
|
|
136
|
+
|
|
137
|
+
if dry_run:
|
|
138
|
+
typer.echo("Dry-run complete. Manifest is valid.")
|
|
139
|
+
return
|
|
140
|
+
|
|
141
|
+
# Determine image tag
|
|
142
|
+
image_tag = image or tag
|
|
143
|
+
if not image_tag and not skip_build:
|
|
144
|
+
image_tag = f"{manifest.name}:latest"
|
|
145
|
+
|
|
146
|
+
# Build if needed
|
|
147
|
+
if not skip_build and not image:
|
|
148
|
+
from gatewaysdk.platform.builder import build_image, detect_build_strategy
|
|
149
|
+
|
|
150
|
+
proj = Path(project_dir)
|
|
151
|
+
strategy = detect_build_strategy(proj, dockerfile)
|
|
152
|
+
typer.echo(f"\nBuilding image '{image_tag}' (strategy: {strategy.value})...")
|
|
153
|
+
|
|
154
|
+
try:
|
|
155
|
+
image_tag = build_image(proj, image_tag, dockerfile=dockerfile)
|
|
156
|
+
typer.echo(f" Built successfully!")
|
|
157
|
+
except (RuntimeError, FileNotFoundError) as e:
|
|
158
|
+
typer.echo(f"Build failed: {e}", err=True)
|
|
159
|
+
raise typer.Exit(1)
|
|
160
|
+
|
|
161
|
+
# Extract image digest for pinning
|
|
162
|
+
from gatewaysdk.platform.builder import get_image_digest
|
|
163
|
+
|
|
164
|
+
image_digest = get_image_digest(image_tag)
|
|
165
|
+
|
|
166
|
+
if not image_tag:
|
|
167
|
+
typer.echo("Error: No image specified. Provide an image argument or use --tag.", err=True)
|
|
168
|
+
raise typer.Exit(1)
|
|
169
|
+
|
|
170
|
+
# Resolve credentials (from flags, env, or config)
|
|
171
|
+
from gatewaysdk.cli.http_client import get_credentials
|
|
172
|
+
|
|
173
|
+
resolved_host, resolved_public, resolved_secret = get_credentials(host, api_key)
|
|
174
|
+
|
|
175
|
+
typer.echo(f"\nRegistering agent '{manifest.name}' with image '{image_tag}'...")
|
|
176
|
+
|
|
177
|
+
# Build request payload (includes runner/resource config from agent.yml)
|
|
178
|
+
from dataclasses import asdict
|
|
179
|
+
|
|
180
|
+
# image_digest is set during build; not available for --skip-build
|
|
181
|
+
_digest = locals().get("image_digest")
|
|
182
|
+
|
|
183
|
+
payload: dict = {
|
|
184
|
+
"name": manifest.name,
|
|
185
|
+
"image": image_tag,
|
|
186
|
+
"manifest": {
|
|
187
|
+
"version": manifest.version,
|
|
188
|
+
"capabilities": manifest.capabilities,
|
|
189
|
+
"resources": manifest.resources,
|
|
190
|
+
"description": manifest.description,
|
|
191
|
+
"runner": asdict(manifest.runner),
|
|
192
|
+
"resource_limits": asdict(manifest.resource_limits),
|
|
193
|
+
"env": manifest.env,
|
|
194
|
+
},
|
|
195
|
+
"capabilities": manifest.capabilities,
|
|
196
|
+
"resources": {
|
|
197
|
+
**manifest.resources,
|
|
198
|
+
"runner": asdict(manifest.runner),
|
|
199
|
+
"resource_limits": asdict(manifest.resource_limits),
|
|
200
|
+
},
|
|
201
|
+
"description": manifest.description,
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if _digest is not None:
|
|
205
|
+
payload["imageDigest"] = _digest
|
|
206
|
+
|
|
207
|
+
if task_schema is not None:
|
|
208
|
+
payload["taskInputSchema"] = task_schema.input_schema
|
|
209
|
+
|
|
210
|
+
# POST to Gateway
|
|
211
|
+
url = f"{resolved_host.rstrip('/')}/api/public/agents/register"
|
|
212
|
+
try:
|
|
213
|
+
response = httpx.post(
|
|
214
|
+
url,
|
|
215
|
+
json=payload,
|
|
216
|
+
auth=(resolved_public, resolved_secret),
|
|
217
|
+
timeout=30.0,
|
|
218
|
+
)
|
|
219
|
+
except httpx.RequestError as e:
|
|
220
|
+
typer.echo(f"Error connecting to {resolved_host}: {e}", err=True)
|
|
221
|
+
raise typer.Exit(1)
|
|
222
|
+
|
|
223
|
+
if response.status_code == 201:
|
|
224
|
+
data = response.json()
|
|
225
|
+
typer.echo(f"Agent registered successfully!")
|
|
226
|
+
typer.echo(f" ID: {data.get('id')}")
|
|
227
|
+
typer.echo(f" Name: {data.get('name')}")
|
|
228
|
+
typer.echo(f" Compatible algorithms: {', '.join(data.get('compatibleAlgorithms', []))}")
|
|
229
|
+
else:
|
|
230
|
+
typer.echo(f"Registration failed (HTTP {response.status_code}):", err=True)
|
|
231
|
+
try:
|
|
232
|
+
typer.echo(f" {json.dumps(response.json(), indent=2)}", err=True)
|
|
233
|
+
except Exception:
|
|
234
|
+
typer.echo(f" {response.text}", err=True)
|
|
235
|
+
raise typer.Exit(1)
|
|
236
|
+
|
|
237
|
+
# Upload secrets from gateway_manifest/.env (if present)
|
|
238
|
+
secrets = parse_secrets(manifest_path)
|
|
239
|
+
if secrets:
|
|
240
|
+
typer.echo(f"\nUploading {len(secrets)} secret(s)...")
|
|
241
|
+
secrets_client = httpx.Client(
|
|
242
|
+
base_url=resolved_host.rstrip("/"),
|
|
243
|
+
auth=(resolved_public, resolved_secret),
|
|
244
|
+
timeout=15.0,
|
|
245
|
+
)
|
|
246
|
+
for secret_name, secret_value in secrets.items():
|
|
247
|
+
try:
|
|
248
|
+
# Try create first
|
|
249
|
+
resp = secrets_client.post(
|
|
250
|
+
"/api/public/secrets",
|
|
251
|
+
json={"name": secret_name, "value": secret_value, "scope": "agent"},
|
|
252
|
+
)
|
|
253
|
+
if resp.status_code in (200, 201):
|
|
254
|
+
typer.echo(f" {secret_name} (created)")
|
|
255
|
+
elif resp.status_code == 409:
|
|
256
|
+
# Already exists — update
|
|
257
|
+
resp = secrets_client.put(
|
|
258
|
+
f"/api/public/secrets/{secret_name}",
|
|
259
|
+
json={"value": secret_value, "scope": "agent"},
|
|
260
|
+
)
|
|
261
|
+
if resp.status_code in (200, 201):
|
|
262
|
+
typer.echo(f" {secret_name} (updated)")
|
|
263
|
+
else:
|
|
264
|
+
typer.echo(f" {secret_name} (update failed: HTTP {resp.status_code})", err=True)
|
|
265
|
+
else:
|
|
266
|
+
typer.echo(f" {secret_name} (failed: HTTP {resp.status_code})", err=True)
|
|
267
|
+
except Exception as e:
|
|
268
|
+
typer.echo(f" {secret_name} (error: {e})", err=True)
|
|
269
|
+
secrets_client.close()
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# =============================================================================
|
|
273
|
+
# init command — scaffold agent project
|
|
274
|
+
# =============================================================================
|
|
275
|
+
|
|
276
|
+
_DOCKERFILE_TEMPLATE = """\
|
|
277
|
+
# Auto-generated by `gateway init`
|
|
278
|
+
FROM python:3.12-slim
|
|
279
|
+
|
|
280
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
281
|
+
build-essential \\
|
|
282
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
283
|
+
|
|
284
|
+
WORKDIR /app
|
|
285
|
+
|
|
286
|
+
# Install Python dependencies
|
|
287
|
+
COPY requirements.txt .
|
|
288
|
+
RUN pip install --no-cache-dir -r requirements.txt
|
|
289
|
+
|
|
290
|
+
# Install gatewaysdk with agent extras (includes gateway-agent CLI)
|
|
291
|
+
RUN pip install --no-cache-dir "gatewaysdk[agent]"
|
|
292
|
+
|
|
293
|
+
# Copy agent code
|
|
294
|
+
COPY . /app/agent/
|
|
295
|
+
|
|
296
|
+
ENV PYTHONPATH="/app:/app/agent"
|
|
297
|
+
|
|
298
|
+
# gateway-agent handles: store client, tracer, runner init, rollout polling
|
|
299
|
+
CMD ["gateway-agent", "--agent", "{agent_ref}"]
|
|
300
|
+
"""
|
|
301
|
+
|
|
302
|
+
_AGENT_YML_TEMPLATE = """\
|
|
303
|
+
# Agent manifest for Gateway Platform
|
|
304
|
+
# Generated by `gateway init`
|
|
305
|
+
|
|
306
|
+
name: {name}
|
|
307
|
+
version: "1.0.0"
|
|
308
|
+
description: >
|
|
309
|
+
{description}
|
|
310
|
+
|
|
311
|
+
capabilities:
|
|
312
|
+
- prompt-optimization
|
|
313
|
+
- tool-use
|
|
314
|
+
|
|
315
|
+
runner:
|
|
316
|
+
n_runners: 1
|
|
317
|
+
poll_interval: 5.0
|
|
318
|
+
|
|
319
|
+
resources:
|
|
320
|
+
memory: "4g"
|
|
321
|
+
cpu: 1.0
|
|
322
|
+
gpu: false
|
|
323
|
+
|
|
324
|
+
env: {{}}
|
|
325
|
+
"""
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
@app.command()
|
|
329
|
+
def init(
|
|
330
|
+
agent_ref: str = typer.Option(
|
|
331
|
+
...,
|
|
332
|
+
"--agent",
|
|
333
|
+
"-a",
|
|
334
|
+
help="Agent reference in module:attribute format (e.g., crm_agent:crm_agent).",
|
|
335
|
+
),
|
|
336
|
+
name: Optional[str] = typer.Option(
|
|
337
|
+
None,
|
|
338
|
+
"--name",
|
|
339
|
+
"-n",
|
|
340
|
+
help="Agent name. Defaults to module name.",
|
|
341
|
+
),
|
|
342
|
+
output_dir: str = typer.Option(
|
|
343
|
+
".",
|
|
344
|
+
"--output-dir",
|
|
345
|
+
"-o",
|
|
346
|
+
help="Output directory for generated files.",
|
|
347
|
+
),
|
|
348
|
+
) -> None:
|
|
349
|
+
"""Scaffold a Gateway agent project with Dockerfile and manifest.
|
|
350
|
+
|
|
351
|
+
Generates:
|
|
352
|
+
- Dockerfile with gateway-agent entrypoint
|
|
353
|
+
- gateway_manifest/agent.yml with default configuration
|
|
354
|
+
|
|
355
|
+
Examples::
|
|
356
|
+
|
|
357
|
+
gateway init --agent crm_agent:crm_agent
|
|
358
|
+
gateway init --agent my_agent:agent --name my-cool-agent
|
|
359
|
+
"""
|
|
360
|
+
output = Path(output_dir)
|
|
361
|
+
|
|
362
|
+
# Derive name from module if not provided
|
|
363
|
+
module_name = agent_ref.split(":")[0]
|
|
364
|
+
agent_name = name or module_name.replace("_", "-")
|
|
365
|
+
description = f"Gateway agent: {agent_name}"
|
|
366
|
+
|
|
367
|
+
# Generate Dockerfile
|
|
368
|
+
dockerfile_path = output / "Dockerfile"
|
|
369
|
+
if dockerfile_path.exists():
|
|
370
|
+
typer.echo(f"Warning: {dockerfile_path} already exists, skipping.")
|
|
371
|
+
else:
|
|
372
|
+
dockerfile_content = _DOCKERFILE_TEMPLATE.format(agent_ref=agent_ref)
|
|
373
|
+
dockerfile_path.write_text(dockerfile_content)
|
|
374
|
+
typer.echo(f"Created {dockerfile_path}")
|
|
375
|
+
|
|
376
|
+
# Generate manifest
|
|
377
|
+
manifest_dir = output / "gateway_manifest"
|
|
378
|
+
manifest_dir.mkdir(parents=True, exist_ok=True)
|
|
379
|
+
agent_yml_path = manifest_dir / "agent.yml"
|
|
380
|
+
if agent_yml_path.exists():
|
|
381
|
+
typer.echo(f"Warning: {agent_yml_path} already exists, skipping.")
|
|
382
|
+
else:
|
|
383
|
+
agent_yml_content = _AGENT_YML_TEMPLATE.format(
|
|
384
|
+
name=agent_name,
|
|
385
|
+
description=description,
|
|
386
|
+
)
|
|
387
|
+
agent_yml_path.write_text(agent_yml_content)
|
|
388
|
+
typer.echo(f"Created {agent_yml_path}")
|
|
389
|
+
|
|
390
|
+
# Generate .gitignore (includes gateway_manifest/.env)
|
|
391
|
+
gitignore_path = output / ".gitignore"
|
|
392
|
+
gitignore_entry = "gateway_manifest/.env"
|
|
393
|
+
if gitignore_path.exists():
|
|
394
|
+
existing = gitignore_path.read_text()
|
|
395
|
+
if gitignore_entry not in existing:
|
|
396
|
+
with open(gitignore_path, "a") as f:
|
|
397
|
+
f.write(f"\n# Gateway secrets (API keys)\n{gitignore_entry}\n")
|
|
398
|
+
typer.echo(f"Updated {gitignore_path} (added {gitignore_entry})")
|
|
399
|
+
else:
|
|
400
|
+
gitignore_path.write_text(f"# Gateway secrets (API keys — never commit)\n{gitignore_entry}\n")
|
|
401
|
+
typer.echo(f"Created {gitignore_path}")
|
|
402
|
+
|
|
403
|
+
# Generate sample .env in gateway_manifest/
|
|
404
|
+
env_path = manifest_dir / ".env"
|
|
405
|
+
if not env_path.exists():
|
|
406
|
+
env_path.write_text(
|
|
407
|
+
"# API keys for the agent container\n"
|
|
408
|
+
"# These are uploaded to the platform as encrypted secrets during `gateway deploy`\n"
|
|
409
|
+
"# ANTHROPIC_API_KEY=sk-ant-...\n"
|
|
410
|
+
"# OPENAI_API_KEY=sk-...\n"
|
|
411
|
+
)
|
|
412
|
+
typer.echo(f"Created {env_path} (sample — add your API keys)")
|
|
413
|
+
|
|
414
|
+
typer.echo(f"\nAgent project scaffolded!")
|
|
415
|
+
typer.echo(f" Agent ref: {agent_ref}")
|
|
416
|
+
typer.echo(f" Name: {agent_name}")
|
|
417
|
+
typer.echo(f"\nNext steps:")
|
|
418
|
+
typer.echo(f" 1. Add your agent code and requirements.txt")
|
|
419
|
+
typer.echo(f" 2. Add API keys to gateway_manifest/.env")
|
|
420
|
+
typer.echo(f" 3. Build & deploy: gateway deploy --tag {agent_name}:latest")
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
# =============================================================================
|
|
424
|
+
# algorithm subcommands
|
|
425
|
+
# =============================================================================
|
|
426
|
+
|
|
427
|
+
algorithm_app = typer.Typer(
|
|
428
|
+
name="algorithm",
|
|
429
|
+
help="List and manage algorithms.",
|
|
430
|
+
no_args_is_help=True,
|
|
431
|
+
)
|
|
432
|
+
app.add_typer(algorithm_app, name="algorithm")
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
@algorithm_app.command("list")
|
|
436
|
+
def algorithm_list() -> None:
|
|
437
|
+
"""List all available algorithms from class metadata."""
|
|
438
|
+
from gatewaysdk.algorithm.base import Algorithm
|
|
439
|
+
|
|
440
|
+
subclasses = Algorithm.__subclasses__()
|
|
441
|
+
if not subclasses:
|
|
442
|
+
typer.echo("No algorithms found. Import algorithm modules first.")
|
|
443
|
+
return
|
|
444
|
+
|
|
445
|
+
typer.echo(f"{'Name':<12} {'Display':<16} {'GPU':<6} {'Description'}")
|
|
446
|
+
typer.echo("-" * 70)
|
|
447
|
+
for cls in subclasses:
|
|
448
|
+
gpu = cls.requirements.get("gpu", False) if cls.requirements else False
|
|
449
|
+
typer.echo(f"{cls.name:<12} {cls.display_name:<16} {str(gpu):<6} {cls.description}")
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
@algorithm_app.command("new")
|
|
453
|
+
def algorithm_new(
|
|
454
|
+
name: str = typer.Argument(..., help="Algorithm name (e.g., my_algo)"),
|
|
455
|
+
output_dir: str = typer.Option(".", help="Directory to create the algorithm in"),
|
|
456
|
+
) -> None:
|
|
457
|
+
"""Scaffold a new algorithm class."""
|
|
458
|
+
output_path = Path(output_dir) / f"{name}.py"
|
|
459
|
+
if output_path.exists():
|
|
460
|
+
typer.echo(f"Error: {output_path} already exists.", err=True)
|
|
461
|
+
raise typer.Exit(1)
|
|
462
|
+
|
|
463
|
+
template = f'''"""Custom algorithm: {name}."""
|
|
464
|
+
|
|
465
|
+
from __future__ import annotations
|
|
466
|
+
|
|
467
|
+
from typing import Any, ClassVar, Dict, Optional
|
|
468
|
+
|
|
469
|
+
from gatewaysdk.algorithm.base import Algorithm
|
|
470
|
+
from gatewaysdk.types import Dataset
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
class {name.title().replace("_", "")}(Algorithm):
|
|
474
|
+
"""TODO: Describe your algorithm."""
|
|
475
|
+
|
|
476
|
+
name: ClassVar[str] = "{name}"
|
|
477
|
+
display_name: ClassVar[str] = "{name.title().replace("_", " ")}"
|
|
478
|
+
description: ClassVar[str] = "TODO: Add description"
|
|
479
|
+
requirements: ClassVar[Dict[str, Any]] = {{"gpu": False, "min_runners": 1}}
|
|
480
|
+
|
|
481
|
+
def run(
|
|
482
|
+
self,
|
|
483
|
+
train_dataset: Optional[Dataset[Any]] = None,
|
|
484
|
+
val_dataset: Optional[Dataset[Any]] = None,
|
|
485
|
+
) -> None:
|
|
486
|
+
trainer = self.get_trainer()
|
|
487
|
+
store = self.get_store()
|
|
488
|
+
|
|
489
|
+
# TODO: Implement your algorithm logic here
|
|
490
|
+
raise NotImplementedError("Implement run() for {name}")
|
|
491
|
+
'''
|
|
492
|
+
|
|
493
|
+
output_path.write_text(template)
|
|
494
|
+
typer.echo(f"Created algorithm scaffold: {output_path}")
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
# =============================================================================
|
|
498
|
+
# auth commands
|
|
499
|
+
# =============================================================================
|
|
500
|
+
|
|
501
|
+
auth_app = typer.Typer(
|
|
502
|
+
name="auth",
|
|
503
|
+
help="Manage authentication credentials.",
|
|
504
|
+
no_args_is_help=True,
|
|
505
|
+
)
|
|
506
|
+
app.add_typer(auth_app, name="auth")
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
@auth_app.command("login")
|
|
510
|
+
def auth_login(
|
|
511
|
+
host: str = typer.Option(..., prompt=True, help="Gateway server URL"),
|
|
512
|
+
public_key: str = typer.Option(..., prompt=True, help="Project public key (pk-lf-...)"),
|
|
513
|
+
api_key: str = typer.Option(..., prompt=True, hide_input=True, help="Project secret key (sk-lf-...)"),
|
|
514
|
+
) -> None:
|
|
515
|
+
"""Store credentials for the gateway CLI."""
|
|
516
|
+
from gatewaysdk.cli.http_client import save_config
|
|
517
|
+
|
|
518
|
+
save_config({"host": host.rstrip("/"), "public_key": public_key, "api_key": api_key})
|
|
519
|
+
typer.echo("Credentials saved to ~/.gateway/config.json")
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
@auth_app.command("logout")
|
|
523
|
+
def auth_logout() -> None:
|
|
524
|
+
"""Clear stored credentials."""
|
|
525
|
+
from gatewaysdk.cli.http_client import CONFIG_FILE
|
|
526
|
+
|
|
527
|
+
if CONFIG_FILE.exists():
|
|
528
|
+
CONFIG_FILE.unlink()
|
|
529
|
+
typer.echo("Credentials cleared.")
|
|
530
|
+
else:
|
|
531
|
+
typer.echo("No credentials stored.")
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
@auth_app.command("status")
|
|
535
|
+
def auth_status() -> None:
|
|
536
|
+
"""Show current authentication status."""
|
|
537
|
+
from gatewaysdk.cli.http_client import get_config
|
|
538
|
+
|
|
539
|
+
config = get_config()
|
|
540
|
+
host = config.get("host")
|
|
541
|
+
has_key = bool(config.get("api_key"))
|
|
542
|
+
|
|
543
|
+
if host:
|
|
544
|
+
typer.echo(f"Host: {host}")
|
|
545
|
+
typer.echo(f"API key: {'configured' if has_key else 'not set'}")
|
|
546
|
+
else:
|
|
547
|
+
typer.echo("No credentials stored. Run `gateway auth login`.")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
# =============================================================================
|
|
551
|
+
# secret commands
|
|
552
|
+
# =============================================================================
|
|
553
|
+
|
|
554
|
+
secret_app = typer.Typer(
|
|
555
|
+
name="secret",
|
|
556
|
+
help="Manage project secrets (encrypted env vars for agent containers).",
|
|
557
|
+
no_args_is_help=True,
|
|
558
|
+
)
|
|
559
|
+
app.add_typer(secret_app, name="secret")
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
@secret_app.command("set")
|
|
563
|
+
def secret_set(
|
|
564
|
+
name: str = typer.Argument(..., help="Secret name (UPPER_SNAKE_CASE, e.g. TAVILY_API_KEY)"),
|
|
565
|
+
value: Optional[str] = typer.Option(
|
|
566
|
+
None,
|
|
567
|
+
"--value",
|
|
568
|
+
"-v",
|
|
569
|
+
help="Secret value. If omitted, you will be prompted.",
|
|
570
|
+
),
|
|
571
|
+
description: Optional[str] = typer.Option(None, "--description", "-d", help="Human-readable description"),
|
|
572
|
+
scope: str = typer.Option("agent", help="Scope: agent, worker, or both"),
|
|
573
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
574
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
575
|
+
) -> None:
|
|
576
|
+
"""Create or update a project secret."""
|
|
577
|
+
from gatewaysdk.cli.http_client import create_client
|
|
578
|
+
|
|
579
|
+
if value is None:
|
|
580
|
+
value = typer.prompt("Secret value", hide_input=True)
|
|
581
|
+
|
|
582
|
+
client = create_client(host, api_key)
|
|
583
|
+
payload: dict = {"name": name, "value": value, "scope": scope}
|
|
584
|
+
if description:
|
|
585
|
+
payload["description"] = description
|
|
586
|
+
|
|
587
|
+
response = client.post("/api/public/secrets", json=payload)
|
|
588
|
+
|
|
589
|
+
if response.status_code in (200, 201):
|
|
590
|
+
data = response.json()
|
|
591
|
+
typer.echo(f"Secret '{data.get('name', name)}' saved ({data.get('displayValue', '****')})")
|
|
592
|
+
elif response.status_code == 409:
|
|
593
|
+
# Already exists -- update
|
|
594
|
+
response = client.put(f"/api/public/secrets/{name}", json=payload)
|
|
595
|
+
if response.status_code in (200, 201):
|
|
596
|
+
typer.echo(f"Secret '{name}' updated.")
|
|
597
|
+
else:
|
|
598
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
599
|
+
raise typer.Exit(1)
|
|
600
|
+
else:
|
|
601
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
602
|
+
raise typer.Exit(1)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
@secret_app.command("list")
|
|
606
|
+
def secret_list(
|
|
607
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
608
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
609
|
+
) -> None:
|
|
610
|
+
"""List project secrets (names and descriptions only, not values)."""
|
|
611
|
+
from gatewaysdk.cli.http_client import create_client
|
|
612
|
+
|
|
613
|
+
client = create_client(host, api_key)
|
|
614
|
+
response = client.get("/api/public/secrets")
|
|
615
|
+
|
|
616
|
+
if response.status_code == 200:
|
|
617
|
+
secrets = response.json().get("secrets", [])
|
|
618
|
+
if not secrets:
|
|
619
|
+
typer.echo("No secrets configured. Run `gateway secret set <name>` to add one.")
|
|
620
|
+
return
|
|
621
|
+
typer.echo(f"{'Name':<30} {'Scope':<10} {'Description'}")
|
|
622
|
+
typer.echo("-" * 70)
|
|
623
|
+
for s in secrets:
|
|
624
|
+
desc = (s.get("description") or "")[:28]
|
|
625
|
+
typer.echo(f"{s.get('name', ''):<30} {s.get('scope', ''):<10} {desc}")
|
|
626
|
+
else:
|
|
627
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
628
|
+
raise typer.Exit(1)
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
@secret_app.command("delete")
|
|
632
|
+
def secret_delete(
|
|
633
|
+
name: str = typer.Argument(..., help="Secret name to delete"),
|
|
634
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
635
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
636
|
+
) -> None:
|
|
637
|
+
"""Delete a project secret."""
|
|
638
|
+
from gatewaysdk.cli.http_client import create_client
|
|
639
|
+
|
|
640
|
+
client = create_client(host, api_key)
|
|
641
|
+
response = client.delete(f"/api/public/secrets/{name}")
|
|
642
|
+
|
|
643
|
+
if response.status_code in (200, 204):
|
|
644
|
+
typer.echo(f"Secret '{name}' deleted.")
|
|
645
|
+
else:
|
|
646
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
647
|
+
raise typer.Exit(1)
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
# =============================================================================
|
|
651
|
+
# agent commands
|
|
652
|
+
# =============================================================================
|
|
653
|
+
|
|
654
|
+
agent_app = typer.Typer(
|
|
655
|
+
name="agent",
|
|
656
|
+
help="Manage registered agents.",
|
|
657
|
+
no_args_is_help=True,
|
|
658
|
+
)
|
|
659
|
+
app.add_typer(agent_app, name="agent")
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
@agent_app.command("list")
|
|
663
|
+
def agent_list(
|
|
664
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
665
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
666
|
+
) -> None:
|
|
667
|
+
"""List registered agents."""
|
|
668
|
+
from gatewaysdk.cli.http_client import create_client
|
|
669
|
+
|
|
670
|
+
client = create_client(host, api_key)
|
|
671
|
+
response = client.get("/api/public/agents")
|
|
672
|
+
|
|
673
|
+
if response.status_code == 200:
|
|
674
|
+
data = response.json()
|
|
675
|
+
agents = data.get("agents", [])
|
|
676
|
+
if not agents:
|
|
677
|
+
typer.echo("No agents registered. Run `gateway deploy` to register an agent.")
|
|
678
|
+
return
|
|
679
|
+
typer.echo(f"{'Name':<20} {'Image':<30} {'Compatible'}")
|
|
680
|
+
typer.echo("-" * 70)
|
|
681
|
+
for a in agents:
|
|
682
|
+
compat = ", ".join(a.get("compatibleAlgorithms", []))
|
|
683
|
+
typer.echo(f"{a.get('name', ''):<20} {a.get('image', '')[:28]:<30} {compat}")
|
|
684
|
+
else:
|
|
685
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
686
|
+
raise typer.Exit(1)
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
# =============================================================================
|
|
690
|
+
# run commands
|
|
691
|
+
# =============================================================================
|
|
692
|
+
|
|
693
|
+
run_app = typer.Typer(
|
|
694
|
+
name="run",
|
|
695
|
+
help="Manage platform optimization runs.",
|
|
696
|
+
no_args_is_help=True,
|
|
697
|
+
)
|
|
698
|
+
app.add_typer(run_app, name="run")
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
@run_app.command("start")
|
|
702
|
+
def run_start(
|
|
703
|
+
algorithm: str = typer.Argument(..., help="Algorithm name (e.g., gepa)"),
|
|
704
|
+
agent_id: str = typer.Option(..., "--agent", help="Registered agent ID"),
|
|
705
|
+
dataset_id: str = typer.Option(..., "--dataset", help="Dataset ID"),
|
|
706
|
+
name: Optional[str] = typer.Option(None, "--name", help="Run name"),
|
|
707
|
+
config_json: Optional[str] = typer.Option(None, "--config", help="JSON config string"),
|
|
708
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
709
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
710
|
+
) -> None:
|
|
711
|
+
"""Start a platform optimization run."""
|
|
712
|
+
from gatewaysdk.cli.http_client import create_client
|
|
713
|
+
|
|
714
|
+
client = create_client(host, api_key)
|
|
715
|
+
|
|
716
|
+
payload: dict = {
|
|
717
|
+
"algorithmName": algorithm,
|
|
718
|
+
"agentId": agent_id,
|
|
719
|
+
"datasetId": dataset_id,
|
|
720
|
+
}
|
|
721
|
+
if name:
|
|
722
|
+
payload["name"] = name
|
|
723
|
+
if config_json:
|
|
724
|
+
payload["config"] = json.loads(config_json)
|
|
725
|
+
|
|
726
|
+
response = client.post("/api/public/runs/start", json=payload)
|
|
727
|
+
|
|
728
|
+
if response.status_code in (200, 201):
|
|
729
|
+
data = response.json()
|
|
730
|
+
typer.echo(f"Run started!")
|
|
731
|
+
typer.echo(f" Run ID: {data.get('runId')}")
|
|
732
|
+
typer.echo(f" Run #: {data.get('runNumber')}")
|
|
733
|
+
typer.echo(f" Job ID: {data.get('jobId')}")
|
|
734
|
+
typer.echo(f" Status: {data.get('status')}")
|
|
735
|
+
else:
|
|
736
|
+
typer.echo(f"Failed (HTTP {response.status_code}):", err=True)
|
|
737
|
+
typer.echo(f" {response.text}", err=True)
|
|
738
|
+
raise typer.Exit(1)
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
@run_app.command("verify")
|
|
742
|
+
def run_verify(
|
|
743
|
+
algorithm: str = typer.Argument(..., help="Algorithm name"),
|
|
744
|
+
agent_id: str = typer.Option(..., "--agent", help="Agent ID"),
|
|
745
|
+
dataset_id: Optional[str] = typer.Option(None, "--dataset", help="Dataset ID"),
|
|
746
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
747
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
748
|
+
) -> None:
|
|
749
|
+
"""Verify an algorithm configuration."""
|
|
750
|
+
from gatewaysdk.cli.http_client import create_client
|
|
751
|
+
|
|
752
|
+
client = create_client(host, api_key)
|
|
753
|
+
payload: dict = {"algorithmName": algorithm, "agentId": agent_id}
|
|
754
|
+
if dataset_id:
|
|
755
|
+
payload["datasetId"] = dataset_id
|
|
756
|
+
|
|
757
|
+
response = client.post("/api/public/runs/verify", json=payload)
|
|
758
|
+
|
|
759
|
+
if response.status_code in (200, 202):
|
|
760
|
+
data = response.json()
|
|
761
|
+
typer.echo(f"Verify job queued: {data.get('jobId')}")
|
|
762
|
+
else:
|
|
763
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
764
|
+
raise typer.Exit(1)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
@run_app.command("list")
|
|
768
|
+
def run_list(
|
|
769
|
+
source: Optional[str] = typer.Option(None, help="Filter by source (platform/sdk)"),
|
|
770
|
+
limit: int = typer.Option(20, help="Max results"),
|
|
771
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
772
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
773
|
+
) -> None:
|
|
774
|
+
"""List runs."""
|
|
775
|
+
from gatewaysdk.cli.http_client import create_client
|
|
776
|
+
|
|
777
|
+
client = create_client(host, api_key)
|
|
778
|
+
params: dict = {"limit": str(limit)}
|
|
779
|
+
if source:
|
|
780
|
+
params["source"] = source
|
|
781
|
+
|
|
782
|
+
response = client.get("/api/public/runs", params=params)
|
|
783
|
+
|
|
784
|
+
if response.status_code == 200:
|
|
785
|
+
data = response.json()
|
|
786
|
+
runs = data.get("runs", [])
|
|
787
|
+
if not runs:
|
|
788
|
+
typer.echo("No runs found.")
|
|
789
|
+
return
|
|
790
|
+
typer.echo(f"{'#':<6} {'Name':<30} {'State':<12} {'Source':<10}")
|
|
791
|
+
typer.echo("-" * 60)
|
|
792
|
+
for r in runs:
|
|
793
|
+
typer.echo(
|
|
794
|
+
f"{r.get('runNumber', ''):<6} "
|
|
795
|
+
f"{(r.get('name') or '')[:28]:<30} "
|
|
796
|
+
f"{r.get('state', ''):<12} "
|
|
797
|
+
f"{r.get('source', ''):<10}"
|
|
798
|
+
)
|
|
799
|
+
else:
|
|
800
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
801
|
+
raise typer.Exit(1)
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
# =============================================================================
|
|
805
|
+
# dataset commands
|
|
806
|
+
# =============================================================================
|
|
807
|
+
|
|
808
|
+
dataset_app = typer.Typer(
|
|
809
|
+
name="dataset",
|
|
810
|
+
help="Manage datasets.",
|
|
811
|
+
no_args_is_help=True,
|
|
812
|
+
)
|
|
813
|
+
app.add_typer(dataset_app, name="dataset")
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
@dataset_app.command("upload")
|
|
817
|
+
def dataset_upload(
|
|
818
|
+
path: str = typer.Argument(..., help="Path to JSON or JSONL file"),
|
|
819
|
+
name: str = typer.Option(..., "--name", help="Dataset name"),
|
|
820
|
+
agent: Optional[str] = typer.Option(
|
|
821
|
+
None,
|
|
822
|
+
"--agent",
|
|
823
|
+
help="Validate items against this agent's task schema",
|
|
824
|
+
),
|
|
825
|
+
description: Optional[str] = typer.Option(None, help="Description"),
|
|
826
|
+
host: Optional[str] = typer.Option(None, envvar="GATEWAY_HOST"),
|
|
827
|
+
api_key: Optional[str] = typer.Option(None, envvar="GATEWAY_SECRET_KEY"),
|
|
828
|
+
) -> None:
|
|
829
|
+
"""Upload a JSON or JSONL file as a dataset.
|
|
830
|
+
|
|
831
|
+
Supports:
|
|
832
|
+
- JSON with {"tasks": [...]} or {"items": [...]} wrapper
|
|
833
|
+
- JSON bare array [item1, item2, ...]
|
|
834
|
+
- JSONL (one JSON object per line)
|
|
835
|
+
|
|
836
|
+
Examples::
|
|
837
|
+
|
|
838
|
+
gateway dataset upload tasks.json --name crm-benchmark-v1
|
|
839
|
+
gateway dataset upload tasks.json --name crm-v1 --agent crm-analyst
|
|
840
|
+
gateway dataset upload tasks.jsonl --name eval-set
|
|
841
|
+
"""
|
|
842
|
+
from gatewaysdk.cli.http_client import create_client
|
|
843
|
+
|
|
844
|
+
file_path = Path(path)
|
|
845
|
+
if not file_path.exists():
|
|
846
|
+
typer.echo(f"Error: File not found: {path}", err=True)
|
|
847
|
+
raise typer.Exit(1)
|
|
848
|
+
|
|
849
|
+
# Read items -- auto-detect JSON vs JSONL
|
|
850
|
+
with open(file_path) as f:
|
|
851
|
+
raw = f.read().strip()
|
|
852
|
+
|
|
853
|
+
if file_path.suffix == ".json" or raw.startswith("{") or raw.startswith("["):
|
|
854
|
+
data = json.loads(raw)
|
|
855
|
+
if isinstance(data, list):
|
|
856
|
+
items = data
|
|
857
|
+
elif isinstance(data, dict):
|
|
858
|
+
items = data.get("tasks") or data.get("items") or []
|
|
859
|
+
else:
|
|
860
|
+
items = []
|
|
861
|
+
else:
|
|
862
|
+
# JSONL: one item per line
|
|
863
|
+
items = [json.loads(line) for line in raw.splitlines() if line.strip()]
|
|
864
|
+
|
|
865
|
+
if not items:
|
|
866
|
+
typer.echo("Error: No items found in file.", err=True)
|
|
867
|
+
raise typer.Exit(1)
|
|
868
|
+
|
|
869
|
+
typer.echo(f"Found {len(items)} items in {file_path.name}")
|
|
870
|
+
|
|
871
|
+
# Schema validation against agent's taskInputSchema
|
|
872
|
+
if agent:
|
|
873
|
+
client = create_client(host, api_key)
|
|
874
|
+
resp = client.get("/api/public/agents")
|
|
875
|
+
if resp.status_code != 200:
|
|
876
|
+
typer.echo(f"Warning: Could not fetch agents for validation: HTTP {resp.status_code}", err=True)
|
|
877
|
+
else:
|
|
878
|
+
agents_list = resp.json().get("agents", [])
|
|
879
|
+
matched = [a for a in agents_list if a.get("name") == agent]
|
|
880
|
+
if not matched:
|
|
881
|
+
typer.echo(f"Warning: Agent '{agent}' not found, skipping validation", err=True)
|
|
882
|
+
else:
|
|
883
|
+
schema = matched[0].get("taskInputSchema")
|
|
884
|
+
if schema:
|
|
885
|
+
try:
|
|
886
|
+
import jsonschema
|
|
887
|
+
except ImportError:
|
|
888
|
+
typer.echo(
|
|
889
|
+
"Warning: jsonschema not installed, skipping validation. "
|
|
890
|
+
"Install with: pip install jsonschema",
|
|
891
|
+
err=True,
|
|
892
|
+
)
|
|
893
|
+
schema = None
|
|
894
|
+
|
|
895
|
+
if schema:
|
|
896
|
+
typer.echo(f"Validating against agent '{agent}' schema...")
|
|
897
|
+
errors = []
|
|
898
|
+
for i, item in enumerate(items):
|
|
899
|
+
try:
|
|
900
|
+
jsonschema.validate(item, schema)
|
|
901
|
+
except jsonschema.ValidationError as e:
|
|
902
|
+
item_name = item.get("name", f"item {i}") if isinstance(item, dict) else f"item {i}"
|
|
903
|
+
errors.append(f" [{i+1}/{len(items)}] {item_name}: {e.message}")
|
|
904
|
+
if errors:
|
|
905
|
+
typer.echo(f"Validation failed ({len(errors)} errors):", err=True)
|
|
906
|
+
for e in errors[:10]:
|
|
907
|
+
typer.echo(e, err=True)
|
|
908
|
+
if len(errors) > 10:
|
|
909
|
+
typer.echo(f" ... and {len(errors) - 10} more", err=True)
|
|
910
|
+
raise typer.Exit(1)
|
|
911
|
+
typer.echo(f" All {len(items)} items valid.")
|
|
912
|
+
else:
|
|
913
|
+
typer.echo(f"Agent '{agent}' has no taskInputSchema, skipping validation")
|
|
914
|
+
|
|
915
|
+
# Upload
|
|
916
|
+
typer.echo(f"Uploading {len(items)} items as dataset '{name}'...")
|
|
917
|
+
|
|
918
|
+
client = create_client(host, api_key)
|
|
919
|
+
payload: dict = {
|
|
920
|
+
"name": name,
|
|
921
|
+
"items": [{"input": item} for item in items],
|
|
922
|
+
}
|
|
923
|
+
if description:
|
|
924
|
+
payload["description"] = description
|
|
925
|
+
|
|
926
|
+
response = client.post("/api/public/datasets/import", json=payload)
|
|
927
|
+
|
|
928
|
+
if response.status_code in (200, 201):
|
|
929
|
+
data = response.json()
|
|
930
|
+
typer.echo(f"Dataset created!")
|
|
931
|
+
typer.echo(f" Dataset ID: {data.get('datasetId')}")
|
|
932
|
+
typer.echo(f" Name: {data.get('datasetName')}")
|
|
933
|
+
typer.echo(f" Items: {data.get('itemCount')}")
|
|
934
|
+
else:
|
|
935
|
+
typer.echo(f"Failed (HTTP {response.status_code}): {response.text}", err=True)
|
|
936
|
+
raise typer.Exit(1)
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
bench_app = typer.Typer(
|
|
940
|
+
name="bench",
|
|
941
|
+
help="Benchmark container hub — version and push benchmark environments to Gateway.",
|
|
942
|
+
no_args_is_help=True,
|
|
943
|
+
)
|
|
944
|
+
app.add_typer(bench_app, name="bench")
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
def _ci_finish(
|
|
948
|
+
result: dict,
|
|
949
|
+
*,
|
|
950
|
+
fail_under: Optional[float],
|
|
951
|
+
bar_source: str = "--fail-under",
|
|
952
|
+
title: str,
|
|
953
|
+
facts: Optional[List[tuple]] = None,
|
|
954
|
+
) -> None:
|
|
955
|
+
"""Shared CI affordances for every eval command: a GitHub Actions step
|
|
956
|
+
summary whenever Actions provides one ($GITHUB_STEP_SUMMARY), then the
|
|
957
|
+
reward gate. The gate is honest by construction: asking for a bar on a
|
|
958
|
+
result that carries no aggregate reward exits 1, same as a miss — a gate
|
|
959
|
+
that cannot measure must not pass."""
|
|
960
|
+
import os
|
|
961
|
+
|
|
962
|
+
metrics = result.get("metrics") if isinstance(result.get("metrics"), dict) else {}
|
|
963
|
+
avg = metrics.get("avg_reward") if metrics else None
|
|
964
|
+
if avg is None and isinstance(result.get("reward"), (int, float)):
|
|
965
|
+
avg = result["reward"]
|
|
966
|
+
|
|
967
|
+
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
968
|
+
if summary_path:
|
|
969
|
+
try:
|
|
970
|
+
rows: List[tuple] = [("avg reward", "n/a" if avg is None else f"{avg:.3f}")]
|
|
971
|
+
n = (metrics or {}).get("n") or result.get("samples")
|
|
972
|
+
if n is not None:
|
|
973
|
+
rows.append(("samples", str(n)))
|
|
974
|
+
if result.get("evaluationId"):
|
|
975
|
+
rows.append(("evaluation", str(result["evaluationId"])))
|
|
976
|
+
rows.extend(facts or [])
|
|
977
|
+
if fail_under is not None:
|
|
978
|
+
verdict = "PASS" if (avg is not None and avg >= fail_under) else "FAIL"
|
|
979
|
+
rows.append(("gate", f"{verdict} — bar {fail_under:g} ({bar_source})"))
|
|
980
|
+
lines = [f"### {title}", "", "| | |", "|---|---|"]
|
|
981
|
+
lines += [f"| {k} | {v} |" for k, v in rows]
|
|
982
|
+
with open(summary_path, "a") as fh:
|
|
983
|
+
fh.write("\n".join(lines) + "\n\n")
|
|
984
|
+
except OSError:
|
|
985
|
+
pass # the summary is a nicety; the gate below is the contract
|
|
986
|
+
|
|
987
|
+
if fail_under is None:
|
|
988
|
+
return
|
|
989
|
+
if avg is None:
|
|
990
|
+
typer.echo(
|
|
991
|
+
f"gate: the result carries no aggregate reward — cannot verify "
|
|
992
|
+
f"the {fail_under:g} bar ({bar_source}), failing.",
|
|
993
|
+
err=True,
|
|
994
|
+
)
|
|
995
|
+
raise typer.Exit(1)
|
|
996
|
+
if avg < fail_under:
|
|
997
|
+
typer.echo(
|
|
998
|
+
f"gate: avg reward {avg:.3f} is below the {fail_under:g} bar " f"({bar_source}).",
|
|
999
|
+
err=True,
|
|
1000
|
+
)
|
|
1001
|
+
raise typer.Exit(1)
|
|
1002
|
+
typer.echo(
|
|
1003
|
+
f"gate: avg reward {avg:.3f} clears the {fail_under:g} bar " f"({bar_source}).",
|
|
1004
|
+
err=True,
|
|
1005
|
+
)
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
worlds_app = typer.Typer(
|
|
1009
|
+
help="Run agents against simulated worlds (world-over-tools dev loop).",
|
|
1010
|
+
no_args_is_help=True,
|
|
1011
|
+
)
|
|
1012
|
+
app.add_typer(worlds_app, name="worlds")
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def _resolve_agent_callable(spec: str):
|
|
1016
|
+
"""``file.py[:fn]`` or ``module.path[:fn]``; the attr defaults to
|
|
1017
|
+
``agent``. Returns the callable or raises typer.Exit with the reason."""
|
|
1018
|
+
import importlib
|
|
1019
|
+
import importlib.util
|
|
1020
|
+
import os
|
|
1021
|
+
|
|
1022
|
+
target, _, attr = spec.partition(":")
|
|
1023
|
+
attr = attr or "agent"
|
|
1024
|
+
looks_like_file = target.endswith(".py") or os.sep in target
|
|
1025
|
+
if looks_like_file or Path(target).is_file():
|
|
1026
|
+
file_path = Path(target).resolve()
|
|
1027
|
+
if not file_path.is_file():
|
|
1028
|
+
typer.echo(f"agent file not found: {target}", err=True)
|
|
1029
|
+
raise typer.Exit(1)
|
|
1030
|
+
module_spec = importlib.util.spec_from_file_location(file_path.stem, file_path)
|
|
1031
|
+
if module_spec is None or module_spec.loader is None:
|
|
1032
|
+
typer.echo(f"could not load agent file: {target}", err=True)
|
|
1033
|
+
raise typer.Exit(1)
|
|
1034
|
+
module = importlib.util.module_from_spec(module_spec)
|
|
1035
|
+
module_spec.loader.exec_module(module)
|
|
1036
|
+
else:
|
|
1037
|
+
try:
|
|
1038
|
+
module = importlib.import_module(target)
|
|
1039
|
+
except ImportError as exc:
|
|
1040
|
+
typer.echo(f"could not import agent module '{target}': {exc}", err=True)
|
|
1041
|
+
raise typer.Exit(1)
|
|
1042
|
+
fn = getattr(module, attr, None)
|
|
1043
|
+
if not callable(fn):
|
|
1044
|
+
typer.echo(
|
|
1045
|
+
f"'{spec}' does not expose a callable '{attr}' — point --agent at "
|
|
1046
|
+
"a function taking the world (file.py:fn or module:fn; fn "
|
|
1047
|
+
"defaults to 'agent')",
|
|
1048
|
+
err=True,
|
|
1049
|
+
)
|
|
1050
|
+
raise typer.Exit(1)
|
|
1051
|
+
return fn
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
@worlds_app.command("ci")
|
|
1055
|
+
def worlds_ci(
|
|
1056
|
+
path: str = typer.Option(".", "--path", "-p", help="World directory inside the checked-out repo."),
|
|
1057
|
+
model: str = typer.Option(..., "--model", "-m", help="Model id for the runner."),
|
|
1058
|
+
agent: str = typer.Option(
|
|
1059
|
+
"mini-swe-agent",
|
|
1060
|
+
"--agent",
|
|
1061
|
+
"-a",
|
|
1062
|
+
help="harbor agent that runs the tasks (use `oracle` for a deterministic regression check of the world itself).",
|
|
1063
|
+
),
|
|
1064
|
+
branch: Optional[str] = typer.Option(
|
|
1065
|
+
None,
|
|
1066
|
+
"--branch",
|
|
1067
|
+
"-b",
|
|
1068
|
+
help="Hub branch to push and run. Defaults to GITHUB_HEAD_REF (PRs), then GITHUB_REF_NAME (pushes), then the dir's recorded branch.",
|
|
1069
|
+
),
|
|
1070
|
+
fail_under: Optional[float] = typer.Option(
|
|
1071
|
+
None,
|
|
1072
|
+
"--fail-under",
|
|
1073
|
+
min=0.0,
|
|
1074
|
+
max=1.0,
|
|
1075
|
+
help="Reward gate. Defaults to the world's own pass bar, then 0.7.",
|
|
1076
|
+
),
|
|
1077
|
+
no_push: bool = typer.Option(
|
|
1078
|
+
False,
|
|
1079
|
+
"--no-push",
|
|
1080
|
+
help="Run the dir's recorded base version as-is (requires a clean checkout) instead of pushing the working tree.",
|
|
1081
|
+
),
|
|
1082
|
+
num_examples: int = typer.Option(-1, "--num-examples", "-n"),
|
|
1083
|
+
rollouts: int = typer.Option(1, "--rollouts-per-example", "-r"),
|
|
1084
|
+
message: Optional[str] = typer.Option(
|
|
1085
|
+
None, "--message", "-M", help="Push change reason (default: a CI commit line)."
|
|
1086
|
+
),
|
|
1087
|
+
output_dir: str = typer.Option("./outputs", "--output-dir", "-o"),
|
|
1088
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1089
|
+
) -> None:
|
|
1090
|
+
"""The PR-to-simulation gate in ONE command, built for GitHub Actions:
|
|
1091
|
+
|
|
1092
|
+
gateway worlds ci --path environments/drive-sim -m openai/gpt-4o-mini
|
|
1093
|
+
|
|
1094
|
+
Pushes the checked-out world onto a hub branch named after the PR
|
|
1095
|
+
(GITHUB_HEAD_REF), runs its tasks as a harbor world (docker), files
|
|
1096
|
+
the run pinned to the pushed version — branch-tagged, PR chip included
|
|
1097
|
+
when the world links a repo — writes a $GITHUB_STEP_SUMMARY table, and
|
|
1098
|
+
exits 1 when the average reward lands below the bar. The bar defaults to
|
|
1099
|
+
the world's own pass threshold, so the job fails exactly when the
|
|
1100
|
+
platform would mark the run failed.
|
|
1101
|
+
|
|
1102
|
+
Needs GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY and
|
|
1103
|
+
OPENROUTER_API_KEY in the environment (repo secrets), plus docker.
|
|
1104
|
+
"""
|
|
1105
|
+
import os
|
|
1106
|
+
|
|
1107
|
+
from gatewaysdk.benchmark_hub import (
|
|
1108
|
+
GatewayBenchmarkClient,
|
|
1109
|
+
GatewayEvalsClient,
|
|
1110
|
+
detect_env_kind,
|
|
1111
|
+
find_mcp_config,
|
|
1112
|
+
harvest_job_dir,
|
|
1113
|
+
local_tree_state,
|
|
1114
|
+
read_local_meta,
|
|
1115
|
+
resolve_harbor_model,
|
|
1116
|
+
run_harbor_job,
|
|
1117
|
+
tasks_root,
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
src = os.path.abspath(path)
|
|
1121
|
+
if not os.path.isdir(src):
|
|
1122
|
+
typer.echo(f"--path is not a directory: {path}", err=True)
|
|
1123
|
+
raise typer.Exit(1)
|
|
1124
|
+
resolved_kind = detect_env_kind(src)
|
|
1125
|
+
if resolved_kind == "mock-tools":
|
|
1126
|
+
_reject_mock_tools()
|
|
1127
|
+
if resolved_kind in ("verifiers", "verifiers-v1"):
|
|
1128
|
+
typer.echo(
|
|
1129
|
+
"`worlds ci` runs harbor worlds (self-grading task dirs); this "
|
|
1130
|
+
"dir is a verifiers package — use `gateway bench eval` instead.",
|
|
1131
|
+
err=True,
|
|
1132
|
+
)
|
|
1133
|
+
raise typer.Exit(1)
|
|
1134
|
+
|
|
1135
|
+
api_key = os.environ.get("OPENROUTER_API_KEY")
|
|
1136
|
+
if not api_key:
|
|
1137
|
+
typer.echo("OPENROUTER_API_KEY is required for `worlds ci`.", err=True)
|
|
1138
|
+
raise typer.Exit(1)
|
|
1139
|
+
|
|
1140
|
+
meta = read_local_meta(Path(src)) or {}
|
|
1141
|
+
ci_branch = (
|
|
1142
|
+
branch
|
|
1143
|
+
or os.environ.get("GITHUB_HEAD_REF")
|
|
1144
|
+
or os.environ.get("GITHUB_REF_NAME")
|
|
1145
|
+
or meta.get("branch")
|
|
1146
|
+
or meta.get("ref")
|
|
1147
|
+
)
|
|
1148
|
+
if not ci_branch:
|
|
1149
|
+
typer.echo(
|
|
1150
|
+
"no branch to run: pass --branch, or run under GitHub Actions "
|
|
1151
|
+
"(GITHUB_HEAD_REF / GITHUB_REF_NAME), or push the dir once so "
|
|
1152
|
+
".gateway records one.",
|
|
1153
|
+
err=True,
|
|
1154
|
+
)
|
|
1155
|
+
raise typer.Exit(1)
|
|
1156
|
+
|
|
1157
|
+
bench = GatewayBenchmarkClient.from_env(host)
|
|
1158
|
+
if no_push:
|
|
1159
|
+
# Run exactly what the dir's base names — refuse a drifted tree, a
|
|
1160
|
+
# pin to code that did not run would fabricate provenance.
|
|
1161
|
+
state = local_tree_state(Path(src))
|
|
1162
|
+
if not (state["clean"] and meta.get("versionId")):
|
|
1163
|
+
typer.echo(
|
|
1164
|
+
"--no-push needs a clean checkout with a recorded base " "(.gateway) — push first, or drop --no-push.",
|
|
1165
|
+
err=True,
|
|
1166
|
+
)
|
|
1167
|
+
raise typer.Exit(1)
|
|
1168
|
+
version_id = str(meta["versionId"])
|
|
1169
|
+
slug = str(meta.get("slug") or os.path.basename(src))
|
|
1170
|
+
content_hash = str(meta.get("contentHash") or "")
|
|
1171
|
+
else:
|
|
1172
|
+
sha = os.environ.get("GITHUB_SHA", "")[:12]
|
|
1173
|
+
repo = os.environ.get("GITHUB_REPOSITORY", "")
|
|
1174
|
+
push_message = message or (f"[ci] {repo}@{sha}" if repo and sha else "[ci] working tree")
|
|
1175
|
+
# Force: the hub branch MIRRORS this checkout (the PR's code). A
|
|
1176
|
+
# concurrent tip on the same branch is by definition stale here.
|
|
1177
|
+
pushed = bench.push(src, branch=ci_branch, force=True, message=push_message)
|
|
1178
|
+
version_id = str(pushed["versionId"])
|
|
1179
|
+
slug = str(pushed["slug"])
|
|
1180
|
+
content_hash = str(pushed.get("contentHash") or "")
|
|
1181
|
+
typer.echo(f"pushed {slug}@{ci_branch} ({content_hash[:8]})", err=True)
|
|
1182
|
+
|
|
1183
|
+
bar_source = "--fail-under"
|
|
1184
|
+
if fail_under is None:
|
|
1185
|
+
try:
|
|
1186
|
+
resolved = bench.resolve_version(slug, ci_branch)
|
|
1187
|
+
world_bar = resolved.get("passThreshold")
|
|
1188
|
+
except Exception:
|
|
1189
|
+
world_bar = None
|
|
1190
|
+
if isinstance(world_bar, (int, float)):
|
|
1191
|
+
fail_under, bar_source = float(world_bar), "world pass bar"
|
|
1192
|
+
else:
|
|
1193
|
+
fail_under, bar_source = 0.7, "default bar"
|
|
1194
|
+
|
|
1195
|
+
harbor_model = resolve_harbor_model(model, agent)
|
|
1196
|
+
jobs_dir = os.path.join(output_dir, "harbor-jobs", f"ci--{slug}--{ci_branch.replace('/', '-')}")
|
|
1197
|
+
run_harbor_job(
|
|
1198
|
+
tasks_root(src),
|
|
1199
|
+
jobs_dir,
|
|
1200
|
+
model=harbor_model,
|
|
1201
|
+
agent=agent,
|
|
1202
|
+
num_examples=num_examples,
|
|
1203
|
+
rollouts_per_example=rollouts,
|
|
1204
|
+
extra_env={"OPENROUTER_API_KEY": api_key},
|
|
1205
|
+
mcp_config=find_mcp_config(src),
|
|
1206
|
+
)
|
|
1207
|
+
result = harvest_job_dir(
|
|
1208
|
+
jobs_dir,
|
|
1209
|
+
version_id,
|
|
1210
|
+
GatewayEvalsClient.from_env(host),
|
|
1211
|
+
model=harbor_model,
|
|
1212
|
+
rollouts_per_example=rollouts,
|
|
1213
|
+
)
|
|
1214
|
+
result["branch"] = ci_branch
|
|
1215
|
+
result["versionId"] = version_id
|
|
1216
|
+
typer.echo(json.dumps(result, indent=2))
|
|
1217
|
+
_ci_finish(
|
|
1218
|
+
result,
|
|
1219
|
+
fail_under=fail_under,
|
|
1220
|
+
bar_source=bar_source,
|
|
1221
|
+
title=f"World sim · {slug}@{ci_branch}",
|
|
1222
|
+
facts=[
|
|
1223
|
+
("model", harbor_model),
|
|
1224
|
+
("agent", agent),
|
|
1225
|
+
("version", content_hash[:8] or version_id[:8]),
|
|
1226
|
+
],
|
|
1227
|
+
)
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
@worlds_app.command("run")
|
|
1231
|
+
def worlds_run(
|
|
1232
|
+
slug: Optional[str] = typer.Argument(
|
|
1233
|
+
None,
|
|
1234
|
+
help="World slug@ref to pull and run against; omit when using --path.",
|
|
1235
|
+
),
|
|
1236
|
+
path: Optional[str] = typer.Option(
|
|
1237
|
+
None,
|
|
1238
|
+
"--path",
|
|
1239
|
+
"-p",
|
|
1240
|
+
help="Local world directory instead of a hub pull (a `bench checkout`, or any bundle dir). A clean checkout files runs pinned to its base version; a drifted or unlinked dir runs unfiled.",
|
|
1241
|
+
),
|
|
1242
|
+
variant: Optional[str] = typer.Option(None, "--variant", "-v", help="World variation (task variant) to load."),
|
|
1243
|
+
agent: str = typer.Option(
|
|
1244
|
+
...,
|
|
1245
|
+
"--agent",
|
|
1246
|
+
"-a",
|
|
1247
|
+
help="Agent callable: file.py[:fn] or module:fn (fn defaults to 'agent'). Called as fn(world); use world.tools / world.wrap inside.",
|
|
1248
|
+
),
|
|
1249
|
+
model: str = typer.Option(
|
|
1250
|
+
"sdk-dev-loop",
|
|
1251
|
+
"--model",
|
|
1252
|
+
"-m",
|
|
1253
|
+
help="Model label recorded on the filed run.",
|
|
1254
|
+
),
|
|
1255
|
+
no_file: bool = typer.Option(False, "--no-file", help="Run and grade locally without filing an evaluation."),
|
|
1256
|
+
fail_under: Optional[float] = typer.Option(
|
|
1257
|
+
None,
|
|
1258
|
+
"--fail-under",
|
|
1259
|
+
min=0.0,
|
|
1260
|
+
max=1.0,
|
|
1261
|
+
help="CI gate: exit 1 when the graded reward lands below this bar. Writes a $GITHUB_STEP_SUMMARY table under GitHub Actions.",
|
|
1262
|
+
),
|
|
1263
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1264
|
+
) -> None:
|
|
1265
|
+
"""Run an agent function against one variation of a world and (by
|
|
1266
|
+
default) file the graded result as a version-pinned, branch-tagged run —
|
|
1267
|
+
the CLI face of ``gatewaysdk.worlds``:
|
|
1268
|
+
|
|
1269
|
+
gateway worlds run google-drive-sim@fix/retention-lock \
|
|
1270
|
+
--variant alice --agent my_agent.py -m "claude-sonnet-4-5"
|
|
1271
|
+
|
|
1272
|
+
gateway worlds run --path ./google-drive-sim --variant alice \
|
|
1273
|
+
--agent my_agent.py
|
|
1274
|
+
|
|
1275
|
+
The agent is any callable taking the loaded world; its tool calls go
|
|
1276
|
+
through the world's handlers over the variation's seeded database, and
|
|
1277
|
+
the task's own grader scores the end state.
|
|
1278
|
+
"""
|
|
1279
|
+
from gatewaysdk import worlds
|
|
1280
|
+
from gatewaysdk.worlds import WorldError
|
|
1281
|
+
|
|
1282
|
+
if (slug is None) == (path is None):
|
|
1283
|
+
typer.echo("pass exactly one of SLUG or --path", err=True)
|
|
1284
|
+
raise typer.Exit(1)
|
|
1285
|
+
|
|
1286
|
+
agent_fn = _resolve_agent_callable(agent)
|
|
1287
|
+
try:
|
|
1288
|
+
world = (
|
|
1289
|
+
worlds.load(slug, variant=variant, host=host)
|
|
1290
|
+
if slug is not None
|
|
1291
|
+
else worlds.load_path(path, variant=variant)
|
|
1292
|
+
)
|
|
1293
|
+
except WorldError as exc:
|
|
1294
|
+
typer.echo(str(exc), err=True)
|
|
1295
|
+
raise typer.Exit(1)
|
|
1296
|
+
|
|
1297
|
+
if world.file_block and not no_file:
|
|
1298
|
+
# Say it BEFORE the agent runs, not after: the run still executes and
|
|
1299
|
+
# grades, only the filing is off.
|
|
1300
|
+
typer.echo(f"note: running unfiled — {world.file_block}", err=True)
|
|
1301
|
+
try:
|
|
1302
|
+
result = world.run(
|
|
1303
|
+
agent_fn,
|
|
1304
|
+
model=model,
|
|
1305
|
+
file=not no_file and world.version_id is not None,
|
|
1306
|
+
host=host,
|
|
1307
|
+
)
|
|
1308
|
+
except WorldError as exc:
|
|
1309
|
+
typer.echo(str(exc), err=True)
|
|
1310
|
+
raise typer.Exit(1)
|
|
1311
|
+
result["world"] = {
|
|
1312
|
+
"slug": world.slug,
|
|
1313
|
+
"branch": world.branch,
|
|
1314
|
+
"versionId": world.version_id,
|
|
1315
|
+
"variant": world.variant,
|
|
1316
|
+
"filed": "evaluationId" in result,
|
|
1317
|
+
}
|
|
1318
|
+
typer.echo(json.dumps(result, indent=2))
|
|
1319
|
+
_ci_finish(
|
|
1320
|
+
result,
|
|
1321
|
+
fail_under=fail_under,
|
|
1322
|
+
title=f"World run · {world.slug or 'local'}"
|
|
1323
|
+
+ (f"@{world.branch}" if world.branch else "")
|
|
1324
|
+
+ f" · {world.variant}",
|
|
1325
|
+
facts=[("model", model)],
|
|
1326
|
+
)
|
|
1327
|
+
|
|
1328
|
+
|
|
1329
|
+
session_app = typer.Typer(
|
|
1330
|
+
help="Live world sessions with api/ui/browser surfaces (the CLI face of gatewaysdk.world_sessions).",
|
|
1331
|
+
no_args_is_help=True,
|
|
1332
|
+
)
|
|
1333
|
+
worlds_app.add_typer(session_app, name="session")
|
|
1334
|
+
|
|
1335
|
+
schema_app = typer.Typer(help="Compile and check offline executable world schemas")
|
|
1336
|
+
worlds_app.add_typer(schema_app, name="schema")
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
@schema_app.command("init")
|
|
1340
|
+
def schema_init(
|
|
1341
|
+
path: Path = typer.Argument(..., help="New or empty directory for the world"),
|
|
1342
|
+
connector: str = typer.Option(
|
|
1343
|
+
"custom",
|
|
1344
|
+
help="Connector template slug; `custom` is the blank contract, `workspace:<slug>` a workspace connector "
|
|
1345
|
+
"pulled from the platform. `gateway worlds schema templates` lists what is installed.",
|
|
1346
|
+
),
|
|
1347
|
+
name: Optional[str] = typer.Option(None, help="Environment name and package id"),
|
|
1348
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST", help="For workspace: connectors"),
|
|
1349
|
+
):
|
|
1350
|
+
"""Initialize a schema world from a connector template, then compile it."""
|
|
1351
|
+
from gatewaysdk.environment.schema.__main__ import main
|
|
1352
|
+
|
|
1353
|
+
if connector.startswith("workspace:"):
|
|
1354
|
+
_schema_init_from_workspace(path, connector[len("workspace:") :], name, host)
|
|
1355
|
+
return
|
|
1356
|
+
argv = ["init", str(path), "--connector", connector]
|
|
1357
|
+
if name:
|
|
1358
|
+
argv += ["--name", name]
|
|
1359
|
+
main(argv)
|
|
1360
|
+
|
|
1361
|
+
|
|
1362
|
+
def _schema_init_from_workspace(path: Path, slug: str, name: Optional[str], host: Optional[str]) -> None:
|
|
1363
|
+
"""The template comes from the platform's copy of the connector, files inline."""
|
|
1364
|
+
from gatewaysdk.connectors import GatewayConnectorsError
|
|
1365
|
+
from gatewaysdk.environment.schema import ContractError
|
|
1366
|
+
from gatewaysdk.environment.schema.scaffold import init_bundle_from_files
|
|
1367
|
+
|
|
1368
|
+
if not slug:
|
|
1369
|
+
raise typer.BadParameter("workspace:<slug> needs the connector's slug")
|
|
1370
|
+
try:
|
|
1371
|
+
pulled = _connectors_client(host).pull(slug)
|
|
1372
|
+
except GatewayConnectorsError as error:
|
|
1373
|
+
typer.echo(str(error), err=True)
|
|
1374
|
+
raise typer.Exit(1)
|
|
1375
|
+
files = {relative: content.encode("utf-8") for relative, content in pulled["files"].items()}
|
|
1376
|
+
try:
|
|
1377
|
+
contract = init_bundle_from_files(path, files, connector=slug, name=name)
|
|
1378
|
+
except ContractError as error:
|
|
1379
|
+
typer.echo(json.dumps({"status": "rejected", "error": error.detail}, sort_keys=True))
|
|
1380
|
+
raise typer.Exit(1)
|
|
1381
|
+
commit = pulled["commit"].get("commitHash") or pulled["commit"].get("id")
|
|
1382
|
+
typer.echo(
|
|
1383
|
+
json.dumps(
|
|
1384
|
+
{"status": "ok", "domain": contract.target, "connector": {"slug": slug, "commit": commit}}, sort_keys=True
|
|
1385
|
+
)
|
|
1386
|
+
)
|
|
1387
|
+
|
|
1388
|
+
|
|
1389
|
+
@schema_app.command("templates")
|
|
1390
|
+
def schema_templates():
|
|
1391
|
+
"""List the installed connector templates; `custom` is always offered last."""
|
|
1392
|
+
from gatewaysdk.environment.schema import list_templates
|
|
1393
|
+
|
|
1394
|
+
for slug in list_templates():
|
|
1395
|
+
typer.echo(slug)
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
@schema_app.command("compile")
|
|
1399
|
+
def schema_compile(path: Path = typer.Argument(...)):
|
|
1400
|
+
from gatewaysdk.environment.schema.__main__ import main
|
|
1401
|
+
|
|
1402
|
+
main(["compile", str(path)])
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
@schema_app.command("check")
|
|
1406
|
+
def schema_check(path: Path = typer.Argument(...)):
|
|
1407
|
+
from gatewaysdk.environment.schema.__main__ import main
|
|
1408
|
+
|
|
1409
|
+
main(["check", str(path)])
|
|
1410
|
+
|
|
1411
|
+
|
|
1412
|
+
@schema_app.command("describe")
|
|
1413
|
+
def schema_describe(path: Path = typer.Argument(...)):
|
|
1414
|
+
from gatewaysdk.environment.schema.__main__ import main
|
|
1415
|
+
|
|
1416
|
+
main(["describe", str(path)])
|
|
1417
|
+
|
|
1418
|
+
|
|
1419
|
+
connector_app = typer.Typer(help="Describe a vendor connection, capture what it returns, seed the world from it")
|
|
1420
|
+
worlds_app.add_typer(connector_app, name="connector")
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
def _pairs(values: List[str], what: str) -> Dict[str, str]:
|
|
1424
|
+
out: Dict[str, str] = {}
|
|
1425
|
+
for item in values:
|
|
1426
|
+
if "=" not in item:
|
|
1427
|
+
raise typer.BadParameter(f"{what} must look like name=value, got {item!r}")
|
|
1428
|
+
key, value = item.split("=", 1)
|
|
1429
|
+
out[key.strip()] = value
|
|
1430
|
+
return out
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
@connector_app.command("validate")
|
|
1434
|
+
def connector_validate(path: Path = typer.Argument(..., help="World directory holding connector.toml")):
|
|
1435
|
+
"""Parse connector.toml and list its operations."""
|
|
1436
|
+
from gatewaysdk.environment.schema.connector import load_spec
|
|
1437
|
+
|
|
1438
|
+
spec = load_spec(path)
|
|
1439
|
+
typer.echo(f"{spec.slug} {spec.base_url} auth={spec.scheme}")
|
|
1440
|
+
for op in spec.operations.values():
|
|
1441
|
+
target = f" -> {op.entity}" if op.entity else ""
|
|
1442
|
+
typer.echo(f" {op.name}: {op.method} {op.path}{target}")
|
|
1443
|
+
|
|
1444
|
+
|
|
1445
|
+
@connector_app.command("capture")
|
|
1446
|
+
def connector_capture(
|
|
1447
|
+
path: Path = typer.Argument(..., help="World directory holding connector.toml"),
|
|
1448
|
+
op: str = typer.Option(..., "--op", help="Operation name from connector.toml"),
|
|
1449
|
+
arg: List[str] = typer.Option([], "--arg", help="Path argument, name=value (repeatable)"),
|
|
1450
|
+
param: List[str] = typer.Option([], "--param", help="Query parameter, name=value (repeatable)"),
|
|
1451
|
+
max_pages: Optional[int] = typer.Option(None, help="Override [capture].max_pages"),
|
|
1452
|
+
remote: bool = typer.Option(
|
|
1453
|
+
False,
|
|
1454
|
+
"--remote",
|
|
1455
|
+
help="Run the capture on the platform (from its allow-listed egress, with the environment's "
|
|
1456
|
+
"secret) and pull the pages into captures/ here.",
|
|
1457
|
+
),
|
|
1458
|
+
environment: Optional[str] = typer.Option(
|
|
1459
|
+
None, "--environment", help="Environment slug or id for --remote (default: the world's directory name)"
|
|
1460
|
+
),
|
|
1461
|
+
timeout: int = typer.Option(900, "--timeout", help="Seconds to wait for a --remote capture"),
|
|
1462
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1463
|
+
):
|
|
1464
|
+
"""Call the live connection once (following its cursor) and record every page under captures/.
|
|
1465
|
+
|
|
1466
|
+
The credential comes from the environment variable named in [auth].secret_env and is
|
|
1467
|
+
never written to disk; recorded bodies are redacted per [redact]. With --remote the
|
|
1468
|
+
platform makes the calls instead — from the pod whose IP the vendor allow-listed, with
|
|
1469
|
+
the credential held as the environment's secret (`gateway bench secrets set`) — and
|
|
1470
|
+
the same redacted pages land in captures/ here, ready for `connector ingest`.
|
|
1471
|
+
"""
|
|
1472
|
+
if remote:
|
|
1473
|
+
written = _remote_capture(
|
|
1474
|
+
path, op, _pairs(arg, "--arg"), _pairs(param, "--param"), max_pages, environment, timeout, host
|
|
1475
|
+
)
|
|
1476
|
+
else:
|
|
1477
|
+
from gatewaysdk.environment.schema.connector import capture
|
|
1478
|
+
|
|
1479
|
+
written = capture(path, op, _pairs(arg, "--arg"), _pairs(param, "--param"), max_pages=max_pages)
|
|
1480
|
+
for file in written:
|
|
1481
|
+
typer.echo(file.relative_to(path).as_posix())
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
def _remote_capture(
|
|
1485
|
+
world: Path,
|
|
1486
|
+
op: str,
|
|
1487
|
+
args: Dict[str, str],
|
|
1488
|
+
params: Dict[str, str],
|
|
1489
|
+
max_pages: Optional[int],
|
|
1490
|
+
environment: Optional[str],
|
|
1491
|
+
timeout: int,
|
|
1492
|
+
host: Optional[str],
|
|
1493
|
+
) -> List[Path]:
|
|
1494
|
+
"""The platform runs the operation; the pages come home through the file drive."""
|
|
1495
|
+
import time as _time
|
|
1496
|
+
|
|
1497
|
+
from gatewaysdk.environment.schema.connector import CAPTURES, MANIFEST, load_spec
|
|
1498
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
1499
|
+
|
|
1500
|
+
load_spec(world) # the same refusals the platform would give, before a round trip
|
|
1501
|
+
spec_text = (world / "connector.toml").read_text(encoding="utf-8")
|
|
1502
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
1503
|
+
container_id = _resolve_environment_id(client, environment or world.resolve().name)
|
|
1504
|
+
request_id = client.start_connector_capture(
|
|
1505
|
+
container_id, connector_toml=spec_text, op=op, args=args, params=params, max_pages=max_pages
|
|
1506
|
+
)
|
|
1507
|
+
typer.echo(f"# remote capture {request_id} on {container_id}", err=True)
|
|
1508
|
+
deadline = _time.time() + timeout
|
|
1509
|
+
while True:
|
|
1510
|
+
view = client.get_connector_capture(container_id, request_id)
|
|
1511
|
+
stage = view.get("stage")
|
|
1512
|
+
if stage == "complete":
|
|
1513
|
+
break
|
|
1514
|
+
if stage in ("failed", "cancelled"):
|
|
1515
|
+
typer.echo(f"Remote capture {stage}: {view.get('error') or 'no detail'}", err=True)
|
|
1516
|
+
raise typer.Exit(1)
|
|
1517
|
+
if _time.time() >= deadline:
|
|
1518
|
+
typer.echo(
|
|
1519
|
+
f"Remote capture still {stage} after {timeout}s; poll it with the request id {request_id}", err=True
|
|
1520
|
+
)
|
|
1521
|
+
raise typer.Exit(1)
|
|
1522
|
+
_time.sleep(5)
|
|
1523
|
+
slug_prefix = f"worlds/{view['worldSlug']}/"
|
|
1524
|
+
written: List[Path] = []
|
|
1525
|
+
for file in view.get("files", []):
|
|
1526
|
+
drive_path = str(file["path"])
|
|
1527
|
+
relative = drive_path[len(slug_prefix) :] if drive_path.startswith(slug_prefix) else drive_path
|
|
1528
|
+
if not relative.startswith(f"{CAPTURES}/") or ".." in relative:
|
|
1529
|
+
typer.echo(f"Refusing a capture outside captures/: {drive_path}", err=True)
|
|
1530
|
+
raise typer.Exit(1)
|
|
1531
|
+
destination = world / relative
|
|
1532
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
1533
|
+
destination.write_bytes(client.read_file(container_id, drive_path))
|
|
1534
|
+
written.append(destination)
|
|
1535
|
+
manifest_path = world / CAPTURES / MANIFEST
|
|
1536
|
+
existing = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.is_file() else []
|
|
1537
|
+
known = {entry.get("file") for entry in existing if isinstance(entry, dict)}
|
|
1538
|
+
for entry in view.get("manifest", []):
|
|
1539
|
+
if isinstance(entry, dict) and entry.get("file") not in known:
|
|
1540
|
+
existing.append(entry)
|
|
1541
|
+
known.add(entry.get("file"))
|
|
1542
|
+
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1543
|
+
manifest_path.write_text(json.dumps(existing, indent=1, sort_keys=True) + "\n", encoding="utf-8")
|
|
1544
|
+
return written
|
|
1545
|
+
|
|
1546
|
+
|
|
1547
|
+
@connector_app.command("conform")
|
|
1548
|
+
def connector_conform(
|
|
1549
|
+
path: Path = typer.Argument(..., help="World directory holding connector.toml"),
|
|
1550
|
+
openapi: list[Path] = typer.Option(
|
|
1551
|
+
..., "--openapi", help="The vendor's OpenAPI document (JSON or YAML); repeatable"
|
|
1552
|
+
),
|
|
1553
|
+
out: Optional[Path] = typer.Option(None, "--out", help="Write the full report as JSON here"),
|
|
1554
|
+
limit: Optional[int] = typer.Option(None, "--limit", help="Check only the first N operations"),
|
|
1555
|
+
):
|
|
1556
|
+
"""Serve the world and check every operation's answer against the vendor's OpenAPI:
|
|
1557
|
+
the documented status, and a body the vendor's response schema accepts."""
|
|
1558
|
+
from gatewaysdk.environment.schema.conform import conform
|
|
1559
|
+
|
|
1560
|
+
report = conform(path, [str(p) for p in openapi], max_operations=limit)
|
|
1561
|
+
for outcome in report.outcomes:
|
|
1562
|
+
if not outcome.documented:
|
|
1563
|
+
mark = " ? "
|
|
1564
|
+
elif outcome.ok:
|
|
1565
|
+
mark = " ok "
|
|
1566
|
+
else:
|
|
1567
|
+
mark = " FAIL"
|
|
1568
|
+
detail = "" if outcome.documented else "not in the document"
|
|
1569
|
+
if outcome.documented and outcome.status is not None and str(outcome.status) not in outcome.expected:
|
|
1570
|
+
detail = f"status {outcome.status}, vendor documents {', '.join(outcome.expected)}"
|
|
1571
|
+
elif outcome.problems:
|
|
1572
|
+
detail = outcome.problems[0] + (f" (+{len(outcome.problems) - 1})" if len(outcome.problems) > 1 else "")
|
|
1573
|
+
typer.echo(f"{mark} {outcome.method:6} {outcome.path:60} {outcome.operation} {detail}")
|
|
1574
|
+
typer.echo(json.dumps(report.summary))
|
|
1575
|
+
if out is not None:
|
|
1576
|
+
out.write_text(json.dumps(report.to_json(), indent=1) + "\n")
|
|
1577
|
+
typer.echo(f"report: {out}")
|
|
1578
|
+
summary = report.summary
|
|
1579
|
+
raise typer.Exit(1 if summary["status_mismatch"] or summary["schema_violations"] else 0)
|
|
1580
|
+
|
|
1581
|
+
|
|
1582
|
+
@connector_app.command("ingest")
|
|
1583
|
+
def connector_ingest(
|
|
1584
|
+
path: Path = typer.Argument(..., help="World directory holding captures/"),
|
|
1585
|
+
batch_id: Optional[str] = typer.Option(None, help="Batch id; defaults to the next initial version"),
|
|
1586
|
+
):
|
|
1587
|
+
"""Seed data/initial.json from the successful captures, gated by the world's contract."""
|
|
1588
|
+
from gatewaysdk.environment.schema.connector import ingest
|
|
1589
|
+
|
|
1590
|
+
result = ingest(path, batch_id=batch_id)
|
|
1591
|
+
typer.echo(f"batch {result['batch']}: {result['mutations']} mutations")
|
|
1592
|
+
for entity, count in sorted(result["entity_counts"].items()):
|
|
1593
|
+
typer.echo(f" {entity}: {count}")
|
|
1594
|
+
|
|
1595
|
+
|
|
1596
|
+
# ── workspace connectors: the template as an object on the platform ──────────
|
|
1597
|
+
|
|
1598
|
+
|
|
1599
|
+
def _connectors_client(host: Optional[str]):
|
|
1600
|
+
from gatewaysdk.connectors import GatewayConnectorsClient
|
|
1601
|
+
|
|
1602
|
+
return GatewayConnectorsClient.from_env(host)
|
|
1603
|
+
|
|
1604
|
+
|
|
1605
|
+
@connector_app.command("publish")
|
|
1606
|
+
def connector_publish(
|
|
1607
|
+
path: Path = typer.Argument(..., help="A connector template directory, or the world directory it was authored in"),
|
|
1608
|
+
slug: Optional[str] = typer.Option(None, "--slug", help="Connector slug (default: the directory name)"),
|
|
1609
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Commit message"),
|
|
1610
|
+
description: Optional[str] = typer.Option(None, "--description", help="One-line description shown in listings"),
|
|
1611
|
+
tag: List[str] = typer.Option([], "--tag", help="Tag (repeatable)"),
|
|
1612
|
+
workspace_visible: bool = typer.Option(
|
|
1613
|
+
False, "--workspace-visible", help="Visible to the whole workspace, not only this project's members"
|
|
1614
|
+
),
|
|
1615
|
+
project_scope: bool = typer.Option(False, "--project-scope", help="Own it in this project rather than the org"),
|
|
1616
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1617
|
+
):
|
|
1618
|
+
"""Save the template as a workspace connector: created on first publish, a new commit after.
|
|
1619
|
+
|
|
1620
|
+
A world directory is mapped back into the template layout (schema/world.json → world.json,
|
|
1621
|
+
tools/handlers.py → handlers.py, …); data/, captures/, db/ and generated files stay behind.
|
|
1622
|
+
The platform compiles the template before accepting it and refuses one that does not make a
|
|
1623
|
+
world. The published connector equals the directory: paths no longer present are removed.
|
|
1624
|
+
"""
|
|
1625
|
+
from gatewaysdk.connectors import GatewayConnectorsError, collect_template
|
|
1626
|
+
|
|
1627
|
+
try:
|
|
1628
|
+
gathered = collect_template(path)
|
|
1629
|
+
except (FileNotFoundError, ValueError) as error:
|
|
1630
|
+
typer.echo(str(error), err=True)
|
|
1631
|
+
raise typer.Exit(1)
|
|
1632
|
+
slug = slug or path.resolve().name
|
|
1633
|
+
client = _connectors_client(host)
|
|
1634
|
+
files: Dict[str, Optional[str]] = dict(gathered.files)
|
|
1635
|
+
parent: Optional[str] = None
|
|
1636
|
+
try:
|
|
1637
|
+
if client.exists(slug):
|
|
1638
|
+
head = client.pull(slug)
|
|
1639
|
+
for stale in sorted(set(head["files"]) - set(files)):
|
|
1640
|
+
files[stale] = None
|
|
1641
|
+
typer.echo(f"# removing {stale}", err=True)
|
|
1642
|
+
head_hash = head["commit"].get("commitHash")
|
|
1643
|
+
parent = head_hash if isinstance(head_hash, str) and len(head_hash) >= 8 else None
|
|
1644
|
+
for left in gathered.skipped:
|
|
1645
|
+
typer.echo(f"# left behind: {left}", err=True)
|
|
1646
|
+
result = client.push(
|
|
1647
|
+
slug,
|
|
1648
|
+
files,
|
|
1649
|
+
message=message or f"publish {slug}",
|
|
1650
|
+
parent_commit=parent,
|
|
1651
|
+
description=description,
|
|
1652
|
+
tags=tag or None,
|
|
1653
|
+
workspace_visible=workspace_visible,
|
|
1654
|
+
scope="project" if project_scope else "org",
|
|
1655
|
+
)
|
|
1656
|
+
except GatewayConnectorsError as error:
|
|
1657
|
+
typer.echo(str(error), err=True)
|
|
1658
|
+
raise typer.Exit(1)
|
|
1659
|
+
commit = result.get("commitHash") or result.get("commitId") or ""
|
|
1660
|
+
typer.echo(f"{result.get('status', 'committed')} {slug} {commit} {result.get('url', '')}".rstrip())
|
|
1661
|
+
|
|
1662
|
+
|
|
1663
|
+
def _table(rows: List[List[str]], header: List[str]) -> str:
|
|
1664
|
+
widths = [max(len(str(cell)) for cell in column) for column in zip(header, *rows)]
|
|
1665
|
+
lines = [header] + rows
|
|
1666
|
+
return "\n".join(" ".join(str(cell).ljust(width) for cell, width in zip(line, widths)).rstrip() for line in lines)
|
|
1667
|
+
|
|
1668
|
+
|
|
1669
|
+
@connector_app.command("list")
|
|
1670
|
+
def connector_list(
|
|
1671
|
+
hub: bool = typer.Option(False, "--hub", help="Every connector this organization can use, published ones included"),
|
|
1672
|
+
query: Optional[str] = typer.Option(None, "--query", help="Filter by slug, name or description"),
|
|
1673
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1674
|
+
):
|
|
1675
|
+
"""List this organization's workspace connectors (--hub adds the ones other organizations published)."""
|
|
1676
|
+
from gatewaysdk.connectors import GatewayConnectorsError
|
|
1677
|
+
|
|
1678
|
+
try:
|
|
1679
|
+
rows = _connectors_client(host).hub(query)
|
|
1680
|
+
except GatewayConnectorsError as error:
|
|
1681
|
+
typer.echo(str(error), err=True)
|
|
1682
|
+
raise typer.Exit(1)
|
|
1683
|
+
if not hub:
|
|
1684
|
+
rows = [row for row in rows if row.get("isOwn")]
|
|
1685
|
+
if not rows:
|
|
1686
|
+
typer.echo(
|
|
1687
|
+
"no connectors" + (" in the hub" if hub else "; `gateway worlds connector publish` adds one"), err=True
|
|
1688
|
+
)
|
|
1689
|
+
return
|
|
1690
|
+
header = ["slug", "name", "vendor", "auth", "entities", "operations", "visibility"] + (["org"] if hub else [])
|
|
1691
|
+
table = []
|
|
1692
|
+
for row in rows:
|
|
1693
|
+
meta = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
|
|
1694
|
+
line = [
|
|
1695
|
+
row.get("slug", ""),
|
|
1696
|
+
row.get("name", ""),
|
|
1697
|
+
meta.get("vendor") or "-",
|
|
1698
|
+
meta.get("auth") or "-",
|
|
1699
|
+
str(meta.get("entityCount", "-")),
|
|
1700
|
+
str(meta.get("operationCount", "-")),
|
|
1701
|
+
row.get("visibility", ""),
|
|
1702
|
+
]
|
|
1703
|
+
if hub:
|
|
1704
|
+
line.append(f"{row.get('orgName', '')}{' (own)' if row.get('isOwn') else ''}")
|
|
1705
|
+
table.append(line)
|
|
1706
|
+
typer.echo(_table(table, header))
|
|
1707
|
+
|
|
1708
|
+
|
|
1709
|
+
@connector_app.command("pull")
|
|
1710
|
+
def connector_pull(
|
|
1711
|
+
slug: str = typer.Argument(..., help="Workspace connector slug"),
|
|
1712
|
+
out: Optional[Path] = typer.Option(None, "--out", help="New or empty directory (default: ./<slug>)"),
|
|
1713
|
+
version: Optional[str] = typer.Option(None, "--version", help="A label or commit hash; the head by default"),
|
|
1714
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1715
|
+
):
|
|
1716
|
+
"""Write the connector's template files to a directory."""
|
|
1717
|
+
from gatewaysdk.connectors import GatewayConnectorsError
|
|
1718
|
+
|
|
1719
|
+
out = out or Path(slug)
|
|
1720
|
+
if out.exists() and (not out.is_dir() or any(out.iterdir())):
|
|
1721
|
+
typer.echo(f"{out}: must be a new or empty directory", err=True)
|
|
1722
|
+
raise typer.Exit(1)
|
|
1723
|
+
try:
|
|
1724
|
+
pulled = _connectors_client(host).pull(slug, version)
|
|
1725
|
+
except GatewayConnectorsError as error:
|
|
1726
|
+
typer.echo(str(error), err=True)
|
|
1727
|
+
raise typer.Exit(1)
|
|
1728
|
+
for relative, content in sorted(pulled["files"].items()):
|
|
1729
|
+
if relative.startswith("/") or ".." in relative.split("/"):
|
|
1730
|
+
typer.echo(f"Refusing a template path outside the directory: {relative}", err=True)
|
|
1731
|
+
raise typer.Exit(1)
|
|
1732
|
+
target = out / relative
|
|
1733
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1734
|
+
target.write_text(content, encoding="utf-8")
|
|
1735
|
+
typer.echo(relative)
|
|
1736
|
+
typer.echo(f"# {slug} @ {pulled['commit'].get('commitHash') or pulled['commit'].get('id', '')} -> {out}", err=True)
|
|
1737
|
+
|
|
1738
|
+
|
|
1739
|
+
@connector_app.command("clone")
|
|
1740
|
+
def connector_clone(
|
|
1741
|
+
source: str = typer.Argument(..., help="A hub connector's slug or repo id"),
|
|
1742
|
+
name: Optional[str] = typer.Option(None, "--name", help="Name for the clone (default: the source's)"),
|
|
1743
|
+
slug: Optional[str] = typer.Option(None, "--slug", help="Slug for the clone (default: the source's)"),
|
|
1744
|
+
project_scope: bool = typer.Option(False, "--project-scope", help="Own it in this project rather than the org"),
|
|
1745
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
1746
|
+
):
|
|
1747
|
+
"""Copy a connector another organization published into this organization, with provenance."""
|
|
1748
|
+
from gatewaysdk.connectors import GatewayConnectorsError
|
|
1749
|
+
|
|
1750
|
+
client = _connectors_client(host)
|
|
1751
|
+
try:
|
|
1752
|
+
rows = client.hub()
|
|
1753
|
+
by_id = [row for row in rows if row.get("id") == source]
|
|
1754
|
+
if by_id:
|
|
1755
|
+
chosen = by_id[0]
|
|
1756
|
+
else:
|
|
1757
|
+
matches = [row for row in rows if row.get("slug") == source]
|
|
1758
|
+
candidates = [row for row in matches if not row.get("isOwn")] or matches
|
|
1759
|
+
if not candidates:
|
|
1760
|
+
typer.echo(f"{source}: not in the hub; `gateway worlds connector list --hub` shows what is", err=True)
|
|
1761
|
+
raise typer.Exit(1)
|
|
1762
|
+
if len(candidates) > 1:
|
|
1763
|
+
typer.echo(f"{source} names {len(candidates)} hub connectors; clone by repo id instead:", err=True)
|
|
1764
|
+
for row in candidates:
|
|
1765
|
+
typer.echo(f" {row.get('id')} {row.get('orgName')}", err=True)
|
|
1766
|
+
raise typer.Exit(1)
|
|
1767
|
+
chosen = candidates[0]
|
|
1768
|
+
result = client.clone(chosen["id"], name=name, slug=slug, scope="project" if project_scope else "org")
|
|
1769
|
+
except GatewayConnectorsError as error:
|
|
1770
|
+
typer.echo(str(error), err=True)
|
|
1771
|
+
raise typer.Exit(1)
|
|
1772
|
+
typer.echo(
|
|
1773
|
+
f"cloned {chosen.get('slug')} from {chosen.get('orgName')} -> {result.get('slug')} "
|
|
1774
|
+
f"({result.get('id')}) commit {result.get('commitId')}"
|
|
1775
|
+
)
|
|
1776
|
+
|
|
1777
|
+
|
|
1778
|
+
def _print_or_install_skill(name: str, install: Optional[Path], force: bool) -> None:
|
|
1779
|
+
from gatewaysdk.connectors.skill import install_skill, next_step, packaged_skills, skill_text
|
|
1780
|
+
|
|
1781
|
+
try:
|
|
1782
|
+
text = skill_text(name)
|
|
1783
|
+
except KeyError:
|
|
1784
|
+
typer.echo(f"No packaged skill {name!r}; the package ships: {', '.join(packaged_skills())}", err=True)
|
|
1785
|
+
raise typer.Exit(1)
|
|
1786
|
+
if install is None:
|
|
1787
|
+
typer.echo(text, nl=False)
|
|
1788
|
+
return
|
|
1789
|
+
report = install_skill(install, name=name, force=force)
|
|
1790
|
+
if report.conflicts:
|
|
1791
|
+
typer.echo("Refusing to overwrite files that differ from the packaged skill:", err=True)
|
|
1792
|
+
for relative in report.conflicts:
|
|
1793
|
+
typer.echo(f" {install / relative}", err=True)
|
|
1794
|
+
typer.echo("Pass --force to replace them.", err=True)
|
|
1795
|
+
raise typer.Exit(1)
|
|
1796
|
+
for relative in report.written:
|
|
1797
|
+
typer.echo(f"wrote {install / relative}")
|
|
1798
|
+
for relative in report.unchanged:
|
|
1799
|
+
typer.echo(f"unchanged {install / relative}")
|
|
1800
|
+
step = next_step(name)
|
|
1801
|
+
if step:
|
|
1802
|
+
typer.echo(f"next: {step}")
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
@connector_app.command("skill")
|
|
1806
|
+
def connector_skill(
|
|
1807
|
+
install: Optional[Path] = typer.Option(
|
|
1808
|
+
None, "--install", help="Repository to install the skill and agent into (under its .claude/)"
|
|
1809
|
+
),
|
|
1810
|
+
force: bool = typer.Option(False, "--force", help="Overwrite an existing file whose content differs"),
|
|
1811
|
+
):
|
|
1812
|
+
"""The connector authoring skill for agents: print it, or --install it into a repository's .claude/.
|
|
1813
|
+
|
|
1814
|
+
An alias of `gateway worlds skill connector-schema-authoring`. Installed, Claude Code, Cursor and
|
|
1815
|
+
Codex pick up the checklist and the connector-template-author agent; the finished template becomes
|
|
1816
|
+
a workspace connector with `gateway worlds connector publish`.
|
|
1817
|
+
"""
|
|
1818
|
+
from gatewaysdk.connectors.skill import SKILL_NAME
|
|
1819
|
+
|
|
1820
|
+
_print_or_install_skill(SKILL_NAME, install, force)
|
|
1821
|
+
|
|
1822
|
+
|
|
1823
|
+
@worlds_app.command("skill")
|
|
1824
|
+
def worlds_skill(
|
|
1825
|
+
name: Optional[str] = typer.Argument(
|
|
1826
|
+
None, help="Packaged skill: connector-schema-authoring, world-data-ingestion, worlds-getting-started"
|
|
1827
|
+
),
|
|
1828
|
+
list_skills: bool = typer.Option(False, "--list", help="List the skills this SDK ships and exit"),
|
|
1829
|
+
install: Optional[Path] = typer.Option(
|
|
1830
|
+
None,
|
|
1831
|
+
"--install",
|
|
1832
|
+
help="Repository to install the skill (and any agent it ships) into, under .claude/; every skill when no NAME",
|
|
1833
|
+
),
|
|
1834
|
+
force: bool = typer.Option(False, "--force", help="Overwrite an existing file whose content differs"),
|
|
1835
|
+
):
|
|
1836
|
+
"""Print a packaged skill for an agent, or --install it into a repository's .claude/:
|
|
1837
|
+
|
|
1838
|
+
gateway worlds skill --list
|
|
1839
|
+
gateway worlds skill worlds-getting-started
|
|
1840
|
+
gateway worlds skill world-data-ingestion --install .
|
|
1841
|
+
gateway worlds skill --install . # every packaged skill
|
|
1842
|
+
"""
|
|
1843
|
+
from gatewaysdk.connectors.skill import packaged_skills
|
|
1844
|
+
|
|
1845
|
+
if list_skills or (name is None and install is None):
|
|
1846
|
+
for skill in packaged_skills():
|
|
1847
|
+
typer.echo(skill)
|
|
1848
|
+
if name is None and not list_skills:
|
|
1849
|
+
typer.echo("Name one to print it, or --install DIR to land every skill in a repository.", err=True)
|
|
1850
|
+
return
|
|
1851
|
+
if name is None:
|
|
1852
|
+
# `--install DIR` alone: every packaged skill, so an agent needs one command.
|
|
1853
|
+
for skill in packaged_skills():
|
|
1854
|
+
_print_or_install_skill(skill, install, force)
|
|
1855
|
+
return
|
|
1856
|
+
_print_or_install_skill(name, install, force)
|
|
1857
|
+
|
|
1858
|
+
|
|
1859
|
+
data_app = typer.Typer(help="Put rows into a world, or see what it holds — local directory or platform slug")
|
|
1860
|
+
worlds_app.add_typer(data_app, name="data")
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
def _with_world(world: str, host: Optional[str]):
|
|
1864
|
+
"""Yield a directory for ``world``: itself when it is one, else the platform world of that
|
|
1865
|
+
slug pulled into a temporary directory (so a slug never needs a checkout to work on).
|
|
1866
|
+
"""
|
|
1867
|
+
import contextlib
|
|
1868
|
+
import tempfile as _tempfile
|
|
1869
|
+
|
|
1870
|
+
@contextlib.contextmanager
|
|
1871
|
+
def local():
|
|
1872
|
+
yield Path(world), None
|
|
1873
|
+
|
|
1874
|
+
@contextlib.contextmanager
|
|
1875
|
+
def pulled():
|
|
1876
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
1877
|
+
|
|
1878
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
1879
|
+
with _tempfile.TemporaryDirectory(prefix="gateway-world-data-") as directory:
|
|
1880
|
+
slug = world.split("@", 1)[0]
|
|
1881
|
+
pulled_dir = Path(directory) / slug
|
|
1882
|
+
client.pull(world, target=str(pulled_dir))
|
|
1883
|
+
yield pulled_dir, client
|
|
1884
|
+
|
|
1885
|
+
return local() if Path(world).is_dir() else pulled()
|
|
1886
|
+
|
|
1887
|
+
|
|
1888
|
+
def _is_directory(world: str) -> bool:
|
|
1889
|
+
return Path(world).is_dir()
|
|
1890
|
+
|
|
1891
|
+
|
|
1892
|
+
def _world_slug(root: Path) -> str:
|
|
1893
|
+
"""The platform slug a local world would import into: its pulled metadata, else its name."""
|
|
1894
|
+
meta = root / ".gateway" / ".env-metadata.json"
|
|
1895
|
+
if meta.is_file():
|
|
1896
|
+
try:
|
|
1897
|
+
slug = json.loads(meta.read_text(encoding="utf-8")).get("slug")
|
|
1898
|
+
if isinstance(slug, str) and slug:
|
|
1899
|
+
return slug
|
|
1900
|
+
except (OSError, ValueError):
|
|
1901
|
+
pass
|
|
1902
|
+
return root.resolve().name
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
def _data_client(host: Optional[str]):
|
|
1906
|
+
from gatewaysdk.world_data import WorldDataClient
|
|
1907
|
+
|
|
1908
|
+
return WorldDataClient.from_env(host)
|
|
1909
|
+
|
|
1910
|
+
|
|
1911
|
+
def _local_check(
|
|
1912
|
+
root: Path, rows: List[Path], entity: Optional[str], mode: str, merge: str, atomic: bool
|
|
1913
|
+
) -> Dict[str, Any]:
|
|
1914
|
+
"""Validate rows files against a world directory's contract in a scratch snapshot:
|
|
1915
|
+
the shape of every row, then references and keys at apply time. Applies nothing."""
|
|
1916
|
+
import shutil as _shutil
|
|
1917
|
+
import tempfile as _tempfile
|
|
1918
|
+
|
|
1919
|
+
from gatewaysdk.environment.schema.batch import ChunkSpec, build_batch, chunk_rows, iter_rows
|
|
1920
|
+
from gatewaysdk.environment.schema.compiler import check_bundle
|
|
1921
|
+
from gatewaysdk.environment.schema.snapshot import EMPTY_DIGEST, build_default, default_snapshot
|
|
1922
|
+
|
|
1923
|
+
contract = check_bundle(root)
|
|
1924
|
+
with _tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
|
|
1925
|
+
work = Path(scratch)
|
|
1926
|
+
chunks = list(chunk_rows(iter_rows([str(r) for r in rows], entity=entity), work / "chunks", entity=entity))
|
|
1927
|
+
if not chunks:
|
|
1928
|
+
return {"rows": 0, "refusals": [], "refusalCount": 0, "entityCounts": {}}
|
|
1929
|
+
build_default(root, contract)
|
|
1930
|
+
parent = work / "state.sqlite"
|
|
1931
|
+
_shutil.copyfile(default_snapshot(root), parent)
|
|
1932
|
+
result = build_batch(
|
|
1933
|
+
contract,
|
|
1934
|
+
parent,
|
|
1935
|
+
[ChunkSpec(chunk.ordinal, chunk.path, chunk.entity) for chunk in chunks],
|
|
1936
|
+
mode=mode,
|
|
1937
|
+
merge=merge,
|
|
1938
|
+
atomic=atomic,
|
|
1939
|
+
batch_id="check",
|
|
1940
|
+
parent_digest=EMPTY_DIGEST,
|
|
1941
|
+
dry_run=True,
|
|
1942
|
+
)
|
|
1943
|
+
return {
|
|
1944
|
+
"rows": sum(chunk.rows for chunk in chunks),
|
|
1945
|
+
"refusals": result["refusals"],
|
|
1946
|
+
"refusalCount": result["refusalCount"],
|
|
1947
|
+
"entityCounts": result["entityCounts"],
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
|
|
1951
|
+
def _write_refusal_report(records: List[Dict[str, Any]], destination: Path) -> None:
|
|
1952
|
+
from gatewaysdk.environment.schema.batch import write_report
|
|
1953
|
+
|
|
1954
|
+
with open(destination, "w", encoding="utf-8") as out:
|
|
1955
|
+
write_report(records, out)
|
|
1956
|
+
|
|
1957
|
+
|
|
1958
|
+
def _pulled_world(world: str, host: Optional[str]):
|
|
1959
|
+
"""A platform world's tree pulled into a temp dir (its contract is what a local
|
|
1960
|
+
check needs); the context yields the directory."""
|
|
1961
|
+
import contextlib
|
|
1962
|
+
import tempfile as _tempfile
|
|
1963
|
+
|
|
1964
|
+
@contextlib.contextmanager
|
|
1965
|
+
def pulled():
|
|
1966
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
1967
|
+
|
|
1968
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
1969
|
+
with _tempfile.TemporaryDirectory(prefix="gateway-world-data-") as directory:
|
|
1970
|
+
pulled_dir = Path(directory) / world.split("@", 1)[0]
|
|
1971
|
+
client.pull(world, target=str(pulled_dir))
|
|
1972
|
+
yield pulled_dir
|
|
1973
|
+
|
|
1974
|
+
return pulled()
|
|
1975
|
+
|
|
1976
|
+
|
|
1977
|
+
@data_app.command("import")
|
|
1978
|
+
def data_import(
|
|
1979
|
+
world: str = typer.Argument(..., help="World directory, or a platform world slug (optionally slug@ref)."),
|
|
1980
|
+
rows: List[str] = typer.Argument(
|
|
1981
|
+
..., help="Rows files ({entity: [rows]} JSON, {entity,row} JSONL, bare rows), or - for stdin."
|
|
1982
|
+
),
|
|
1983
|
+
entity: Optional[str] = typer.Option(None, "--entity", help="The entity bare-row files hold."),
|
|
1984
|
+
mode: str = typer.Option("append", "--mode", help="append (upsert by primary key) | replace (named entities)."),
|
|
1985
|
+
merge: str = typer.Option(
|
|
1986
|
+
"upsert", "--merge", help="Inside the batch, by key: upsert (last wins) | fill (first wins, later add fields)."
|
|
1987
|
+
),
|
|
1988
|
+
chunk_rows: Optional[int] = typer.Option(
|
|
1989
|
+
None, "--chunk-rows", help="Rows per uploaded chunk (instead of --chunk-bytes)."
|
|
1990
|
+
),
|
|
1991
|
+
chunk_bytes: Optional[str] = typer.Option(
|
|
1992
|
+
None, "--chunk-bytes", help="JSONL bytes per uploaded chunk before gzip (default 64MiB)."
|
|
1993
|
+
),
|
|
1994
|
+
parallel: int = typer.Option(4, "--parallel", help="Chunk uploads in flight."),
|
|
1995
|
+
resume: bool = typer.Option(
|
|
1996
|
+
False, "--resume", help="Pick up the newest incomplete upload of this world and these files."
|
|
1997
|
+
),
|
|
1998
|
+
atomic: bool = typer.Option(
|
|
1999
|
+
True, "--atomic/--per-chunk", help="One refusal refuses the batch (atomic) or only its chunk."
|
|
2000
|
+
),
|
|
2001
|
+
publish: str = typer.Option("now", "--publish", help="now: a version per batch | later: fold with `data publish`."),
|
|
2002
|
+
wait_: bool = typer.Option(
|
|
2003
|
+
True, "--wait/--no-wait", help="Wait for the batch to apply (or print {batchId, state})."
|
|
2004
|
+
),
|
|
2005
|
+
report: Optional[Path] = typer.Option(None, "--report", help="Write the refusals as JSONL here."),
|
|
2006
|
+
dry_run: bool = typer.Option(
|
|
2007
|
+
False, "--dry-run", help="Validate locally against the contract; upload and write nothing."
|
|
2008
|
+
),
|
|
2009
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Change reason for the version."),
|
|
2010
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2011
|
+
) -> None:
|
|
2012
|
+
"""Put rows into a world through its own gate:
|
|
2013
|
+
|
|
2014
|
+
gateway worlds data import ./acme-world customers.json --mode append
|
|
2015
|
+
gateway worlds data import acme-world export.jsonl --entity customers --chunk-bytes 64MiB -m "Q3 customers"
|
|
2016
|
+
|
|
2017
|
+
A directory: the rows go into the tree (data/initial.json for a schema world, db/seed.sql for
|
|
2018
|
+
mock-tools), small inputs only. A slug: the rows stream up as gzipped chunks into a data batch
|
|
2019
|
+
the platform validates, applies onto the world's snapshot and (with --publish now) turns into a
|
|
2020
|
+
data-only version. Exit 0 applied or published, 2 refused (report written), 1 error.
|
|
2021
|
+
"""
|
|
2022
|
+
from gatewaysdk.environment.errors import WorldStartError
|
|
2023
|
+
from gatewaysdk.environment.schema import ContractError
|
|
2024
|
+
from gatewaysdk.environment.schema.data import INLINE_BYTES, INLINE_ROWS, MERGES, MODES, import_rows, parse_rows
|
|
2025
|
+
from gatewaysdk.world_data import Flags, WorldDataError, tty_progress
|
|
2026
|
+
|
|
2027
|
+
if mode not in MODES or merge not in MERGES or publish not in ("now", "later"):
|
|
2028
|
+
typer.echo("--mode is append|replace, --merge is upsert|fill, --publish is now|later", err=True)
|
|
2029
|
+
raise typer.Exit(1)
|
|
2030
|
+
try:
|
|
2031
|
+
size = _parse_size(chunk_bytes) if chunk_bytes else None
|
|
2032
|
+
if _is_directory(world) or dry_run:
|
|
2033
|
+
root_ctx = _local_root(world, host)
|
|
2034
|
+
with root_ctx as root:
|
|
2035
|
+
if dry_run:
|
|
2036
|
+
checked = _local_check(root, [Path(r) for r in rows], entity, mode, merge, atomic)
|
|
2037
|
+
if report is not None:
|
|
2038
|
+
_write_refusal_report(checked["refusals"], report)
|
|
2039
|
+
typer.echo(json.dumps({"world": world, "dryRun": True, **_summary(checked)}, indent=2))
|
|
2040
|
+
raise typer.Exit(2 if checked["refusalCount"] else 0)
|
|
2041
|
+
_refuse_large_local(root, [Path(r) for r in rows], entity, INLINE_ROWS, INLINE_BYTES)
|
|
2042
|
+
parsed = parse_rows([Path(r) for r in rows], entity=entity)
|
|
2043
|
+
result = import_rows(root, parsed, mode=mode)
|
|
2044
|
+
typer.echo(json.dumps({"world": world, **result}, indent=2))
|
|
2045
|
+
return
|
|
2046
|
+
client = _data_client(host)
|
|
2047
|
+
progress = tty_progress()
|
|
2048
|
+
from gatewaysdk.world_data import import_rows as upload_rows
|
|
2049
|
+
from gatewaysdk.world_data import wait as wait_batch
|
|
2050
|
+
from gatewaysdk.world_data import write_status_report
|
|
2051
|
+
|
|
2052
|
+
flags = Flags(
|
|
2053
|
+
mode=mode,
|
|
2054
|
+
merge=merge,
|
|
2055
|
+
atomic=atomic,
|
|
2056
|
+
publish=publish,
|
|
2057
|
+
dryRun=False,
|
|
2058
|
+
chunkBytes=size or Flags().chunkBytes,
|
|
2059
|
+
chunkRows=chunk_rows,
|
|
2060
|
+
entity=entity,
|
|
2061
|
+
)
|
|
2062
|
+
manifest = upload_rows(
|
|
2063
|
+
client, world, list(rows), flags=flags, parallel=parallel, resume=resume, message=message, progress=progress
|
|
2064
|
+
)
|
|
2065
|
+
if not wait_:
|
|
2066
|
+
typer.echo(json.dumps({"batchId": manifest.batch_id, "state": "queued"}))
|
|
2067
|
+
return
|
|
2068
|
+
status = wait_batch(client, world, manifest.batch_id, progress=progress)
|
|
2069
|
+
if report is not None:
|
|
2070
|
+
write_status_report(client, status, report)
|
|
2071
|
+
typer.echo(json.dumps({"world": world, **status}, indent=2))
|
|
2072
|
+
state = status.get("state")
|
|
2073
|
+
raise typer.Exit(0 if state in ("applied", "published") else 2 if state == "refused" else 1)
|
|
2074
|
+
except (ContractError, WorldStartError, WorldDataError, OSError, ValueError) as exc:
|
|
2075
|
+
typer.echo(str(exc), err=True)
|
|
2076
|
+
raise typer.Exit(1)
|
|
2077
|
+
|
|
2078
|
+
|
|
2079
|
+
def _local_root(world: str, host: Optional[str]):
|
|
2080
|
+
import contextlib
|
|
2081
|
+
|
|
2082
|
+
@contextlib.contextmanager
|
|
2083
|
+
def local():
|
|
2084
|
+
yield Path(world)
|
|
2085
|
+
|
|
2086
|
+
return local() if _is_directory(world) else _pulled_world(world, host)
|
|
2087
|
+
|
|
2088
|
+
|
|
2089
|
+
def _parse_size(text: str) -> int:
|
|
2090
|
+
units = {
|
|
2091
|
+
"": 1,
|
|
2092
|
+
"b": 1,
|
|
2093
|
+
"k": 1024,
|
|
2094
|
+
"kib": 1024,
|
|
2095
|
+
"kb": 1000,
|
|
2096
|
+
"m": 1024**2,
|
|
2097
|
+
"mib": 1024**2,
|
|
2098
|
+
"mb": 1000**2,
|
|
2099
|
+
"g": 1024**3,
|
|
2100
|
+
"gib": 1024**3,
|
|
2101
|
+
"gb": 1000**3,
|
|
2102
|
+
}
|
|
2103
|
+
raw = text.strip().lower().replace(" ", "")
|
|
2104
|
+
digits = raw.rstrip("abcdefghijklmnopqrstuvwxyz")
|
|
2105
|
+
unit = raw[len(digits) :]
|
|
2106
|
+
if not digits.isdigit() or unit not in units:
|
|
2107
|
+
raise ValueError(f"--chunk-bytes takes a size like 64MiB or 1000000, not {text!r}")
|
|
2108
|
+
return int(digits) * units[unit]
|
|
2109
|
+
|
|
2110
|
+
|
|
2111
|
+
def _summary(checked: Dict[str, Any]) -> Dict[str, Any]:
|
|
2112
|
+
return {
|
|
2113
|
+
"rows": checked["rows"],
|
|
2114
|
+
"refusalCount": checked["refusalCount"],
|
|
2115
|
+
"refusals": checked["refusals"][:200],
|
|
2116
|
+
"entityCounts": checked["entityCounts"],
|
|
2117
|
+
}
|
|
2118
|
+
|
|
2119
|
+
|
|
2120
|
+
def _refuse_large_local(root: Path, rows: List[Path], entity: Optional[str], max_rows: int, max_bytes: int) -> None:
|
|
2121
|
+
from gatewaysdk.environment.schema.batch import count_rows
|
|
2122
|
+
|
|
2123
|
+
size = sum(path.stat().st_size for path in rows if str(path) != "-" and path.is_file())
|
|
2124
|
+
if size > max_bytes or count_rows([str(r) for r in rows], entity=entity, cap=max_rows) > max_rows:
|
|
2125
|
+
raise ValueError(
|
|
2126
|
+
f"{len(rows)} file(s) exceed the inline cap ({max_rows} rows / {max_bytes} bytes) for a directory; "
|
|
2127
|
+
f"import into the platform world `{_world_slug(root)}`"
|
|
2128
|
+
)
|
|
2129
|
+
|
|
2130
|
+
|
|
2131
|
+
@data_app.command("check")
|
|
2132
|
+
def data_check(
|
|
2133
|
+
world: str = typer.Argument(..., help="World directory, or a platform world slug (its contract is pulled)."),
|
|
2134
|
+
rows: List[str] = typer.Argument(..., help="Rows files, or - for stdin."),
|
|
2135
|
+
entity: Optional[str] = typer.Option(None, "--entity", help="The entity bare-row files hold."),
|
|
2136
|
+
mode: str = typer.Option("append", "--mode", help="append | replace, as the import would run."),
|
|
2137
|
+
merge: str = typer.Option("upsert", "--merge", help="upsert | fill, as the import would run."),
|
|
2138
|
+
report: Optional[Path] = typer.Option(None, "--report", help="Write the refusals as JSONL here."),
|
|
2139
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2140
|
+
) -> None:
|
|
2141
|
+
"""Dry run: every row checked against the world's contract (shape, keys, references) in a
|
|
2142
|
+
scratch snapshot; nothing uploaded, nothing written. Exit 2 when any row is refused.
|
|
2143
|
+
|
|
2144
|
+
gateway worlds data check acme-world export.jsonl --report refusals.jsonl
|
|
2145
|
+
"""
|
|
2146
|
+
from gatewaysdk.environment.schema import ContractError
|
|
2147
|
+
from gatewaysdk.world_data import WorldDataError
|
|
2148
|
+
|
|
2149
|
+
try:
|
|
2150
|
+
with _local_root(world, host) as root:
|
|
2151
|
+
checked = _local_check(root, [Path(r) for r in rows], entity, mode, merge, True)
|
|
2152
|
+
except (ContractError, WorldDataError, OSError, ValueError) as exc:
|
|
2153
|
+
typer.echo(str(exc), err=True)
|
|
2154
|
+
raise typer.Exit(1)
|
|
2155
|
+
if report is not None:
|
|
2156
|
+
_write_refusal_report(checked["refusals"], report)
|
|
2157
|
+
typer.echo(json.dumps({"world": world, "dryRun": True, **_summary(checked)}, indent=2))
|
|
2158
|
+
raise typer.Exit(2 if checked["refusalCount"] else 0)
|
|
2159
|
+
|
|
2160
|
+
|
|
2161
|
+
@data_app.command("extract")
|
|
2162
|
+
def data_extract(
|
|
2163
|
+
world: str = typer.Argument(..., help="World directory, or a platform world slug (its contract is pulled)."),
|
|
2164
|
+
source: str = typer.Argument(..., help="Input file, or - for stdin."),
|
|
2165
|
+
shape: str = typer.Option(..., "--shape", help="tool-results | vendor-envelope | session-export."),
|
|
2166
|
+
entity: str = typer.Option(..., "--entity", help="The entity the rows belong to."),
|
|
2167
|
+
out: Optional[str] = typer.Option(None, "--out", help="Write tagged JSONL here (default stdout, also -)."),
|
|
2168
|
+
merge: str = typer.Option("fill", "--merge", help="Dedupe by primary key: fill (first wins) | upsert (last wins)."),
|
|
2169
|
+
no_redact: bool = typer.Option(False, "--no-redact", help="Skip the world's [redact] policy."),
|
|
2170
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2171
|
+
) -> None:
|
|
2172
|
+
"""Rows out of a tool-results export, a vendor response or a session export, shaped for
|
|
2173
|
+
`data import`: null and empty fields dropped, undeclared fields stripped (and counted on
|
|
2174
|
+
stderr), deduped by primary key, the world's [redact] policy applied.
|
|
2175
|
+
|
|
2176
|
+
gateway worlds data extract ./spycloud-world calls.jsonl --shape session-export --entity records --out rows.jsonl
|
|
2177
|
+
"""
|
|
2178
|
+
import contextlib
|
|
2179
|
+
|
|
2180
|
+
from gatewaysdk.environment.schema import ContractError
|
|
2181
|
+
from gatewaysdk.environment.schema.extract import extract
|
|
2182
|
+
from gatewaysdk.world_data import WorldDataError
|
|
2183
|
+
|
|
2184
|
+
try:
|
|
2185
|
+
with _local_root(world, host) as root:
|
|
2186
|
+
sink = contextlib.nullcontext(sys.stdout) if out in (None, "-") else open(out, "w", encoding="utf-8")
|
|
2187
|
+
with sink as stream:
|
|
2188
|
+
summary = extract(
|
|
2189
|
+
root, [source], shape=shape, entity=entity, merge=merge, redaction=not no_redact, out=stream
|
|
2190
|
+
)
|
|
2191
|
+
except (ContractError, WorldDataError, OSError, ValueError) as exc:
|
|
2192
|
+
typer.echo(str(exc), err=True)
|
|
2193
|
+
raise typer.Exit(1)
|
|
2194
|
+
typer.echo(json.dumps(summary.to_json()), err=True)
|
|
2195
|
+
|
|
2196
|
+
|
|
2197
|
+
@data_app.command("status")
|
|
2198
|
+
def data_status(
|
|
2199
|
+
batch_id: str = typer.Argument(..., help="The batch id `data import` printed or recorded in .gateway/imports/."),
|
|
2200
|
+
world: Optional[str] = typer.Option(None, "--world", help="The world slug, when no manifest names it."),
|
|
2201
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2202
|
+
) -> None:
|
|
2203
|
+
"""One batch's state, progress, counts and first refusals."""
|
|
2204
|
+
from gatewaysdk.world_data import MANIFEST_DIR, Manifest, WorldDataError
|
|
2205
|
+
|
|
2206
|
+
slug = world
|
|
2207
|
+
if slug is None:
|
|
2208
|
+
manifest_path = Path.cwd() / MANIFEST_DIR / f"{batch_id}.json"
|
|
2209
|
+
if manifest_path.is_file():
|
|
2210
|
+
slug = Manifest.load(manifest_path).data.get("slug")
|
|
2211
|
+
if not slug:
|
|
2212
|
+
typer.echo(f"no manifest for {batch_id} under {MANIFEST_DIR}; pass --world SLUG", err=True)
|
|
2213
|
+
raise typer.Exit(1)
|
|
2214
|
+
try:
|
|
2215
|
+
typer.echo(json.dumps(_data_client(host).status(slug, batch_id), indent=2))
|
|
2216
|
+
except (WorldDataError, ValueError) as exc:
|
|
2217
|
+
typer.echo(str(exc), err=True)
|
|
2218
|
+
raise typer.Exit(1)
|
|
2219
|
+
|
|
2220
|
+
|
|
2221
|
+
@data_app.command("batches")
|
|
2222
|
+
def data_batches(
|
|
2223
|
+
world: str = typer.Argument(..., help="World slug."),
|
|
2224
|
+
limit: Optional[int] = typer.Option(None, "--limit", help="At most this many, newest first."),
|
|
2225
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2226
|
+
) -> None:
|
|
2227
|
+
"""The world's data batches, newest first."""
|
|
2228
|
+
from gatewaysdk.world_data import WorldDataError
|
|
2229
|
+
|
|
2230
|
+
try:
|
|
2231
|
+
typer.echo(json.dumps(_data_client(host).batches(world, limit=limit), indent=2))
|
|
2232
|
+
except (WorldDataError, ValueError) as exc:
|
|
2233
|
+
typer.echo(str(exc), err=True)
|
|
2234
|
+
raise typer.Exit(1)
|
|
2235
|
+
|
|
2236
|
+
|
|
2237
|
+
@data_app.command("publish")
|
|
2238
|
+
def data_publish(
|
|
2239
|
+
world: str = typer.Argument(..., help="World slug."),
|
|
2240
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Change reason for the version."),
|
|
2241
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2242
|
+
) -> None:
|
|
2243
|
+
"""Fold every applied `--publish later` batch into one data-only version."""
|
|
2244
|
+
from gatewaysdk.world_data import WorldDataError
|
|
2245
|
+
|
|
2246
|
+
try:
|
|
2247
|
+
typer.echo(json.dumps(_data_client(host).publish(world, message), indent=2))
|
|
2248
|
+
except (WorldDataError, ValueError) as exc:
|
|
2249
|
+
typer.echo(str(exc), err=True)
|
|
2250
|
+
raise typer.Exit(1)
|
|
2251
|
+
|
|
2252
|
+
|
|
2253
|
+
@data_app.command("show")
|
|
2254
|
+
def data_show(
|
|
2255
|
+
world: str = typer.Argument(..., help="World directory, or a platform world slug (optionally slug@ref)."),
|
|
2256
|
+
entity: Optional[str] = typer.Option(None, "--entity", help="Print this entity's rows instead of counts."),
|
|
2257
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2258
|
+
) -> None:
|
|
2259
|
+
"""What a world seeds with: every entity's row count, or one entity's rows:
|
|
2260
|
+
|
|
2261
|
+
gateway worlds data show ./acme-world
|
|
2262
|
+
gateway worlds data show acme-world --entity customers
|
|
2263
|
+
|
|
2264
|
+
A directory answers from its compiled snapshot when it has one; a slug asks the platform,
|
|
2265
|
+
which reads the world's snapshot.
|
|
2266
|
+
"""
|
|
2267
|
+
from gatewaysdk.environment.errors import WorldStartError
|
|
2268
|
+
from gatewaysdk.environment.schema import ContractError
|
|
2269
|
+
from gatewaysdk.environment.schema.data import entity_counts, entity_rows, world_kind
|
|
2270
|
+
from gatewaysdk.world_data import WorldDataError
|
|
2271
|
+
|
|
2272
|
+
try:
|
|
2273
|
+
if not _is_directory(world):
|
|
2274
|
+
answer = _data_client(host).counts(world, entity)
|
|
2275
|
+
out: Dict[str, Any] = {
|
|
2276
|
+
"world": world,
|
|
2277
|
+
"kind": answer.get("kind"),
|
|
2278
|
+
"entity_counts": answer.get("entityCounts"),
|
|
2279
|
+
}
|
|
2280
|
+
if entity is not None:
|
|
2281
|
+
out["entity"] = entity
|
|
2282
|
+
out["rows"] = answer.get("rows")
|
|
2283
|
+
typer.echo(json.dumps(out, indent=2))
|
|
2284
|
+
return
|
|
2285
|
+
root = Path(world)
|
|
2286
|
+
out = {"world": world, "kind": world_kind(root)}
|
|
2287
|
+
if entity is None:
|
|
2288
|
+
out["entity_counts"] = entity_counts(root)
|
|
2289
|
+
else:
|
|
2290
|
+
out["entity"] = entity
|
|
2291
|
+
out["rows"] = entity_rows(root, entity)
|
|
2292
|
+
except (ContractError, WorldStartError, WorldDataError, OSError, ValueError) as exc:
|
|
2293
|
+
typer.echo(str(exc), err=True)
|
|
2294
|
+
raise typer.Exit(1)
|
|
2295
|
+
typer.echo(json.dumps(out, indent=2))
|
|
2296
|
+
|
|
2297
|
+
|
|
2298
|
+
@worlds_app.command("serve")
|
|
2299
|
+
def worlds_serve(
|
|
2300
|
+
path: Path = typer.Argument(..., help="World directory holding connector.toml"),
|
|
2301
|
+
port: int = typer.Option(8080, help="Port to listen on"),
|
|
2302
|
+
api_key: Optional[str] = typer.Option(
|
|
2303
|
+
None, "--api-key", help="Require this credential; default accepts any non-empty one"
|
|
2304
|
+
),
|
|
2305
|
+
scenario: Optional[str] = typer.Option(None, "--scenario", help="Start in one of the world's scenarios/<name>"),
|
|
2306
|
+
):
|
|
2307
|
+
"""Serve the world's vendor API locally: its routes answered from its store, no platform needed."""
|
|
2308
|
+
from gatewaysdk.environment.schema.api import serve_forever
|
|
2309
|
+
|
|
2310
|
+
serve_forever(path, port=port, api_key=api_key, scenario=scenario)
|
|
2311
|
+
|
|
2312
|
+
|
|
2313
|
+
@schema_app.command("api-bundle")
|
|
2314
|
+
def schema_api_bundle(
|
|
2315
|
+
path: Path = typer.Argument(..., help="World directory holding connector.toml"),
|
|
2316
|
+
out: Path = typer.Option(..., "--out", help="Empty directory for the api-kind bundle"),
|
|
2317
|
+
):
|
|
2318
|
+
"""Write the api-kind bundle the platform hosts for this world. `gateway bench push` does this for you."""
|
|
2319
|
+
from gatewaysdk.environment.schema.api import build_api_bundle
|
|
2320
|
+
|
|
2321
|
+
typer.echo(f"api bundle: {build_api_bundle(path, out)}")
|
|
2322
|
+
|
|
2323
|
+
|
|
2324
|
+
def _read_task_spec(file: Path, flag: str = "--spec") -> Dict[str, Any]:
|
|
2325
|
+
"""A task spec from a JSON file: ``{instruction, seed?, grader?, metadata?, name?}``."""
|
|
2326
|
+
try:
|
|
2327
|
+
parsed = json.loads(file.read_text(encoding="utf-8"))
|
|
2328
|
+
except (OSError, ValueError) as exc:
|
|
2329
|
+
typer.echo(f"{flag} {file}: {exc}", err=True)
|
|
2330
|
+
raise typer.Exit(2)
|
|
2331
|
+
if not isinstance(parsed, dict) or not isinstance(parsed.get("instruction"), str):
|
|
2332
|
+
typer.echo(f'{flag} {file}: a task spec is a JSON object with an "instruction" string', err=True)
|
|
2333
|
+
raise typer.Exit(2)
|
|
2334
|
+
return parsed
|
|
2335
|
+
|
|
2336
|
+
|
|
2337
|
+
def _task_from_flags(
|
|
2338
|
+
task: Optional[str],
|
|
2339
|
+
task_id: Optional[str],
|
|
2340
|
+
task_version: Optional[int],
|
|
2341
|
+
task_file: Optional[Path],
|
|
2342
|
+
) -> Any:
|
|
2343
|
+
"""Exactly one task source: a bundled name (``--task``), a stored task
|
|
2344
|
+
(``--task-id``, optionally ``--task-version``) or an inline spec read from
|
|
2345
|
+
a JSON file (``--task-file``). Same refusals as the TS CLI."""
|
|
2346
|
+
if sum(1 for given in (task, task_id, task_file) if given) > 1:
|
|
2347
|
+
typer.echo("worlds session open takes one of --task, --task-id or --task-file", err=True)
|
|
2348
|
+
raise typer.Exit(2)
|
|
2349
|
+
if task_version is not None and not task_id:
|
|
2350
|
+
typer.echo("--task-version needs --task-id", err=True)
|
|
2351
|
+
raise typer.Exit(2)
|
|
2352
|
+
if task:
|
|
2353
|
+
return task
|
|
2354
|
+
if task_id:
|
|
2355
|
+
return {"id": task_id} if task_version is None else {"id": task_id, "version": task_version}
|
|
2356
|
+
if task_file is None:
|
|
2357
|
+
typer.echo("worlds session open needs --task, --task-id or --task-file", err=True)
|
|
2358
|
+
raise typer.Exit(2)
|
|
2359
|
+
return _read_task_spec(task_file, "--task-file")
|
|
2360
|
+
|
|
2361
|
+
|
|
2362
|
+
def _task_json(session: Any) -> Dict[str, Any]:
|
|
2363
|
+
task = session.task_info
|
|
2364
|
+
out: Dict[str, Any] = {"source": task.source, "name": task.name, "hash": task.hash, "metadata": task.metadata}
|
|
2365
|
+
if task.source == "stored":
|
|
2366
|
+
out["id"] = task.id
|
|
2367
|
+
out["version"] = task.version
|
|
2368
|
+
return out
|
|
2369
|
+
|
|
2370
|
+
|
|
2371
|
+
def _world_from_flags(world: Optional[str], no_world: bool) -> Any:
|
|
2372
|
+
"""``--world <slug>`` / ``--no-world`` → the API's ``world``: slug, ``None``
|
|
2373
|
+
to unpin, or the ``_UNSET`` marker when neither was given."""
|
|
2374
|
+
from gatewaysdk.world_sessions import _UNSET
|
|
2375
|
+
|
|
2376
|
+
if world and no_world:
|
|
2377
|
+
typer.echo("--world and --no-world exclude each other", err=True)
|
|
2378
|
+
raise typer.Exit(2)
|
|
2379
|
+
if no_world:
|
|
2380
|
+
return None
|
|
2381
|
+
return world if world else _UNSET
|
|
2382
|
+
|
|
2383
|
+
|
|
2384
|
+
def _camel_keys(value: Any) -> Any:
|
|
2385
|
+
"""``asdict`` output with snake_case field names back in the wire's camelCase.
|
|
2386
|
+
Only dataclass fields are renamed; a ``spec`` (customer JSON) is left alone."""
|
|
2387
|
+
|
|
2388
|
+
def camel(name: str) -> str:
|
|
2389
|
+
head, *rest = name.split("_")
|
|
2390
|
+
return head + "".join(part.title() for part in rest)
|
|
2391
|
+
|
|
2392
|
+
if isinstance(value, list):
|
|
2393
|
+
return [_camel_keys(item) for item in value]
|
|
2394
|
+
if isinstance(value, dict):
|
|
2395
|
+
return {camel(key): (value[key] if key == "spec" else _camel_keys(value[key])) for key in value}
|
|
2396
|
+
return value
|
|
2397
|
+
|
|
2398
|
+
|
|
2399
|
+
def _task_result_json(result: Any) -> Dict[str, Any]:
|
|
2400
|
+
"""A ``world_tasks`` dataclass as the platform's JSON."""
|
|
2401
|
+
return _camel_keys(asdict(result))
|
|
2402
|
+
|
|
2403
|
+
|
|
2404
|
+
def _surfaces_json(session: Any) -> Dict[str, Any]:
|
|
2405
|
+
s = session.surfaces
|
|
2406
|
+
out: Dict[str, Any] = {"sessionId": session.session_id, "surfaces": s.kinds}
|
|
2407
|
+
if s.api is not None:
|
|
2408
|
+
out["api"] = {"url": s.api.url}
|
|
2409
|
+
if s.api.token:
|
|
2410
|
+
out["api"]["token"] = s.api.token
|
|
2411
|
+
if s.ui is not None:
|
|
2412
|
+
out["ui"] = {"url": s.ui.url, "entry": s.ui.entry_url, "apiPath": s.ui.api_path}
|
|
2413
|
+
if s.browser is not None:
|
|
2414
|
+
out["browser"] = {"wsUrl": s.browser.ws_url, "viewport": s.browser.viewport}
|
|
2415
|
+
return out
|
|
2416
|
+
|
|
2417
|
+
|
|
2418
|
+
@session_app.command("open")
|
|
2419
|
+
def session_open(
|
|
2420
|
+
slug: str = typer.Argument(..., help="World slug (optionally slug@ref)."),
|
|
2421
|
+
task: Optional[str] = typer.Option(None, "--task", "-t", help="Bundled task / scenario to open the session on."),
|
|
2422
|
+
task_id: Optional[str] = typer.Option(
|
|
2423
|
+
None, "--task-id", help="Stored task (see `worlds task`) to open the session on instead of --task."
|
|
2424
|
+
),
|
|
2425
|
+
task_version: Optional[int] = typer.Option(
|
|
2426
|
+
None, "--task-version", help="With --task-id: open that version of the task instead of the current one."
|
|
2427
|
+
),
|
|
2428
|
+
task_file: Optional[Path] = typer.Option(
|
|
2429
|
+
None,
|
|
2430
|
+
"--task-file",
|
|
2431
|
+
help="JSON file holding an inline task spec {instruction, seed?, grader?, metadata?} (World Host worlds).",
|
|
2432
|
+
),
|
|
2433
|
+
surface: List[str] = typer.Option(
|
|
2434
|
+
[],
|
|
2435
|
+
"--surface",
|
|
2436
|
+
"-s",
|
|
2437
|
+
help="Surface to ask for besides tools: api, ui, browser (repeatable).",
|
|
2438
|
+
),
|
|
2439
|
+
version_id: Optional[str] = typer.Option(None, "--version-id", help="Pin an exact world version."),
|
|
2440
|
+
no_wait: bool = typer.Option(False, "--no-wait", help="Return as soon as the platform accepts the session."),
|
|
2441
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2442
|
+
) -> None:
|
|
2443
|
+
"""Open a live session and print where it answers:
|
|
2444
|
+
|
|
2445
|
+
gateway worlds session open payments-like -t refund-double-charge \\
|
|
2446
|
+
-s api -s ui -s browser
|
|
2447
|
+
|
|
2448
|
+
``--task-file task.json`` opens on an inline task instead of a bundled
|
|
2449
|
+
one: ``{instruction, seed?, grader?, metadata?}``, the session's own.
|
|
2450
|
+
``--task-id <id>`` opens a copy of a stored task's current (or
|
|
2451
|
+
``--task-version``) spec.
|
|
2452
|
+
|
|
2453
|
+
Prints ``{sessionId, ready, surfaces, api.url, ui.entry, browser.wsUrl,
|
|
2454
|
+
task}`` — the same descriptor ``WorldSession.surfaces`` parses, plus the
|
|
2455
|
+
platform's task block (source bundled|stored|inline, name, hash, id/version). The
|
|
2456
|
+
platform refuses (HTTP 400) rather than serving fewer surfaces than asked.
|
|
2457
|
+
"""
|
|
2458
|
+
from gatewaysdk.world_sessions import WorldSessionError, WorldSessionTimeout, open_session
|
|
2459
|
+
|
|
2460
|
+
task_ref = _task_from_flags(task, task_id, task_version, task_file)
|
|
2461
|
+
try:
|
|
2462
|
+
session = open_session(slug, task_ref, version_id=version_id, surfaces=surface or None, host=host)
|
|
2463
|
+
except (WorldSessionError, ValueError) as exc:
|
|
2464
|
+
typer.echo(str(exc), err=True)
|
|
2465
|
+
raise typer.Exit(1)
|
|
2466
|
+
|
|
2467
|
+
def descriptor(ready: bool) -> Dict[str, Any]:
|
|
2468
|
+
return {**_surfaces_json(session), "ready": ready, "task": _task_json(session)}
|
|
2469
|
+
|
|
2470
|
+
if not no_wait:
|
|
2471
|
+
try:
|
|
2472
|
+
session.ready()
|
|
2473
|
+
except WorldSessionTimeout as exc:
|
|
2474
|
+
# The session is still coming (a cold pod takes longer than this
|
|
2475
|
+
# wait): print where it will answer so the caller polls instead of
|
|
2476
|
+
# losing the id and opening another one.
|
|
2477
|
+
typer.echo(json.dumps(descriptor(False), indent=2))
|
|
2478
|
+
typer.echo(
|
|
2479
|
+
f"{exc} Poll `gateway worlds session status {session.session_id}` until ready.",
|
|
2480
|
+
err=True,
|
|
2481
|
+
)
|
|
2482
|
+
raise typer.Exit(1)
|
|
2483
|
+
except WorldSessionError as exc:
|
|
2484
|
+
typer.echo(str(exc), err=True)
|
|
2485
|
+
raise typer.Exit(1)
|
|
2486
|
+
typer.echo(json.dumps(descriptor(session.is_ready), indent=2))
|
|
2487
|
+
|
|
2488
|
+
|
|
2489
|
+
@session_app.command("status")
|
|
2490
|
+
def session_status(
|
|
2491
|
+
session_id: str = typer.Argument(...),
|
|
2492
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2493
|
+
) -> None:
|
|
2494
|
+
"""Print a session's state, tools and surfaces."""
|
|
2495
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2496
|
+
|
|
2497
|
+
try:
|
|
2498
|
+
session = attach_session(session_id, host=host)
|
|
2499
|
+
out = dict(session.status())
|
|
2500
|
+
out.update(_surfaces_json(session))
|
|
2501
|
+
typer.echo(json.dumps(out, indent=2))
|
|
2502
|
+
except WorldSessionError as exc:
|
|
2503
|
+
typer.echo(str(exc), err=True)
|
|
2504
|
+
raise typer.Exit(1)
|
|
2505
|
+
|
|
2506
|
+
|
|
2507
|
+
@session_app.command("state")
|
|
2508
|
+
def session_state(
|
|
2509
|
+
session_id: str = typer.Argument(...),
|
|
2510
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2511
|
+
) -> None:
|
|
2512
|
+
"""Dump the world's state right now — every entity's rows, what a grader reads."""
|
|
2513
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2514
|
+
|
|
2515
|
+
try:
|
|
2516
|
+
typer.echo(json.dumps(attach_session(session_id, host=host).state(), indent=2))
|
|
2517
|
+
except WorldSessionError as exc:
|
|
2518
|
+
typer.echo(str(exc), err=True)
|
|
2519
|
+
raise typer.Exit(1)
|
|
2520
|
+
|
|
2521
|
+
|
|
2522
|
+
@session_app.command("reset")
|
|
2523
|
+
def session_reset(
|
|
2524
|
+
session_id: str = typer.Argument(...),
|
|
2525
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2526
|
+
) -> None:
|
|
2527
|
+
"""Put the world back to its opening state (snapshot plus the task's seed)."""
|
|
2528
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2529
|
+
|
|
2530
|
+
try:
|
|
2531
|
+
attach_session(session_id, host=host).reset()
|
|
2532
|
+
except WorldSessionError as exc:
|
|
2533
|
+
typer.echo(str(exc), err=True)
|
|
2534
|
+
raise typer.Exit(1)
|
|
2535
|
+
typer.echo(json.dumps({"sessionId": session_id, "reset": True}, indent=2))
|
|
2536
|
+
|
|
2537
|
+
|
|
2538
|
+
@session_app.command("grade")
|
|
2539
|
+
def session_grade(
|
|
2540
|
+
session_id: str = typer.Argument(...),
|
|
2541
|
+
timeout: Optional[float] = typer.Option(
|
|
2542
|
+
None,
|
|
2543
|
+
"--timeout",
|
|
2544
|
+
help="Seconds to wait for the grade; defaults to the grader's budget (a rubric judge gets 15 min).",
|
|
2545
|
+
),
|
|
2546
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2547
|
+
) -> None:
|
|
2548
|
+
"""Run the session's task grader against the world's current state and print {reward, rewards, raw}.
|
|
2549
|
+
|
|
2550
|
+
Assertions and python graders answer from the engine in seconds. A rubric grader is
|
|
2551
|
+
judged by the platform's worker; the command waits for that verdict.
|
|
2552
|
+
"""
|
|
2553
|
+
from gatewaysdk.world_sessions import (
|
|
2554
|
+
WorldSessionError,
|
|
2555
|
+
attach_session,
|
|
2556
|
+
named_rewards,
|
|
2557
|
+
)
|
|
2558
|
+
|
|
2559
|
+
if timeout is not None and timeout <= 0:
|
|
2560
|
+
typer.echo("--timeout must be a positive number of seconds", err=True)
|
|
2561
|
+
raise typer.Exit(1)
|
|
2562
|
+
try:
|
|
2563
|
+
raw = attach_session(session_id, host=host).grade(timeout=timeout)
|
|
2564
|
+
except WorldSessionError as exc:
|
|
2565
|
+
typer.echo(str(exc), err=True)
|
|
2566
|
+
raise typer.Exit(1)
|
|
2567
|
+
reward = raw.get("reward")
|
|
2568
|
+
out = {
|
|
2569
|
+
"sessionId": session_id,
|
|
2570
|
+
"reward": reward if isinstance(reward, (int, float)) and not isinstance(reward, bool) else None,
|
|
2571
|
+
"rewards": named_rewards(raw),
|
|
2572
|
+
"raw": raw,
|
|
2573
|
+
}
|
|
2574
|
+
typer.echo(json.dumps(out, indent=2))
|
|
2575
|
+
|
|
2576
|
+
|
|
2577
|
+
@session_app.command("browse")
|
|
2578
|
+
def session_browse(
|
|
2579
|
+
session_id: str = typer.Argument(...),
|
|
2580
|
+
mode: str = typer.Option("auto", "--mode", help="hosted | local | auto (prefers hosted)."),
|
|
2581
|
+
path: Optional[str] = typer.Option(None, "--path", help="UI path to open instead of the declared entry."),
|
|
2582
|
+
screenshot: Optional[str] = typer.Option(None, "--screenshot", help="Write a PNG of the page here."),
|
|
2583
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2584
|
+
) -> None:
|
|
2585
|
+
"""Open the session's UI in its browser surface and print what the agent
|
|
2586
|
+
would read — a smoke test for the ui/browser surfaces:
|
|
2587
|
+
|
|
2588
|
+
gateway worlds session browse ws_123 --path /customers --screenshot out.png
|
|
2589
|
+
"""
|
|
2590
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2591
|
+
|
|
2592
|
+
try:
|
|
2593
|
+
session = attach_session(session_id, host=host)
|
|
2594
|
+
with session.browser(mode=mode) as browser:
|
|
2595
|
+
url = browser.goto(path) if path else browser.goto()
|
|
2596
|
+
if screenshot:
|
|
2597
|
+
Path(screenshot).write_bytes(browser.screenshot())
|
|
2598
|
+
typer.echo(json.dumps({"url": url, "title": browser.title(), "text": browser.text()}, indent=2))
|
|
2599
|
+
except WorldSessionError as exc:
|
|
2600
|
+
typer.echo(str(exc), err=True)
|
|
2601
|
+
raise typer.Exit(1)
|
|
2602
|
+
|
|
2603
|
+
|
|
2604
|
+
@session_app.command("close")
|
|
2605
|
+
def session_close(
|
|
2606
|
+
session_id: str = typer.Argument(...),
|
|
2607
|
+
keep_warm: bool = typer.Option(False, "--keep-warm", help="Hand the container back to the warm pool."),
|
|
2608
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2609
|
+
) -> None:
|
|
2610
|
+
"""End a session."""
|
|
2611
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2612
|
+
|
|
2613
|
+
try:
|
|
2614
|
+
typer.echo(json.dumps(attach_session(session_id, host=host).close(keep_warm=keep_warm), indent=2))
|
|
2615
|
+
except WorldSessionError as exc:
|
|
2616
|
+
typer.echo(str(exc), err=True)
|
|
2617
|
+
raise typer.Exit(1)
|
|
2618
|
+
|
|
2619
|
+
|
|
2620
|
+
@session_app.command("seed")
|
|
2621
|
+
def session_seed(
|
|
2622
|
+
session_id: str = typer.Argument(...),
|
|
2623
|
+
rows: List[Path] = typer.Argument(..., help="Rows files: {entity: [rows]} JSON, {entity,row} JSONL, or bare rows."),
|
|
2624
|
+
entity: Optional[str] = typer.Option(None, "--entity", help="The entity bare-row files hold."),
|
|
2625
|
+
mode: str = typer.Option("append", "--mode", help="append (upsert by primary key) | replace (named entities)."),
|
|
2626
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2627
|
+
) -> None:
|
|
2628
|
+
"""Put rows into a LIVE session's world and wait for the engine's answer:
|
|
2629
|
+
|
|
2630
|
+
gateway worlds session seed ws_123 customers.json --mode replace
|
|
2631
|
+
|
|
2632
|
+
Posts ``{kind: "seed", args: {mode, rows}}``; the engine applies the rows through the
|
|
2633
|
+
world's contract, so a refused row leaves the session unchanged and the error names it.
|
|
2634
|
+
"""
|
|
2635
|
+
from gatewaysdk.environment.schema import ContractError
|
|
2636
|
+
from gatewaysdk.environment.schema.data import parse_rows
|
|
2637
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2638
|
+
|
|
2639
|
+
try:
|
|
2640
|
+
parsed = parse_rows(rows, entity=entity)
|
|
2641
|
+
result = attach_session(session_id, host=host).seed(parsed, mode=mode)
|
|
2642
|
+
except (ContractError, WorldSessionError, ValueError, OSError) as exc:
|
|
2643
|
+
typer.echo(str(exc), err=True)
|
|
2644
|
+
raise typer.Exit(1)
|
|
2645
|
+
typer.echo(json.dumps({"sessionId": session_id, "mode": mode, "result": result}, indent=2))
|
|
2646
|
+
|
|
2647
|
+
|
|
2648
|
+
@session_app.command("export")
|
|
2649
|
+
def session_export(
|
|
2650
|
+
session_id: str = typer.Argument(...),
|
|
2651
|
+
out: Optional[Path] = typer.Option(None, "--out", help="Write the JSONL here instead of stdout."),
|
|
2652
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2653
|
+
) -> None:
|
|
2654
|
+
"""The session's call log as JSONL, one ``{seq, kind, tool, args, result, error, completedAt}`` per
|
|
2655
|
+
line, oldest first — turn a real run into rows for `gateway worlds data import`:
|
|
2656
|
+
|
|
2657
|
+
gateway worlds session export ws_123 --out run.jsonl
|
|
2658
|
+
"""
|
|
2659
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2660
|
+
|
|
2661
|
+
try:
|
|
2662
|
+
calls = attach_session(session_id, host=host).calls()
|
|
2663
|
+
except WorldSessionError as exc:
|
|
2664
|
+
typer.echo(str(exc), err=True)
|
|
2665
|
+
raise typer.Exit(1)
|
|
2666
|
+
keys = ("seq", "kind", "tool", "args", "result", "error", "completedAt")
|
|
2667
|
+
lines = [json.dumps({key: call.get(key) for key in keys}, sort_keys=True) for call in calls]
|
|
2668
|
+
text = "\n".join(lines) + ("\n" if lines else "")
|
|
2669
|
+
if out is None:
|
|
2670
|
+
typer.echo(text, nl=False)
|
|
2671
|
+
return
|
|
2672
|
+
out.write_text(text, encoding="utf-8")
|
|
2673
|
+
typer.echo(f"wrote {len(lines)} calls to {out}")
|
|
2674
|
+
|
|
2675
|
+
|
|
2676
|
+
session_task_app = typer.Typer(help="A session's task: promote an inline one into a stored task.", no_args_is_help=True)
|
|
2677
|
+
session_app.add_typer(session_task_app, name="task")
|
|
2678
|
+
|
|
2679
|
+
|
|
2680
|
+
@session_task_app.command("save")
|
|
2681
|
+
def session_task_save(
|
|
2682
|
+
session_id: str = typer.Argument(...),
|
|
2683
|
+
name: Optional[str] = typer.Option(None, "--name", "-n", help="Stored task name (default: the spec's own)."),
|
|
2684
|
+
world: Optional[str] = typer.Option(None, "--world", "-w", help="Pin the task to this world slug."),
|
|
2685
|
+
no_world: bool = typer.Option(False, "--no-world", help="Do not pin the task to the session's world."),
|
|
2686
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Version note."),
|
|
2687
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2688
|
+
) -> None:
|
|
2689
|
+
"""Store the session's inline task as a reusable project task (version 1):
|
|
2690
|
+
|
|
2691
|
+
gateway worlds session task save ws_123 -n refund-late-invoice
|
|
2692
|
+
|
|
2693
|
+
The session itself keeps running unchanged; later sessions open the
|
|
2694
|
+
stored copy with ``session open <world> --task-id <id>``.
|
|
2695
|
+
"""
|
|
2696
|
+
from gatewaysdk.world_sessions import WorldSessionError, attach_session
|
|
2697
|
+
|
|
2698
|
+
pin = _world_from_flags(world, no_world)
|
|
2699
|
+
try:
|
|
2700
|
+
saved = attach_session(session_id, host=host).save_task(name=name, world=pin, message=message)
|
|
2701
|
+
except WorldSessionError as exc:
|
|
2702
|
+
typer.echo(str(exc), err=True)
|
|
2703
|
+
raise typer.Exit(1)
|
|
2704
|
+
typer.echo(json.dumps(_task_result_json(saved), indent=2))
|
|
2705
|
+
|
|
2706
|
+
|
|
2707
|
+
task_app = typer.Typer(
|
|
2708
|
+
help="Stored world tasks: task specs kept by the project outside any bundle, versioned, opened by id.",
|
|
2709
|
+
no_args_is_help=True,
|
|
2710
|
+
)
|
|
2711
|
+
worlds_app.add_typer(task_app, name="task")
|
|
2712
|
+
|
|
2713
|
+
|
|
2714
|
+
@task_app.command("create")
|
|
2715
|
+
def task_create(
|
|
2716
|
+
spec: Path = typer.Argument(..., help="JSON file: {instruction, seed?, grader?, metadata?, name?}."),
|
|
2717
|
+
name: Optional[str] = typer.Option(None, "--name", "-n", help="Task name (default: the spec's name)."),
|
|
2718
|
+
world: Optional[str] = typer.Option(None, "--world", "-w", help="Pin the task to this world slug."),
|
|
2719
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Version note."),
|
|
2720
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2721
|
+
) -> None:
|
|
2722
|
+
"""Store a task spec as version 1 of a new project task:
|
|
2723
|
+
|
|
2724
|
+
gateway worlds task create task.json -n refund-late-invoice -w acme-billing
|
|
2725
|
+
"""
|
|
2726
|
+
from gatewaysdk.world_sessions import WorldSessionError
|
|
2727
|
+
from gatewaysdk.world_tasks import create_task
|
|
2728
|
+
|
|
2729
|
+
parsed = _read_task_spec(spec, "spec")
|
|
2730
|
+
try:
|
|
2731
|
+
created = create_task(parsed, name=name, world=world, message=message, host=host)
|
|
2732
|
+
except WorldSessionError as exc:
|
|
2733
|
+
typer.echo(str(exc), err=True)
|
|
2734
|
+
raise typer.Exit(1)
|
|
2735
|
+
typer.echo(json.dumps(_task_result_json(created), indent=2))
|
|
2736
|
+
|
|
2737
|
+
|
|
2738
|
+
@task_app.command("update")
|
|
2739
|
+
def task_update(
|
|
2740
|
+
task_id: str = typer.Argument(...),
|
|
2741
|
+
spec: Optional[Path] = typer.Option(None, "--spec", help="JSON file with the new spec → the next version."),
|
|
2742
|
+
name: Optional[str] = typer.Option(None, "--name", "-n", help="Rename the task."),
|
|
2743
|
+
world: Optional[str] = typer.Option(None, "--world", "-w", help="Pin the task to this world slug."),
|
|
2744
|
+
no_world: bool = typer.Option(False, "--no-world", help="Unpin the task from its world."),
|
|
2745
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Version note."),
|
|
2746
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2747
|
+
) -> None:
|
|
2748
|
+
"""Change a task. A new ``--spec`` appends a version (old ones stay openable
|
|
2749
|
+
with ``--task-version``); ``--name`` / ``--world`` / ``--no-world`` change in place."""
|
|
2750
|
+
from gatewaysdk.world_sessions import WorldSessionError
|
|
2751
|
+
from gatewaysdk.world_tasks import update_task
|
|
2752
|
+
|
|
2753
|
+
pin = _world_from_flags(world, no_world)
|
|
2754
|
+
if spec is None and name is None and no_world is False and world is None:
|
|
2755
|
+
typer.echo("worlds task update needs --spec, --name, --world or --no-world", err=True)
|
|
2756
|
+
raise typer.Exit(2)
|
|
2757
|
+
parsed = _read_task_spec(spec) if spec is not None else None
|
|
2758
|
+
try:
|
|
2759
|
+
updated = update_task(task_id, spec=parsed, name=name, world=pin, message=message, host=host)
|
|
2760
|
+
except WorldSessionError as exc:
|
|
2761
|
+
typer.echo(str(exc), err=True)
|
|
2762
|
+
raise typer.Exit(1)
|
|
2763
|
+
typer.echo(json.dumps(_task_result_json(updated), indent=2))
|
|
2764
|
+
|
|
2765
|
+
|
|
2766
|
+
@task_app.command("list")
|
|
2767
|
+
def task_list(
|
|
2768
|
+
world: Optional[str] = typer.Option(None, "--world", "-w", help="Only tasks pinned to this world."),
|
|
2769
|
+
metadata: List[str] = typer.Option(
|
|
2770
|
+
[], "--metadata", help="key=value the task's spec metadata must carry (repeatable, all must match)."
|
|
2771
|
+
),
|
|
2772
|
+
page: Optional[int] = typer.Option(None, "--page"),
|
|
2773
|
+
limit: Optional[int] = typer.Option(None, "--limit"),
|
|
2774
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2775
|
+
) -> None:
|
|
2776
|
+
"""The project's stored tasks, newest change first:
|
|
2777
|
+
|
|
2778
|
+
gateway worlds task list -w acme-billing --metadata externalUserId=u_42
|
|
2779
|
+
"""
|
|
2780
|
+
from gatewaysdk.world_sessions import WorldSessionError
|
|
2781
|
+
from gatewaysdk.world_tasks import list_tasks
|
|
2782
|
+
|
|
2783
|
+
pairs: Dict[str, str] = {}
|
|
2784
|
+
for pair in metadata:
|
|
2785
|
+
key, sep, value = pair.partition("=")
|
|
2786
|
+
if not sep or not key:
|
|
2787
|
+
typer.echo(f"--metadata {pair}: expected key=value", err=True)
|
|
2788
|
+
raise typer.Exit(2)
|
|
2789
|
+
pairs[key] = value
|
|
2790
|
+
try:
|
|
2791
|
+
listed = list_tasks(world=world, metadata=pairs or None, page=page, limit=limit, host=host)
|
|
2792
|
+
except WorldSessionError as exc:
|
|
2793
|
+
typer.echo(str(exc), err=True)
|
|
2794
|
+
raise typer.Exit(1)
|
|
2795
|
+
out = {
|
|
2796
|
+
"tasks": [_task_result_json(task) for task in listed.tasks],
|
|
2797
|
+
"meta": {
|
|
2798
|
+
"page": listed.page,
|
|
2799
|
+
"limit": listed.limit,
|
|
2800
|
+
"totalItems": listed.total_items,
|
|
2801
|
+
"totalPages": listed.total_pages,
|
|
2802
|
+
},
|
|
2803
|
+
}
|
|
2804
|
+
typer.echo(json.dumps(out, indent=2))
|
|
2805
|
+
|
|
2806
|
+
|
|
2807
|
+
@task_app.command("show")
|
|
2808
|
+
def task_show(
|
|
2809
|
+
task_id: str = typer.Argument(...),
|
|
2810
|
+
version: Optional[int] = typer.Option(None, "--version", help="Show this version's spec instead of the current."),
|
|
2811
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2812
|
+
) -> None:
|
|
2813
|
+
"""One task: its current (or ``--version``) spec and its version history."""
|
|
2814
|
+
from gatewaysdk.world_sessions import WorldSessionError
|
|
2815
|
+
from gatewaysdk.world_tasks import get_task
|
|
2816
|
+
|
|
2817
|
+
try:
|
|
2818
|
+
detail = get_task(task_id, version=version, host=host)
|
|
2819
|
+
except WorldSessionError as exc:
|
|
2820
|
+
typer.echo(str(exc), err=True)
|
|
2821
|
+
raise typer.Exit(1)
|
|
2822
|
+
typer.echo(json.dumps(_task_result_json(detail), indent=2))
|
|
2823
|
+
|
|
2824
|
+
|
|
2825
|
+
@task_app.command("validate")
|
|
2826
|
+
def task_validate(
|
|
2827
|
+
task_id: str = typer.Argument(...),
|
|
2828
|
+
version: Optional[int] = typer.Option(None, "--version", help="Validate this task version (default: current)."),
|
|
2829
|
+
world: Optional[str] = typer.Option(None, "--world", "-w", help="Against this world's head version."),
|
|
2830
|
+
version_id: Optional[str] = typer.Option(None, "--version-id", help="Against this exact world version."),
|
|
2831
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2832
|
+
) -> None:
|
|
2833
|
+
"""Check a task fits a world version — every seed and check entity exists
|
|
2834
|
+
in its contract — without opening a session. Exit 1 when it does not."""
|
|
2835
|
+
from gatewaysdk.world_sessions import WorldSessionError
|
|
2836
|
+
from gatewaysdk.world_tasks import validate_task
|
|
2837
|
+
|
|
2838
|
+
if world and version_id:
|
|
2839
|
+
typer.echo("--world and --version-id exclude each other", err=True)
|
|
2840
|
+
raise typer.Exit(2)
|
|
2841
|
+
try:
|
|
2842
|
+
result = validate_task(task_id, version=version, slug=world, version_id=version_id, host=host)
|
|
2843
|
+
except WorldSessionError as exc:
|
|
2844
|
+
typer.echo(str(exc), err=True)
|
|
2845
|
+
raise typer.Exit(1)
|
|
2846
|
+
typer.echo(json.dumps(_task_result_json(result), indent=2))
|
|
2847
|
+
if not result.ok:
|
|
2848
|
+
raise typer.Exit(1)
|
|
2849
|
+
|
|
2850
|
+
|
|
2851
|
+
@task_app.command("delete")
|
|
2852
|
+
def task_delete(
|
|
2853
|
+
task_id: str = typer.Argument(...),
|
|
2854
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2855
|
+
) -> None:
|
|
2856
|
+
"""Remove a stored task and all its versions. Open sessions keep their copy."""
|
|
2857
|
+
from gatewaysdk.world_sessions import WorldSessionError
|
|
2858
|
+
from gatewaysdk.world_tasks import delete_task
|
|
2859
|
+
|
|
2860
|
+
try:
|
|
2861
|
+
typer.echo(json.dumps(delete_task(task_id, host=host), indent=2))
|
|
2862
|
+
except WorldSessionError as exc:
|
|
2863
|
+
typer.echo(str(exc), err=True)
|
|
2864
|
+
raise typer.Exit(1)
|
|
2865
|
+
|
|
2866
|
+
|
|
2867
|
+
@bench_app.command("push")
|
|
2868
|
+
def bench_push(
|
|
2869
|
+
path: str = typer.Argument(..., help="Path to the verifiers-format environment directory."),
|
|
2870
|
+
slug: Optional[str] = typer.Option(None, "--slug", help="Container slug (defaults to the directory name)."),
|
|
2871
|
+
name: Optional[str] = typer.Option(None, "--name", help="Human-readable container name."),
|
|
2872
|
+
description: Optional[str] = typer.Option(None, "--description", help="Container description."),
|
|
2873
|
+
tag: Optional[List[str]] = typer.Option(None, "--tag", help="Tag (repeatable)."),
|
|
2874
|
+
visibility: Optional[str] = typer.Option(None, "--visibility", help="PRIVATE or PUBLIC."),
|
|
2875
|
+
auto_bump: bool = typer.Option(False, "--auto-bump", help="Patch-bump pyproject version before pushing."),
|
|
2876
|
+
rc: bool = typer.Option(False, "--rc", help="Bump/create an .rc pre-release before pushing."),
|
|
2877
|
+
post: bool = typer.Option(False, "--post", help="Bump/create a .post release before pushing."),
|
|
2878
|
+
image: Optional[str] = typer.Option(
|
|
2879
|
+
None,
|
|
2880
|
+
"--image",
|
|
2881
|
+
help="Pre-built OCI image ref for image/agentic envs (recorded on the version).",
|
|
2882
|
+
),
|
|
2883
|
+
build_wheel: bool = typer.Option(
|
|
2884
|
+
False,
|
|
2885
|
+
"--build-wheel",
|
|
2886
|
+
help="Build a wheel (uv build --wheel) + upload it for pip-install-by-name.",
|
|
2887
|
+
),
|
|
2888
|
+
branch: Optional[str] = typer.Option(
|
|
2889
|
+
None,
|
|
2890
|
+
"--branch",
|
|
2891
|
+
"-b",
|
|
2892
|
+
help="Branch to push to (default: the dir's tracked branch, else main).",
|
|
2893
|
+
),
|
|
2894
|
+
force: bool = typer.Option(
|
|
2895
|
+
False,
|
|
2896
|
+
"--force",
|
|
2897
|
+
help="Skip the fast-forward check (move the branch even if it diverged).",
|
|
2898
|
+
),
|
|
2899
|
+
message: Optional[str] = typer.Option(None, "--message", "-m", help="Change reason, shown in History."),
|
|
2900
|
+
author: Optional[str] = typer.Option(
|
|
2901
|
+
None, "--author", help="Attribution shown in History (default: GATEWAY_AUTHOR env)."
|
|
2902
|
+
),
|
|
2903
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2904
|
+
) -> None:
|
|
2905
|
+
"""Push a benchmark container to Gateway as a content-hashed source bundle.
|
|
2906
|
+
|
|
2907
|
+
The dir's ``.gateway/.env-metadata.json`` (written by pull/push) supplies the
|
|
2908
|
+
BASE version — pushing onto a branch whose tip moved refuses with a 409
|
|
2909
|
+
("pull/rebase, use another --branch, or --force"). ``--auto-bump/--rc/--post``
|
|
2910
|
+
(mutually exclusive) rewrite ``[project].version`` in pyproject.toml before
|
|
2911
|
+
hashing, so the bump lands in the bundle.
|
|
2912
|
+
|
|
2913
|
+
Examples::
|
|
2914
|
+
|
|
2915
|
+
gateway bench push ./automationbench -m "tighten grader"
|
|
2916
|
+
gateway bench push ./automationbench --branch wip-alice
|
|
2917
|
+
gateway bench push ./automationbench --auto-bump
|
|
2918
|
+
"""
|
|
2919
|
+
import os as _os
|
|
2920
|
+
import shutil as _shutil
|
|
2921
|
+
import tempfile as _tempfile
|
|
2922
|
+
|
|
2923
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
2924
|
+
from gatewaysdk.environment.schema import bundle_mode
|
|
2925
|
+
from gatewaysdk.environment.schema.api import build_api_bundle
|
|
2926
|
+
|
|
2927
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
2928
|
+
source = Path(path)
|
|
2929
|
+
# A schema world that describes a vendor connection is pushed AS the api-kind bundle
|
|
2930
|
+
# that serves it: the world stays the authority, the packaging is not the user's job.
|
|
2931
|
+
serves_api = (source / "connector.toml").is_file() and bundle_mode(source) is not None
|
|
2932
|
+
staging = _tempfile.TemporaryDirectory(prefix="gateway-world-api-") if serves_api else None
|
|
2933
|
+
if serves_api:
|
|
2934
|
+
bundle = build_api_bundle(source, Path(staging.name) / "bundle")
|
|
2935
|
+
if (source / ".gateway").is_dir():
|
|
2936
|
+
_shutil.copytree(source / ".gateway", bundle / ".gateway")
|
|
2937
|
+
path = str(bundle)
|
|
2938
|
+
slug = slug or source.resolve().name
|
|
2939
|
+
result = client.push(
|
|
2940
|
+
path,
|
|
2941
|
+
slug=slug,
|
|
2942
|
+
name=name,
|
|
2943
|
+
description=description,
|
|
2944
|
+
tags=tag,
|
|
2945
|
+
visibility=visibility,
|
|
2946
|
+
auto_bump=auto_bump,
|
|
2947
|
+
rc=rc,
|
|
2948
|
+
post=post,
|
|
2949
|
+
image_ref=image,
|
|
2950
|
+
wheel=build_wheel,
|
|
2951
|
+
branch=branch,
|
|
2952
|
+
force=force,
|
|
2953
|
+
message=message,
|
|
2954
|
+
author=author or _os.environ.get("GATEWAY_AUTHOR"),
|
|
2955
|
+
)
|
|
2956
|
+
if serves_api:
|
|
2957
|
+
meta = Path(path) / ".gateway"
|
|
2958
|
+
if meta.is_dir():
|
|
2959
|
+
_shutil.rmtree(source / ".gateway", ignore_errors=True)
|
|
2960
|
+
_shutil.copytree(meta, source / ".gateway")
|
|
2961
|
+
staging.cleanup()
|
|
2962
|
+
typer.echo(json.dumps(result, indent=2))
|
|
2963
|
+
|
|
2964
|
+
|
|
2965
|
+
@bench_app.command("install")
|
|
2966
|
+
def bench_install(
|
|
2967
|
+
slug: str = typer.Argument(..., help="Container slug, optionally slug@latest | @<semver> | @<hash>."),
|
|
2968
|
+
tool: str = typer.Option("uv", "--tool", help="Installer: uv or pip."),
|
|
2969
|
+
no_upgrade: bool = typer.Option(False, "--no-upgrade", help="Don't reinstall/upgrade if already present."),
|
|
2970
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
2971
|
+
) -> None:
|
|
2972
|
+
"""pip-install a container version's built wheel by name (wheel-url-direct).
|
|
2973
|
+
|
|
2974
|
+
The version must have been pushed with ``--build-wheel``. Runs in the current
|
|
2975
|
+
Python environment (activate a venv first for uv).
|
|
2976
|
+
|
|
2977
|
+
Examples::
|
|
2978
|
+
|
|
2979
|
+
gateway bench install automationbench@1.2.0
|
|
2980
|
+
"""
|
|
2981
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
2982
|
+
|
|
2983
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
2984
|
+
result = client.install(slug, tool=tool, no_upgrade=no_upgrade)
|
|
2985
|
+
typer.echo(json.dumps(result, indent=2))
|
|
2986
|
+
|
|
2987
|
+
|
|
2988
|
+
def _reject_mock_tools() -> None:
|
|
2989
|
+
"""`bench eval` runs verifiers and harbor world types; a ``mock-tools``
|
|
2990
|
+
bundle (a ``gateway-env.toml`` world) is executed by the local runtime
|
|
2991
|
+
instead."""
|
|
2992
|
+
typer.echo(
|
|
2993
|
+
"this container is a mock-tools world (gateway-env.toml) — "
|
|
2994
|
+
"`gateway bench eval` runs the verifiers and harbor world types only. "
|
|
2995
|
+
"Run it locally with the environment runtime "
|
|
2996
|
+
"(`import gatewaysdk.environment as gwenv; env = gwenv.init(<slug>)`), "
|
|
2997
|
+
"or from the platform, whose hosted runner serves mock-tools worlds.",
|
|
2998
|
+
err=True,
|
|
2999
|
+
)
|
|
3000
|
+
raise typer.Exit(1)
|
|
3001
|
+
|
|
3002
|
+
|
|
3003
|
+
def _reject_verifiers_v1() -> None:
|
|
3004
|
+
"""`bench eval` only runs the legacy v0 in-process path locally; verifiers
|
|
3005
|
+
v1 Taskset packages run via the platform's verifiers-v1 world type."""
|
|
3006
|
+
typer.echo(
|
|
3007
|
+
"this container is a verifiers v1 Taskset package (verifiers.v1 "
|
|
3008
|
+
"namespace) — `gateway bench eval` currently only runs the legacy v0 "
|
|
3009
|
+
"in-process path locally. Run it from the platform (its hosted runner "
|
|
3010
|
+
"handles the verifiers-v1 world type, including Anthropic-dialect "
|
|
3011
|
+
"endpoints), or pin the env to verifiers<1.0.0 and expose a v0 "
|
|
3012
|
+
"load_environment() for local eval.",
|
|
3013
|
+
err=True,
|
|
3014
|
+
)
|
|
3015
|
+
raise typer.Exit(1)
|
|
3016
|
+
|
|
3017
|
+
|
|
3018
|
+
@bench_app.command("eval")
|
|
3019
|
+
def bench_eval(
|
|
3020
|
+
slug: Optional[str] = typer.Argument(
|
|
3021
|
+
None,
|
|
3022
|
+
help="Container slug@ref to eval (install it first: `bench install`). Omit when using --path.",
|
|
3023
|
+
),
|
|
3024
|
+
path: Optional[str] = typer.Option(
|
|
3025
|
+
None,
|
|
3026
|
+
"--path",
|
|
3027
|
+
"-p",
|
|
3028
|
+
help="Run a LOCAL world directory (harbor kind) instead of pulling a ref. A clean `bench checkout` files the run pinned to its base version; a drifted or unlinked dir runs without filing.",
|
|
3029
|
+
),
|
|
3030
|
+
model: str = typer.Option(..., "--model", "-m", help="Model id for OpenRouter."),
|
|
3031
|
+
env_id: Optional[str] = typer.Option(
|
|
3032
|
+
None,
|
|
3033
|
+
"--env-id",
|
|
3034
|
+
help="verifiers env id to load (defaults to the slug).",
|
|
3035
|
+
),
|
|
3036
|
+
kind: str = typer.Option(
|
|
3037
|
+
"auto",
|
|
3038
|
+
"--kind",
|
|
3039
|
+
help="Environment kind: auto (pull + detect) | verifiers | verifiers-v1 | harbor.",
|
|
3040
|
+
),
|
|
3041
|
+
agent: str = typer.Option(
|
|
3042
|
+
"mini-swe-agent",
|
|
3043
|
+
"--agent",
|
|
3044
|
+
"-a",
|
|
3045
|
+
help="harbor kind only: agent that runs the tasks (harbor agent name or acp: shorthand).",
|
|
3046
|
+
),
|
|
3047
|
+
num_examples: int = typer.Option(-1, "--num-examples", "-n"),
|
|
3048
|
+
rollouts: int = typer.Option(1, "--rollouts-per-example", "-r"),
|
|
3049
|
+
save_results: bool = typer.Option(False, "--save-results", "-s", help="Write a resumable results dir."),
|
|
3050
|
+
output_dir: str = typer.Option("./outputs", "--output-dir", "-o"),
|
|
3051
|
+
resume: Optional[str] = typer.Option(None, "--resume", "-R", help="Resume from a results dir path."),
|
|
3052
|
+
fail_under: Optional[float] = typer.Option(
|
|
3053
|
+
None,
|
|
3054
|
+
"--fail-under",
|
|
3055
|
+
min=0.0,
|
|
3056
|
+
max=1.0,
|
|
3057
|
+
help="CI gate: exit 1 when the average reward lands below this bar (also fails when no aggregate is available). Writes a $GITHUB_STEP_SUMMARY table under GitHub Actions.",
|
|
3058
|
+
),
|
|
3059
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3060
|
+
) -> None:
|
|
3061
|
+
"""Run a local eval against a container version and harvest a version-pinned
|
|
3062
|
+
evaluation to Gateway. Auto-detects the env kind from the bundle:
|
|
3063
|
+
|
|
3064
|
+
- verifiers kind: in-process rollout loop (requires ``verifiers`` + ``openai``
|
|
3065
|
+
installed and the container installed via ``gateway bench install``).
|
|
3066
|
+
``--save-results`` writes ``outputs/evals/<env>--<model>/<run>/`` for
|
|
3067
|
+
``--resume``.
|
|
3068
|
+
- harbor kind: runs the task dirs as a harbor world
|
|
3069
|
+
(docker sandbox by default; each task grades itself via its tests/), then
|
|
3070
|
+
harvests the jobs dir.
|
|
3071
|
+
|
|
3072
|
+
``--path`` runs your working copy: the harbor runner executes the LOCAL
|
|
3073
|
+
task dirs, and the result files only when the dir is a clean checkout
|
|
3074
|
+
(its ``.gateway`` base matches the tree) — pinned to that base version,
|
|
3075
|
+
branch tag included. A drifted tree runs but stays local, with the push
|
|
3076
|
+
hint printed up front.
|
|
3077
|
+
|
|
3078
|
+
Requires ``OPENROUTER_API_KEY`` either way.
|
|
3079
|
+
"""
|
|
3080
|
+
import os
|
|
3081
|
+
import tempfile
|
|
3082
|
+
|
|
3083
|
+
from gatewaysdk.benchmark_hub import (
|
|
3084
|
+
EvalConfig,
|
|
3085
|
+
GatewayBenchmarkClient,
|
|
3086
|
+
GatewayEvalsClient,
|
|
3087
|
+
detect_env_kind,
|
|
3088
|
+
find_mcp_config,
|
|
3089
|
+
harvest_job_dir,
|
|
3090
|
+
local_tree_state,
|
|
3091
|
+
resolve_harbor_model,
|
|
3092
|
+
run_harbor_job,
|
|
3093
|
+
run_local_eval,
|
|
3094
|
+
tasks_root,
|
|
3095
|
+
)
|
|
3096
|
+
|
|
3097
|
+
if (slug is None) == (path is None):
|
|
3098
|
+
typer.echo("pass exactly one of SLUG or --path", err=True)
|
|
3099
|
+
raise typer.Exit(1)
|
|
3100
|
+
|
|
3101
|
+
api_key = os.environ.get("OPENROUTER_API_KEY")
|
|
3102
|
+
if not api_key:
|
|
3103
|
+
typer.echo("OPENROUTER_API_KEY is required for `bench eval`.", err=True)
|
|
3104
|
+
raise typer.Exit(1)
|
|
3105
|
+
|
|
3106
|
+
if kind not in ("auto", "verifiers", "verifiers-v1", "harbor"):
|
|
3107
|
+
typer.echo(
|
|
3108
|
+
f"unknown --kind '{kind}' (auto|verifiers|verifiers-v1|harbor)",
|
|
3109
|
+
err=True,
|
|
3110
|
+
)
|
|
3111
|
+
raise typer.Exit(1)
|
|
3112
|
+
if kind == "verifiers-v1":
|
|
3113
|
+
_reject_verifiers_v1()
|
|
3114
|
+
|
|
3115
|
+
if path is not None:
|
|
3116
|
+
src = os.path.abspath(path)
|
|
3117
|
+
if not os.path.isdir(src):
|
|
3118
|
+
typer.echo(f"--path is not a directory: {path}", err=True)
|
|
3119
|
+
raise typer.Exit(1)
|
|
3120
|
+
resolved_kind = detect_env_kind(src) if kind == "auto" else kind
|
|
3121
|
+
if resolved_kind == "mock-tools":
|
|
3122
|
+
_reject_mock_tools()
|
|
3123
|
+
if resolved_kind == "verifiers-v1":
|
|
3124
|
+
_reject_verifiers_v1()
|
|
3125
|
+
if resolved_kind == "verifiers":
|
|
3126
|
+
typer.echo(
|
|
3127
|
+
"--path runs harbor worlds; verifiers packages run from the " "hub (`gateway bench eval <slug>`).",
|
|
3128
|
+
err=True,
|
|
3129
|
+
)
|
|
3130
|
+
raise typer.Exit(1)
|
|
3131
|
+
|
|
3132
|
+
# Filing is honest-or-off: only a CLEAN checkout pins its base
|
|
3133
|
+
# version. Said up front — not after a long docker run.
|
|
3134
|
+
state = local_tree_state(Path(src))
|
|
3135
|
+
meta = state["meta"] or {}
|
|
3136
|
+
file_version_id = meta.get("versionId") if state["clean"] else None
|
|
3137
|
+
if file_version_id:
|
|
3138
|
+
typer.echo(
|
|
3139
|
+
f"filing to {meta.get('slug')}@{meta.get('branch') or meta.get('ref')} "
|
|
3140
|
+
f"({str(meta.get('contentHash') or '')[:8]})",
|
|
3141
|
+
err=True,
|
|
3142
|
+
)
|
|
3143
|
+
elif meta.get("versionId"):
|
|
3144
|
+
dirty = state["dirty"]
|
|
3145
|
+
shown = ", ".join(dirty[:5]) + (", …" if len(dirty) > 5 else "")
|
|
3146
|
+
typer.echo(
|
|
3147
|
+
f"note: {len(dirty)} file(s) differ from "
|
|
3148
|
+
f"{meta.get('branch') or meta.get('ref')}@"
|
|
3149
|
+
f"{str(meta.get('contentHash') or '')[:8]} ({shown}) — the run "
|
|
3150
|
+
"executes your local code but will NOT be filed. Push first "
|
|
3151
|
+
"(`gateway bench push`) to file it.",
|
|
3152
|
+
err=True,
|
|
3153
|
+
)
|
|
3154
|
+
else:
|
|
3155
|
+
typer.echo(
|
|
3156
|
+
"note: this dir has no hub link (.gateway) — the run stays "
|
|
3157
|
+
"local. Push it (`gateway bench push`) to file runs.",
|
|
3158
|
+
err=True,
|
|
3159
|
+
)
|
|
3160
|
+
|
|
3161
|
+
harbor_model = resolve_harbor_model(model, agent)
|
|
3162
|
+
jobs_dir = os.path.join(output_dir, "harbor-jobs", f"local--{os.path.basename(src) or 'world'}")
|
|
3163
|
+
run_harbor_job(
|
|
3164
|
+
tasks_root(src),
|
|
3165
|
+
jobs_dir,
|
|
3166
|
+
model=harbor_model,
|
|
3167
|
+
agent=agent,
|
|
3168
|
+
num_examples=num_examples,
|
|
3169
|
+
rollouts_per_example=rollouts,
|
|
3170
|
+
extra_env={"OPENROUTER_API_KEY": api_key},
|
|
3171
|
+
mcp_config=find_mcp_config(src),
|
|
3172
|
+
)
|
|
3173
|
+
if file_version_id:
|
|
3174
|
+
result = harvest_job_dir(
|
|
3175
|
+
jobs_dir,
|
|
3176
|
+
file_version_id,
|
|
3177
|
+
GatewayEvalsClient.from_env(host),
|
|
3178
|
+
model=harbor_model,
|
|
3179
|
+
rollouts_per_example=rollouts,
|
|
3180
|
+
)
|
|
3181
|
+
else:
|
|
3182
|
+
# Unfiled runs still aggregate locally — the CI gate and the
|
|
3183
|
+
# printed result must not depend on filing.
|
|
3184
|
+
from gatewaysdk.benchmark_hub.harbor_adapter import (
|
|
3185
|
+
metrics_from_samples,
|
|
3186
|
+
samples_from_job_dir,
|
|
3187
|
+
)
|
|
3188
|
+
|
|
3189
|
+
local_samples = samples_from_job_dir(jobs_dir)
|
|
3190
|
+
result = {
|
|
3191
|
+
"jobsDir": jobs_dir,
|
|
3192
|
+
"filed": False,
|
|
3193
|
+
"metrics": metrics_from_samples(local_samples) if local_samples else {},
|
|
3194
|
+
}
|
|
3195
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3196
|
+
_ci_finish(
|
|
3197
|
+
result,
|
|
3198
|
+
fail_under=fail_under,
|
|
3199
|
+
title=f"World eval · {os.path.basename(src)} (local)",
|
|
3200
|
+
facts=[("model", harbor_model), ("agent", agent)],
|
|
3201
|
+
)
|
|
3202
|
+
return
|
|
3203
|
+
|
|
3204
|
+
bench = GatewayBenchmarkClient.from_env(host)
|
|
3205
|
+
ref_slug, ref = slug.rsplit("@", 1) if "@" in slug else (slug, "latest")
|
|
3206
|
+
details = bench.resolve_version(ref_slug, ref)
|
|
3207
|
+
version_id = details["versionId"]
|
|
3208
|
+
|
|
3209
|
+
with tempfile.TemporaryDirectory(prefix="gateway-bench-eval-") as td:
|
|
3210
|
+
src: Optional[str] = None
|
|
3211
|
+
if kind in ("auto", "harbor"):
|
|
3212
|
+
src = os.path.join(td, "src")
|
|
3213
|
+
bench.pull(f"{ref_slug}@{ref}", target=src)
|
|
3214
|
+
resolved_kind = detect_env_kind(src) if kind == "auto" else "harbor"
|
|
3215
|
+
else:
|
|
3216
|
+
resolved_kind = "verifiers"
|
|
3217
|
+
|
|
3218
|
+
if resolved_kind == "mock-tools":
|
|
3219
|
+
_reject_mock_tools()
|
|
3220
|
+
if resolved_kind == "verifiers-v1":
|
|
3221
|
+
_reject_verifiers_v1()
|
|
3222
|
+
|
|
3223
|
+
if resolved_kind == "harbor":
|
|
3224
|
+
assert src is not None
|
|
3225
|
+
harbor_model = resolve_harbor_model(model, agent)
|
|
3226
|
+
jobs_dir = os.path.join(output_dir, "harbor-jobs", f"{ref_slug}--{ref}")
|
|
3227
|
+
run_harbor_job(
|
|
3228
|
+
tasks_root(src),
|
|
3229
|
+
jobs_dir,
|
|
3230
|
+
model=harbor_model,
|
|
3231
|
+
agent=agent,
|
|
3232
|
+
num_examples=num_examples,
|
|
3233
|
+
rollouts_per_example=rollouts,
|
|
3234
|
+
extra_env={"OPENROUTER_API_KEY": api_key},
|
|
3235
|
+
# Env-bundled Claude-style MCP config (e.g. enterpriseops-*:
|
|
3236
|
+
# the gym's tools + x-database-id header) — auto-detected.
|
|
3237
|
+
mcp_config=find_mcp_config(src),
|
|
3238
|
+
)
|
|
3239
|
+
result = harvest_job_dir(
|
|
3240
|
+
jobs_dir,
|
|
3241
|
+
version_id,
|
|
3242
|
+
GatewayEvalsClient.from_env(host),
|
|
3243
|
+
model=harbor_model,
|
|
3244
|
+
rollouts_per_example=rollouts,
|
|
3245
|
+
)
|
|
3246
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3247
|
+
_ci_finish(
|
|
3248
|
+
result,
|
|
3249
|
+
fail_under=fail_under,
|
|
3250
|
+
title=f"World eval · {ref_slug}@{ref}",
|
|
3251
|
+
facts=[("model", harbor_model), ("agent", agent)],
|
|
3252
|
+
)
|
|
3253
|
+
return
|
|
3254
|
+
|
|
3255
|
+
try:
|
|
3256
|
+
# verifiers' rollout loop is async — newer releases reject a sync
|
|
3257
|
+
# OpenAI client outright ("Unsupported client type").
|
|
3258
|
+
from openai import AsyncOpenAI
|
|
3259
|
+
except ImportError:
|
|
3260
|
+
typer.echo("openai is required: pip install openai", err=True)
|
|
3261
|
+
raise typer.Exit(1)
|
|
3262
|
+
|
|
3263
|
+
client = AsyncOpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key)
|
|
3264
|
+
config = EvalConfig(
|
|
3265
|
+
env_id=env_id or ref_slug,
|
|
3266
|
+
model=model,
|
|
3267
|
+
num_examples=num_examples,
|
|
3268
|
+
rollouts_per_example=rollouts,
|
|
3269
|
+
save_results=save_results or resume is not None,
|
|
3270
|
+
output_dir=output_dir,
|
|
3271
|
+
resume_path=resume,
|
|
3272
|
+
)
|
|
3273
|
+
result = run_local_eval(
|
|
3274
|
+
config.env_id,
|
|
3275
|
+
version_id,
|
|
3276
|
+
GatewayEvalsClient.from_env(host),
|
|
3277
|
+
client=client,
|
|
3278
|
+
config=config,
|
|
3279
|
+
)
|
|
3280
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3281
|
+
_ci_finish(
|
|
3282
|
+
result,
|
|
3283
|
+
fail_under=fail_under,
|
|
3284
|
+
title=f"World eval · {ref_slug}@{ref}",
|
|
3285
|
+
facts=[("model", model)],
|
|
3286
|
+
)
|
|
3287
|
+
|
|
3288
|
+
|
|
3289
|
+
@bench_app.command("harvest-harbor")
|
|
3290
|
+
def bench_harvest_harbor(
|
|
3291
|
+
jobs_dir: str = typer.Argument(..., help="A completed harbor (or pier) jobs directory."),
|
|
3292
|
+
slug: str = typer.Argument(..., help="Container slug@ref the evaluation pins to."),
|
|
3293
|
+
model: str = typer.Option(..., "--model", "-m", help="Model id the job ran with."),
|
|
3294
|
+
rollouts: int = typer.Option(1, "--rollouts-per-example", "-r"),
|
|
3295
|
+
run_id: Optional[str] = typer.Option(None, "--run-id", help="Environment RunResult id to back-link."),
|
|
3296
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3297
|
+
) -> None:
|
|
3298
|
+
"""Harvest an EXISTING harbor jobs directory into a version-pinned Gateway
|
|
3299
|
+
evaluation (harvest-only twin of `bench eval --kind harbor`)."""
|
|
3300
|
+
from gatewaysdk.benchmark_hub import (
|
|
3301
|
+
GatewayBenchmarkClient,
|
|
3302
|
+
GatewayEvalsClient,
|
|
3303
|
+
harvest_job_dir,
|
|
3304
|
+
)
|
|
3305
|
+
|
|
3306
|
+
bench = GatewayBenchmarkClient.from_env(host)
|
|
3307
|
+
ref_slug, ref = slug.rsplit("@", 1) if "@" in slug else (slug, "latest")
|
|
3308
|
+
version_id = bench.resolve_version(ref_slug, ref)["versionId"]
|
|
3309
|
+
result = harvest_job_dir(
|
|
3310
|
+
jobs_dir,
|
|
3311
|
+
version_id,
|
|
3312
|
+
GatewayEvalsClient.from_env(host),
|
|
3313
|
+
model=model,
|
|
3314
|
+
run_id=run_id,
|
|
3315
|
+
rollouts_per_example=rollouts,
|
|
3316
|
+
)
|
|
3317
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3318
|
+
|
|
3319
|
+
|
|
3320
|
+
@bench_app.command("run-configs")
|
|
3321
|
+
def bench_run_configs(
|
|
3322
|
+
action: str = typer.Argument("list", help="list | create | delete"),
|
|
3323
|
+
name: Optional[str] = typer.Option(None, "--name", help="Config name (create)."),
|
|
3324
|
+
models: Optional[str] = typer.Option(None, "--models", help="Comma-separated model list (create)."),
|
|
3325
|
+
parallelism: int = typer.Option(1, "--parallelism", help="Containers per model, 1-16 (create)."),
|
|
3326
|
+
tasks: Optional[str] = typer.Option(None, "--tasks", help="Comma-separated bundled task names (create)."),
|
|
3327
|
+
world_tasks: Optional[str] = typer.Option(
|
|
3328
|
+
None,
|
|
3329
|
+
"--world-tasks",
|
|
3330
|
+
help="Comma-separated stored task refs, <id> or <id>@<version> (create; api worlds, not with --tasks).",
|
|
3331
|
+
),
|
|
3332
|
+
agent: Optional[str] = typer.Option(None, "--agent", help="harbor agent (create)."),
|
|
3333
|
+
auto_run_on_push: bool = typer.Option(
|
|
3334
|
+
False, "--auto-run-on-push", help="Dispatch on every pushed READY version (create)."
|
|
3335
|
+
),
|
|
3336
|
+
config_id: Optional[str] = typer.Option(None, "--id", help="Config id (delete/update)."),
|
|
3337
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3338
|
+
) -> None:
|
|
3339
|
+
"""Run configs from the CLI — list them, create one for CI, delete one.
|
|
3340
|
+
|
|
3341
|
+
gateway bench run-configs list
|
|
3342
|
+
gateway bench run-configs create --name ci-suite --models openai/gpt-5-mini --parallelism 4
|
|
3343
|
+
gateway bench run-configs create --name stored --models openai/gpt-5-mini --world-tasks wt_a,wt_b@2
|
|
3344
|
+
gateway bench run-configs delete --id cmrc123
|
|
3345
|
+
"""
|
|
3346
|
+
from ..benchmark_hub.client import GatewayBenchmarkClient
|
|
3347
|
+
from ..world_tasks import parse_world_task_refs
|
|
3348
|
+
|
|
3349
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3350
|
+
if action == "list":
|
|
3351
|
+
rows = client.list_run_configs().get("runConfigs", [])
|
|
3352
|
+
if not rows:
|
|
3353
|
+
typer.echo("No run configs in this project.")
|
|
3354
|
+
return
|
|
3355
|
+
for row in rows:
|
|
3356
|
+
flags = []
|
|
3357
|
+
if row.get("parallelism", 1) > 1:
|
|
3358
|
+
flags.append(f"x{row['parallelism']} containers")
|
|
3359
|
+
if row.get("autoRunOnPush"):
|
|
3360
|
+
flags.append("auto-on-push")
|
|
3361
|
+
suffix = f" [{', '.join(flags)}]" if flags else ""
|
|
3362
|
+
typer.echo(f"{row['id']} {row['name']} ({', '.join(row.get('models', []) or ['no models'])}){suffix}")
|
|
3363
|
+
return
|
|
3364
|
+
if action == "delete":
|
|
3365
|
+
if not config_id:
|
|
3366
|
+
raise typer.BadParameter("--id is required for delete")
|
|
3367
|
+
client.delete_run_config(config_id)
|
|
3368
|
+
typer.echo(f"Deleted {config_id}.")
|
|
3369
|
+
return
|
|
3370
|
+
if action != "create":
|
|
3371
|
+
raise typer.BadParameter("action must be one of: list | create | delete")
|
|
3372
|
+
if not name or not models:
|
|
3373
|
+
raise typer.BadParameter("--name and --models are required for create")
|
|
3374
|
+
fields: Dict[str, Any] = {
|
|
3375
|
+
"name": name,
|
|
3376
|
+
"models": [m.strip() for m in models.split(",") if m.strip()],
|
|
3377
|
+
"parallelism": parallelism,
|
|
3378
|
+
}
|
|
3379
|
+
if agent:
|
|
3380
|
+
fields["agent"] = agent
|
|
3381
|
+
if tasks:
|
|
3382
|
+
fields["taskNames"] = [t.strip() for t in tasks.split(",") if t.strip()]
|
|
3383
|
+
if world_tasks:
|
|
3384
|
+
if tasks:
|
|
3385
|
+
raise typer.BadParameter(
|
|
3386
|
+
"--tasks and --world-tasks are exclusive: a config runs bundled scenarios or stored tasks"
|
|
3387
|
+
)
|
|
3388
|
+
try:
|
|
3389
|
+
fields["worldTasks"] = parse_world_task_refs(world_tasks)
|
|
3390
|
+
except ValueError as exc:
|
|
3391
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
3392
|
+
if auto_run_on_push:
|
|
3393
|
+
fields["autoRunOnPush"] = True
|
|
3394
|
+
if config_id:
|
|
3395
|
+
fields["id"] = config_id
|
|
3396
|
+
row = client.upsert_run_config(**fields)
|
|
3397
|
+
typer.echo(f"{row['id']} {row['name']} ({', '.join(row.get('models', []))})")
|
|
3398
|
+
|
|
3399
|
+
|
|
3400
|
+
@bench_app.command("secrets")
|
|
3401
|
+
def bench_secrets(
|
|
3402
|
+
action: str = typer.Argument(..., help="set | list | unset"),
|
|
3403
|
+
env: str = typer.Option(..., "--env", help="World slug the secret belongs to."),
|
|
3404
|
+
key: Optional[str] = typer.Option(None, "--key", help="Secret name (UPPER_SNAKE_CASE) — required for set/unset."),
|
|
3405
|
+
value: Optional[str] = typer.Option(
|
|
3406
|
+
None, "--value", help="Secret value for `set`; omit to read from stdin (keeps it out of shell history)."
|
|
3407
|
+
),
|
|
3408
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3409
|
+
) -> None:
|
|
3410
|
+
"""Environment secrets — env vars every HOSTED run of the world receives.
|
|
3411
|
+
|
|
3412
|
+
Write-only: `list` shows names and rotation times, never values.
|
|
3413
|
+
|
|
3414
|
+
gateway bench secrets set --env blockchain-investigator --key ETHERSCAN_API_KEY
|
|
3415
|
+
gateway bench secrets list --env blockchain-investigator
|
|
3416
|
+
gateway bench secrets unset --env blockchain-investigator --key ETHERSCAN_API_KEY
|
|
3417
|
+
"""
|
|
3418
|
+
import sys
|
|
3419
|
+
|
|
3420
|
+
from ..benchmark_hub.client import GatewayBenchmarkClient
|
|
3421
|
+
|
|
3422
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3423
|
+
if action == "list":
|
|
3424
|
+
rows = client.list_env_secret_keys(env).get("secrets", [])
|
|
3425
|
+
if not rows:
|
|
3426
|
+
typer.echo(f"No secrets on {env}.")
|
|
3427
|
+
return
|
|
3428
|
+
for row in rows:
|
|
3429
|
+
typer.echo(f"{row['key']} (rotated {row['updatedAt']})")
|
|
3430
|
+
return
|
|
3431
|
+
if action not in ("set", "unset"):
|
|
3432
|
+
raise typer.BadParameter("action must be one of: set | list | unset")
|
|
3433
|
+
if not key:
|
|
3434
|
+
raise typer.BadParameter(f"--key is required for `{action}`")
|
|
3435
|
+
if action == "unset":
|
|
3436
|
+
client.delete_env_secret(env, key)
|
|
3437
|
+
typer.echo(f"Deleted {key} from {env}.")
|
|
3438
|
+
return
|
|
3439
|
+
secret = value
|
|
3440
|
+
if secret is None:
|
|
3441
|
+
typer.echo(f"Value for {key} (input hidden): ", nl=False, err=True)
|
|
3442
|
+
secret = sys.stdin.readline().strip() if not sys.stdin.isatty() else __import__("getpass").getpass("")
|
|
3443
|
+
if not secret:
|
|
3444
|
+
raise typer.BadParameter("empty value — nothing stored")
|
|
3445
|
+
res = client.set_env_secret(env, key, secret)
|
|
3446
|
+
typer.echo(f"{'Added' if res.get('created') else 'Rotated'} {key} on {env} — next hosted run receives it.")
|
|
3447
|
+
|
|
3448
|
+
|
|
3449
|
+
@bench_app.command("dispatch")
|
|
3450
|
+
def bench_dispatch(
|
|
3451
|
+
run_config_id: str = typer.Argument(..., help="Run config id (world page -> Runs -> Configs)."),
|
|
3452
|
+
container_id: Optional[str] = typer.Option(
|
|
3453
|
+
None, "--container-id", help="Which environment, when the project hosts several."
|
|
3454
|
+
),
|
|
3455
|
+
version_id: Optional[str] = typer.Option(
|
|
3456
|
+
None, "--version-id", help="Pin to a specific READY version (default: head)."
|
|
3457
|
+
),
|
|
3458
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3459
|
+
) -> None:
|
|
3460
|
+
"""Start a hosted run of a run config — the same dispatch the Run button
|
|
3461
|
+
uses, parallelism included: the config's `parallelism` fans the task list
|
|
3462
|
+
across that many containers per model.
|
|
3463
|
+
|
|
3464
|
+
Examples::
|
|
3465
|
+
|
|
3466
|
+
gateway bench dispatch cmrunconfig123
|
|
3467
|
+
gateway bench dispatch cmrunconfig123 --version-id cmversion456
|
|
3468
|
+
"""
|
|
3469
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3470
|
+
|
|
3471
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3472
|
+
result = client.dispatch_run_config(run_config_id, container_id=container_id, version_id=version_id)
|
|
3473
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3474
|
+
|
|
3475
|
+
|
|
3476
|
+
@bench_app.command("pull")
|
|
3477
|
+
def bench_pull(
|
|
3478
|
+
slug: str = typer.Argument(
|
|
3479
|
+
...,
|
|
3480
|
+
help="Container slug, optionally slug@latest | @<semver> | @<hash-prefix>.",
|
|
3481
|
+
),
|
|
3482
|
+
target: Optional[str] = typer.Option(None, "--target", "-t", help="Target dir (defaults to ./<slug>)."),
|
|
3483
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3484
|
+
) -> None:
|
|
3485
|
+
"""Reconstruct a container version's source directory, addressed by name.
|
|
3486
|
+
|
|
3487
|
+
Examples::
|
|
3488
|
+
|
|
3489
|
+
gateway bench pull automationbench
|
|
3490
|
+
gateway bench pull automationbench@1.2.0 --target ./ab
|
|
3491
|
+
"""
|
|
3492
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3493
|
+
|
|
3494
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3495
|
+
result = client.pull(slug, target=target)
|
|
3496
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3497
|
+
|
|
3498
|
+
|
|
3499
|
+
@bench_app.command("fork")
|
|
3500
|
+
def bench_fork(
|
|
3501
|
+
slug: str = typer.Argument(..., help="Source container slug (forks the head; omit @version)."),
|
|
3502
|
+
into: Optional[str] = typer.Option(None, "--into", help="Target slug (defaults to <slug>-fork)."),
|
|
3503
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3504
|
+
) -> None:
|
|
3505
|
+
"""Fork a container into a new one (shares the immutable, content-addressed
|
|
3506
|
+
bundle; records provenance via forkedFromVersionId).
|
|
3507
|
+
|
|
3508
|
+
Examples::
|
|
3509
|
+
|
|
3510
|
+
gateway bench fork automationbench --into automationbench-v2
|
|
3511
|
+
"""
|
|
3512
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3513
|
+
|
|
3514
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3515
|
+
result = client.fork(slug, target_slug=into)
|
|
3516
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3517
|
+
|
|
3518
|
+
|
|
3519
|
+
@bench_app.command("status")
|
|
3520
|
+
def bench_status(
|
|
3521
|
+
path: str = typer.Argument(".", help="Working dir (defaults to cwd)."),
|
|
3522
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3523
|
+
) -> None:
|
|
3524
|
+
"""Where this dir stands: tracked branch, base version, local edits
|
|
3525
|
+
(dirty), and whether the remote branch tip moved past the base (behind).
|
|
3526
|
+
|
|
3527
|
+
Examples::
|
|
3528
|
+
|
|
3529
|
+
gateway bench status
|
|
3530
|
+
gateway bench status ./automationbench
|
|
3531
|
+
"""
|
|
3532
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3533
|
+
|
|
3534
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3535
|
+
result = client.status(path)
|
|
3536
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3537
|
+
if result.get("dirty"):
|
|
3538
|
+
typer.echo("# local edits not pushed", err=True)
|
|
3539
|
+
if result.get("behind"):
|
|
3540
|
+
typer.echo(
|
|
3541
|
+
"# branch tip moved past your base — `gateway bench pull` before pushing",
|
|
3542
|
+
err=True,
|
|
3543
|
+
)
|
|
3544
|
+
|
|
3545
|
+
|
|
3546
|
+
@bench_app.command("log")
|
|
3547
|
+
def bench_log(
|
|
3548
|
+
slug: str = typer.Argument(..., help="Container slug."),
|
|
3549
|
+
branch: Optional[str] = typer.Option(None, "--branch", "-b", help="Branch (default main)."),
|
|
3550
|
+
limit: int = typer.Option(20, "--limit", "-n", help="Max entries."),
|
|
3551
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3552
|
+
) -> None:
|
|
3553
|
+
"""A branch's version lineage, tip first — hash, semver, author, reason.
|
|
3554
|
+
|
|
3555
|
+
Examples::
|
|
3556
|
+
|
|
3557
|
+
gateway bench log automationbench
|
|
3558
|
+
gateway bench log automationbench --branch wip-alice -n 50
|
|
3559
|
+
"""
|
|
3560
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3561
|
+
|
|
3562
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3563
|
+
result = client.log(slug, branch=branch, limit=limit)
|
|
3564
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3565
|
+
|
|
3566
|
+
|
|
3567
|
+
@bench_app.command("branch")
|
|
3568
|
+
def bench_branch(
|
|
3569
|
+
name: Optional[str] = typer.Argument(None, help="Create a branch with this name (omit to list)."),
|
|
3570
|
+
path: str = typer.Option(".", "--path", help="Working dir whose container/branch to use."),
|
|
3571
|
+
slug: Optional[str] = typer.Option(None, "--slug", help="Container slug (default: the dir's tracked slug)."),
|
|
3572
|
+
from_version: Optional[str] = typer.Option(None, "--from", help="Start version id (default: the main tip)."),
|
|
3573
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3574
|
+
) -> None:
|
|
3575
|
+
"""List branches/tags (the dir's current branch marked with *), or create one.
|
|
3576
|
+
|
|
3577
|
+
Examples::
|
|
3578
|
+
|
|
3579
|
+
gateway bench branch # list, current marked *
|
|
3580
|
+
gateway bench branch wip-alice # create from main tip
|
|
3581
|
+
gateway bench branch hotfix --from <versionId>
|
|
3582
|
+
"""
|
|
3583
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient, read_local_meta
|
|
3584
|
+
|
|
3585
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3586
|
+
meta = read_local_meta(Path(path)) or {}
|
|
3587
|
+
resolved_slug = slug or meta.get("slug")
|
|
3588
|
+
if not resolved_slug:
|
|
3589
|
+
typer.echo("Error: no --slug and this dir tracks no container (pull first).", err=True)
|
|
3590
|
+
raise typer.Exit(1)
|
|
3591
|
+
|
|
3592
|
+
if name:
|
|
3593
|
+
result = client.create_ref(resolved_slug, name, kind="branch", version_id=from_version)
|
|
3594
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3595
|
+
return
|
|
3596
|
+
|
|
3597
|
+
refs = client.list_refs(resolved_slug)
|
|
3598
|
+
current = meta.get("branch") if meta.get("slug") == resolved_slug else None
|
|
3599
|
+
for ref in refs.get("refs", []):
|
|
3600
|
+
marker = "*" if ref.get("kind") == "branch" and ref.get("name") == current else " "
|
|
3601
|
+
semver = ref.get("semanticVersion") or ""
|
|
3602
|
+
typer.echo(
|
|
3603
|
+
f"{marker} {ref.get('kind'):<6} {ref.get('name'):<24} "
|
|
3604
|
+
f"{(ref.get('contentHash') or '')[:12]:<12} {semver}"
|
|
3605
|
+
)
|
|
3606
|
+
|
|
3607
|
+
|
|
3608
|
+
@bench_app.command("checkout")
|
|
3609
|
+
def bench_checkout(
|
|
3610
|
+
slug_ref: str = typer.Argument(..., help="slug@<branch|tag|semver|hash> (bare slug = main)."),
|
|
3611
|
+
target: Optional[str] = typer.Option(None, "--target", "-t", help="Target dir (defaults to ./<slug>)."),
|
|
3612
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3613
|
+
) -> None:
|
|
3614
|
+
"""Materialize a ref into a working dir and track it (pull + set base).
|
|
3615
|
+
|
|
3616
|
+
Examples::
|
|
3617
|
+
|
|
3618
|
+
gateway bench checkout automationbench@wip-alice
|
|
3619
|
+
gateway bench checkout automationbench@v1.2.0 -t ./ab-1.2
|
|
3620
|
+
"""
|
|
3621
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3622
|
+
|
|
3623
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3624
|
+
result = client.pull(slug_ref, target=target)
|
|
3625
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3626
|
+
|
|
3627
|
+
|
|
3628
|
+
@bench_app.command("tag")
|
|
3629
|
+
def bench_tag(
|
|
3630
|
+
name: str = typer.Argument(..., help="Tag name (immutable)."),
|
|
3631
|
+
path: str = typer.Option(".", "--path", help="Working dir whose container to use."),
|
|
3632
|
+
slug: Optional[str] = typer.Option(None, "--slug", help="Container slug (default: the dir's tracked slug)."),
|
|
3633
|
+
version: Optional[str] = typer.Option(None, "--version", help="Version id to tag (default: main tip)."),
|
|
3634
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3635
|
+
) -> None:
|
|
3636
|
+
"""Create an immutable tag pointing at a version.
|
|
3637
|
+
|
|
3638
|
+
Examples::
|
|
3639
|
+
|
|
3640
|
+
gateway bench tag v1.0.0
|
|
3641
|
+
gateway bench tag baseline --version <versionId>
|
|
3642
|
+
"""
|
|
3643
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient, read_local_meta
|
|
3644
|
+
|
|
3645
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3646
|
+
resolved_slug = slug or (read_local_meta(Path(path)) or {}).get("slug")
|
|
3647
|
+
if not resolved_slug:
|
|
3648
|
+
typer.echo("Error: no --slug and this dir tracks no container (pull first).", err=True)
|
|
3649
|
+
raise typer.Exit(1)
|
|
3650
|
+
result = client.create_ref(resolved_slug, name, kind="tag", version_id=version)
|
|
3651
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3652
|
+
|
|
3653
|
+
|
|
3654
|
+
@bench_app.command("propose")
|
|
3655
|
+
def bench_propose(
|
|
3656
|
+
path: str = typer.Argument(".", help="Working dir with local edits."),
|
|
3657
|
+
title: str = typer.Option(..., "--title", help="Proposal title, shown to the reviewer."),
|
|
3658
|
+
message: str = typer.Option(..., "--message", "-m", help="Why this changeset."),
|
|
3659
|
+
branch: Optional[str] = typer.Option(
|
|
3660
|
+
None, "--branch", "-b", help="Branch the proposal merges into (default main)."
|
|
3661
|
+
),
|
|
3662
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3663
|
+
) -> None:
|
|
3664
|
+
"""File local edits as a change proposal (a PR to the env) instead of
|
|
3665
|
+
pushing to the branch — review + apply happen on the environment homepage.
|
|
3666
|
+
|
|
3667
|
+
Examples::
|
|
3668
|
+
|
|
3669
|
+
gateway bench propose --title "Harden verifier" -m "SQL check missed nulls"
|
|
3670
|
+
"""
|
|
3671
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3672
|
+
|
|
3673
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3674
|
+
result = client.propose(path, title=title, message=message, target_branch=branch)
|
|
3675
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3676
|
+
|
|
3677
|
+
|
|
3678
|
+
@bench_app.command("diff")
|
|
3679
|
+
def bench_diff(
|
|
3680
|
+
base: str = typer.Argument(..., help="Base version id."),
|
|
3681
|
+
head: str = typer.Argument(..., help="Head version id."),
|
|
3682
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3683
|
+
) -> None:
|
|
3684
|
+
"""Per-file diff (added/removed/modified/unchanged) between two version ids."""
|
|
3685
|
+
from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
|
|
3686
|
+
|
|
3687
|
+
client = GatewayBenchmarkClient.from_env(host)
|
|
3688
|
+
result = client.diff(base, head)
|
|
3689
|
+
typer.echo(json.dumps(result, indent=2))
|
|
3690
|
+
|
|
3691
|
+
|
|
3692
|
+
environment_app = typer.Typer(
|
|
3693
|
+
name="environment",
|
|
3694
|
+
help="Environment hub — inspect environments, task sets, rollouts, and the Environment Agent.",
|
|
3695
|
+
no_args_is_help=True,
|
|
3696
|
+
)
|
|
3697
|
+
app.add_typer(environment_app, name="environment")
|
|
3698
|
+
|
|
3699
|
+
|
|
3700
|
+
@environment_app.command("list")
|
|
3701
|
+
def environment_list(
|
|
3702
|
+
slug: Optional[str] = typer.Option(None, "--slug", help="Filter to one environment slug."),
|
|
3703
|
+
limit: int = typer.Option(50, "--limit", "-n", help="Max environments to list."),
|
|
3704
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3705
|
+
) -> None:
|
|
3706
|
+
"""List the project's environments (id, slug, latest version, task-set link)."""
|
|
3707
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3708
|
+
|
|
3709
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3710
|
+
result = client.list_environments(slug=slug, limit=limit)
|
|
3711
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3712
|
+
|
|
3713
|
+
|
|
3714
|
+
@environment_app.command("get")
|
|
3715
|
+
def environment_get(
|
|
3716
|
+
container_id: str = typer.Argument(..., help="Environment (container) id."),
|
|
3717
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3718
|
+
) -> None:
|
|
3719
|
+
"""Show one environment with its version history and rollout count."""
|
|
3720
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3721
|
+
|
|
3722
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3723
|
+
result = client.get_environment(container_id)
|
|
3724
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3725
|
+
|
|
3726
|
+
|
|
3727
|
+
@environment_app.command("tasks")
|
|
3728
|
+
def environment_tasks(
|
|
3729
|
+
container_id: str = typer.Argument(..., help="Environment (container) id."),
|
|
3730
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3731
|
+
) -> None:
|
|
3732
|
+
"""Show the environment's linked task set and its tasks."""
|
|
3733
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3734
|
+
|
|
3735
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3736
|
+
result = client.get_task_set(container_id)
|
|
3737
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3738
|
+
|
|
3739
|
+
|
|
3740
|
+
def _resolve_environment_id(client: Any, ref: str) -> str:
|
|
3741
|
+
"""Accept a container id OR a slug everywhere an environment is named.
|
|
3742
|
+
|
|
3743
|
+
Ids are opaque cuids; slugs are what the UI and bundle manifests show —
|
|
3744
|
+
a CLI that only takes the one you don't have is a lookup errand."""
|
|
3745
|
+
listing = client.list_environments(slug=ref, limit=5)
|
|
3746
|
+
for env in listing.environments:
|
|
3747
|
+
if env.slug == ref:
|
|
3748
|
+
return env.id
|
|
3749
|
+
return ref
|
|
3750
|
+
|
|
3751
|
+
|
|
3752
|
+
@environment_app.command("run")
|
|
3753
|
+
def environment_run(
|
|
3754
|
+
environment: str = typer.Argument(..., help="Environment slug or container id."),
|
|
3755
|
+
model: List[str] = typer.Option(
|
|
3756
|
+
...,
|
|
3757
|
+
"--model",
|
|
3758
|
+
"-m",
|
|
3759
|
+
help=(
|
|
3760
|
+
"Model to run, repeatable. Use a provider-prefixed id — the "
|
|
3761
|
+
"dispatch credential ladder resolves by prefix: `anthropic/...`, "
|
|
3762
|
+
"`openrouter/vendor/model`, `gateway/vendor/model` "
|
|
3763
|
+
"(platform-funded), or `bedrock/...`. Example: "
|
|
3764
|
+
"-m openrouter/z-ai/glm-5.2 -m anthropic/claude-sonnet-4-6"
|
|
3765
|
+
),
|
|
3766
|
+
),
|
|
3767
|
+
version_id: Optional[str] = typer.Option(None, "--version-id", help="Pin a READY version (default: newest)."),
|
|
3768
|
+
num_examples: Optional[int] = typer.Option(None, "--num-examples", "-n", help="Cap scenarios (default: all)."),
|
|
3769
|
+
rollouts: Optional[int] = typer.Option(None, "--rollouts", "-k", help="Rollouts per scenario (default 1)."),
|
|
3770
|
+
wait: bool = typer.Option(True, "--wait/--no-wait", help="Block until every run settles and print verdicts."),
|
|
3771
|
+
wait_timeout: int = typer.Option(1200, "--wait-timeout", help="Seconds to wait before giving up (with --wait)."),
|
|
3772
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3773
|
+
) -> None:
|
|
3774
|
+
"""Dispatch hosted rollouts — one runner job per model — and (by default)
|
|
3775
|
+
wait for the verdicts. The same dispatch the world page's Run button uses.
|
|
3776
|
+
|
|
3777
|
+
Examples::
|
|
3778
|
+
|
|
3779
|
+
gateway environment run slack-legal -m openrouter/z-ai/glm-5.2
|
|
3780
|
+
gateway environment run slack-legal -m gateway/openai/gpt-5-mini --no-wait
|
|
3781
|
+
"""
|
|
3782
|
+
import time as _time
|
|
3783
|
+
|
|
3784
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3785
|
+
|
|
3786
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3787
|
+
container_id = _resolve_environment_id(client, environment)
|
|
3788
|
+
dispatch = client.run_evaluation(
|
|
3789
|
+
container_id,
|
|
3790
|
+
models=list(model),
|
|
3791
|
+
version_id=version_id,
|
|
3792
|
+
num_examples=num_examples,
|
|
3793
|
+
rollouts_per_example=rollouts,
|
|
3794
|
+
)
|
|
3795
|
+
typer.echo(json.dumps(dispatch.model_dump(), indent=2))
|
|
3796
|
+
dispatched = [r for r in dispatch.runs if r.ok and r.runId]
|
|
3797
|
+
if any(not r.ok for r in dispatch.runs):
|
|
3798
|
+
for r in dispatch.runs:
|
|
3799
|
+
if not r.ok:
|
|
3800
|
+
typer.echo(f"# {r.model}: {r.error}", err=True)
|
|
3801
|
+
if not wait or not dispatched:
|
|
3802
|
+
raise typer.Exit(0 if dispatched else 1)
|
|
3803
|
+
|
|
3804
|
+
deadline = _time.time() + wait_timeout
|
|
3805
|
+
failed = False
|
|
3806
|
+
for r in dispatched:
|
|
3807
|
+
while True:
|
|
3808
|
+
budget = int(min(45, max(0, deadline - _time.time())))
|
|
3809
|
+
verdict = client.get_run(r.runId, wait_seconds=budget)
|
|
3810
|
+
if verdict.settled:
|
|
3811
|
+
typer.echo(f"{r.model}: {verdict.status} avgReward={verdict.avgReward}")
|
|
3812
|
+
for rollout in verdict.rollouts:
|
|
3813
|
+
typer.echo(f" {rollout.task}: {rollout.reward} {json.dumps(rollout.verifiers)}")
|
|
3814
|
+
failed = failed or verdict.status.upper() not in ("COMPLETED", "SUCCEEDED")
|
|
3815
|
+
break
|
|
3816
|
+
if _time.time() >= deadline:
|
|
3817
|
+
typer.echo(f"{r.model}: still running after {wait_timeout}s (runId={r.runId})", err=True)
|
|
3818
|
+
failed = True
|
|
3819
|
+
break
|
|
3820
|
+
raise typer.Exit(1 if failed else 0)
|
|
3821
|
+
|
|
3822
|
+
|
|
3823
|
+
@environment_app.command("run-status")
|
|
3824
|
+
def environment_run_status(
|
|
3825
|
+
run_id: str = typer.Argument(..., help="Run id from `environment run` / dispatch."),
|
|
3826
|
+
wait: int = typer.Option(0, "--wait", "-w", help="Block server-side up to N seconds (max 50) for it to settle."),
|
|
3827
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3828
|
+
) -> None:
|
|
3829
|
+
"""One run's verdict: status, avgReward, settled, per-task rollouts.
|
|
3830
|
+
|
|
3831
|
+
`settled: false` after a wait means keep polling — it is not failure.
|
|
3832
|
+
"""
|
|
3833
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3834
|
+
|
|
3835
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3836
|
+
result = client.get_run(run_id, wait_seconds=wait)
|
|
3837
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3838
|
+
|
|
3839
|
+
|
|
3840
|
+
@environment_app.command("link-task-set")
|
|
3841
|
+
def environment_link_task_set(
|
|
3842
|
+
environment: str = typer.Argument(..., help="Environment slug or container id."),
|
|
3843
|
+
tasks: Optional[str] = typer.Option(
|
|
3844
|
+
None, "--tasks", help="tasks.jsonl to fill the set from (rows: id, prompt, metadata, worlds)."
|
|
3845
|
+
),
|
|
3846
|
+
name: Optional[str] = typer.Option(None, "--name", help="Task-set name (default: '<slug> scenarios')."),
|
|
3847
|
+
description: Optional[str] = typer.Option(None, "--description", help="Task-set description."),
|
|
3848
|
+
fresh: bool = typer.Option(False, "--fresh", help="Always create a new set instead of reusing the linked one."),
|
|
3849
|
+
detach: bool = typer.Option(False, "--detach", help="Unlink the environment's task set and exit."),
|
|
3850
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3851
|
+
) -> None:
|
|
3852
|
+
"""Create/fill a scenario set and attach it — the command the world page's
|
|
3853
|
+
empty state names. Idempotent: reuses the linked set and skips tasks whose
|
|
3854
|
+
names already exist.
|
|
3855
|
+
|
|
3856
|
+
Examples::
|
|
3857
|
+
|
|
3858
|
+
gateway environment link-task-set qualtrics-suite --tasks tasks.jsonl
|
|
3859
|
+
gateway environment link-task-set qualtrics-suite --detach
|
|
3860
|
+
"""
|
|
3861
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3862
|
+
|
|
3863
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3864
|
+
container_id = _resolve_environment_id(client, environment)
|
|
3865
|
+
|
|
3866
|
+
if detach:
|
|
3867
|
+
result = client.link_task_set(container_id, None)
|
|
3868
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3869
|
+
return
|
|
3870
|
+
|
|
3871
|
+
existing: set = set()
|
|
3872
|
+
dataset_id: Optional[str] = None
|
|
3873
|
+
if not fresh:
|
|
3874
|
+
try:
|
|
3875
|
+
current = client.get_task_set(container_id)
|
|
3876
|
+
dataset_id = current.datasetId
|
|
3877
|
+
existing = {t.name for t in current.tasks}
|
|
3878
|
+
typer.echo(f"# reusing linked task set {dataset_id}", err=True)
|
|
3879
|
+
except Exception:
|
|
3880
|
+
dataset_id = None
|
|
3881
|
+
if dataset_id is None:
|
|
3882
|
+
set_name = name or f"{environment} scenarios"
|
|
3883
|
+
created = client.create_task_set(set_name, description=description)
|
|
3884
|
+
dataset_id = created.datasetId
|
|
3885
|
+
typer.echo(f"# created task set {dataset_id} ({set_name})", err=True)
|
|
3886
|
+
|
|
3887
|
+
if tasks:
|
|
3888
|
+
for line in Path(tasks).read_text().splitlines():
|
|
3889
|
+
if not line.strip():
|
|
3890
|
+
continue
|
|
3891
|
+
row = json.loads(line)
|
|
3892
|
+
task_name = str(row.get("id") or row.get("task") or "")
|
|
3893
|
+
if not task_name or task_name in existing:
|
|
3894
|
+
continue
|
|
3895
|
+
metadata = dict(row.get("metadata") or {})
|
|
3896
|
+
if row.get("worlds") is not None:
|
|
3897
|
+
metadata["worlds"] = row["worlds"]
|
|
3898
|
+
client.create_task(dataset_id, task_name, str(row.get("prompt") or ""), metadata=metadata or None)
|
|
3899
|
+
typer.echo(f"# task {task_name} created", err=True)
|
|
3900
|
+
|
|
3901
|
+
result = client.link_task_set(container_id, dataset_id)
|
|
3902
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3903
|
+
|
|
3904
|
+
|
|
3905
|
+
@environment_app.command("performance")
|
|
3906
|
+
def environment_performance(
|
|
3907
|
+
container_id: str = typer.Argument(..., help="Environment (container) id."),
|
|
3908
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3909
|
+
) -> None:
|
|
3910
|
+
"""Show per-task performance across rollouts (nulls kept as their own category)."""
|
|
3911
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3912
|
+
|
|
3913
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3914
|
+
result = client.get_performance(container_id)
|
|
3915
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3916
|
+
|
|
3917
|
+
|
|
3918
|
+
@environment_app.command("rollouts")
|
|
3919
|
+
def environment_rollouts(
|
|
3920
|
+
container_id: str = typer.Argument(..., help="Environment (container) id."),
|
|
3921
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3922
|
+
) -> None:
|
|
3923
|
+
"""List the environment's rollouts (evaluations), newest first."""
|
|
3924
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3925
|
+
|
|
3926
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3927
|
+
result = client.list_rollouts(container_id)
|
|
3928
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3929
|
+
|
|
3930
|
+
|
|
3931
|
+
@environment_app.command("samples")
|
|
3932
|
+
def environment_samples(
|
|
3933
|
+
container_id: str = typer.Argument(..., help="Environment (container) id."),
|
|
3934
|
+
evaluation_id: str = typer.Argument(..., help="Rollout (evaluation) id."),
|
|
3935
|
+
limit: int = typer.Option(200, "--limit", "-n", help="Max samples to return."),
|
|
3936
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3937
|
+
) -> None:
|
|
3938
|
+
"""Show the scored samples of one rollout (reward/score/correct + sub-rewards)."""
|
|
3939
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3940
|
+
|
|
3941
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3942
|
+
result = client.get_rollout_samples(container_id, evaluation_id, limit=limit)
|
|
3943
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3944
|
+
|
|
3945
|
+
|
|
3946
|
+
@environment_app.command("create-task")
|
|
3947
|
+
def environment_create_task(
|
|
3948
|
+
dataset_id: str = typer.Argument(..., help="Task set dataset id (kind='tasks')."),
|
|
3949
|
+
name: str = typer.Option(..., "--name", help="Task name."),
|
|
3950
|
+
prompt: str = typer.Option(..., "--prompt", help="Task prompt."),
|
|
3951
|
+
expected_output: Optional[str] = typer.Option(None, "--expected-output", help="Optional expected output."),
|
|
3952
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3953
|
+
) -> None:
|
|
3954
|
+
"""Create a task in a task set (a kind='tasks' dataset item). Requires a secret key.
|
|
3955
|
+
|
|
3956
|
+
The task set dataset id is on the environment (``environment get`` →
|
|
3957
|
+
``taskSetDatasetId``) or on ``environment tasks`` (→ ``datasetId``).
|
|
3958
|
+
"""
|
|
3959
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3960
|
+
|
|
3961
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3962
|
+
result = client.create_task(dataset_id, name, prompt, expected_output=expected_output)
|
|
3963
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3964
|
+
|
|
3965
|
+
|
|
3966
|
+
@environment_app.command("agent-status")
|
|
3967
|
+
def environment_agent_status(
|
|
3968
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3969
|
+
) -> None:
|
|
3970
|
+
"""Show the project's Environment Agent config and recent runs."""
|
|
3971
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3972
|
+
|
|
3973
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3974
|
+
result = client.get_agent()
|
|
3975
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3976
|
+
|
|
3977
|
+
|
|
3978
|
+
@environment_app.command("agent-trigger")
|
|
3979
|
+
def environment_agent_trigger(
|
|
3980
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
3981
|
+
) -> None:
|
|
3982
|
+
"""Enqueue a manual Environment Agent run. Requires a saved config and a secret key."""
|
|
3983
|
+
from gatewaysdk.environments import GatewayEnvironmentClient
|
|
3984
|
+
|
|
3985
|
+
client = GatewayEnvironmentClient.from_env(host)
|
|
3986
|
+
result = client.trigger_agent()
|
|
3987
|
+
typer.echo(json.dumps(result.model_dump(), indent=2))
|
|
3988
|
+
|
|
3989
|
+
|
|
3990
|
+
# =============================================================================
|
|
3991
|
+
# release commands (the CI/CD half of Releases)
|
|
3992
|
+
# =============================================================================
|
|
3993
|
+
|
|
3994
|
+
release_app = typer.Typer(
|
|
3995
|
+
name="release",
|
|
3996
|
+
help="Release gate for agent builds — ask the platform whether a build may ship.",
|
|
3997
|
+
no_args_is_help=True,
|
|
3998
|
+
)
|
|
3999
|
+
app.add_typer(release_app, name="release")
|
|
4000
|
+
|
|
4001
|
+
|
|
4002
|
+
@release_app.command("gate")
|
|
4003
|
+
def release_gate(
|
|
4004
|
+
build: Optional[str] = typer.Option(None, "--build", help="Agent build id."),
|
|
4005
|
+
agent: Optional[str] = typer.Option(None, "--agent", help="Agent id (with --version)."),
|
|
4006
|
+
version: Optional[str] = typer.Option(None, "--version", help="Declared build version (with --agent)."),
|
|
4007
|
+
commit: Optional[str] = typer.Option(
|
|
4008
|
+
None, "--commit", help="Commit sha the build was registered from (full, or a 7+ char prefix)."
|
|
4009
|
+
),
|
|
4010
|
+
wait: bool = typer.Option(False, "--wait", help="Poll until every run attributed to the build is terminal."),
|
|
4011
|
+
timeout: float = typer.Option(
|
|
4012
|
+
1800.0, "--timeout", min=1.0, help="Seconds to wait for runs to settle (with --wait)."
|
|
4013
|
+
),
|
|
4014
|
+
interval: float = typer.Option(15.0, "--interval", min=1.0, help="Seconds between polls (with --wait)."),
|
|
4015
|
+
as_json: bool = typer.Option(False, "--json", help="Print the endpoint's JSON instead of a table."),
|
|
4016
|
+
host: Optional[str] = typer.Option(None, "--host", envvar="GATEWAY_HOST"),
|
|
4017
|
+
api_key: Optional[str] = typer.Option(None, "--api-key", envvar="GATEWAY_SECRET_KEY"),
|
|
4018
|
+
) -> None:
|
|
4019
|
+
"""The release gate's verdict on a build, as an exit code CI can branch on.
|
|
4020
|
+
|
|
4021
|
+
Exit codes::
|
|
4022
|
+
|
|
4023
|
+
0 pass the gate is open on the build's latest run
|
|
4024
|
+
1 blocked the latest run failed the project's rule (reasons printed)
|
|
4025
|
+
2 untested no run recorded this build, or its latest run is still going
|
|
4026
|
+
|
|
4027
|
+
The verdict is computed on the server with the same rule the Releases
|
|
4028
|
+
page enforces on Promote and Rollback; this command never re-derives it.
|
|
4029
|
+
|
|
4030
|
+
Examples::
|
|
4031
|
+
|
|
4032
|
+
gateway release gate --commit $GITHUB_SHA --wait
|
|
4033
|
+
gateway release gate --agent checkout-bot --version 2.4.0 --json
|
|
4034
|
+
gateway release gate --build cm1abc... --wait --timeout 900
|
|
4035
|
+
"""
|
|
4036
|
+
import os
|
|
4037
|
+
|
|
4038
|
+
from gatewaysdk.cli import release_gate as gate_mod
|
|
4039
|
+
from gatewaysdk.cli.http_client import create_client
|
|
4040
|
+
|
|
4041
|
+
try:
|
|
4042
|
+
query = gate_mod.build_query(build=build, agent=agent, version=version, commit=commit)
|
|
4043
|
+
except gate_mod.GateLookupError as exc:
|
|
4044
|
+
typer.echo(f"Error: {exc}", err=True)
|
|
4045
|
+
raise typer.Exit(2)
|
|
4046
|
+
|
|
4047
|
+
client = create_client(host, api_key)
|
|
4048
|
+
|
|
4049
|
+
def _progress(result: dict) -> None:
|
|
4050
|
+
if as_json:
|
|
4051
|
+
return
|
|
4052
|
+
runs = result.get("evaluatedRuns") or []
|
|
4053
|
+
pending = [r for r in runs if str(r.get("state", "")).upper() not in gate_mod.TERMINAL_STATES]
|
|
4054
|
+
if not runs:
|
|
4055
|
+
typer.echo("waiting: no run has recorded this build yet", err=True)
|
|
4056
|
+
elif pending:
|
|
4057
|
+
typer.echo(f"waiting: {len(pending)} of {len(runs)} run(s) still going", err=True)
|
|
4058
|
+
|
|
4059
|
+
try:
|
|
4060
|
+
if wait:
|
|
4061
|
+
result, settled = gate_mod.wait_for_gate(
|
|
4062
|
+
client, query, timeout_s=timeout, interval_s=interval, on_poll=_progress
|
|
4063
|
+
)
|
|
4064
|
+
if not settled:
|
|
4065
|
+
typer.echo(f"timed out after {timeout:g}s; reporting the latest verdict", err=True)
|
|
4066
|
+
else:
|
|
4067
|
+
result = gate_mod.fetch_gate(client, query)
|
|
4068
|
+
except LookupError as exc:
|
|
4069
|
+
typer.echo(f"Error: {exc}", err=True)
|
|
4070
|
+
raise typer.Exit(2)
|
|
4071
|
+
except httpx.HTTPStatusError as exc:
|
|
4072
|
+
typer.echo(f"Failed (HTTP {exc.response.status_code}): {exc.response.text}", err=True)
|
|
4073
|
+
raise typer.Exit(2)
|
|
4074
|
+
|
|
4075
|
+
typer.echo(gate_mod.render_json(result) if as_json else gate_mod.render_table(result))
|
|
4076
|
+
gate_mod.write_step_summary(result, os.environ.get("GITHUB_STEP_SUMMARY"))
|
|
4077
|
+
raise typer.Exit(gate_mod.exit_code_for(result))
|
|
4078
|
+
|
|
4079
|
+
|
|
4080
|
+
def main() -> None:
|
|
4081
|
+
"""Entry point for the `gateway` CLI command."""
|
|
4082
|
+
app()
|
|
4083
|
+
|
|
4084
|
+
|
|
4085
|
+
if __name__ == "__main__":
|
|
4086
|
+
main()
|