apsimo 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- apsimo/__init__.py +38 -0
- apsimo/__main__.py +6 -0
- apsimo/agent/__init__.py +6 -0
- apsimo/agent/client.py +276 -0
- apsimo/agent/models.py +46 -0
- apsimo/agents/__init__.py +20 -0
- apsimo/agents/models.py +264 -0
- apsimo/agents/store.py +861 -0
- apsimo/agents/websocket.py +522 -0
- apsimo/api/__init__.py +1 -0
- apsimo/api/auth_telemetry.py +287 -0
- apsimo/api/authority.py +1203 -0
- apsimo/api/contact_grants.py +347 -0
- apsimo/api/middleware.py +483 -0
- apsimo/api/routers/__init__.py +1 -0
- apsimo/api/routers/commitment_work.py +265 -0
- apsimo/api/routers/context_gate.py +123 -0
- apsimo/api/routers/executions.py +140 -0
- apsimo/api/routers/followup_plans.py +147 -0
- apsimo/api/routers/governed_actions.py +162 -0
- apsimo/api/routers/host.py +14473 -0
- apsimo/api/routers/initiative_work.py +115 -0
- apsimo/api/routers/mining.py +104 -0
- apsimo/api/routers/observations.py +110 -0
- apsimo/api/routers/social_state.py +225 -0
- apsimo/api/routers/task_queue.py +2715 -0
- apsimo/api/routers/temporal_followups.py +251 -0
- apsimo/api/routers/transport.py +110 -0
- apsimo/api/routers/transport_ingress_api.py +240 -0
- apsimo/api/schemas/__init__.py +1 -0
- apsimo/api/schemas/host.py +1949 -0
- apsimo/autonomy/cli.py +110 -0
- apsimo/autonomy/condition_worker.py +437 -0
- apsimo/autonomy/config.py +424 -0
- apsimo/autonomy/loop.py +4316 -0
- apsimo/autonomy/registry.py +339 -0
- apsimo/autonomy/scheduler.py +1822 -0
- apsimo/autonomy/synthesis.py +449 -0
- apsimo/backup.py +962 -0
- apsimo/beliefs/__init__.py +23 -0
- apsimo/beliefs/contradictions.py +109 -0
- apsimo/beliefs/decay.py +61 -0
- apsimo/beliefs/engine.py +479 -0
- apsimo/beliefs/models.py +67 -0
- apsimo/beliefs/promotion.py +41 -0
- apsimo/beliefs/resolve.py +58 -0
- apsimo/beliefs/source_claims.py +690 -0
- apsimo/beliefs/source_projection.py +883 -0
- apsimo/beliefs/source_time.py +208 -0
- apsimo/beliefs/store.py +133 -0
- apsimo/briefings/aggregators.py +824 -0
- apsimo/briefings/composer.py +420 -0
- apsimo/briefings/config.py +55 -0
- apsimo/briefings/delivery.py +439 -0
- apsimo/briefings/engagement.py +97 -0
- apsimo/briefings/engine.py +274 -0
- apsimo/briefings/enhancer.py +99 -0
- apsimo/briefings/models.py +183 -0
- apsimo/briefings/scheduler.py +382 -0
- apsimo/briefings/store.py +435 -0
- apsimo/chain/__init__.py +48 -0
- apsimo/chain/block.py +100 -0
- apsimo/chain/cli.py +704 -0
- apsimo/chain/genesis.py +443 -0
- apsimo/chain/identity.py +416 -0
- apsimo/chain/keys.py +1025 -0
- apsimo/chain/local_keys.py +187 -0
- apsimo/chain/manager.py +290 -0
- apsimo/chain/node.py +163 -0
- apsimo/chain/plugin_transactions.py +371 -0
- apsimo/chain/protocol.py +220 -0
- apsimo/chain/state_machine.py +676 -0
- apsimo/chain/storage.py +503 -0
- apsimo/chain/transactions.py +250 -0
- apsimo/chain/validation.py +397 -0
- apsimo/channels/__init__.py +1 -0
- apsimo/channels/manifest.py +31 -0
- apsimo/channels/migrations/001_channels_schema.sql +12 -0
- apsimo/channels/phone_gateways.py +42 -0
- apsimo/channels/presence.py +188 -0
- apsimo/channels/router.py +235 -0
- apsimo/channels/store.py +231 -0
- apsimo/cli.py +2688 -0
- apsimo/cognition/__init__.py +11 -0
- apsimo/cognition/charter.py +398 -0
- apsimo/cognition/drive_governance.py +3530 -0
- apsimo/cognition/evidence_pipeline.py +1627 -0
- apsimo/cognition/external_events.py +932 -0
- apsimo/cognition/goal_spine.py +3488 -0
- apsimo/cognition/introspection.py +214 -0
- apsimo/cognition/prompt.py +150 -0
- apsimo/cognition/runtime.py +108 -0
- apsimo/cognition/trigger.py +154 -0
- apsimo/commitments/__init__.py +18 -0
- apsimo/commitments/local_work.py +355 -0
- apsimo/commitments/store.py +1052 -0
- apsimo/commitments/work.py +91 -0
- apsimo/compat.py +53 -0
- apsimo/compression/__init__.py +467 -0
- apsimo/connectors/__init__.py +21 -0
- apsimo/connectors/base.py +152 -0
- apsimo/connectors/caldav_calendar.py +125 -0
- apsimo/connectors/fs_documents.py +85 -0
- apsimo/connectors/imap_email.py +138 -0
- apsimo/connectors/manager.py +218 -0
- apsimo/connectors/webhook_pull.py +88 -0
- apsimo/contacts/__init__.py +33 -0
- apsimo/contacts/comms.py +357 -0
- apsimo/contacts/config.py +79 -0
- apsimo/contacts/exporters/__init__.py +1 -0
- apsimo/contacts/exporters/vcard.py +71 -0
- apsimo/contacts/identity_links.py +251 -0
- apsimo/contacts/importer.py +280 -0
- apsimo/contacts/importers/__init__.py +1 -0
- apsimo/contacts/importers/batch.py +43 -0
- apsimo/contacts/importers/macos_contacts.py +101 -0
- apsimo/contacts/migrations/001_contacts_schema.sql +141 -0
- apsimo/contacts/migrations/002_trust_scopes.sql +36 -0
- apsimo/contacts/migrations/003_open_gateway_enum.sql +32 -0
- apsimo/contacts/migrations/004_contact_provision_operations.sql +18 -0
- apsimo/contacts/migrations/005_identity_links.sql +27 -0
- apsimo/contacts/models.py +308 -0
- apsimo/contacts/scoring.py +16 -0
- apsimo/contacts/store.py +1623 -0
- apsimo/contacts/transport_ingress.py +252 -0
- apsimo/contacts/world_bridge.py +314 -0
- apsimo/contextgate/__init__.py +69 -0
- apsimo/contextgate/chunker.py +169 -0
- apsimo/contextgate/estimate.py +54 -0
- apsimo/contextgate/gate.py +313 -0
- apsimo/contextgate/retrieve.py +115 -0
- apsimo/delivery/__init__.py +16 -0
- apsimo/delivery/bridge.py +1260 -0
- apsimo/delivery/channels.py +526 -0
- apsimo/delivery/classification.py +50 -0
- apsimo/delivery/rate_limiter.py +268 -0
- apsimo/delivery/reachout_policy.py +206 -0
- apsimo/directed/__init__.py +22 -0
- apsimo/directed/audit.py +167 -0
- apsimo/directed/intake.py +95 -0
- apsimo/directed/models.py +191 -0
- apsimo/directed/service.py +509 -0
- apsimo/directives/__init__.py +25 -0
- apsimo/directives/evidence.py +87 -0
- apsimo/directives/extractor.py +188 -0
- apsimo/directives/guard.py +364 -0
- apsimo/directives/models.py +206 -0
- apsimo/directives/service.py +372 -0
- apsimo/directives/store.py +167 -0
- apsimo/doctor.py +2173 -0
- apsimo/environment.py +43 -0
- apsimo/events/__init__.py +33 -0
- apsimo/events/broadcaster.py +98 -0
- apsimo/events/bus.py +217 -0
- apsimo/events/journal.py +863 -0
- apsimo/events/stream.py +131 -0
- apsimo/events/types.py +150 -0
- apsimo/execution_results.py +357 -0
- apsimo/feedback/__init__.py +5 -0
- apsimo/feedback/store.py +76 -0
- apsimo/feeds/__init__.py +19 -0
- apsimo/feeds/cli.py +84 -0
- apsimo/feeds/engine.py +437 -0
- apsimo/feeds/example-feed.yaml +77 -0
- apsimo/feeds/hermes_cron.py +126 -0
- apsimo/feeds/manager.py +235 -0
- apsimo/feeds/spec.py +250 -0
- apsimo/feeds/template.py +202 -0
- apsimo/gate/__init__.py +18 -0
- apsimo/gate/audit.py +61 -0
- apsimo/gate/communication_policy.py +166 -0
- apsimo/gate/config.py +72 -0
- apsimo/gate/context_provenance.py +170 -0
- apsimo/gate/env_risk.py +226 -0
- apsimo/gate/guard_audit.py +353 -0
- apsimo/gate/layers/__init__.py +1 -0
- apsimo/gate/layers/base.py +15 -0
- apsimo/gate/layers/l1_recipient.py +66 -0
- apsimo/gate/layers/l2_pii.py +134 -0
- apsimo/gate/layers/l3_cross_context.py +50 -0
- apsimo/gate/layers/l4_trust_tier.py +78 -0
- apsimo/gate/layers/l5_injection.py +199 -0
- apsimo/gate/layers/l6_review.py +86 -0
- apsimo/gate/layers/l7_delay.py +100 -0
- apsimo/gate/layers/tom2_epistemic.py +185 -0
- apsimo/gate/models.py +64 -0
- apsimo/gate/pending_dispatch.py +5 -0
- apsimo/gate/pipeline.py +206 -0
- apsimo/gate/rejection.py +259 -0
- apsimo/gate/response_guard.py +700 -0
- apsimo/gate/rulesets/injection_v1.yaml +51 -0
- apsimo/gate/surface_policy.py +189 -0
- apsimo/gate/taint.py +226 -0
- apsimo/genesis.json +9 -0
- apsimo/goals/__init__.py +100 -0
- apsimo/goals/config.py +38 -0
- apsimo/goals/decomposer.py +421 -0
- apsimo/goals/engine.py +617 -0
- apsimo/goals/inference.py +354 -0
- apsimo/goals/models.py +302 -0
- apsimo/goals/priority.py +270 -0
- apsimo/goals/queue_bridge.py +149 -0
- apsimo/goals/replan.py +450 -0
- apsimo/goals/schema.sql +89 -0
- apsimo/goals/store.py +692 -0
- apsimo/governed_actions.py +1708 -0
- apsimo/harness_integration/__init__.py +45 -0
- apsimo/harness_integration/context.py +41 -0
- apsimo/harness_integration/skills.py +231 -0
- apsimo/identity/__init__.py +26 -0
- apsimo/identity/participants.py +181 -0
- apsimo/identity/resolver.py +329 -0
- apsimo/identity_bootstrap/__init__.py +5 -0
- apsimo/identity_bootstrap/builder.py +208 -0
- apsimo/identity_bootstrap/corpus.py +443 -0
- apsimo/identity_bootstrap/models.py +54 -0
- apsimo/identity_bootstrap/runner.py +353 -0
- apsimo/identity_bootstrap/seeders/__init__.py +25 -0
- apsimo/identity_bootstrap/seeders/briefings.py +109 -0
- apsimo/identity_bootstrap/seeders/chain.py +57 -0
- apsimo/identity_bootstrap/seeders/goals.py +128 -0
- apsimo/identity_bootstrap/seeders/memory.py +191 -0
- apsimo/identity_bootstrap/seeders/neo4j_cognition.py +79 -0
- apsimo/identity_bootstrap/seeders/relationship.py +152 -0
- apsimo/identity_bootstrap/seeders/sessions.py +67 -0
- apsimo/identity_bootstrap/seeders/skills.py +92 -0
- apsimo/identity_bootstrap/seeders/task_queue.py +72 -0
- apsimo/identity_bootstrap/seeders/world_model.py +143 -0
- apsimo/identity_bootstrap/self_query.py +92 -0
- apsimo/identity_bootstrap/self_reflection.py +155 -0
- apsimo/identity_bootstrap/skill.py +37 -0
- apsimo/identity_bootstrap/verifier.py +436 -0
- apsimo/initiatives/__init__.py +20 -0
- apsimo/initiatives/action_registry.py +454 -0
- apsimo/initiatives/approval_authority.py +2105 -0
- apsimo/initiatives/approval_policy.py +123 -0
- apsimo/initiatives/assignment.py +263 -0
- apsimo/initiatives/backup_evidence.py +100 -0
- apsimo/initiatives/context_freshness.py +103 -0
- apsimo/initiatives/models.py +318 -0
- apsimo/initiatives/native_work.py +270 -0
- apsimo/initiatives/standing_approvals.py +232 -0
- apsimo/initiatives/store.py +1081 -0
- apsimo/initiatives/temporal_followup.py +410 -0
- apsimo/intelligence/__init__.py +1 -0
- apsimo/intelligence/cognition/__init__.py +24 -0
- apsimo/intelligence/cognition/gap_detector.py +148 -0
- apsimo/intelligence/cognition/metalearner.py +547 -0
- apsimo/intelligence/cognition/metrics_collector.py +217 -0
- apsimo/intelligence/cognition/performance_index.py +299 -0
- apsimo/intelligence/cognition/registry.py +192 -0
- apsimo/intelligence/cognition/strategy_adjuster.py +222 -0
- apsimo/intelligence/cognition/types.py +16 -0
- apsimo/intelligence/components/__init__.py +66 -0
- apsimo/intelligence/components/anomaly_detector.py +413 -0
- apsimo/intelligence/components/initiative_engine.py +2643 -0
- apsimo/intelligence/components/preference_learner.py +521 -0
- apsimo/intelligence/components/research_orchestrator.py +358 -0
- apsimo/intelligence/components/self_directed_thinker.py +221 -0
- apsimo/intelligence/components/self_reflector.py +252 -0
- apsimo/intelligence/components/session_continuity.py +154 -0
- apsimo/intelligence/components/task_planner.py +320 -0
- apsimo/intelligence/components/tool_learner.py +217 -0
- apsimo/intelligence/graph/__init__.py +79 -0
- apsimo/intelligence/graph/client.py +2483 -0
- apsimo/intelligence/graph/consolidator.py +405 -0
- apsimo/intelligence/graph/distiller.py +312 -0
- apsimo/intelligence/graph/migrations.py +129 -0
- apsimo/intelligence/graph/queries.py +248 -0
- apsimo/intelligence/graph/recall.py +281 -0
- apsimo/intelligence/graph/reconciler.py +144 -0
- apsimo/intelligence/graph/schema.py +337 -0
- apsimo/intelligence/graph/selection.py +252 -0
- apsimo/intelligence/learning/__init__.py +17 -0
- apsimo/intelligence/learning/continuous_learner.py +245 -0
- apsimo/intelligence/learning/feedback_store.py +321 -0
- apsimo/intelligence/mind_model/__init__.py +1 -0
- apsimo/intelligence/mind_model/graph_baseline.py +136 -0
- apsimo/intelligence/mind_model/signal_collector.py +361 -0
- apsimo/intelligence/relationships/__init__.py +11 -0
- apsimo/intelligence/relationships/profiler.py +389 -0
- apsimo/intelligence/relationships/scorer.py +560 -0
- apsimo/intelligence/relationships/signal_floor.py +66 -0
- apsimo/intelligence/relationships/trust_tiers.py +300 -0
- apsimo/intelligence/synthesis/__init__.py +40 -0
- apsimo/intelligence/synthesis/connection_discoverer.py +379 -0
- apsimo/intelligence/synthesis/cross_domain_analyzer.py +287 -0
- apsimo/intelligence/synthesis/insight_deliverer.py +171 -0
- apsimo/intelligence/synthesis/insight_store.py +79 -0
- apsimo/intelligence/synthesis/insight_validator.py +183 -0
- apsimo/intelligence/synthesis/novelty_scorer.py +267 -0
- apsimo/intelligence/turn_middleware/__init__.py +15 -0
- apsimo/intelligence/turn_middleware/memory_sync.py +119 -0
- apsimo/mcp/__init__.py +41 -0
- apsimo/mcp/__main__.py +6 -0
- apsimo/mcp/config.py +287 -0
- apsimo/mcp/server.py +501 -0
- apsimo/migrations.py +187 -0
- apsimo/mining/__init__.py +27 -0
- apsimo/mining/corpus.py +239 -0
- apsimo/mining/escalations.py +289 -0
- apsimo/mining/models.py +169 -0
- apsimo/mining/store.py +210 -0
- apsimo/models/__init__.py +30 -0
- apsimo/models/memory.py +80 -0
- apsimo/models/mesh.py +72 -0
- apsimo/models/person.py +104 -0
- apsimo/models/signal.py +108 -0
- apsimo/observations/__init__.py +15 -0
- apsimo/observations/store.py +277 -0
- apsimo/patterns/__init__.py +6 -0
- apsimo/patterns/extract.py +187 -0
- apsimo/patterns/store.py +227 -0
- apsimo/persona/__init__.py +1 -0
- apsimo/persona/engine.py +611 -0
- apsimo/persona/manifest.py +140 -0
- apsimo/projects/__init__.py +28 -0
- apsimo/projects/engine.py +1681 -0
- apsimo/projects/event_outbox.py +188 -0
- apsimo/projects/models.py +216 -0
- apsimo/projects/planner.py +181 -0
- apsimo/projects/store.py +1446 -0
- apsimo/proposals/__init__.py +12 -0
- apsimo/proposals/engine.py +114 -0
- apsimo/proposals/models.py +207 -0
- apsimo/qualification/__init__.py +1 -0
- apsimo/qualification/cases.py +75 -0
- apsimo/qualification/cli.py +51 -0
- apsimo/qualification/memory_cases.py +209 -0
- apsimo/qualification/records.py +92 -0
- apsimo/qualification/report.py +87 -0
- apsimo/qualification/runner.py +311 -0
- apsimo/qualification/structured_cases.py +131 -0
- apsimo/reasoning/__init__.py +13 -0
- apsimo/reasoning/executor.py +506 -0
- apsimo/reasoning/loop.py +373 -0
- apsimo/reasoning/native_tools/__init__.py +16 -0
- apsimo/reasoning/native_tools/calculate.py +141 -0
- apsimo/reasoning/native_tools/file_ops.py +150 -0
- apsimo/reasoning/native_tools/web_search.py +49 -0
- apsimo/reasoning/tool_policy.py +182 -0
- apsimo/redact/__init__.py +176 -0
- apsimo/repos/__init__.py +5 -0
- apsimo/repos/mirrors.py +204 -0
- apsimo/research/__init__.py +41 -0
- apsimo/research/artifact.py +482 -0
- apsimo/research/gatherer.py +387 -0
- apsimo/research/pipeline.py +513 -0
- apsimo/research/search/__init__.py +7 -0
- apsimo/research/search/base.py +41 -0
- apsimo/research/search/brave.py +59 -0
- apsimo/research/search/cache.py +51 -0
- apsimo/research/search/duckduckgo.py +103 -0
- apsimo/research/search/orchestrator.py +119 -0
- apsimo/research/search/serpapi.py +59 -0
- apsimo/research/search/tavily.py +59 -0
- apsimo/research/synthesizer.py +309 -0
- apsimo/router/__init__.py +30 -0
- apsimo/router/complexity_scorer.py +148 -0
- apsimo/router/endpoints.py +153 -0
- apsimo/router/fallback.py +58 -0
- apsimo/router/functions.py +243 -0
- apsimo/router/native_policy.py +52 -0
- apsimo/router/router.py +762 -0
- apsimo/router/self_learning.py +174 -0
- apsimo/router/tiers.py +677 -0
- apsimo/sandbox/__init__.py +21 -0
- apsimo/sandbox/backend.py +195 -0
- apsimo/sandbox/manager.py +173 -0
- apsimo/scope_bounds.py +7 -0
- apsimo/secrets/__init__.py +6 -0
- apsimo/secrets/backends/__init__.py +8 -0
- apsimo/secrets/backends/base.py +42 -0
- apsimo/secrets/backends/env.py +110 -0
- apsimo/secrets/backends/keyring.py +72 -0
- apsimo/secrets/backends/onepassword.py +232 -0
- apsimo/secrets/cli.py +191 -0
- apsimo/secrets/manager.py +160 -0
- apsimo/secrets/migration.py +101 -0
- apsimo/secrets/types.py +98 -0
- apsimo/seed.py +41 -0
- apsimo/self_model/__init__.py +37 -0
- apsimo/self_model/appraisals.py +673 -0
- apsimo/self_model/benchmark.py +1314 -0
- apsimo/self_model/brief.py +40 -0
- apsimo/self_model/event_concerns.py +1128 -0
- apsimo/self_model/execution_forecasts.py +353 -0
- apsimo/self_model/expectations.py +1595 -0
- apsimo/self_model/experiments.py +1150 -0
- apsimo/self_model/journal.py +148 -0
- apsimo/self_model/judgments.py +705 -0
- apsimo/self_model/native_outcomes.py +55 -0
- apsimo/self_model/params.py +220 -0
- apsimo/self_model/perspective.py +246 -0
- apsimo/self_model/reconcile.py +183 -0
- apsimo/self_model/reply_forecasts.py +381 -0
- apsimo/self_model/runtime_forecasts.py +296 -0
- apsimo/self_model/runtime_models.py +67 -0
- apsimo/self_model/settlement.py +207 -0
- apsimo/self_model/situation.py +1731 -0
- apsimo/self_model/store.py +883 -0
- apsimo/self_model/supervised.py +137 -0
- apsimo/self_model/thinker.py +99 -0
- apsimo/self_model/trust.py +388 -0
- apsimo/self_model/workspace.py +2388 -0
- apsimo/server.py +4197 -0
- apsimo/services/__init__.py +1 -0
- apsimo/services/agent_bridge.py +474 -0
- apsimo/services/initiative_executor.py +914 -0
- apsimo/services/instance.py +297 -0
- apsimo/sessions/__init__.py +22 -0
- apsimo/sessions/config.py +13 -0
- apsimo/sessions/context_loader.py +88 -0
- apsimo/sessions/federation_session.py +75 -0
- apsimo/sessions/isolated_session.py +98 -0
- apsimo/sessions/reports.py +84 -0
- apsimo/sessions/store.py +148 -0
- apsimo/setup.py +2818 -0
- apsimo/setup_hermes.py +879 -0
- apsimo/setup_local_work.py +218 -0
- apsimo/setup_native_goals.py +134 -0
- apsimo/setup_native_reviews.py +115 -0
- apsimo/skills/__init__.py +10 -0
- apsimo/skills/base.py +108 -0
- apsimo/skills/budget.py +28 -0
- apsimo/skills/executor.py +493 -0
- apsimo/skills/executors/__init__.py +1 -0
- apsimo/skills/executors/behavioral_correction.py +75 -0
- apsimo/skills/executors/capability_gap.py +38 -0
- apsimo/skills/executors/data_quality.py +163 -0
- apsimo/skills/executors/knowledge_acquisition.py +41 -0
- apsimo/skills/executors/operational_hygiene.py +185 -0
- apsimo/skills/executors/subsystem_health.py +169 -0
- apsimo/skills/hermes_export.py +431 -0
- apsimo/skills/index.py +123 -0
- apsimo/skills/learning/__init__.py +21 -0
- apsimo/skills/learning/novelty_detector.py +206 -0
- apsimo/skills/learning/pattern_extractor.py +199 -0
- apsimo/skills/learning/triggers.py +159 -0
- apsimo/skills/loader.py +246 -0
- apsimo/skills/migrations/002_progressive_loading.sql +6 -0
- apsimo/skills/migrations/backfill_triggers.py +20 -0
- apsimo/skills/models.py +202 -0
- apsimo/skills/packager.py +128 -0
- apsimo/skills/protocols.py +70 -0
- apsimo/skills/registry.py +191 -0
- apsimo/skills/runtime.py +58 -0
- apsimo/skills/sandbox_runner.py +229 -0
- apsimo/skills/scheduler.py +129 -0
- apsimo/skills/schema.py +79 -0
- apsimo/skills/security/__init__.py +12 -0
- apsimo/skills/security/guards.py +53 -0
- apsimo/skills/security/scanner.py +223 -0
- apsimo/skills_memory/__init__.py +26 -0
- apsimo/skills_memory/distill.py +159 -0
- apsimo/skills_memory/models.py +85 -0
- apsimo/skills_memory/retrieve.py +62 -0
- apsimo/skills_memory/store.py +172 -0
- apsimo/surprise/__init__.py +6 -0
- apsimo/surprise/accumulation.py +57 -0
- apsimo/surprise/scorer.py +102 -0
- apsimo/surprise/store.py +203 -0
- apsimo/task_queue/__init__.py +69 -0
- apsimo/task_queue/action_receipts.py +148 -0
- apsimo/task_queue/approval_relay_canary.py +108 -0
- apsimo/task_queue/config.py +85 -0
- apsimo/task_queue/contract.py +361 -0
- apsimo/task_queue/events.py +130 -0
- apsimo/task_queue/governor.py +1031 -0
- apsimo/task_queue/handlers/__init__.py +16 -0
- apsimo/task_queue/handlers/base.py +37 -0
- apsimo/task_queue/handlers/inference.py +640 -0
- apsimo/task_queue/handlers/monitoring.py +116 -0
- apsimo/task_queue/handlers/registry.py +75 -0
- apsimo/task_queue/handlers/subtask_handler.py +173 -0
- apsimo/task_queue/handlers/system_maintenance.py +147 -0
- apsimo/task_queue/mesh_integration.py +111 -0
- apsimo/task_queue/models.py +317 -0
- apsimo/task_queue/queue_manager.py +8286 -0
- apsimo/task_queue/routing.py +287 -0
- apsimo/task_queue/scheduler.py +252 -0
- apsimo/task_queue/schema.sql +197 -0
- apsimo/task_queue/work_control.py +342 -0
- apsimo/task_queue/worker.py +993 -0
- apsimo/telemetry.py +145 -0
- apsimo/tom/__init__.py +6 -0
- apsimo/tom/affect.py +387 -0
- apsimo/tom/approvals.py +171 -0
- apsimo/tom/arcs.py +896 -0
- apsimo/tom/asymmetry.py +131 -0
- apsimo/tom/eligibility.py +248 -0
- apsimo/tom/engagement.py +214 -0
- apsimo/tom/exposure.py +214 -0
- apsimo/tom/extractor.py +306 -0
- apsimo/tom/fact_adapters.py +144 -0
- apsimo/tom/facts.py +326 -0
- apsimo/tom/integration.py +592 -0
- apsimo/tom/leveled.py +118 -0
- apsimo/tom/levels.py +247 -0
- apsimo/tom/recipient_audit.py +995 -0
- apsimo/tom/recipient_simulator.py +593 -0
- apsimo/tom/source_lineage.py +93 -0
- apsimo/tom/tom2.py +277 -0
- apsimo/tom/visibility.py +559 -0
- apsimo/tom/visibility_store.py +414 -0
- apsimo/tools/__init__.py +0 -0
- apsimo/tools/definitions.py +740 -0
- apsimo/tools/handlers.py +943 -0
- apsimo/toolsmith/__init__.py +26 -0
- apsimo/toolsmith/authority.py +166 -0
- apsimo/toolsmith/engine.py +559 -0
- apsimo/toolsmith/integrity.py +100 -0
- apsimo/toolsmith/miner.py +145 -0
- apsimo/toolsmith/policy.py +110 -0
- apsimo/toolsmith/registry.py +635 -0
- apsimo/turns/__init__.py +17 -0
- apsimo/turns/audio.py +134 -0
- apsimo/turns/documents.py +235 -0
- apsimo/turns/executions.py +486 -0
- apsimo/turns/hermes_history.py +245 -0
- apsimo/turns/hermes_kanban.py +268 -0
- apsimo/turns/hermes_work.py +96 -0
- apsimo/turns/idempotency.py +752 -0
- apsimo/turns/local_work.py +115 -0
- apsimo/turns/media.py +581 -0
- apsimo/turns/reported_workers.py +196 -0
- apsimo/turns/source_annotations.py +283 -0
- apsimo/turns/source_attribution.py +154 -0
- apsimo/turns/source_read.py +351 -0
- apsimo/turns/source_vectors.py +263 -0
- apsimo/turns/video.py +210 -0
- apsimo/util/autonomy_preset.py +220 -0
- apsimo/util/instance.py +92 -0
- apsimo/util/model_output.py +25 -0
- apsimo/util/quiet_hours.py +27 -0
- apsimo/util/session_safety.py +37 -0
- apsimo/util/temporal.py +343 -0
- apsimo/vector/__init__.py +75 -0
- apsimo/vector/backfill.py +171 -0
- apsimo/vector/caption.py +114 -0
- apsimo/vector/collections.py +51 -0
- apsimo/vector/config.py +102 -0
- apsimo/vector/embedder.py +670 -0
- apsimo/vector/image_preprocess.py +406 -0
- apsimo/vector/image_store.py +296 -0
- apsimo/vector/indexes.py +162 -0
- apsimo/vector/migrate.py +334 -0
- apsimo/vector/multimodal_provider.py +417 -0
- apsimo/vector/multimodal_types.py +87 -0
- apsimo/vector/openai_provider.py +119 -0
- apsimo/vector/query.py +49 -0
- apsimo/vector/reranker.py +565 -0
- apsimo/vector/safety_image.py +159 -0
- apsimo/vector/scanner.py +197 -0
- apsimo/vector/setup.py +289 -0
- apsimo/vector/store.py +533 -0
- apsimo/vector/tiers.py +263 -0
- apsimo/work_orders.py +925 -0
- apsimo/workers/__init__.py +21 -0
- apsimo/workers/agent_bridge.py +640 -0
- apsimo/workers/colony_worker.py +382 -0
- apsimo/workers/queue_worker.py +441 -0
- apsimo/workers/skills_sync.py +152 -0
- apsimo/world_model/__init__.py +71 -0
- apsimo/world_model/causal_maintenance.py +131 -0
- apsimo/world_model/causal_policy.py +43 -0
- apsimo/world_model/causal_query.py +125 -0
- apsimo/world_model/confidence.py +54 -0
- apsimo/world_model/config.py +64 -0
- apsimo/world_model/constants.py +97 -0
- apsimo/world_model/entities.py +145 -0
- apsimo/world_model/expectation_resolvers.py +177 -0
- apsimo/world_model/extraction/__init__.py +7 -0
- apsimo/world_model/extraction/base.py +62 -0
- apsimo/world_model/extraction/conversation_extractor.py +262 -0
- apsimo/world_model/extraction/detector.py +74 -0
- apsimo/world_model/extraction/document_extractor.py +78 -0
- apsimo/world_model/extraction/formats/__init__.py +24 -0
- apsimo/world_model/extraction/formats/csv_fmt.py +68 -0
- apsimo/world_model/extraction/formats/html_fmt.py +72 -0
- apsimo/world_model/extraction/formats/json_fmt.py +68 -0
- apsimo/world_model/extraction/formats/pdf.py +43 -0
- apsimo/world_model/extraction/formats/text.py +27 -0
- apsimo/world_model/extraction/llm_extractor.py +164 -0
- apsimo/world_model/extraction/pipeline.py +73 -0
- apsimo/world_model/integrations/__init__.py +5 -0
- apsimo/world_model/integrations/mind_model_bridge.py +115 -0
- apsimo/world_model/integrations/social_intel_bridge.py +120 -0
- apsimo/world_model/jobs/__init__.py +4 -0
- apsimo/world_model/jobs/extraction_job.py +168 -0
- apsimo/world_model/llm_extract.py +572 -0
- apsimo/world_model/neo4j/__init__.py +5 -0
- apsimo/world_model/neo4j/backend.py +654 -0
- apsimo/world_model/observations.py +155 -0
- apsimo/world_model/populator.py +307 -0
- apsimo/world_model/postgres/__init__.py +1 -0
- apsimo/world_model/postgres/backend.py +683 -0
- apsimo/world_model/relationships.py +25 -0
- apsimo/world_model/resolution/__init__.py +13 -0
- apsimo/world_model/resolution/entity_resolver.py +232 -0
- apsimo/world_model/resolution/merge_audit.py +16 -0
- apsimo/world_model/resolution/merge_workflow.py +117 -0
- apsimo/world_model/source_reports.py +121 -0
- apsimo/world_model/sqlite/__init__.py +4 -0
- apsimo/world_model/sqlite/backend.py +855 -0
- apsimo/world_model/sqlite/schema.sql +132 -0
- apsimo/world_model/store.py +545 -0
- apsimo-1.3.0.dist-info/METADATA +78 -0
- apsimo-1.3.0.dist-info/RECORD +614 -0
- apsimo-1.3.0.dist-info/WHEEL +5 -0
- apsimo-1.3.0.dist-info/entry_points.txt +11 -0
- apsimo-1.3.0.dist-info/licenses/LICENSE +21 -0
- apsimo-1.3.0.dist-info/top_level.txt +2 -0
- colony_sidecar/__init__.py +4 -0
|
@@ -0,0 +1,2715 @@
|
|
|
1
|
+
"""Task Queue API — ``/v1/host/queue`` endpoints for distributed job scheduling.
|
|
2
|
+
|
|
3
|
+
Exposes the TaskQueueManager / QueueManager surface to external workers
|
|
4
|
+
(including the host agent's cron-driven worker).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import math
|
|
11
|
+
import os
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
from datetime import datetime, timezone, timedelta
|
|
15
|
+
from typing import Any, Dict, List, Optional
|
|
16
|
+
|
|
17
|
+
from fastapi import APIRouter, HTTPException, Query, Request, Response
|
|
18
|
+
from pydantic import BaseModel, Field, model_validator
|
|
19
|
+
|
|
20
|
+
from apsimo.api.authority import (
|
|
21
|
+
WorkerGrant,
|
|
22
|
+
request_authority,
|
|
23
|
+
worker_authority_mode,
|
|
24
|
+
)
|
|
25
|
+
from apsimo.initiatives.approval_authority import (
|
|
26
|
+
AUTHORIZATION_PROJECTION_SCHEMA,
|
|
27
|
+
ApprovalAuthorityError,
|
|
28
|
+
ApprovalAuthorityStore,
|
|
29
|
+
DEFAULT_GRANT_MAX_USES,
|
|
30
|
+
DEFAULT_GRANT_TTL_SECONDS,
|
|
31
|
+
authority_mode,
|
|
32
|
+
approval_binding_digest,
|
|
33
|
+
approval_presentation_digest,
|
|
34
|
+
build_action_binding,
|
|
35
|
+
build_approval_presentation,
|
|
36
|
+
resolve_grant_envelope,
|
|
37
|
+
)
|
|
38
|
+
from apsimo.task_queue.models import (
|
|
39
|
+
Job,
|
|
40
|
+
JobCapabilityRequirement,
|
|
41
|
+
JobPriority,
|
|
42
|
+
JobStatus,
|
|
43
|
+
JobType,
|
|
44
|
+
WorkerCapabilities,
|
|
45
|
+
is_canonical_job_id,
|
|
46
|
+
)
|
|
47
|
+
from apsimo.task_queue.queue_manager import (
|
|
48
|
+
QueueExecutionUnavailable,
|
|
49
|
+
TaskQueueManager,
|
|
50
|
+
)
|
|
51
|
+
from apsimo.util.session_safety import (
|
|
52
|
+
load_last_user_message_at,
|
|
53
|
+
save_last_user_message_at,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
|
+
router = APIRouter(prefix="/v1/host/queue", tags=["task_queue"])
|
|
59
|
+
|
|
60
|
+
_RESERVED_JOB_TAGS = frozenset({
|
|
61
|
+
"approved_by", "approved_at", "auto_approved_by_policy",
|
|
62
|
+
"action_digest", "bounded_grant_id", "bounded_grant_expires_at",
|
|
63
|
+
"bounded_grant_ttl_state", "bounded_grant_uses_state",
|
|
64
|
+
"rejected_by", "rejected_at",
|
|
65
|
+
"rejected_reason", "hold_kind", "blocked_reason",
|
|
66
|
+
"agent_action_route", "agent_action_route_node",
|
|
67
|
+
"thought_route", "thought_route_node",
|
|
68
|
+
"outbound_target",
|
|
69
|
+
"action_result_contract", "operational_completion_only",
|
|
70
|
+
"verification_pending", "worker_completion_terminalized",
|
|
71
|
+
})
|
|
72
|
+
_RESERVED_JOB_TAG_PREFIXES = (
|
|
73
|
+
"approval_", "governor_", "worker_authority_", "success_",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _reserved_job_tags(tags: Optional[Dict[str, str]]) -> List[str]:
|
|
78
|
+
return sorted(
|
|
79
|
+
key for key in (tags or {})
|
|
80
|
+
if key in _RESERVED_JOB_TAGS
|
|
81
|
+
or any(key.startswith(prefix) for prefix in _RESERVED_JOB_TAG_PREFIXES)
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
class WorkerRegisterRequest(BaseModel):
|
|
85
|
+
node_id: str
|
|
86
|
+
capabilities: Optional[List[str]] = None
|
|
87
|
+
capacity: Optional[Dict[str, float]] = None
|
|
88
|
+
max_concurrent: Optional[int] = Field(None, ge=1, le=1024)
|
|
89
|
+
job_types: Optional[List[str]] = None
|
|
90
|
+
available: bool = True
|
|
91
|
+
load: float = Field(0.0, ge=0.0, le=1.0)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class WorkerHeartbeatRequest(BaseModel):
|
|
95
|
+
job_ids: List[str] = []
|
|
96
|
+
progress: Optional[Dict[str, float]] = None
|
|
97
|
+
claim_attempt_ids: Dict[str, str] = {}
|
|
98
|
+
load: Optional[float] = None
|
|
99
|
+
|
|
100
|
+
@model_validator(mode="after")
|
|
101
|
+
def exact_attempts_for_jobs(self):
|
|
102
|
+
job_ids = set(self.job_ids)
|
|
103
|
+
attempt_ids = set(self.claim_attempt_ids)
|
|
104
|
+
if job_ids != attempt_ids or any(
|
|
105
|
+
not str(value).strip()
|
|
106
|
+
for value in self.claim_attempt_ids.values()
|
|
107
|
+
):
|
|
108
|
+
raise ValueError(
|
|
109
|
+
"claim_attempt_ids must contain one non-empty exact attempt "
|
|
110
|
+
"for every job_id and no other entries"
|
|
111
|
+
)
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class JobPostRequest(BaseModel):
|
|
116
|
+
job_type: str = "agent_action"
|
|
117
|
+
payload: Dict[str, Any] = {}
|
|
118
|
+
priority: str = "normal"
|
|
119
|
+
capabilities: Optional[List[Dict[str, Any]]] = None
|
|
120
|
+
deadline: Optional[str] = None
|
|
121
|
+
max_retries: int = 3
|
|
122
|
+
timeout_secs: float = 3600.0
|
|
123
|
+
depends_on: List[str] = []
|
|
124
|
+
tags: Optional[Dict[str, str]] = None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class JobClaimRequest(BaseModel):
|
|
128
|
+
node_id: str
|
|
129
|
+
capabilities: Optional[List[str]] = None
|
|
130
|
+
capacity: Optional[Dict[str, float]] = None
|
|
131
|
+
max_concurrent: Optional[int] = Field(None, ge=1, le=1024)
|
|
132
|
+
job_types: Optional[List[str]] = None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class JobCompleteRequest(BaseModel):
|
|
136
|
+
output: Dict[str, Any] = {}
|
|
137
|
+
claim_attempt_id: str = Field(min_length=1, max_length=128)
|
|
138
|
+
# Deprecated compatibility telemetry. Queue timing is always derived from
|
|
139
|
+
# the durable server claim/start ledger and this value is ignored.
|
|
140
|
+
started_at: Optional[str] = None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class ActionSuccessAttestationRequest(BaseModel):
|
|
144
|
+
"""Independent evidence for one exact generic Action Plane attempt."""
|
|
145
|
+
|
|
146
|
+
model_config = {"extra": "forbid", "strict": True}
|
|
147
|
+
|
|
148
|
+
schema_name: str = Field(alias="schema")
|
|
149
|
+
version: int
|
|
150
|
+
job_id: str = Field(min_length=1, max_length=128)
|
|
151
|
+
claim_attempt_id: str = Field(min_length=1, max_length=128)
|
|
152
|
+
action_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
|
153
|
+
effect_class: str
|
|
154
|
+
terminal_outcome: str
|
|
155
|
+
receipt_refs: List[Any] = Field(min_length=1, max_length=32)
|
|
156
|
+
observed_at: str
|
|
157
|
+
summary: str = Field("", max_length=500)
|
|
158
|
+
|
|
159
|
+
@model_validator(mode="after")
|
|
160
|
+
def exact_action_receipt(self):
|
|
161
|
+
from apsimo.task_queue.action_receipts import (
|
|
162
|
+
ActionReceiptAttestationV1,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
ActionReceiptAttestationV1.from_payload(
|
|
166
|
+
self.model_dump(by_alias=True)
|
|
167
|
+
)
|
|
168
|
+
return self
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class JobFailRequest(BaseModel):
|
|
172
|
+
error: str
|
|
173
|
+
claim_attempt_id: str = Field(min_length=1, max_length=128)
|
|
174
|
+
# Deprecated and non-authoritative; retained so older workers still parse.
|
|
175
|
+
started_at: Optional[str] = None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class JobHeartbeatRequest(BaseModel):
|
|
179
|
+
progress: Optional[float] = None
|
|
180
|
+
log_lines: Optional[List[str]] = None
|
|
181
|
+
claim_attempt_id: str = Field(min_length=1, max_length=128)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class JobStartRequest(BaseModel):
|
|
185
|
+
claim_attempt_id: str = Field(min_length=1, max_length=128)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class JobReleaseRequest(BaseModel):
|
|
189
|
+
claim_attempt_id: str = Field(min_length=1, max_length=128)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class WorkControlOperationRequest(BaseModel):
|
|
193
|
+
"""Strict caller-authored CAS command for one durable work target."""
|
|
194
|
+
|
|
195
|
+
model_config = {"extra": "forbid", "strict": True}
|
|
196
|
+
|
|
197
|
+
schema_name: str = Field(alias="schema", pattern=r"^WorkControlOperationV1$")
|
|
198
|
+
version: int = Field(ge=1, le=1)
|
|
199
|
+
operation_id: str = Field(
|
|
200
|
+
min_length=1,
|
|
201
|
+
max_length=128,
|
|
202
|
+
pattern=r"^[A-Za-z0-9][A-Za-z0-9_.:@/+\-]{0,127}$",
|
|
203
|
+
)
|
|
204
|
+
operation: str
|
|
205
|
+
target_id: str = Field(min_length=1, max_length=192)
|
|
206
|
+
run_id: str = Field(min_length=1, max_length=128)
|
|
207
|
+
attempt_id: Optional[str] = Field(None, max_length=128)
|
|
208
|
+
expected_revision: int = Field(ge=1)
|
|
209
|
+
expected_state_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
|
210
|
+
parameters: Dict[str, Any] = {}
|
|
211
|
+
reason: str = Field("", max_length=500)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class WorkControlAckRequest(BaseModel):
|
|
215
|
+
"""Exact claimant acknowledgement; identity never comes from this body."""
|
|
216
|
+
|
|
217
|
+
model_config = {"extra": "forbid", "strict": True}
|
|
218
|
+
|
|
219
|
+
schema_name: str = Field(alias="schema", pattern=r"^WorkControlAckV1$")
|
|
220
|
+
version: int = Field(ge=1, le=1)
|
|
221
|
+
attempt_id: str = Field(min_length=1, max_length=128)
|
|
222
|
+
outcome: str
|
|
223
|
+
details: Dict[str, Any] = {}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class WorkControlAckEnvelopeRequest(WorkControlAckRequest):
|
|
227
|
+
"""Slash-safe worker acknowledgement representation."""
|
|
228
|
+
|
|
229
|
+
node_id: str = Field(min_length=1, max_length=192)
|
|
230
|
+
operation_id: str = Field(
|
|
231
|
+
min_length=1,
|
|
232
|
+
max_length=128,
|
|
233
|
+
pattern=r"^[A-Za-z0-9][A-Za-z0-9_.:@/+-]{0,127}$",
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class WorkControlWorkerOutcomeRequest(BaseModel):
|
|
238
|
+
"""Durable slash-safe steer outcome recorded before acknowledgement."""
|
|
239
|
+
|
|
240
|
+
model_config = {"extra": "forbid", "strict": True}
|
|
241
|
+
|
|
242
|
+
schema_name: str = Field(
|
|
243
|
+
alias="schema", pattern=r"^WorkControlWorkerOutcomeV1$",
|
|
244
|
+
)
|
|
245
|
+
version: int = Field(ge=1, le=1)
|
|
246
|
+
node_id: str = Field(min_length=1, max_length=192)
|
|
247
|
+
operation_id: str = Field(
|
|
248
|
+
min_length=1,
|
|
249
|
+
max_length=128,
|
|
250
|
+
pattern=r"^[A-Za-z0-9][A-Za-z0-9_.:@/+-]{0,127}$",
|
|
251
|
+
)
|
|
252
|
+
attempt_id: str = Field(min_length=1, max_length=128)
|
|
253
|
+
outcome: str
|
|
254
|
+
details: Dict[str, Any] = {}
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class WorkEffectReconciliationRequest(BaseModel):
|
|
258
|
+
"""Independent exact-attempt applied/not-applied evidence."""
|
|
259
|
+
|
|
260
|
+
model_config = {"extra": "forbid", "strict": True}
|
|
261
|
+
|
|
262
|
+
schema_name: str = Field(
|
|
263
|
+
alias="schema", pattern=r"^WorkEffectReconciliationV1$",
|
|
264
|
+
)
|
|
265
|
+
version: int = Field(ge=1, le=1)
|
|
266
|
+
reconciliation_id: str = Field(
|
|
267
|
+
min_length=1,
|
|
268
|
+
max_length=128,
|
|
269
|
+
pattern=r"^[A-Za-z0-9][A-Za-z0-9_.:@/+-]{0,127}$",
|
|
270
|
+
)
|
|
271
|
+
target_id: str = Field(min_length=1, max_length=192)
|
|
272
|
+
attempt_id: str = Field(min_length=1, max_length=128)
|
|
273
|
+
authority_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
|
274
|
+
finding: str
|
|
275
|
+
evidence_refs: List[str] = Field(min_length=1, max_length=32)
|
|
276
|
+
observed_at: str
|
|
277
|
+
summary: str = Field("", max_length=1000)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _grant_request_ttl_default() -> int:
|
|
281
|
+
maximum = resolve_grant_envelope().max_ttl_seconds
|
|
282
|
+
return (
|
|
283
|
+
DEFAULT_GRANT_TTL_SECONDS
|
|
284
|
+
if maximum is None else min(DEFAULT_GRANT_TTL_SECONDS, maximum)
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _grant_request_uses_default() -> int:
|
|
289
|
+
maximum = resolve_grant_envelope().max_uses
|
|
290
|
+
return (
|
|
291
|
+
DEFAULT_GRANT_MAX_USES
|
|
292
|
+
if maximum is None else min(DEFAULT_GRANT_MAX_USES, maximum)
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class BoundedGrantRequest(BaseModel):
|
|
297
|
+
"""An exact-scope grant constrained by the server's grant envelope."""
|
|
298
|
+
|
|
299
|
+
expires_in_seconds: int = Field(
|
|
300
|
+
default_factory=_grant_request_ttl_default, ge=60,
|
|
301
|
+
)
|
|
302
|
+
max_uses: int = Field(default_factory=_grant_request_uses_default, ge=1)
|
|
303
|
+
# Optional only so clients can echo the scope they displayed. The server
|
|
304
|
+
# rejects any difference; omitting it means "the exact displayed scope".
|
|
305
|
+
exact_scope: Optional[Dict[str, Any]] = None
|
|
306
|
+
|
|
307
|
+
@model_validator(mode="after")
|
|
308
|
+
def within_configured_envelope(self):
|
|
309
|
+
"""Keep over-limit API requests at validation-time HTTP 422."""
|
|
310
|
+
|
|
311
|
+
envelope = resolve_grant_envelope()
|
|
312
|
+
if (
|
|
313
|
+
envelope.max_ttl_seconds is not None
|
|
314
|
+
and self.expires_in_seconds > envelope.max_ttl_seconds
|
|
315
|
+
):
|
|
316
|
+
raise ValueError(
|
|
317
|
+
"expires_in_seconds exceeds COLONY_GRANT_MAX_TTL_SECONDS"
|
|
318
|
+
)
|
|
319
|
+
if (
|
|
320
|
+
envelope.max_uses is not None
|
|
321
|
+
and self.max_uses > envelope.max_uses
|
|
322
|
+
):
|
|
323
|
+
raise ValueError("max_uses exceeds COLONY_GRANT_MAX_USES")
|
|
324
|
+
return self
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
class JobApproveRequest(BaseModel):
|
|
328
|
+
# Deprecated compatibility input. It is intentionally ignored: authority
|
|
329
|
+
# comes from the authenticated request principal, never caller prose.
|
|
330
|
+
approved_by: Optional[str] = None
|
|
331
|
+
# Deprecated spelling. In shadow migration mode it maps to an exact-scope
|
|
332
|
+
# grant with safe request defaults and the deployment's configured envelope;
|
|
333
|
+
# it never creates action-name-only authority.
|
|
334
|
+
always: bool = False
|
|
335
|
+
approval_request_id: Optional[str] = None
|
|
336
|
+
expected_action_digest: Optional[str] = None
|
|
337
|
+
decision_id: Optional[str] = None
|
|
338
|
+
grant: Optional[BoundedGrantRequest] = None
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class JobRejectRequest(BaseModel):
|
|
342
|
+
# Deprecated and ignored for authority; retained so old clients parse.
|
|
343
|
+
rejected_by: Optional[str] = None
|
|
344
|
+
reason: str = "rejected_by_owner"
|
|
345
|
+
approval_request_id: Optional[str] = None
|
|
346
|
+
expected_action_digest: Optional[str] = None
|
|
347
|
+
decision_id: Optional[str] = None
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class ApprovalDecisionRequest(BaseModel):
|
|
351
|
+
decision: str
|
|
352
|
+
decision_id: str
|
|
353
|
+
expected_action_digest: str
|
|
354
|
+
grant: Optional[BoundedGrantRequest] = None
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
class ApprovalRelayCanaryRequest(BaseModel):
|
|
358
|
+
"""The only caller-controlled canary value is an opaque retry token."""
|
|
359
|
+
|
|
360
|
+
model_config = {"extra": "forbid", "strict": True}
|
|
361
|
+
|
|
362
|
+
schema_name: str = Field(alias="schema", pattern=r"^ApprovalRelayCanaryV1$")
|
|
363
|
+
version: int = Field(ge=1, le=1)
|
|
364
|
+
idempotency_key: str = Field(
|
|
365
|
+
min_length=16,
|
|
366
|
+
max_length=128,
|
|
367
|
+
pattern=r"^[A-Za-z0-9._:@/+-]+$",
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
# ---------------------------------------------------------------------------
|
|
372
|
+
# Helpers
|
|
373
|
+
# ---------------------------------------------------------------------------
|
|
374
|
+
|
|
375
|
+
def _get_queue() -> TaskQueueManager:
|
|
376
|
+
try:
|
|
377
|
+
return TaskQueueManager.get_instance()
|
|
378
|
+
except RuntimeError:
|
|
379
|
+
raise HTTPException(status_code=503, detail="Task queue not initialized")
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _work_control_http_error(exc: Exception) -> HTTPException:
|
|
383
|
+
from apsimo.task_queue.work_control import WorkControlError
|
|
384
|
+
|
|
385
|
+
if isinstance(exc, WorkControlError):
|
|
386
|
+
return HTTPException(
|
|
387
|
+
status_code=exc.status_code,
|
|
388
|
+
detail=exc.detail(),
|
|
389
|
+
)
|
|
390
|
+
raise exc
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@router.get("/contract")
|
|
394
|
+
async def queue_contract(request: Request = None) -> Dict[str, Any]:
|
|
395
|
+
"""Return the authenticated, deploy-pinned worker protocol identity."""
|
|
396
|
+
|
|
397
|
+
from apsimo.task_queue.contract import (
|
|
398
|
+
QueueContractIdentityError,
|
|
399
|
+
queue_contract_identity,
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
try:
|
|
403
|
+
contract = queue_contract_identity()
|
|
404
|
+
except QueueContractIdentityError as exc:
|
|
405
|
+
raise HTTPException(status_code=503, detail={
|
|
406
|
+
"code": "queue_contract_identity_unavailable",
|
|
407
|
+
"message": str(exc),
|
|
408
|
+
}) from exc
|
|
409
|
+
try:
|
|
410
|
+
runtime = _get_queue().queue.execution_readiness()
|
|
411
|
+
except HTTPException:
|
|
412
|
+
contract["runtime_readiness"] = {
|
|
413
|
+
"queue_initialized": False,
|
|
414
|
+
"thought": {
|
|
415
|
+
"ready": False,
|
|
416
|
+
"node_id": None,
|
|
417
|
+
"reason": "queue_not_initialized",
|
|
418
|
+
},
|
|
419
|
+
}
|
|
420
|
+
else:
|
|
421
|
+
contract["runtime_readiness"] = {
|
|
422
|
+
"queue_initialized": True,
|
|
423
|
+
"queue_execution_ready": runtime["ready"],
|
|
424
|
+
"queue_execution_reason": runtime["reason"],
|
|
425
|
+
"thought": runtime["typed_routes"]["thought"],
|
|
426
|
+
}
|
|
427
|
+
return contract
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _governor() -> Any:
|
|
431
|
+
"""The server-side WorkerGovernor (item 5), or None if not wired."""
|
|
432
|
+
try:
|
|
433
|
+
from apsimo.api.routers.host import _worker_governor
|
|
434
|
+
return _worker_governor
|
|
435
|
+
except Exception:
|
|
436
|
+
return None
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _worker_authority_error(code: str, message: str, *, status: int = 403):
|
|
440
|
+
return HTTPException(status_code=status, detail={"code": code, "message": message})
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _worker_request_context(
|
|
444
|
+
request: Optional[Request],
|
|
445
|
+
*,
|
|
446
|
+
node_id: str,
|
|
447
|
+
required_scope: str,
|
|
448
|
+
claimant: bool = False,
|
|
449
|
+
) -> Dict[str, Any]:
|
|
450
|
+
"""Resolve worker authority from transport authentication, never a body claim."""
|
|
451
|
+
|
|
452
|
+
mode = worker_authority_mode()
|
|
453
|
+
if mode == "invalid":
|
|
454
|
+
raise _worker_authority_error(
|
|
455
|
+
"worker_authority_configuration_invalid",
|
|
456
|
+
"COLONY_WORKER_AUTHORITY_MODE must be shadow or enforce",
|
|
457
|
+
status=503,
|
|
458
|
+
)
|
|
459
|
+
if request is None:
|
|
460
|
+
# Embedded/in-process queue integrations are a separate trusted lane.
|
|
461
|
+
return {
|
|
462
|
+
"mode": "internal",
|
|
463
|
+
"principal": "trusted-internal",
|
|
464
|
+
"credential": "in_process",
|
|
465
|
+
"grant": None,
|
|
466
|
+
"would_deny": False,
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
authority = request_authority(request)
|
|
470
|
+
grant = next(
|
|
471
|
+
(item for item in authority.worker_grants if item.node_id == node_id),
|
|
472
|
+
None,
|
|
473
|
+
)
|
|
474
|
+
scoped = (
|
|
475
|
+
authority.authenticated
|
|
476
|
+
and not authority.legacy
|
|
477
|
+
and not authority.anonymous
|
|
478
|
+
and authority.has_scope(required_scope)
|
|
479
|
+
and grant is not None
|
|
480
|
+
)
|
|
481
|
+
if mode == "enforce" and not scoped:
|
|
482
|
+
if not authority.authenticated or authority.legacy or authority.anonymous:
|
|
483
|
+
raise _worker_authority_error(
|
|
484
|
+
"scoped_worker_authority_required",
|
|
485
|
+
"a scoped authenticated worker principal is required",
|
|
486
|
+
)
|
|
487
|
+
if not authority.has_scope(required_scope):
|
|
488
|
+
raise _worker_authority_error(
|
|
489
|
+
"worker_scope_required",
|
|
490
|
+
f"worker principal requires exact scope {required_scope}",
|
|
491
|
+
)
|
|
492
|
+
raise _worker_authority_error(
|
|
493
|
+
"worker_claimant_mismatch" if claimant else "worker_node_not_granted",
|
|
494
|
+
(
|
|
495
|
+
"authenticated worker principal does not own this job claim"
|
|
496
|
+
if claimant
|
|
497
|
+
else "authenticated worker principal is not granted this node_id"
|
|
498
|
+
),
|
|
499
|
+
)
|
|
500
|
+
return {
|
|
501
|
+
"mode": mode,
|
|
502
|
+
"principal": authority.principal_id,
|
|
503
|
+
"credential": authority.credential_id or "none",
|
|
504
|
+
"grant": grant,
|
|
505
|
+
"would_deny": not scoped,
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _worker_tags(context: Dict[str, Any]) -> Dict[str, str]:
|
|
510
|
+
return {
|
|
511
|
+
"worker_authority_mode": str(context["mode"]),
|
|
512
|
+
"worker_authority_principal": str(context["principal"]),
|
|
513
|
+
"worker_authority_credential": str(context["credential"]),
|
|
514
|
+
"worker_authority_would_deny": (
|
|
515
|
+
"true" if context.get("would_deny") else "false"
|
|
516
|
+
),
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _job_type_set(values: List[str], *, field: str) -> set[JobType]:
|
|
521
|
+
result: set[JobType] = set()
|
|
522
|
+
for value in values:
|
|
523
|
+
try:
|
|
524
|
+
result.add(JobType(value))
|
|
525
|
+
except (TypeError, ValueError) as exc:
|
|
526
|
+
raise HTTPException(
|
|
527
|
+
status_code=422,
|
|
528
|
+
detail={"code": "invalid_job_type", "field": field, "value": value},
|
|
529
|
+
) from exc
|
|
530
|
+
return result
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _body_exceeds_worker_grant(
|
|
534
|
+
body: WorkerRegisterRequest | JobClaimRequest,
|
|
535
|
+
grant: WorkerGrant,
|
|
536
|
+
) -> bool:
|
|
537
|
+
grant_capacity = grant.capacity_map()
|
|
538
|
+
if (
|
|
539
|
+
body.capabilities is not None
|
|
540
|
+
and not set(body.capabilities).issubset(grant.capabilities)
|
|
541
|
+
):
|
|
542
|
+
return True
|
|
543
|
+
if body.job_types is not None:
|
|
544
|
+
requested_types = set(body.job_types)
|
|
545
|
+
if not requested_types or not requested_types.issubset(grant.job_types):
|
|
546
|
+
return True
|
|
547
|
+
if body.max_concurrent is not None and body.max_concurrent > grant.max_concurrent:
|
|
548
|
+
return True
|
|
549
|
+
if body.capacity is not None:
|
|
550
|
+
for key, value in body.capacity.items():
|
|
551
|
+
amount = float(value)
|
|
552
|
+
if (
|
|
553
|
+
not math.isfinite(amount)
|
|
554
|
+
or amount < 0
|
|
555
|
+
or key not in grant_capacity
|
|
556
|
+
or amount > grant_capacity.get(key, -1.0)
|
|
557
|
+
):
|
|
558
|
+
return True
|
|
559
|
+
return False
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _bounded_worker_capabilities(
|
|
563
|
+
body: WorkerRegisterRequest | JobClaimRequest,
|
|
564
|
+
context: Dict[str, Any],
|
|
565
|
+
) -> WorkerCapabilities:
|
|
566
|
+
"""Build effective caps; enforce bodies may omit or narrow keyring ceilings."""
|
|
567
|
+
|
|
568
|
+
grant: WorkerGrant | None = context.get("grant")
|
|
569
|
+
if context["mode"] != "enforce":
|
|
570
|
+
exceeds = grant is not None and _body_exceeds_worker_grant(body, grant)
|
|
571
|
+
if exceeds:
|
|
572
|
+
context["would_deny"] = True
|
|
573
|
+
# A correctly scoped shadow consumer should exercise the exact
|
|
574
|
+
# enforcement shape: omissions derive from its server grant. Legacy,
|
|
575
|
+
# unprovisioned, and intentionally over-broad shadow traffic retains
|
|
576
|
+
# historical body behavior so migration cannot silently break it.
|
|
577
|
+
use_grant = grant is not None and not exceeds and not context.get("would_deny")
|
|
578
|
+
return WorkerCapabilities(
|
|
579
|
+
node_id=body.node_id,
|
|
580
|
+
capabilities=(
|
|
581
|
+
set(grant.capabilities)
|
|
582
|
+
if use_grant and body.capabilities is None
|
|
583
|
+
else set(body.capabilities or [])
|
|
584
|
+
),
|
|
585
|
+
capacity=(
|
|
586
|
+
grant.capacity_map()
|
|
587
|
+
if use_grant and body.capacity is None
|
|
588
|
+
else body.capacity or {}
|
|
589
|
+
),
|
|
590
|
+
max_concurrent=(
|
|
591
|
+
grant.max_concurrent
|
|
592
|
+
if use_grant and body.max_concurrent is None
|
|
593
|
+
else body.max_concurrent or 4
|
|
594
|
+
),
|
|
595
|
+
job_types=_job_type_set(
|
|
596
|
+
sorted(grant.job_types)
|
|
597
|
+
if use_grant and body.job_types is None
|
|
598
|
+
else body.job_types or [],
|
|
599
|
+
field="job_types",
|
|
600
|
+
),
|
|
601
|
+
available=bool(getattr(body, "available", True)),
|
|
602
|
+
load=float(getattr(body, "load", 0.0)),
|
|
603
|
+
)
|
|
604
|
+
assert grant is not None
|
|
605
|
+
|
|
606
|
+
capabilities = (
|
|
607
|
+
set(grant.capabilities)
|
|
608
|
+
if body.capabilities is None
|
|
609
|
+
else set(body.capabilities)
|
|
610
|
+
)
|
|
611
|
+
job_type_names = (
|
|
612
|
+
set(grant.job_types)
|
|
613
|
+
if body.job_types is None
|
|
614
|
+
else set(body.job_types)
|
|
615
|
+
)
|
|
616
|
+
capacity = grant.capacity_map() if body.capacity is None else dict(body.capacity)
|
|
617
|
+
max_concurrent = body.max_concurrent or grant.max_concurrent
|
|
618
|
+
if _body_exceeds_worker_grant(body, grant):
|
|
619
|
+
raise _worker_authority_error(
|
|
620
|
+
"worker_grant_exceeded",
|
|
621
|
+
"worker request may only narrow its server-owned keyring grant",
|
|
622
|
+
)
|
|
623
|
+
return WorkerCapabilities(
|
|
624
|
+
node_id=body.node_id,
|
|
625
|
+
capabilities=capabilities,
|
|
626
|
+
capacity={key: float(value) for key, value in capacity.items()},
|
|
627
|
+
max_concurrent=max_concurrent,
|
|
628
|
+
job_types=_job_type_set(sorted(job_type_names), field="job_types"),
|
|
629
|
+
available=bool(getattr(body, "available", True)),
|
|
630
|
+
load=float(getattr(body, "load", 0.0)),
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def _parse_dt(s: Optional[str]) -> Optional[datetime]:
|
|
635
|
+
if not s:
|
|
636
|
+
return None
|
|
637
|
+
try:
|
|
638
|
+
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
|
639
|
+
except (ValueError, TypeError):
|
|
640
|
+
return None
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _approval_store() -> ApprovalAuthorityStore:
|
|
644
|
+
return ApprovalAuthorityStore()
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _approval_error(exc: ApprovalAuthorityError) -> HTTPException:
|
|
648
|
+
status = 404 if exc.code == "request_not_found" else 409
|
|
649
|
+
if exc.code in {"authority_required", "approval_scope_required"}:
|
|
650
|
+
status = 403
|
|
651
|
+
return HTTPException(
|
|
652
|
+
status_code=status,
|
|
653
|
+
detail={"code": exc.code, "message": exc.message},
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _decision_authority(request: Optional[Request]) -> tuple[str, str, str]:
|
|
658
|
+
"""Return server-derived actor/evidence/mode for an approval decision."""
|
|
659
|
+
|
|
660
|
+
mode = authority_mode()
|
|
661
|
+
if mode == "invalid":
|
|
662
|
+
raise HTTPException(
|
|
663
|
+
status_code=503,
|
|
664
|
+
detail={
|
|
665
|
+
"code": "approval_authority_mode_invalid",
|
|
666
|
+
"message": (
|
|
667
|
+
"COLONY_APPROVAL_AUTHORITY_MODE must be shadow or enforce"
|
|
668
|
+
),
|
|
669
|
+
},
|
|
670
|
+
)
|
|
671
|
+
if request is None:
|
|
672
|
+
# Direct in-process calls are an explicit trusted integration surface,
|
|
673
|
+
# not an HTTP body claim. They remain for embedded Colony deployments.
|
|
674
|
+
return "trusted-internal", "in_process", mode
|
|
675
|
+
|
|
676
|
+
authority = request_authority(request)
|
|
677
|
+
allowed = (
|
|
678
|
+
authority.authenticated
|
|
679
|
+
and not authority.legacy
|
|
680
|
+
and authority.has_scope("api:access")
|
|
681
|
+
and authority.has_scope("approvals:decide")
|
|
682
|
+
)
|
|
683
|
+
if mode == "enforce" and not allowed:
|
|
684
|
+
raise HTTPException(
|
|
685
|
+
status_code=403,
|
|
686
|
+
detail={
|
|
687
|
+
"code": "approval_scope_required",
|
|
688
|
+
"message": "a scoped authenticated approvals:decide principal is required",
|
|
689
|
+
},
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
actor = authority.principal_id
|
|
693
|
+
credential = authority.credential_id or "none"
|
|
694
|
+
if allowed:
|
|
695
|
+
evidence = f"scoped_principal:{actor}:{credential}"
|
|
696
|
+
else:
|
|
697
|
+
# Shadow mode preserves legacy bearer/dev traffic during consumer
|
|
698
|
+
# migration, but records that it would fail enforcement. The body's
|
|
699
|
+
# approved_by/rejected_by value still has no effect.
|
|
700
|
+
evidence = f"shadow_compat:{actor}:{credential}"
|
|
701
|
+
return actor, evidence, mode
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def _approval_relay_canary_authority(request: Request) -> str:
|
|
705
|
+
"""Require the dedicated scoped bridge even while migration is shadow."""
|
|
706
|
+
|
|
707
|
+
authority = request_authority(request)
|
|
708
|
+
allowed = bool(
|
|
709
|
+
authority.authenticated
|
|
710
|
+
and not authority.legacy
|
|
711
|
+
and not authority.anonymous
|
|
712
|
+
and authority.has_scope("api:access")
|
|
713
|
+
and authority.has_scope("approvals:decide")
|
|
714
|
+
)
|
|
715
|
+
if not allowed:
|
|
716
|
+
raise HTTPException(status_code=403, detail={
|
|
717
|
+
"code": "approval_relay_canary_scope_required",
|
|
718
|
+
"message": (
|
|
719
|
+
"a scoped authenticated api:access + approvals:decide "
|
|
720
|
+
"principal is required"
|
|
721
|
+
),
|
|
722
|
+
})
|
|
723
|
+
return authority.principal_id
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def _job_binding(job: Job):
|
|
727
|
+
return build_action_binding(
|
|
728
|
+
job_id=job.job_id,
|
|
729
|
+
job_type=job.job_type.value,
|
|
730
|
+
payload=job.payload,
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
async def _ensure_job_approval_request(
|
|
735
|
+
queue: TaskQueueManager,
|
|
736
|
+
job: Job,
|
|
737
|
+
) -> tuple[ApprovalAuthorityStore, Dict[str, Any], Any]:
|
|
738
|
+
binding = _job_binding(job)
|
|
739
|
+
store = _approval_store()
|
|
740
|
+
presentation = build_approval_presentation(
|
|
741
|
+
job_id=job.job_id,
|
|
742
|
+
job_type=job.job_type.value,
|
|
743
|
+
payload=job.payload,
|
|
744
|
+
deadline=job.deadline,
|
|
745
|
+
)
|
|
746
|
+
approval_request = store.ensure_request(
|
|
747
|
+
job_id=job.job_id,
|
|
748
|
+
binding=binding,
|
|
749
|
+
presentation=presentation,
|
|
750
|
+
)
|
|
751
|
+
await queue.queue.merge_job_tags(job.job_id, {
|
|
752
|
+
"approval_request_id": approval_request["request_id"],
|
|
753
|
+
"action_digest": binding.action_digest,
|
|
754
|
+
"approval_scope_digest": binding.scope_digest,
|
|
755
|
+
"approval_binding_digest": approval_request["binding_digest"],
|
|
756
|
+
"approval_request_digest": approval_request["request_digest"],
|
|
757
|
+
"approval_presentation_digest": approval_request[
|
|
758
|
+
"presentation_digest"
|
|
759
|
+
],
|
|
760
|
+
"approval_expires_at": approval_request["expires_at"],
|
|
761
|
+
})
|
|
762
|
+
return store, approval_request, binding
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
async def _decide_job(
|
|
766
|
+
*,
|
|
767
|
+
job: Job,
|
|
768
|
+
decision: str,
|
|
769
|
+
decision_id: Optional[str],
|
|
770
|
+
approval_request_id: Optional[str],
|
|
771
|
+
expected_action_digest: Optional[str],
|
|
772
|
+
grant: Optional[BoundedGrantRequest],
|
|
773
|
+
request: Optional[Request],
|
|
774
|
+
rejection_reason: str = "rejected_by_owner",
|
|
775
|
+
) -> Dict[str, Any]:
|
|
776
|
+
queue = _get_queue()
|
|
777
|
+
actor, evidence, mode = _decision_authority(request)
|
|
778
|
+
from apsimo.task_queue.approval_relay_canary import (
|
|
779
|
+
is_exact_job as is_exact_approval_relay_canary,
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
relay_canary = is_exact_approval_relay_canary(job)
|
|
783
|
+
if relay_canary and grant is not None:
|
|
784
|
+
raise HTTPException(status_code=409, detail={
|
|
785
|
+
"code": "approval_relay_canary_grant_forbidden",
|
|
786
|
+
"message": "the inert approval relay canary cannot create a grant",
|
|
787
|
+
})
|
|
788
|
+
# An exact client retry is a read of the durable winner after the queue
|
|
789
|
+
# transition. This includes approvals that remain BLOCKED solely on a
|
|
790
|
+
# separate dependency; they must not be mistaken for a new approval gate.
|
|
791
|
+
if approval_request_id and expected_action_digest and decision_id:
|
|
792
|
+
store = _approval_store()
|
|
793
|
+
stored = store.get_request(approval_request_id)
|
|
794
|
+
if (
|
|
795
|
+
stored is not None
|
|
796
|
+
and stored.get("job_id") == job.job_id
|
|
797
|
+
and stored.get("status") in {"approved", "rejected"}
|
|
798
|
+
):
|
|
799
|
+
try:
|
|
800
|
+
replay = store.decide(
|
|
801
|
+
approval_request_id,
|
|
802
|
+
decision=decision,
|
|
803
|
+
decision_id=decision_id,
|
|
804
|
+
expected_action_digest=expected_action_digest,
|
|
805
|
+
decided_by=actor,
|
|
806
|
+
authority_evidence=evidence,
|
|
807
|
+
)
|
|
808
|
+
except ApprovalAuthorityError as exc:
|
|
809
|
+
raise _approval_error(exc) from exc
|
|
810
|
+
if relay_canary and replay["replayed"]:
|
|
811
|
+
# A pre-fix scheduler could have projected an APPROVE winner
|
|
812
|
+
# as QUEUED in the cross-database crash window. An exact
|
|
813
|
+
# replay succeeds only after the server repairs the canonical
|
|
814
|
+
# canary to its sole valid terminal state.
|
|
815
|
+
repaired = await (
|
|
816
|
+
queue.queue.repair_approval_relay_canary_terminal(
|
|
817
|
+
job.job_id
|
|
818
|
+
)
|
|
819
|
+
)
|
|
820
|
+
terminal = await queue.queue.get_job(job.job_id)
|
|
821
|
+
if (
|
|
822
|
+
not repaired
|
|
823
|
+
or terminal is None
|
|
824
|
+
or terminal.status is not JobStatus.CANCELLED
|
|
825
|
+
):
|
|
826
|
+
raise HTTPException(status_code=409, detail={
|
|
827
|
+
"code": "approval_relay_canary_not_cancelled",
|
|
828
|
+
"message": (
|
|
829
|
+
"the durable canary winner has not converged "
|
|
830
|
+
"to CANCELLED"
|
|
831
|
+
),
|
|
832
|
+
})
|
|
833
|
+
winner = replay["request"]
|
|
834
|
+
return {
|
|
835
|
+
"success": True,
|
|
836
|
+
"job_id": terminal.job_id,
|
|
837
|
+
"status": JobStatus.CANCELLED.value,
|
|
838
|
+
"decision": decision,
|
|
839
|
+
"decided_by": winner["decided_by"],
|
|
840
|
+
"decided_at": winner["decided_at"],
|
|
841
|
+
"approval_request": winner,
|
|
842
|
+
"bounded_grant": None,
|
|
843
|
+
"replayed": True,
|
|
844
|
+
"authority_mode": mode,
|
|
845
|
+
}
|
|
846
|
+
queue_transition_applied = bool(
|
|
847
|
+
replay["replayed"]
|
|
848
|
+
and (
|
|
849
|
+
(
|
|
850
|
+
decision == "approve"
|
|
851
|
+
and (
|
|
852
|
+
job.status is not JobStatus.BLOCKED
|
|
853
|
+
or (
|
|
854
|
+
job.tags.get("blocked_reason")
|
|
855
|
+
== "dependencies_pending"
|
|
856
|
+
and job.tags.get("approval_request_id")
|
|
857
|
+
== approval_request_id
|
|
858
|
+
and job.tags.get("approval_decision_id")
|
|
859
|
+
== decision_id
|
|
860
|
+
)
|
|
861
|
+
)
|
|
862
|
+
)
|
|
863
|
+
or (
|
|
864
|
+
decision == "reject"
|
|
865
|
+
and job.status is JobStatus.CANCELLED
|
|
866
|
+
)
|
|
867
|
+
)
|
|
868
|
+
)
|
|
869
|
+
if queue_transition_applied:
|
|
870
|
+
winner = replay["request"]
|
|
871
|
+
return {
|
|
872
|
+
"success": True,
|
|
873
|
+
"job_id": job.job_id,
|
|
874
|
+
"status": job.status.value,
|
|
875
|
+
"decision": decision,
|
|
876
|
+
"decided_by": winner["decided_by"],
|
|
877
|
+
"decided_at": winner["decided_at"],
|
|
878
|
+
"approval_request": winner,
|
|
879
|
+
"bounded_grant": replay["grant"],
|
|
880
|
+
"replayed": True,
|
|
881
|
+
"authority_mode": mode,
|
|
882
|
+
}
|
|
883
|
+
if job.status != JobStatus.BLOCKED:
|
|
884
|
+
raise HTTPException(
|
|
885
|
+
status_code=409,
|
|
886
|
+
detail=f"Job is {job.status.value}, not blocked",
|
|
887
|
+
)
|
|
888
|
+
if job.tags.get("blocked_reason") != "awaiting_owner_approval":
|
|
889
|
+
raise HTTPException(
|
|
890
|
+
status_code=409,
|
|
891
|
+
detail={
|
|
892
|
+
"code": "job_not_approval_blocked",
|
|
893
|
+
"message": "only owner-approval-blocked jobs can be decided",
|
|
894
|
+
},
|
|
895
|
+
)
|
|
896
|
+
|
|
897
|
+
try:
|
|
898
|
+
store, approval_request, binding = await _ensure_job_approval_request(queue, job)
|
|
899
|
+
if approval_request_id and approval_request_id != approval_request["request_id"]:
|
|
900
|
+
raise ApprovalAuthorityError(
|
|
901
|
+
"request_superseded", "approval request is not current for this job"
|
|
902
|
+
)
|
|
903
|
+
if mode == "enforce" and not approval_request_id:
|
|
904
|
+
raise ApprovalAuthorityError(
|
|
905
|
+
"request_id_required", "approval_request_id is required in enforce mode"
|
|
906
|
+
)
|
|
907
|
+
if mode == "enforce" and not expected_action_digest:
|
|
908
|
+
raise ApprovalAuthorityError(
|
|
909
|
+
"action_digest_required", "expected_action_digest is required in enforce mode"
|
|
910
|
+
)
|
|
911
|
+
if mode == "enforce" and not decision_id:
|
|
912
|
+
raise ApprovalAuthorityError(
|
|
913
|
+
"decision_id_required", "decision_id is required in enforce mode"
|
|
914
|
+
)
|
|
915
|
+
|
|
916
|
+
normalized_decision_id = decision_id or (
|
|
917
|
+
"compat_" + os.urandom(16).hex()
|
|
918
|
+
)
|
|
919
|
+
effective_grant = grant
|
|
920
|
+
exact_scope = None
|
|
921
|
+
if effective_grant is not None:
|
|
922
|
+
exact_scope = (
|
|
923
|
+
binding.scope
|
|
924
|
+
if effective_grant.exact_scope is None
|
|
925
|
+
else effective_grant.exact_scope
|
|
926
|
+
)
|
|
927
|
+
|
|
928
|
+
result = store.decide(
|
|
929
|
+
approval_request["request_id"],
|
|
930
|
+
decision=decision,
|
|
931
|
+
decision_id=normalized_decision_id,
|
|
932
|
+
expected_action_digest=expected_action_digest or binding.action_digest,
|
|
933
|
+
decided_by=actor,
|
|
934
|
+
authority_evidence=evidence,
|
|
935
|
+
grant_scope=exact_scope,
|
|
936
|
+
grant_ttl_seconds=(
|
|
937
|
+
effective_grant.expires_in_seconds
|
|
938
|
+
if effective_grant else DEFAULT_GRANT_TTL_SECONDS
|
|
939
|
+
),
|
|
940
|
+
grant_max_uses=(
|
|
941
|
+
effective_grant.max_uses if effective_grant else DEFAULT_GRANT_MAX_USES
|
|
942
|
+
),
|
|
943
|
+
)
|
|
944
|
+
except ApprovalAuthorityError as exc:
|
|
945
|
+
raise _approval_error(exc) from exc
|
|
946
|
+
|
|
947
|
+
decided_at = result["request"]["decided_at"]
|
|
948
|
+
tags = {
|
|
949
|
+
"approval_request_id": result["request"]["request_id"],
|
|
950
|
+
"action_digest": result["request"]["action_digest"],
|
|
951
|
+
"approval_decision_id": result["request"]["decision_id"],
|
|
952
|
+
"approval_authority": actor,
|
|
953
|
+
"approval_authority_mode": mode,
|
|
954
|
+
}
|
|
955
|
+
if relay_canary:
|
|
956
|
+
tags.update({
|
|
957
|
+
"approval_relay_canary_decision": decision,
|
|
958
|
+
"approval_relay_canary_terminalized": "true",
|
|
959
|
+
"external_effect": "false",
|
|
960
|
+
})
|
|
961
|
+
if decision == "approve":
|
|
962
|
+
tags.update({"approved_by": actor, "approved_at": decided_at})
|
|
963
|
+
else:
|
|
964
|
+
tags.update({
|
|
965
|
+
"rejected_by": actor,
|
|
966
|
+
"rejected_at": decided_at,
|
|
967
|
+
"rejected_reason": rejection_reason,
|
|
968
|
+
})
|
|
969
|
+
new_status = JobStatus.CANCELLED
|
|
970
|
+
reason = "approval_relay_canary_%s_no_effect" % decision
|
|
971
|
+
elif decision == "approve":
|
|
972
|
+
tags.update({"approved_by": actor, "approved_at": decided_at})
|
|
973
|
+
if result["grant"]:
|
|
974
|
+
tags["bounded_grant_id"] = result["grant"]["grant_id"]
|
|
975
|
+
new_status = JobStatus.QUEUED
|
|
976
|
+
reason = f"approved_by_principal={actor}"
|
|
977
|
+
if job.depends_on:
|
|
978
|
+
dependencies_ready = True
|
|
979
|
+
for dependency_id in job.depends_on:
|
|
980
|
+
dependency = await queue.queue.get_job(dependency_id)
|
|
981
|
+
if (
|
|
982
|
+
dependency is None
|
|
983
|
+
or dependency.status is not JobStatus.COMPLETED
|
|
984
|
+
):
|
|
985
|
+
dependencies_ready = False
|
|
986
|
+
break
|
|
987
|
+
if not dependencies_ready:
|
|
988
|
+
# Approval provenance is durable, but it cannot erase a
|
|
989
|
+
# separate dependency gate. unblock_ready_jobs() will queue
|
|
990
|
+
# the job only after every prerequisite independently closes.
|
|
991
|
+
new_status = JobStatus.BLOCKED
|
|
992
|
+
reason = "approved_waiting_for_dependencies"
|
|
993
|
+
tags.update({
|
|
994
|
+
"hold_kind": "dependency",
|
|
995
|
+
"blocked_reason": "dependencies_pending",
|
|
996
|
+
})
|
|
997
|
+
else:
|
|
998
|
+
tags.update({
|
|
999
|
+
"rejected_by": actor,
|
|
1000
|
+
"rejected_at": decided_at,
|
|
1001
|
+
"rejected_reason": rejection_reason,
|
|
1002
|
+
})
|
|
1003
|
+
new_status = JobStatus.CANCELLED
|
|
1004
|
+
reason = rejection_reason
|
|
1005
|
+
if relay_canary:
|
|
1006
|
+
# Derive and apply the canary terminal state from the durable winner,
|
|
1007
|
+
# including an idempotent scheduler race or historical QUEUED row.
|
|
1008
|
+
changed = await queue.queue.repair_approval_relay_canary_terminal(
|
|
1009
|
+
job.job_id
|
|
1010
|
+
)
|
|
1011
|
+
else:
|
|
1012
|
+
changed = await queue.queue.update_job_status(
|
|
1013
|
+
job.job_id,
|
|
1014
|
+
new_status,
|
|
1015
|
+
reason=reason,
|
|
1016
|
+
tags=tags,
|
|
1017
|
+
remove_tags=[
|
|
1018
|
+
"hold_kind", "blocked_reason", "awaiting_owner_approval",
|
|
1019
|
+
"governor_error", "governor_last_recheck_at",
|
|
1020
|
+
],
|
|
1021
|
+
)
|
|
1022
|
+
if not changed:
|
|
1023
|
+
raise HTTPException(
|
|
1024
|
+
status_code=409,
|
|
1025
|
+
detail={
|
|
1026
|
+
"code": "job_transition_failed",
|
|
1027
|
+
"message": "decision is durable but the job transition must be reconciled",
|
|
1028
|
+
},
|
|
1029
|
+
)
|
|
1030
|
+
return {
|
|
1031
|
+
"success": True,
|
|
1032
|
+
"job_id": job.job_id,
|
|
1033
|
+
"status": new_status.value,
|
|
1034
|
+
"decision": decision,
|
|
1035
|
+
"decided_by": actor,
|
|
1036
|
+
"decided_at": decided_at,
|
|
1037
|
+
"approval_request": result["request"],
|
|
1038
|
+
"bounded_grant": result["grant"],
|
|
1039
|
+
"replayed": result["replayed"],
|
|
1040
|
+
"authority_mode": mode,
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def _job_to_dict(job: Job) -> Dict[str, Any]:
|
|
1045
|
+
return {
|
|
1046
|
+
"job_id": job.job_id,
|
|
1047
|
+
"job_type": job.job_type.value,
|
|
1048
|
+
"payload": job.payload,
|
|
1049
|
+
"priority": job.priority.value,
|
|
1050
|
+
"status": job.status.value,
|
|
1051
|
+
"capabilities": [
|
|
1052
|
+
{
|
|
1053
|
+
"name": capability.name,
|
|
1054
|
+
"minimum": capability.minimum,
|
|
1055
|
+
"preferred": capability.preferred,
|
|
1056
|
+
}
|
|
1057
|
+
for capability in job.capabilities
|
|
1058
|
+
],
|
|
1059
|
+
"claimed_by": job.claimed_by,
|
|
1060
|
+
"claimed_at": job.claimed_at.isoformat() if job.claimed_at else None,
|
|
1061
|
+
"claim_attempt_id": job.claim_attempt_id,
|
|
1062
|
+
"claim_expires_at": (
|
|
1063
|
+
job.claim_expires_at.isoformat() if job.claim_expires_at else None
|
|
1064
|
+
),
|
|
1065
|
+
"posted_at": job.posted_at.isoformat() if job.posted_at else None,
|
|
1066
|
+
"deadline": job.deadline.isoformat() if job.deadline else None,
|
|
1067
|
+
"max_retries": job.max_retries,
|
|
1068
|
+
"retry_count": job.retry_count,
|
|
1069
|
+
"timeout_secs": job.timeout_secs,
|
|
1070
|
+
"depends_on": job.depends_on,
|
|
1071
|
+
"tags": job.tags,
|
|
1072
|
+
"result": {
|
|
1073
|
+
"worker_node_id": job.result.worker_node_id,
|
|
1074
|
+
"status": job.result.status.value,
|
|
1075
|
+
"claim_attempt_id": job.result.claim_attempt_id,
|
|
1076
|
+
"output": job.result.output,
|
|
1077
|
+
"error": job.result.error,
|
|
1078
|
+
"started_at": job.result.started_at.isoformat() if job.result and job.result.started_at else None,
|
|
1079
|
+
"completed_at": job.result.completed_at.isoformat() if job.result and job.result.completed_at else None,
|
|
1080
|
+
"duration_seconds": job.result.duration_seconds if job.result else None,
|
|
1081
|
+
} if job.result else None,
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
def _job_inspection_dict(job: Job) -> Dict[str, Any]:
|
|
1086
|
+
"""Return authority canary fields without payload/result disclosure."""
|
|
1087
|
+
|
|
1088
|
+
import hashlib
|
|
1089
|
+
import json
|
|
1090
|
+
|
|
1091
|
+
route_tag_names = {
|
|
1092
|
+
"agent_action_route", "agent_action_route_node",
|
|
1093
|
+
"thought_route", "thought_route_node",
|
|
1094
|
+
"schema", "risk_class", "idempotency_key",
|
|
1095
|
+
"work_order_digest", "work_order_version", "executor_protocol",
|
|
1096
|
+
"thought_job_digest", "concern_id", "viewer_scope", "shareability",
|
|
1097
|
+
"action_digest", "action_result_contract", "verification_pending",
|
|
1098
|
+
"success_attestation_schema", "success_evidence_digest",
|
|
1099
|
+
"success_receipt_refs_digest", "success_verifier_identity",
|
|
1100
|
+
"success_verifier_type",
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
def digest(value: Any) -> str:
|
|
1104
|
+
encoded = json.dumps(
|
|
1105
|
+
value,
|
|
1106
|
+
sort_keys=True,
|
|
1107
|
+
separators=(",", ":"),
|
|
1108
|
+
ensure_ascii=True,
|
|
1109
|
+
default=str,
|
|
1110
|
+
).encode("utf-8")
|
|
1111
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
1112
|
+
|
|
1113
|
+
result_projection = None
|
|
1114
|
+
if job.result is not None:
|
|
1115
|
+
result_projection = {
|
|
1116
|
+
"status": job.result.status.value,
|
|
1117
|
+
"claim_attempt_id": job.result.claim_attempt_id,
|
|
1118
|
+
"worker_node_id": job.result.worker_node_id,
|
|
1119
|
+
"output": job.result.output,
|
|
1120
|
+
"error": job.result.error,
|
|
1121
|
+
}
|
|
1122
|
+
return {
|
|
1123
|
+
"job_id": job.job_id,
|
|
1124
|
+
"job_type": job.job_type.value,
|
|
1125
|
+
"status": job.status.value,
|
|
1126
|
+
"capabilities": [
|
|
1127
|
+
{
|
|
1128
|
+
"name": capability.name,
|
|
1129
|
+
"minimum": capability.minimum,
|
|
1130
|
+
"preferred": capability.preferred,
|
|
1131
|
+
}
|
|
1132
|
+
for capability in job.capabilities
|
|
1133
|
+
],
|
|
1134
|
+
"claimed_by": job.claimed_by,
|
|
1135
|
+
"claim_attempt_id": job.claim_attempt_id,
|
|
1136
|
+
"claim_expires_at": (
|
|
1137
|
+
job.claim_expires_at.isoformat() if job.claim_expires_at else None
|
|
1138
|
+
),
|
|
1139
|
+
"tags": {
|
|
1140
|
+
key: value for key, value in (job.tags or {}).items()
|
|
1141
|
+
if key in route_tag_names
|
|
1142
|
+
},
|
|
1143
|
+
"payload_sha256": digest(job.payload),
|
|
1144
|
+
"result_sha256": (
|
|
1145
|
+
digest(result_projection) if result_projection is not None else None
|
|
1146
|
+
),
|
|
1147
|
+
"result_status": (
|
|
1148
|
+
job.result.status.value if job.result is not None else None
|
|
1149
|
+
),
|
|
1150
|
+
"result_claim_attempt_id": (
|
|
1151
|
+
job.result.claim_attempt_id if job.result is not None else None
|
|
1152
|
+
),
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
# ---------------------------------------------------------------------------
|
|
1157
|
+
# WorkControlV1 — generic operator control over durable queue work
|
|
1158
|
+
# ---------------------------------------------------------------------------
|
|
1159
|
+
|
|
1160
|
+
@router.get("/work")
|
|
1161
|
+
async def get_work_control_target_query(
|
|
1162
|
+
target_id: str = Query(min_length=1, max_length=192),
|
|
1163
|
+
) -> Dict[str, Any]:
|
|
1164
|
+
"""Slash-safe target inspection; path form remains compatibility-only."""
|
|
1165
|
+
|
|
1166
|
+
return await get_work_control_target(target_id)
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
@router.post("/work/operations")
|
|
1170
|
+
async def apply_work_control_operation_body(
|
|
1171
|
+
body: WorkControlOperationRequest,
|
|
1172
|
+
request: Request,
|
|
1173
|
+
) -> Dict[str, Any]:
|
|
1174
|
+
"""Slash-safe mutation using the exact target ID already in the body."""
|
|
1175
|
+
|
|
1176
|
+
return await apply_work_control_operation(body.target_id, body, request)
|
|
1177
|
+
|
|
1178
|
+
|
|
1179
|
+
@router.get("/work/operations/receipt")
|
|
1180
|
+
async def get_work_control_receipt_query(
|
|
1181
|
+
target_id: str = Query(min_length=1, max_length=192),
|
|
1182
|
+
operation_id: str = Query(min_length=1, max_length=128),
|
|
1183
|
+
) -> Dict[str, Any]:
|
|
1184
|
+
"""Slash-safe receipt lookup for canonical operation identifiers."""
|
|
1185
|
+
|
|
1186
|
+
return await get_work_control_receipt(target_id, operation_id)
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
@router.post("/work/reconciliations")
|
|
1190
|
+
async def reconcile_work_effect(
|
|
1191
|
+
body: WorkEffectReconciliationRequest,
|
|
1192
|
+
request: Request,
|
|
1193
|
+
) -> Dict[str, Any]:
|
|
1194
|
+
"""Close one ambiguous effect using an independent scoped verifier."""
|
|
1195
|
+
|
|
1196
|
+
authority = request_authority(request)
|
|
1197
|
+
if (
|
|
1198
|
+
not authority.authenticated
|
|
1199
|
+
or authority.anonymous
|
|
1200
|
+
or authority.legacy
|
|
1201
|
+
or not authority.has_scope("workers:attest")
|
|
1202
|
+
):
|
|
1203
|
+
raise HTTPException(status_code=403, detail={
|
|
1204
|
+
"code": "independent_verifier_required",
|
|
1205
|
+
"message": "a scoped workers:attest verifier is required",
|
|
1206
|
+
})
|
|
1207
|
+
queue = _get_queue()
|
|
1208
|
+
job = await queue.queue.get_job(body.target_id)
|
|
1209
|
+
if job is None:
|
|
1210
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
1211
|
+
executor_node = str(
|
|
1212
|
+
job.result.worker_node_id if job.result is not None else ""
|
|
1213
|
+
).strip()
|
|
1214
|
+
executor_principal = str(
|
|
1215
|
+
(job.tags or {}).get("worker_authority_principal") or ""
|
|
1216
|
+
).strip()
|
|
1217
|
+
if (
|
|
1218
|
+
authority.principal_id in {executor_node, executor_principal}
|
|
1219
|
+
or any(
|
|
1220
|
+
grant.node_id == executor_node
|
|
1221
|
+
for grant in authority.worker_grants
|
|
1222
|
+
)
|
|
1223
|
+
):
|
|
1224
|
+
raise HTTPException(status_code=403, detail={
|
|
1225
|
+
"code": "independent_verifier_required",
|
|
1226
|
+
"message": "the executor cannot reconcile its own effect",
|
|
1227
|
+
})
|
|
1228
|
+
payload = body.model_dump(by_alias=True)
|
|
1229
|
+
try:
|
|
1230
|
+
return await queue.queue.reconcile_work_effect(
|
|
1231
|
+
reconciliation_id=payload["reconciliation_id"],
|
|
1232
|
+
target_id=payload["target_id"],
|
|
1233
|
+
attempt_id=payload["attempt_id"],
|
|
1234
|
+
authority_digest=payload["authority_digest"],
|
|
1235
|
+
finding=payload["finding"],
|
|
1236
|
+
evidence_refs=payload["evidence_refs"],
|
|
1237
|
+
observed_at=payload["observed_at"],
|
|
1238
|
+
summary=payload.get("summary") or "",
|
|
1239
|
+
verifier_identity=authority.principal_id,
|
|
1240
|
+
verifier_type="scoped_effect_reconciler",
|
|
1241
|
+
verifier_authority={
|
|
1242
|
+
"authority_kind": "scoped_principal",
|
|
1243
|
+
"principal_id": authority.principal_id,
|
|
1244
|
+
"credential_id": authority.credential_id,
|
|
1245
|
+
"required_scope": "workers:attest",
|
|
1246
|
+
},
|
|
1247
|
+
)
|
|
1248
|
+
except Exception as exc:
|
|
1249
|
+
raise _work_control_http_error(exc) from exc
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
@router.get("/work/{target_id}")
|
|
1253
|
+
async def get_work_control_target(target_id: str) -> Dict[str, Any]:
|
|
1254
|
+
"""Read the exact revision/digest and currently allowed operations."""
|
|
1255
|
+
|
|
1256
|
+
try:
|
|
1257
|
+
return await _get_queue().queue.work_control.inspect(target_id)
|
|
1258
|
+
except Exception as exc:
|
|
1259
|
+
raise _work_control_http_error(exc) from exc
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
@router.post("/work/{target_id}/operations")
|
|
1263
|
+
async def apply_work_control_operation(
|
|
1264
|
+
target_id: str,
|
|
1265
|
+
body: WorkControlOperationRequest,
|
|
1266
|
+
request: Request,
|
|
1267
|
+
) -> Dict[str, Any]:
|
|
1268
|
+
"""CAS-apply an idempotent command from an exact scoped principal."""
|
|
1269
|
+
|
|
1270
|
+
authority = request_authority(request)
|
|
1271
|
+
if (
|
|
1272
|
+
not authority.authenticated
|
|
1273
|
+
or authority.anonymous
|
|
1274
|
+
or authority.legacy
|
|
1275
|
+
or not authority.has_scope("work:control")
|
|
1276
|
+
):
|
|
1277
|
+
raise HTTPException(status_code=403, detail={
|
|
1278
|
+
"code": "exact_work_control_principal_required",
|
|
1279
|
+
"message": (
|
|
1280
|
+
"WorkControl mutations require a scoped work:control principal"
|
|
1281
|
+
),
|
|
1282
|
+
})
|
|
1283
|
+
payload = body.model_dump(by_alias=True)
|
|
1284
|
+
if payload["target_id"] != target_id:
|
|
1285
|
+
raise HTTPException(status_code=409, detail={
|
|
1286
|
+
"code": "work_target_path_mismatch",
|
|
1287
|
+
"message": "body target_id must match the path target",
|
|
1288
|
+
})
|
|
1289
|
+
try:
|
|
1290
|
+
return await _get_queue().queue.work_control.operate(
|
|
1291
|
+
operation_id=payload["operation_id"],
|
|
1292
|
+
operation=payload["operation"],
|
|
1293
|
+
target_id=payload["target_id"],
|
|
1294
|
+
run_id=payload["run_id"],
|
|
1295
|
+
attempt_id=payload.get("attempt_id"),
|
|
1296
|
+
expected_revision=payload["expected_revision"],
|
|
1297
|
+
expected_state_digest=payload["expected_state_digest"],
|
|
1298
|
+
parameters=payload.get("parameters") or {},
|
|
1299
|
+
reason=payload.get("reason") or "",
|
|
1300
|
+
requested_by=authority.principal_id,
|
|
1301
|
+
request_authority={
|
|
1302
|
+
"authority_kind": "scoped_principal",
|
|
1303
|
+
"principal_id": authority.principal_id,
|
|
1304
|
+
"credential_id": authority.credential_id,
|
|
1305
|
+
"required_scope": "work:control",
|
|
1306
|
+
},
|
|
1307
|
+
)
|
|
1308
|
+
except Exception as exc:
|
|
1309
|
+
raise _work_control_http_error(exc) from exc
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
@router.get("/work/{target_id}/operations/{operation_id}")
|
|
1313
|
+
async def get_work_control_receipt(
|
|
1314
|
+
target_id: str,
|
|
1315
|
+
operation_id: str,
|
|
1316
|
+
) -> Dict[str, Any]:
|
|
1317
|
+
"""Read accepted and immutable outcome receipts for one operation."""
|
|
1318
|
+
|
|
1319
|
+
try:
|
|
1320
|
+
return await _get_queue().queue.work_control.receipt(
|
|
1321
|
+
target_id, operation_id,
|
|
1322
|
+
)
|
|
1323
|
+
except Exception as exc:
|
|
1324
|
+
raise _work_control_http_error(exc) from exc
|
|
1325
|
+
|
|
1326
|
+
|
|
1327
|
+
@router.get("/workers/{node_id}/controls")
|
|
1328
|
+
async def pending_worker_controls(
|
|
1329
|
+
node_id: str,
|
|
1330
|
+
request: Request,
|
|
1331
|
+
) -> List[Dict[str, Any]]:
|
|
1332
|
+
"""Deliver pending commands only to their transport-attested worker."""
|
|
1333
|
+
|
|
1334
|
+
context = _worker_request_context(
|
|
1335
|
+
request,
|
|
1336
|
+
node_id=node_id,
|
|
1337
|
+
required_scope="workers:lifecycle",
|
|
1338
|
+
claimant=True,
|
|
1339
|
+
)
|
|
1340
|
+
if context.get("would_deny"):
|
|
1341
|
+
raise _worker_authority_error(
|
|
1342
|
+
"exact_worker_grant_required",
|
|
1343
|
+
"WorkControl delivery requires an exact scoped worker grant",
|
|
1344
|
+
)
|
|
1345
|
+
return await _get_queue().queue.work_control.pending_for_worker(node_id)
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
@router.post("/workers/controls/ack")
|
|
1349
|
+
async def acknowledge_worker_control_body(
|
|
1350
|
+
body: WorkControlAckEnvelopeRequest,
|
|
1351
|
+
request: Request,
|
|
1352
|
+
) -> Dict[str, Any]:
|
|
1353
|
+
"""Slash-safe worker acknowledgement for canonical operation IDs."""
|
|
1354
|
+
|
|
1355
|
+
return await acknowledge_worker_control(
|
|
1356
|
+
body.node_id, body.operation_id, body, request,
|
|
1357
|
+
)
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
@router.post("/workers/controls/outcome")
|
|
1361
|
+
async def record_worker_control_outcome(
|
|
1362
|
+
body: WorkControlWorkerOutcomeRequest,
|
|
1363
|
+
request: Request,
|
|
1364
|
+
) -> Dict[str, Any]:
|
|
1365
|
+
"""Persist an idempotent steer result before the worker sends its ack."""
|
|
1366
|
+
|
|
1367
|
+
context = _worker_request_context(
|
|
1368
|
+
request,
|
|
1369
|
+
node_id=body.node_id,
|
|
1370
|
+
required_scope="workers:lifecycle",
|
|
1371
|
+
claimant=True,
|
|
1372
|
+
)
|
|
1373
|
+
if context.get("would_deny"):
|
|
1374
|
+
raise _worker_authority_error(
|
|
1375
|
+
"exact_worker_grant_required",
|
|
1376
|
+
"durable WorkControl outcome requires an exact worker grant",
|
|
1377
|
+
)
|
|
1378
|
+
try:
|
|
1379
|
+
return await _get_queue().queue.record_work_control_worker_outcome(
|
|
1380
|
+
worker_id=body.node_id,
|
|
1381
|
+
operation_id=body.operation_id,
|
|
1382
|
+
attempt_id=body.attempt_id,
|
|
1383
|
+
outcome=body.outcome,
|
|
1384
|
+
details=body.details,
|
|
1385
|
+
)
|
|
1386
|
+
except Exception as exc:
|
|
1387
|
+
raise _work_control_http_error(exc) from exc
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
@router.post("/workers/{node_id}/controls/{operation_id}/ack")
|
|
1391
|
+
async def acknowledge_worker_control(
|
|
1392
|
+
node_id: str,
|
|
1393
|
+
operation_id: str,
|
|
1394
|
+
body: WorkControlAckRequest,
|
|
1395
|
+
request: Request,
|
|
1396
|
+
) -> Dict[str, Any]:
|
|
1397
|
+
"""Persist the exact claimant's cooperative acknowledgement."""
|
|
1398
|
+
|
|
1399
|
+
context = _worker_request_context(
|
|
1400
|
+
request,
|
|
1401
|
+
node_id=node_id,
|
|
1402
|
+
required_scope="workers:lifecycle",
|
|
1403
|
+
claimant=True,
|
|
1404
|
+
)
|
|
1405
|
+
if context.get("would_deny"):
|
|
1406
|
+
raise _worker_authority_error(
|
|
1407
|
+
"exact_worker_grant_required",
|
|
1408
|
+
"WorkControl acknowledgement requires an exact scoped worker grant",
|
|
1409
|
+
)
|
|
1410
|
+
authority = request_authority(request)
|
|
1411
|
+
try:
|
|
1412
|
+
return await _get_queue().queue.work_control.acknowledge(
|
|
1413
|
+
worker_id=node_id,
|
|
1414
|
+
operation_id=operation_id,
|
|
1415
|
+
attempt_id=body.attempt_id,
|
|
1416
|
+
outcome=body.outcome,
|
|
1417
|
+
details=body.details,
|
|
1418
|
+
ack_authority={
|
|
1419
|
+
"authority_kind": "scoped_worker_principal",
|
|
1420
|
+
"principal_id": authority.principal_id,
|
|
1421
|
+
"credential_id": authority.credential_id,
|
|
1422
|
+
"worker_id": node_id,
|
|
1423
|
+
"required_scope": "workers:lifecycle",
|
|
1424
|
+
"worker_authority_mode": context.get("mode"),
|
|
1425
|
+
},
|
|
1426
|
+
)
|
|
1427
|
+
except Exception as exc:
|
|
1428
|
+
raise _work_control_http_error(exc) from exc
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
# ---------------------------------------------------------------------------
|
|
1432
|
+
# Worker endpoints
|
|
1433
|
+
# ---------------------------------------------------------------------------
|
|
1434
|
+
|
|
1435
|
+
@router.post("/workers/register")
|
|
1436
|
+
async def register_worker(
|
|
1437
|
+
body: WorkerRegisterRequest,
|
|
1438
|
+
request: Request = None,
|
|
1439
|
+
) -> Dict[str, Any]:
|
|
1440
|
+
"""Register a worker node with the scheduler."""
|
|
1441
|
+
queue = _get_queue()
|
|
1442
|
+
context = _worker_request_context(
|
|
1443
|
+
request, node_id=body.node_id, required_scope="workers:register",
|
|
1444
|
+
)
|
|
1445
|
+
caps = _bounded_worker_capabilities(body, context)
|
|
1446
|
+
await queue.queue.register_worker(caps)
|
|
1447
|
+
logger.info("Worker registered: %s (types=%s)", body.node_id, body.job_types)
|
|
1448
|
+
return {
|
|
1449
|
+
"success": True,
|
|
1450
|
+
"node_id": body.node_id,
|
|
1451
|
+
"worker_authority": _worker_tags(context),
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
@router.post("/workers/{node_id}/heartbeat")
|
|
1456
|
+
async def worker_heartbeat(
|
|
1457
|
+
node_id: str,
|
|
1458
|
+
body: WorkerHeartbeatRequest,
|
|
1459
|
+
request: Request = None,
|
|
1460
|
+
) -> Dict[str, Any]:
|
|
1461
|
+
"""Receive a worker heartbeat."""
|
|
1462
|
+
queue = _get_queue()
|
|
1463
|
+
context = _worker_request_context(
|
|
1464
|
+
request, node_id=node_id, required_scope="workers:lifecycle",
|
|
1465
|
+
)
|
|
1466
|
+
progress = body.progress or {}
|
|
1467
|
+
updated = await queue.queue.send_heartbeat(
|
|
1468
|
+
worker_id=node_id,
|
|
1469
|
+
job_ids=body.job_ids,
|
|
1470
|
+
progress=progress,
|
|
1471
|
+
claim_attempt_ids=body.claim_attempt_ids,
|
|
1472
|
+
)
|
|
1473
|
+
if updated != len(set(body.job_ids)):
|
|
1474
|
+
raise HTTPException(status_code=409, detail={
|
|
1475
|
+
"code": "worker_heartbeat_claim_mismatch",
|
|
1476
|
+
"message": "one or more heartbeat jobs are not claimed by this worker",
|
|
1477
|
+
})
|
|
1478
|
+
if body.load is not None:
|
|
1479
|
+
await queue.queue.update_worker_load(node_id, body.load)
|
|
1480
|
+
return {
|
|
1481
|
+
"success": True,
|
|
1482
|
+
"node_id": node_id,
|
|
1483
|
+
"jobs_updated": updated,
|
|
1484
|
+
"worker_authority": _worker_tags(context),
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
|
|
1488
|
+
@router.post("/workers/{node_id}/deregister")
|
|
1489
|
+
async def deregister_worker(
|
|
1490
|
+
node_id: str,
|
|
1491
|
+
request: Request = None,
|
|
1492
|
+
) -> Dict[str, Any]:
|
|
1493
|
+
"""Remove a worker from the scheduler."""
|
|
1494
|
+
queue = _get_queue()
|
|
1495
|
+
context = _worker_request_context(
|
|
1496
|
+
request, node_id=node_id, required_scope="workers:register",
|
|
1497
|
+
)
|
|
1498
|
+
await queue.queue.deregister_worker(node_id)
|
|
1499
|
+
logger.info("Worker deregistered: %s", node_id)
|
|
1500
|
+
return {
|
|
1501
|
+
"success": True,
|
|
1502
|
+
"node_id": node_id,
|
|
1503
|
+
"worker_authority": _worker_tags(context),
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
# ---------------------------------------------------------------------------
|
|
1508
|
+
# Job endpoints
|
|
1509
|
+
# ---------------------------------------------------------------------------
|
|
1510
|
+
|
|
1511
|
+
@router.post("/approvals/canary")
|
|
1512
|
+
async def create_approval_relay_canary(
|
|
1513
|
+
body: ApprovalRelayCanaryRequest,
|
|
1514
|
+
request: Request,
|
|
1515
|
+
) -> Dict[str, Any]:
|
|
1516
|
+
"""Ensure one canonical approval request with no executable outcome."""
|
|
1517
|
+
|
|
1518
|
+
principal = _approval_relay_canary_authority(request)
|
|
1519
|
+
from apsimo.task_queue.approval_relay_canary import (
|
|
1520
|
+
SCHEMA,
|
|
1521
|
+
TERMINAL_POLICY,
|
|
1522
|
+
idempotency_digest,
|
|
1523
|
+
)
|
|
1524
|
+
|
|
1525
|
+
digest = idempotency_digest(body.idempotency_key)
|
|
1526
|
+
queue = _get_queue()
|
|
1527
|
+
try:
|
|
1528
|
+
job, created = await queue.queue.ensure_approval_relay_canary(digest)
|
|
1529
|
+
projection = await get_job_approval_projection(job.job_id)
|
|
1530
|
+
except (ApprovalAuthorityError, ValueError, RuntimeError) as exc:
|
|
1531
|
+
raise HTTPException(status_code=409, detail={
|
|
1532
|
+
"code": "approval_relay_canary_conflict",
|
|
1533
|
+
"message": str(exc),
|
|
1534
|
+
}) from exc
|
|
1535
|
+
canonical_request = projection.get("request") or {}
|
|
1536
|
+
if not canonical_request.get("request_id"):
|
|
1537
|
+
raise HTTPException(status_code=503, detail={
|
|
1538
|
+
"code": "approval_relay_canary_authority_unavailable",
|
|
1539
|
+
"message": "canonical approval request was not materialized",
|
|
1540
|
+
})
|
|
1541
|
+
return {
|
|
1542
|
+
"schema": "ApprovalRelayCanaryReceiptV1",
|
|
1543
|
+
"version": 1,
|
|
1544
|
+
"canary_schema": SCHEMA,
|
|
1545
|
+
"created": created,
|
|
1546
|
+
"created_by": principal,
|
|
1547
|
+
"job_id": job.job_id,
|
|
1548
|
+
"job_status": projection["job_status"],
|
|
1549
|
+
"external_effect": False,
|
|
1550
|
+
"terminal_policy": TERMINAL_POLICY,
|
|
1551
|
+
"idempotency_digest": digest,
|
|
1552
|
+
"request_id": canonical_request["request_id"],
|
|
1553
|
+
"action_digest": projection["action_digest"],
|
|
1554
|
+
"scope_digest": projection["scope_digest"],
|
|
1555
|
+
"binding_digest": projection["binding_digest"],
|
|
1556
|
+
"request_digest": projection["request_digest"],
|
|
1557
|
+
"presentation_digest": projection["presentation_digest"],
|
|
1558
|
+
"request_status": canonical_request["status"],
|
|
1559
|
+
"decision": canonical_request.get("decision"),
|
|
1560
|
+
"decision_id": canonical_request.get("decision_id"),
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
@router.post("/jobs")
|
|
1565
|
+
async def create_job(body: JobPostRequest) -> Dict[str, Any]:
|
|
1566
|
+
"""Post a new job to the queue."""
|
|
1567
|
+
queue = _get_queue()
|
|
1568
|
+
reserved = _reserved_job_tags(body.tags)
|
|
1569
|
+
if reserved:
|
|
1570
|
+
raise HTTPException(status_code=400, detail={
|
|
1571
|
+
"code": "reserved_job_tags",
|
|
1572
|
+
"message": "authority tags may only be set by Colony control planes",
|
|
1573
|
+
"tags": reserved,
|
|
1574
|
+
})
|
|
1575
|
+
reserved_canary_hint = str(
|
|
1576
|
+
body.payload.get("action_hint") or ""
|
|
1577
|
+
).strip()
|
|
1578
|
+
if (
|
|
1579
|
+
body.payload.get("schema") == "ApprovalRelayCanaryV1"
|
|
1580
|
+
or reserved_canary_hint == "approval_relay_canary"
|
|
1581
|
+
):
|
|
1582
|
+
raise HTTPException(status_code=400, detail={
|
|
1583
|
+
"code": "approval_relay_canary_authority_reserved",
|
|
1584
|
+
"message": (
|
|
1585
|
+
"ApprovalRelayCanaryV1 may only be issued by Colony's "
|
|
1586
|
+
"server-owned approval canary endpoint"
|
|
1587
|
+
),
|
|
1588
|
+
})
|
|
1589
|
+
if body.payload.get("schema") == "WorkOrderV1":
|
|
1590
|
+
raise HTTPException(status_code=400, detail={
|
|
1591
|
+
"code": "work_order_authority_reserved",
|
|
1592
|
+
"message": (
|
|
1593
|
+
"WorkOrderV1 authority may only be issued by the durable "
|
|
1594
|
+
"ProjectEngine adapter"
|
|
1595
|
+
),
|
|
1596
|
+
})
|
|
1597
|
+
if body.job_type == JobType.THOUGHT.value or (
|
|
1598
|
+
body.payload.get("schema") == "ThoughtJobV1"
|
|
1599
|
+
):
|
|
1600
|
+
raise HTTPException(status_code=400, detail={
|
|
1601
|
+
"code": "thought_job_authority_reserved",
|
|
1602
|
+
"message": (
|
|
1603
|
+
"ThoughtJobV1 may only be issued by Colony's cognition spine"
|
|
1604
|
+
),
|
|
1605
|
+
})
|
|
1606
|
+
from apsimo.task_queue.routing import (
|
|
1607
|
+
AGENT_ACTION_ROUTE_CAPABILITIES,
|
|
1608
|
+
)
|
|
1609
|
+
caller_route_capabilities = sorted({
|
|
1610
|
+
str(capability.get("name") or "").strip()
|
|
1611
|
+
for capability in (body.capabilities or [])
|
|
1612
|
+
} & AGENT_ACTION_ROUTE_CAPABILITIES)
|
|
1613
|
+
if caller_route_capabilities:
|
|
1614
|
+
raise HTTPException(status_code=400, detail={
|
|
1615
|
+
"code": "agent_action_route_authority_reserved",
|
|
1616
|
+
"message": (
|
|
1617
|
+
"agent_action executor routes are derived by Colony and "
|
|
1618
|
+
"cannot be selected by API callers"
|
|
1619
|
+
),
|
|
1620
|
+
"capabilities": caller_route_capabilities,
|
|
1621
|
+
})
|
|
1622
|
+
try:
|
|
1623
|
+
job_type = JobType(body.job_type) if body.job_type else JobType.AGENT_ACTION
|
|
1624
|
+
except ValueError:
|
|
1625
|
+
raise HTTPException(status_code=400, detail={
|
|
1626
|
+
"code": "invalid_job_type",
|
|
1627
|
+
"message": f"unknown job_type {body.job_type!r}",
|
|
1628
|
+
"valid_job_types": sorted(t.value for t in JobType),
|
|
1629
|
+
})
|
|
1630
|
+
payload = dict(body.payload)
|
|
1631
|
+
public_effect = False
|
|
1632
|
+
if job_type is JobType.AGENT_ACTION:
|
|
1633
|
+
from apsimo.initiatives.action_registry import get_action
|
|
1634
|
+
|
|
1635
|
+
action_hint = str(payload.get("action_hint") or "").strip()
|
|
1636
|
+
action_spec = get_action(action_hint)
|
|
1637
|
+
if action_spec is None:
|
|
1638
|
+
raise HTTPException(status_code=400, detail={
|
|
1639
|
+
"code": "unregistered_agent_action",
|
|
1640
|
+
"message": (
|
|
1641
|
+
"public agent_action jobs require a named action_hint "
|
|
1642
|
+
"from Colony's server-owned action registry"
|
|
1643
|
+
),
|
|
1644
|
+
})
|
|
1645
|
+
declared_risk = str(payload.get("risk") or "").strip().lower()
|
|
1646
|
+
canonical_risk = action_spec.risk.value
|
|
1647
|
+
if declared_risk and declared_risk != canonical_risk:
|
|
1648
|
+
raise HTTPException(status_code=400, detail={
|
|
1649
|
+
"code": "agent_action_risk_mismatch",
|
|
1650
|
+
"message": "agent_action risk must match the server registry",
|
|
1651
|
+
})
|
|
1652
|
+
payload["risk"] = canonical_risk
|
|
1653
|
+
public_effect = canonical_risk != "read_only"
|
|
1654
|
+
# JobPriority is an int Enum (NORMAL=50, HIGH=80, ...); look up by NAME, not value —
|
|
1655
|
+
# JobPriority("HIGH") tries to match a member whose value is the string "HIGH" and always
|
|
1656
|
+
# raises (it even 500'd the default "normal"). Accept the name or the numeric value.
|
|
1657
|
+
if body.priority:
|
|
1658
|
+
_p = str(body.priority).upper()
|
|
1659
|
+
priority = JobPriority[_p] if _p in JobPriority.__members__ else JobPriority(int(body.priority))
|
|
1660
|
+
else:
|
|
1661
|
+
priority = JobPriority.NORMAL
|
|
1662
|
+
|
|
1663
|
+
caps: List[JobCapabilityRequirement] = []
|
|
1664
|
+
if body.capabilities:
|
|
1665
|
+
for c in body.capabilities:
|
|
1666
|
+
caps.append(JobCapabilityRequirement(
|
|
1667
|
+
name=c["name"],
|
|
1668
|
+
minimum=c.get("minimum"),
|
|
1669
|
+
preferred=c.get("preferred", False),
|
|
1670
|
+
))
|
|
1671
|
+
|
|
1672
|
+
job = Job(
|
|
1673
|
+
job_type=job_type,
|
|
1674
|
+
payload=payload,
|
|
1675
|
+
priority=priority,
|
|
1676
|
+
capabilities=caps,
|
|
1677
|
+
deadline=_parse_dt(body.deadline),
|
|
1678
|
+
max_retries=body.max_retries,
|
|
1679
|
+
timeout_secs=body.timeout_secs,
|
|
1680
|
+
depends_on=body.depends_on,
|
|
1681
|
+
tags=body.tags or {},
|
|
1682
|
+
posted_by="api",
|
|
1683
|
+
)
|
|
1684
|
+
if public_effect:
|
|
1685
|
+
job.status = JobStatus.BLOCKED
|
|
1686
|
+
job.tags.update({
|
|
1687
|
+
"hold_kind": "approval",
|
|
1688
|
+
"blocked_reason": "awaiting_owner_approval",
|
|
1689
|
+
"awaiting_owner_approval": "true",
|
|
1690
|
+
"approval_requested_at": datetime.now(timezone.utc).isoformat(),
|
|
1691
|
+
})
|
|
1692
|
+
if job_type is JobType.AGENT_ACTION:
|
|
1693
|
+
from apsimo.task_queue.routing import (
|
|
1694
|
+
expected_agent_action_routes,
|
|
1695
|
+
)
|
|
1696
|
+
try:
|
|
1697
|
+
expected_agent_action_routes(job)
|
|
1698
|
+
except ValueError as exc:
|
|
1699
|
+
raise HTTPException(status_code=400, detail={
|
|
1700
|
+
"code": "agent_action_routing_invalid",
|
|
1701
|
+
"message": str(exc),
|
|
1702
|
+
}) from exc
|
|
1703
|
+
job_id = await queue.queue.post(job)
|
|
1704
|
+
return {"success": True, "job_id": job_id}
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
@router.post("/jobs/claim")
|
|
1708
|
+
async def claim_job(
|
|
1709
|
+
body: JobClaimRequest,
|
|
1710
|
+
request: Request = None,
|
|
1711
|
+
) -> Optional[Dict[str, Any]]:
|
|
1712
|
+
"""Claim through QueueManager's mandatory atomic authority boundary."""
|
|
1713
|
+
queue = _get_queue()
|
|
1714
|
+
context = _worker_request_context(
|
|
1715
|
+
request, node_id=body.node_id, required_scope="workers:claim",
|
|
1716
|
+
)
|
|
1717
|
+
caps = _bounded_worker_capabilities(body, context)
|
|
1718
|
+
try:
|
|
1719
|
+
job = await queue.queue.claim_job(
|
|
1720
|
+
body.node_id,
|
|
1721
|
+
caps,
|
|
1722
|
+
authority_tags=_worker_tags(context),
|
|
1723
|
+
)
|
|
1724
|
+
except QueueExecutionUnavailable as exc:
|
|
1725
|
+
raise HTTPException(status_code=503, detail={
|
|
1726
|
+
"code": "queue_execution_unavailable",
|
|
1727
|
+
"message": str(exc),
|
|
1728
|
+
}) from exc
|
|
1729
|
+
if job is None:
|
|
1730
|
+
return None
|
|
1731
|
+
out = _job_to_dict(job)
|
|
1732
|
+
mode = str(job.tags.get("governor_mode") or "")
|
|
1733
|
+
if mode:
|
|
1734
|
+
governor = {
|
|
1735
|
+
"mode": mode,
|
|
1736
|
+
"enforced": str(job.tags.get("governor_enforced") or "false")
|
|
1737
|
+
== "true",
|
|
1738
|
+
"would_refuse": str(
|
|
1739
|
+
job.tags.get("governor_would_refuse") or "false"
|
|
1740
|
+
) == "true",
|
|
1741
|
+
}
|
|
1742
|
+
error = str(job.tags.get("governor_error") or "")
|
|
1743
|
+
if error:
|
|
1744
|
+
governor["error"] = error
|
|
1745
|
+
out["governor"] = governor
|
|
1746
|
+
return out
|
|
1747
|
+
|
|
1748
|
+
|
|
1749
|
+
@router.post("/jobs/{job_id}/start")
|
|
1750
|
+
async def start_job(
|
|
1751
|
+
job_id: str,
|
|
1752
|
+
body: JobStartRequest,
|
|
1753
|
+
request: Request = None,
|
|
1754
|
+
) -> Dict[str, Any]:
|
|
1755
|
+
"""Transition a claimed job to RUNNING."""
|
|
1756
|
+
queue = _get_queue()
|
|
1757
|
+
job = await queue.queue.get_job(job_id)
|
|
1758
|
+
if job is None:
|
|
1759
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
1760
|
+
worker_id = job.claimed_by
|
|
1761
|
+
if not worker_id:
|
|
1762
|
+
raise HTTPException(status_code=409, detail={
|
|
1763
|
+
"code": "job_not_claimed", "message": "job has no active claimant",
|
|
1764
|
+
})
|
|
1765
|
+
context = _worker_request_context(
|
|
1766
|
+
request, node_id=worker_id, required_scope="workers:lifecycle", claimant=True,
|
|
1767
|
+
)
|
|
1768
|
+
claim_attempt_id = body.claim_attempt_id
|
|
1769
|
+
replayed = (
|
|
1770
|
+
job.status is JobStatus.RUNNING
|
|
1771
|
+
and job.claim_attempt_id == claim_attempt_id
|
|
1772
|
+
)
|
|
1773
|
+
if not await queue.queue.start_job(
|
|
1774
|
+
job_id, worker_id, claim_attempt_id=claim_attempt_id,
|
|
1775
|
+
):
|
|
1776
|
+
raise HTTPException(status_code=409, detail={
|
|
1777
|
+
"code": "job_transition_failed",
|
|
1778
|
+
"message": "job is not claimed by this worker in the expected state",
|
|
1779
|
+
})
|
|
1780
|
+
return {
|
|
1781
|
+
"success": True,
|
|
1782
|
+
"job_id": job_id,
|
|
1783
|
+
"claim_attempt_id": claim_attempt_id,
|
|
1784
|
+
"replayed": replayed,
|
|
1785
|
+
"worker_authority": _worker_tags(context),
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
|
|
1789
|
+
@router.post("/jobs/{job_id}/complete")
|
|
1790
|
+
async def complete_job(
|
|
1791
|
+
job_id: str,
|
|
1792
|
+
body: JobCompleteRequest,
|
|
1793
|
+
request: Request = None,
|
|
1794
|
+
) -> Dict[str, Any]:
|
|
1795
|
+
"""Mark a job as completed."""
|
|
1796
|
+
queue = _get_queue()
|
|
1797
|
+
job = await queue.queue.get_job(job_id)
|
|
1798
|
+
if job is None:
|
|
1799
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
1800
|
+
worker_id = job.claimed_by or (
|
|
1801
|
+
job.result.worker_node_id
|
|
1802
|
+
if (
|
|
1803
|
+
job.status in {
|
|
1804
|
+
JobStatus.COMPLETED, JobStatus.NEUTRAL, JobStatus.FAILED,
|
|
1805
|
+
}
|
|
1806
|
+
and job.result is not None
|
|
1807
|
+
and str(job.tags.get("worker_completion_terminalized") or "")
|
|
1808
|
+
== "true"
|
|
1809
|
+
)
|
|
1810
|
+
else None
|
|
1811
|
+
)
|
|
1812
|
+
if not worker_id:
|
|
1813
|
+
raise HTTPException(status_code=409, detail={
|
|
1814
|
+
"code": "job_not_claimed", "message": "job has no active claimant",
|
|
1815
|
+
})
|
|
1816
|
+
context = _worker_request_context(
|
|
1817
|
+
request, node_id=worker_id, required_scope="workers:lifecycle", claimant=True,
|
|
1818
|
+
)
|
|
1819
|
+
|
|
1820
|
+
# Completion auditing and trust evidence live in QueueManager so embedded
|
|
1821
|
+
# and direct consumers cannot bypass the HTTP-only path.
|
|
1822
|
+
audit = await queue.queue.complete_job(
|
|
1823
|
+
job_id=job_id,
|
|
1824
|
+
worker_id=worker_id,
|
|
1825
|
+
output=body.output,
|
|
1826
|
+
started_at=None,
|
|
1827
|
+
claim_attempt_id=body.claim_attempt_id,
|
|
1828
|
+
)
|
|
1829
|
+
|
|
1830
|
+
if not audit.get("transitioned"):
|
|
1831
|
+
raise HTTPException(status_code=409, detail={
|
|
1832
|
+
"code": "job_transition_failed",
|
|
1833
|
+
"message": "job is not running under this worker claim",
|
|
1834
|
+
})
|
|
1835
|
+
|
|
1836
|
+
return {
|
|
1837
|
+
"success": bool(audit.get("transitioned")),
|
|
1838
|
+
"job_id": job_id,
|
|
1839
|
+
"claim_attempt_id": body.claim_attempt_id,
|
|
1840
|
+
"worker_authority": _worker_tags(context),
|
|
1841
|
+
**audit,
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
@router.post("/attestations/jobs/{job_id}")
|
|
1846
|
+
async def attest_action_success(
|
|
1847
|
+
job_id: str,
|
|
1848
|
+
body: ActionSuccessAttestationRequest,
|
|
1849
|
+
request: Request = None,
|
|
1850
|
+
) -> Dict[str, Any]:
|
|
1851
|
+
"""Promote a generic effect only from an independent scoped verifier."""
|
|
1852
|
+
|
|
1853
|
+
queue = _get_queue()
|
|
1854
|
+
job = await queue.queue.get_job(job_id)
|
|
1855
|
+
if job is None:
|
|
1856
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
1857
|
+
authority = request_authority(request)
|
|
1858
|
+
executor_node = str(
|
|
1859
|
+
job.result.worker_node_id if job.result is not None else ""
|
|
1860
|
+
).strip()
|
|
1861
|
+
executor_principal = str(
|
|
1862
|
+
(job.tags or {}).get("worker_authority_principal") or ""
|
|
1863
|
+
).strip()
|
|
1864
|
+
same_executor_grant = any(
|
|
1865
|
+
grant.node_id == executor_node
|
|
1866
|
+
for grant in authority.worker_grants
|
|
1867
|
+
)
|
|
1868
|
+
if (
|
|
1869
|
+
not authority.authenticated
|
|
1870
|
+
or authority.legacy
|
|
1871
|
+
or same_executor_grant
|
|
1872
|
+
or (
|
|
1873
|
+
executor_principal
|
|
1874
|
+
and authority.principal_id == executor_principal
|
|
1875
|
+
)
|
|
1876
|
+
or authority.principal_id == executor_node
|
|
1877
|
+
):
|
|
1878
|
+
raise HTTPException(status_code=403, detail={
|
|
1879
|
+
"code": "independent_verifier_required",
|
|
1880
|
+
"message": (
|
|
1881
|
+
"the executor principal cannot attest its own action result"
|
|
1882
|
+
),
|
|
1883
|
+
})
|
|
1884
|
+
from apsimo.task_queue.action_receipts import (
|
|
1885
|
+
ActionReceiptAttestationV1,
|
|
1886
|
+
)
|
|
1887
|
+
|
|
1888
|
+
receipt = ActionReceiptAttestationV1.from_payload(
|
|
1889
|
+
body.model_dump(by_alias=True)
|
|
1890
|
+
)
|
|
1891
|
+
result = await queue.queue.attest_action_success(
|
|
1892
|
+
job_id,
|
|
1893
|
+
attestation=receipt,
|
|
1894
|
+
verifier_identity=authority.principal_id,
|
|
1895
|
+
verifier_type="scoped_receipt_verifier",
|
|
1896
|
+
)
|
|
1897
|
+
if result is None:
|
|
1898
|
+
raise HTTPException(status_code=409, detail={
|
|
1899
|
+
"code": "action_attestation_rejected",
|
|
1900
|
+
"message": (
|
|
1901
|
+
"receipt does not match the pending action and claim attempt"
|
|
1902
|
+
),
|
|
1903
|
+
})
|
|
1904
|
+
return {
|
|
1905
|
+
"success": True,
|
|
1906
|
+
"job_id": job_id,
|
|
1907
|
+
"claim_attempt_id": body.claim_attempt_id,
|
|
1908
|
+
"action_digest": body.action_digest,
|
|
1909
|
+
"verifier_identity": authority.principal_id,
|
|
1910
|
+
"schema": "ActionReceiptAttestationResultV1",
|
|
1911
|
+
"version": 1,
|
|
1912
|
+
**result,
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
|
|
1916
|
+
@router.post("/jobs/{job_id}/fail")
|
|
1917
|
+
async def fail_job(
|
|
1918
|
+
job_id: str,
|
|
1919
|
+
body: JobFailRequest,
|
|
1920
|
+
request: Request = None,
|
|
1921
|
+
) -> Dict[str, Any]:
|
|
1922
|
+
"""Mark a job as failed."""
|
|
1923
|
+
queue = _get_queue()
|
|
1924
|
+
job = await queue.queue.get_job(job_id)
|
|
1925
|
+
if job is None:
|
|
1926
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
1927
|
+
worker_id = job.claimed_by or (
|
|
1928
|
+
job.result.worker_node_id
|
|
1929
|
+
if (
|
|
1930
|
+
job.status in {JobStatus.QUEUED, JobStatus.FAILED}
|
|
1931
|
+
and job.result is not None
|
|
1932
|
+
and job.result.status is JobStatus.FAILED
|
|
1933
|
+
and job.result.claim_attempt_id == body.claim_attempt_id
|
|
1934
|
+
)
|
|
1935
|
+
else None
|
|
1936
|
+
)
|
|
1937
|
+
if not worker_id:
|
|
1938
|
+
raise HTTPException(status_code=409, detail={
|
|
1939
|
+
"code": "job_not_claimed", "message": "job has no active claimant",
|
|
1940
|
+
})
|
|
1941
|
+
context = _worker_request_context(
|
|
1942
|
+
request, node_id=worker_id, required_scope="workers:lifecycle", claimant=True,
|
|
1943
|
+
)
|
|
1944
|
+
transitioned = await queue.queue.fail_job(
|
|
1945
|
+
job_id=job_id,
|
|
1946
|
+
worker_id=worker_id,
|
|
1947
|
+
error=body.error,
|
|
1948
|
+
started_at=None,
|
|
1949
|
+
claim_attempt_id=body.claim_attempt_id,
|
|
1950
|
+
)
|
|
1951
|
+
|
|
1952
|
+
if not transitioned:
|
|
1953
|
+
raise HTTPException(status_code=409, detail={
|
|
1954
|
+
"code": "job_transition_failed",
|
|
1955
|
+
"message": "job is not claimed by this worker in a fail-able state",
|
|
1956
|
+
})
|
|
1957
|
+
|
|
1958
|
+
stored = await queue.queue.get_job(job_id)
|
|
1959
|
+
|
|
1960
|
+
return {
|
|
1961
|
+
"success": True,
|
|
1962
|
+
"job_id": job_id,
|
|
1963
|
+
"job_status": stored.status.value if stored is not None else "unknown",
|
|
1964
|
+
"replayed": job.claimed_by is None,
|
|
1965
|
+
"claim_attempt_id": body.claim_attempt_id,
|
|
1966
|
+
"worker_authority": _worker_tags(context),
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
@router.post("/jobs/{job_id}/heartbeat")
|
|
1971
|
+
async def job_heartbeat(
|
|
1972
|
+
job_id: str,
|
|
1973
|
+
body: JobHeartbeatRequest,
|
|
1974
|
+
request: Request = None,
|
|
1975
|
+
) -> Dict[str, Any]:
|
|
1976
|
+
"""Update job progress heartbeat."""
|
|
1977
|
+
queue = _get_queue()
|
|
1978
|
+
job = await queue.queue.get_job(job_id)
|
|
1979
|
+
if job is None:
|
|
1980
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
1981
|
+
worker_id = job.claimed_by
|
|
1982
|
+
if not worker_id:
|
|
1983
|
+
raise HTTPException(status_code=409, detail={
|
|
1984
|
+
"code": "job_not_claimed", "message": "job has no active claimant",
|
|
1985
|
+
})
|
|
1986
|
+
context = _worker_request_context(
|
|
1987
|
+
request, node_id=worker_id, required_scope="workers:lifecycle", claimant=True,
|
|
1988
|
+
)
|
|
1989
|
+
progress = {job_id: body.progress} if body.progress is not None else None
|
|
1990
|
+
updated = await queue.queue.send_heartbeat(
|
|
1991
|
+
worker_id,
|
|
1992
|
+
[job_id],
|
|
1993
|
+
progress=progress,
|
|
1994
|
+
claim_attempt_ids=(
|
|
1995
|
+
{job_id: body.claim_attempt_id} if body.claim_attempt_id else {}
|
|
1996
|
+
),
|
|
1997
|
+
)
|
|
1998
|
+
if updated != 1:
|
|
1999
|
+
raise HTTPException(status_code=409, detail={
|
|
2000
|
+
"code": "worker_heartbeat_claim_mismatch",
|
|
2001
|
+
"message": "job is no longer claimed by this worker",
|
|
2002
|
+
})
|
|
2003
|
+
return {
|
|
2004
|
+
"success": True,
|
|
2005
|
+
"job_id": job_id,
|
|
2006
|
+
"claim_attempt_id": body.claim_attempt_id,
|
|
2007
|
+
"worker_authority": _worker_tags(context),
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
|
|
2011
|
+
@router.post("/jobs/{job_id}/release")
|
|
2012
|
+
async def release_job(
|
|
2013
|
+
job_id: str,
|
|
2014
|
+
body: JobReleaseRequest,
|
|
2015
|
+
request: Request = None,
|
|
2016
|
+
) -> Dict[str, Any]:
|
|
2017
|
+
"""Release a claimed job back to the queue."""
|
|
2018
|
+
queue = _get_queue()
|
|
2019
|
+
job = await queue.queue.get_job(job_id)
|
|
2020
|
+
if job is None:
|
|
2021
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
2022
|
+
claim_attempt_id = body.claim_attempt_id
|
|
2023
|
+
worker_id = job.claimed_by
|
|
2024
|
+
replayed = False
|
|
2025
|
+
if not worker_id and claim_attempt_id is not None:
|
|
2026
|
+
worker_id = await queue.queue.worker_for_claim_attempt(
|
|
2027
|
+
job_id, claim_attempt_id,
|
|
2028
|
+
)
|
|
2029
|
+
replayed = worker_id is not None
|
|
2030
|
+
if not worker_id:
|
|
2031
|
+
raise HTTPException(status_code=409, detail={
|
|
2032
|
+
"code": "job_not_claimed", "message": "job has no active claimant",
|
|
2033
|
+
})
|
|
2034
|
+
context = _worker_request_context(
|
|
2035
|
+
request, node_id=worker_id, required_scope="workers:lifecycle", claimant=True,
|
|
2036
|
+
)
|
|
2037
|
+
if not await queue.queue.release_job(
|
|
2038
|
+
job_id, worker_id, claim_attempt_id=claim_attempt_id,
|
|
2039
|
+
):
|
|
2040
|
+
raise HTTPException(status_code=409, detail={
|
|
2041
|
+
"code": "job_transition_failed",
|
|
2042
|
+
"message": "job is not claimed by this worker in a releasable state",
|
|
2043
|
+
})
|
|
2044
|
+
return {
|
|
2045
|
+
"success": True,
|
|
2046
|
+
"job_id": job_id,
|
|
2047
|
+
"claim_attempt_id": claim_attempt_id,
|
|
2048
|
+
"replayed": replayed,
|
|
2049
|
+
"worker_authority": _worker_tags(context),
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2052
|
+
|
|
2053
|
+
@router.get("/jobs/blocked")
|
|
2054
|
+
async def list_blocked_jobs(
|
|
2055
|
+
task_type: Optional[str] = None,
|
|
2056
|
+
limit: int = Query(50, ge=1, le=200),
|
|
2057
|
+
after: Optional[str] = None,
|
|
2058
|
+
response: Response = None,
|
|
2059
|
+
) -> List[Dict[str, Any]]:
|
|
2060
|
+
"""List BLOCKED jobs awaiting owner approval (v0.17.0), optionally by type.
|
|
2061
|
+
|
|
2062
|
+
Dependency-blocked jobs are excluded — they resolve automatically
|
|
2063
|
+
when their dependencies complete.
|
|
2064
|
+
"""
|
|
2065
|
+
if after is not None and not is_canonical_job_id(after):
|
|
2066
|
+
raise HTTPException(status_code=422, detail={
|
|
2067
|
+
"code": "invalid_blocked_jobs_cursor",
|
|
2068
|
+
"message": "after must be a canonical queue job ID",
|
|
2069
|
+
})
|
|
2070
|
+
queue = _get_queue()
|
|
2071
|
+
jobs = await queue.queue.get_jobs_by_status(JobStatus.BLOCKED)
|
|
2072
|
+
canonical_jobs = []
|
|
2073
|
+
legacy_count = 0
|
|
2074
|
+
for job in jobs:
|
|
2075
|
+
if job.tags.get("blocked_reason", "") != "awaiting_owner_approval":
|
|
2076
|
+
continue
|
|
2077
|
+
if task_type and job.job_type.value != task_type:
|
|
2078
|
+
continue
|
|
2079
|
+
if is_canonical_job_id(job.job_id):
|
|
2080
|
+
canonical_jobs.append(job)
|
|
2081
|
+
else:
|
|
2082
|
+
legacy_count += 1
|
|
2083
|
+
if response is not None:
|
|
2084
|
+
response.headers["X-Colony-Blocked-Legacy-Count"] = str(legacy_count)
|
|
2085
|
+
|
|
2086
|
+
items = []
|
|
2087
|
+
for job in sorted(canonical_jobs, key=lambda item: item.job_id):
|
|
2088
|
+
blocked_reason = job.tags.get("blocked_reason", "")
|
|
2089
|
+
if after is not None and job.job_id <= after:
|
|
2090
|
+
continue
|
|
2091
|
+
# Approval-at-birth is owned by QueueManager.post. This GET is a
|
|
2092
|
+
# projection only; polling it can never create or change authority.
|
|
2093
|
+
approval_request = _approval_store().get_request_for_job(job.job_id)
|
|
2094
|
+
presentation = (
|
|
2095
|
+
approval_request.get("presentation")
|
|
2096
|
+
if approval_request is not None else None
|
|
2097
|
+
)
|
|
2098
|
+
items.append({
|
|
2099
|
+
"id": job.job_id,
|
|
2100
|
+
"action_hint": (
|
|
2101
|
+
presentation.get("action_name") if presentation else None
|
|
2102
|
+
),
|
|
2103
|
+
"risk": (
|
|
2104
|
+
presentation.get("risk") if presentation
|
|
2105
|
+
else "projection_unavailable"
|
|
2106
|
+
),
|
|
2107
|
+
"description": (
|
|
2108
|
+
presentation.get("summary") if presentation
|
|
2109
|
+
else "Approval presentation unavailable"
|
|
2110
|
+
),
|
|
2111
|
+
"created_at": job.posted_at.isoformat() if job.posted_at else None,
|
|
2112
|
+
"blocked_reason": blocked_reason,
|
|
2113
|
+
"approval_request_id": (
|
|
2114
|
+
approval_request.get("request_id") if approval_request else None
|
|
2115
|
+
),
|
|
2116
|
+
"action_digest": (
|
|
2117
|
+
approval_request.get("action_digest") if approval_request else None
|
|
2118
|
+
),
|
|
2119
|
+
"approval_expires_at": (
|
|
2120
|
+
approval_request.get("expires_at") if approval_request else None
|
|
2121
|
+
),
|
|
2122
|
+
"approval_scope": (
|
|
2123
|
+
approval_request.get("scope") if approval_request else None
|
|
2124
|
+
),
|
|
2125
|
+
"presentation": presentation,
|
|
2126
|
+
"presentation_digest": (
|
|
2127
|
+
approval_request.get("presentation_digest")
|
|
2128
|
+
if approval_request else None
|
|
2129
|
+
),
|
|
2130
|
+
"binding_digest": (
|
|
2131
|
+
approval_request.get("binding_digest")
|
|
2132
|
+
if approval_request else None
|
|
2133
|
+
),
|
|
2134
|
+
"request_digest": (
|
|
2135
|
+
approval_request.get("request_digest")
|
|
2136
|
+
if approval_request else None
|
|
2137
|
+
),
|
|
2138
|
+
"projection_status": (
|
|
2139
|
+
"available" if presentation else "projection_unavailable"
|
|
2140
|
+
),
|
|
2141
|
+
})
|
|
2142
|
+
if len(items) >= limit:
|
|
2143
|
+
break
|
|
2144
|
+
return items
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
@router.post("/jobs/{job_id}/approve")
|
|
2148
|
+
async def approve_job(
|
|
2149
|
+
job_id: str,
|
|
2150
|
+
body: JobApproveRequest,
|
|
2151
|
+
request: Request = None,
|
|
2152
|
+
) -> Dict[str, Any]:
|
|
2153
|
+
"""Compatibility approval endpoint backed by immutable authority state.
|
|
2154
|
+
|
|
2155
|
+
``approved_by`` is accepted only so existing clients keep parsing; it is
|
|
2156
|
+
ignored. The decision actor is derived from the authenticated principal.
|
|
2157
|
+
``always`` requests the safe grant defaults under the deployment's
|
|
2158
|
+
configured exact-scope envelope; it never creates an action-name bypass.
|
|
2159
|
+
"""
|
|
2160
|
+
queue = _get_queue()
|
|
2161
|
+
job = await queue.queue.get_job(job_id)
|
|
2162
|
+
if job is None:
|
|
2163
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
2164
|
+
grant = body.grant
|
|
2165
|
+
if body.always and grant is None:
|
|
2166
|
+
grant = BoundedGrantRequest()
|
|
2167
|
+
result = await _decide_job(
|
|
2168
|
+
job=job,
|
|
2169
|
+
decision="approve",
|
|
2170
|
+
decision_id=body.decision_id,
|
|
2171
|
+
approval_request_id=body.approval_request_id,
|
|
2172
|
+
expected_action_digest=body.expected_action_digest,
|
|
2173
|
+
grant=grant,
|
|
2174
|
+
request=request,
|
|
2175
|
+
)
|
|
2176
|
+
# Response aliases keep old integrations operational while exposing the
|
|
2177
|
+
# new durable model. Both aliases contain the same exact-scope grant.
|
|
2178
|
+
result["approved_by"] = result["decided_by"]
|
|
2179
|
+
result["approved_at"] = result["decided_at"]
|
|
2180
|
+
result["standing_approval"] = result["bounded_grant"]
|
|
2181
|
+
logger.info("Job %s approved by principal %s", job_id, result["decided_by"])
|
|
2182
|
+
return result
|
|
2183
|
+
|
|
2184
|
+
|
|
2185
|
+
@router.post("/jobs/{job_id}/reject")
|
|
2186
|
+
async def reject_job(
|
|
2187
|
+
job_id: str,
|
|
2188
|
+
body: JobRejectRequest,
|
|
2189
|
+
request: Request = None,
|
|
2190
|
+
) -> Dict[str, Any]:
|
|
2191
|
+
"""Reject a BLOCKED job using server-derived decision authority."""
|
|
2192
|
+
queue = _get_queue()
|
|
2193
|
+
job = await queue.queue.get_job(job_id)
|
|
2194
|
+
if job is None:
|
|
2195
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
2196
|
+
result = await _decide_job(
|
|
2197
|
+
job=job,
|
|
2198
|
+
decision="reject",
|
|
2199
|
+
decision_id=body.decision_id,
|
|
2200
|
+
approval_request_id=body.approval_request_id,
|
|
2201
|
+
expected_action_digest=body.expected_action_digest,
|
|
2202
|
+
grant=None,
|
|
2203
|
+
request=request,
|
|
2204
|
+
rejection_reason=body.reason,
|
|
2205
|
+
)
|
|
2206
|
+
result["rejected_by"] = result["decided_by"]
|
|
2207
|
+
result["reason"] = body.reason
|
|
2208
|
+
logger.info("Job %s rejected by principal %s", job_id, result["decided_by"])
|
|
2209
|
+
return result
|
|
2210
|
+
|
|
2211
|
+
|
|
2212
|
+
# ---------------------------------------------------------------------------
|
|
2213
|
+
# Durable approval requests and exact-scope grants
|
|
2214
|
+
# ---------------------------------------------------------------------------
|
|
2215
|
+
|
|
2216
|
+
@router.get("/approvals/requests")
|
|
2217
|
+
async def list_approval_requests(
|
|
2218
|
+
status: Optional[str] = None,
|
|
2219
|
+
limit: int = Query(100, ge=1, le=500),
|
|
2220
|
+
) -> List[Dict[str, Any]]:
|
|
2221
|
+
# The authority database may also contain non-queue experiment/charter
|
|
2222
|
+
# records, and a cross-database crash may leave a historical orphan. This
|
|
2223
|
+
# queue API exposes only requests that still have a canonical queue job.
|
|
2224
|
+
# The host bridge uses `/jobs/blocked`, not this administrative ledger view.
|
|
2225
|
+
candidates = _approval_store().list_requests(status=status, limit=500)
|
|
2226
|
+
queue = _get_queue()
|
|
2227
|
+
result: List[Dict[str, Any]] = []
|
|
2228
|
+
for candidate in candidates:
|
|
2229
|
+
if await queue.queue.get_job(candidate["job_id"]) is None:
|
|
2230
|
+
continue
|
|
2231
|
+
result.append(candidate)
|
|
2232
|
+
if len(result) >= limit:
|
|
2233
|
+
break
|
|
2234
|
+
return result
|
|
2235
|
+
|
|
2236
|
+
|
|
2237
|
+
@router.get("/approvals/requests/{request_id}")
|
|
2238
|
+
async def get_approval_request(request_id: str) -> Dict[str, Any]:
|
|
2239
|
+
result = _approval_store().get_request(request_id)
|
|
2240
|
+
if result is None:
|
|
2241
|
+
raise HTTPException(status_code=404, detail="Approval request not found")
|
|
2242
|
+
if await _get_queue().queue.get_job(result["job_id"]) is None:
|
|
2243
|
+
raise HTTPException(status_code=404, detail={
|
|
2244
|
+
"code": "queue_approval_request_not_found",
|
|
2245
|
+
"message": "Approval request is not owned by a canonical queue job",
|
|
2246
|
+
})
|
|
2247
|
+
return result
|
|
2248
|
+
|
|
2249
|
+
|
|
2250
|
+
def _request_projection(value: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
|
2251
|
+
if value is None:
|
|
2252
|
+
return None
|
|
2253
|
+
return {
|
|
2254
|
+
"request_id": value["request_id"],
|
|
2255
|
+
"request_digest": value["request_digest"],
|
|
2256
|
+
"job_id": value["job_id"],
|
|
2257
|
+
"action_digest": value["action_digest"],
|
|
2258
|
+
"scope_digest": value["scope_digest"],
|
|
2259
|
+
"binding_digest": value["binding_digest"],
|
|
2260
|
+
"presentation_digest": value["presentation_digest"],
|
|
2261
|
+
"status": value["status"],
|
|
2262
|
+
"created_at": value["created_at"],
|
|
2263
|
+
"expires_at": value["expires_at"],
|
|
2264
|
+
"superseded_by": value.get("superseded_by"),
|
|
2265
|
+
"decision": value.get("decision"),
|
|
2266
|
+
"decision_id": value.get("decision_id"),
|
|
2267
|
+
"decided_at": value.get("decided_at"),
|
|
2268
|
+
"decided_by": value.get("decided_by"),
|
|
2269
|
+
"authority_evidence": value.get("authority_evidence"),
|
|
2270
|
+
"grant_id": value.get("grant_id"),
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
|
|
2274
|
+
async def _queue_approval_state_projection(
|
|
2275
|
+
queue: TaskQueueManager,
|
|
2276
|
+
job: Job,
|
|
2277
|
+
authorization: Dict[str, Any],
|
|
2278
|
+
) -> Dict[str, Optional[str]]:
|
|
2279
|
+
"""Project bounded queue state without trusting mutable job tags."""
|
|
2280
|
+
|
|
2281
|
+
hold_kind: Optional[str] = None
|
|
2282
|
+
blocked_reason: Optional[str] = None
|
|
2283
|
+
if job.status is JobStatus.BLOCKED:
|
|
2284
|
+
auth_kind = str(authorization.get("kind") or "")
|
|
2285
|
+
auth_status = str(authorization.get("status") or "")
|
|
2286
|
+
if auth_kind == "request" and auth_status == "pending":
|
|
2287
|
+
hold_kind = "approval"
|
|
2288
|
+
blocked_reason = "awaiting_owner_approval"
|
|
2289
|
+
elif auth_status == "authorized" and job.depends_on:
|
|
2290
|
+
dependency_failed = False
|
|
2291
|
+
dependency_pending = False
|
|
2292
|
+
for dependency_id in job.depends_on:
|
|
2293
|
+
dependency = await queue.queue.get_job(dependency_id)
|
|
2294
|
+
if dependency is None:
|
|
2295
|
+
dependency_pending = True
|
|
2296
|
+
elif dependency.status in {
|
|
2297
|
+
JobStatus.FAILED,
|
|
2298
|
+
JobStatus.CANCELLED,
|
|
2299
|
+
JobStatus.NEUTRAL,
|
|
2300
|
+
}:
|
|
2301
|
+
dependency_failed = True
|
|
2302
|
+
elif dependency.status is not JobStatus.COMPLETED:
|
|
2303
|
+
dependency_pending = True
|
|
2304
|
+
if dependency_failed or dependency_pending:
|
|
2305
|
+
hold_kind = "dependency"
|
|
2306
|
+
blocked_reason = (
|
|
2307
|
+
"dependency_terminal_transition_pending"
|
|
2308
|
+
if dependency_failed else "dependencies_pending"
|
|
2309
|
+
)
|
|
2310
|
+
if hold_kind is None:
|
|
2311
|
+
# A durable decision and its queue transition live in different
|
|
2312
|
+
# SQLite commits. Any other BLOCKED combination is conservatively
|
|
2313
|
+
# incomplete, rather than copied from caller/worker-controlled tags.
|
|
2314
|
+
hold_kind = "authority_transition"
|
|
2315
|
+
blocked_reason = (
|
|
2316
|
+
"canonical_authority_release_pending"
|
|
2317
|
+
if auth_status == "authorized"
|
|
2318
|
+
else "canonical_authority_terminal_pending"
|
|
2319
|
+
if auth_status in {"rejected", "expired", "superseded"}
|
|
2320
|
+
else "canonical_authority_unavailable"
|
|
2321
|
+
)
|
|
2322
|
+
return {
|
|
2323
|
+
"job_status": job.status.value,
|
|
2324
|
+
"hold_kind": hold_kind,
|
|
2325
|
+
"blocked_reason": blocked_reason,
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
|
|
2329
|
+
@router.get("/approvals/jobs/{job_id}")
|
|
2330
|
+
async def get_job_approval_projection(job_id: str) -> Dict[str, Any]:
|
|
2331
|
+
"""Return the exact transport-neutral authority the host may import.
|
|
2332
|
+
|
|
2333
|
+
This endpoint recomputes the immutable job binding and presentation. It
|
|
2334
|
+
reads decision/grant provenance from the canonical authority database,
|
|
2335
|
+
never from worker-controlled tags, and does not expose the raw job payload.
|
|
2336
|
+
"""
|
|
2337
|
+
|
|
2338
|
+
queue = _get_queue()
|
|
2339
|
+
job = await queue.queue.get_job(job_id)
|
|
2340
|
+
if job is None:
|
|
2341
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
2342
|
+
try:
|
|
2343
|
+
binding = _job_binding(job)
|
|
2344
|
+
presentation = build_approval_presentation(
|
|
2345
|
+
job_id=job.job_id,
|
|
2346
|
+
job_type=job.job_type.value,
|
|
2347
|
+
payload=job.payload,
|
|
2348
|
+
deadline=job.deadline,
|
|
2349
|
+
)
|
|
2350
|
+
except ApprovalAuthorityError as exc:
|
|
2351
|
+
raise _approval_error(exc) from exc
|
|
2352
|
+
presentation_digest = approval_presentation_digest(presentation)
|
|
2353
|
+
binding_digest = approval_binding_digest(
|
|
2354
|
+
job_id=job.job_id,
|
|
2355
|
+
job_type=job.job_type.value,
|
|
2356
|
+
action_digest=binding.action_digest,
|
|
2357
|
+
scope_digest=binding.scope_digest,
|
|
2358
|
+
)
|
|
2359
|
+
store = _approval_store()
|
|
2360
|
+
direct_request = store.get_request_for_job(job.job_id)
|
|
2361
|
+
grant_use = store.get_grant_use(binding.action_digest)
|
|
2362
|
+
source_request = None
|
|
2363
|
+
authorization: Dict[str, Any]
|
|
2364
|
+
expires_at = None
|
|
2365
|
+
if direct_request is not None:
|
|
2366
|
+
source_request = direct_request
|
|
2367
|
+
binding_matches = bool(
|
|
2368
|
+
direct_request["action_digest"] == binding.action_digest
|
|
2369
|
+
and direct_request["scope_digest"] == binding.scope_digest
|
|
2370
|
+
and direct_request["presentation_digest"] == presentation_digest
|
|
2371
|
+
)
|
|
2372
|
+
if (
|
|
2373
|
+
binding_matches
|
|
2374
|
+
and direct_request["status"] == "approved"
|
|
2375
|
+
and direct_request.get("decision") == "approve"
|
|
2376
|
+
):
|
|
2377
|
+
authorization = {
|
|
2378
|
+
"kind": "direct_decision",
|
|
2379
|
+
"status": "authorized",
|
|
2380
|
+
"request_id": direct_request["request_id"],
|
|
2381
|
+
"decision_id": direct_request["decision_id"],
|
|
2382
|
+
"decision": direct_request["decision"],
|
|
2383
|
+
"decided_at": direct_request["decided_at"],
|
|
2384
|
+
"decided_by": direct_request["decided_by"],
|
|
2385
|
+
"authority_evidence": direct_request["authority_evidence"],
|
|
2386
|
+
}
|
|
2387
|
+
else:
|
|
2388
|
+
authorization = {
|
|
2389
|
+
"kind": "request",
|
|
2390
|
+
"status": (
|
|
2391
|
+
direct_request["status"]
|
|
2392
|
+
if binding_matches else "invalid_binding"
|
|
2393
|
+
),
|
|
2394
|
+
"request_id": direct_request["request_id"],
|
|
2395
|
+
"decision_id": direct_request.get("decision_id"),
|
|
2396
|
+
"decision": direct_request.get("decision"),
|
|
2397
|
+
"binding_matches": binding_matches,
|
|
2398
|
+
}
|
|
2399
|
+
expires_at = direct_request["expires_at"]
|
|
2400
|
+
elif (
|
|
2401
|
+
grant_use is not None
|
|
2402
|
+
and grant_use.get("operation_id") == job.job_id
|
|
2403
|
+
and grant_use.get("scope_digest") == binding.scope_digest
|
|
2404
|
+
):
|
|
2405
|
+
source_request = store.get_request(grant_use["source_request_id"])
|
|
2406
|
+
source_request_matches = bool(
|
|
2407
|
+
source_request is not None
|
|
2408
|
+
and source_request["request_id"] == grant_use["source_request_id"]
|
|
2409
|
+
and source_request["status"] == "approved"
|
|
2410
|
+
and source_request.get("decision") == "approve"
|
|
2411
|
+
and source_request.get("decision_id") == grant_use["decision_id"]
|
|
2412
|
+
and source_request.get("grant_id") == grant_use["grant_id"]
|
|
2413
|
+
and source_request.get("decided_by") == grant_use["granted_by"]
|
|
2414
|
+
and source_request["scope_digest"] == grant_use["scope_digest"]
|
|
2415
|
+
)
|
|
2416
|
+
authorization = {
|
|
2417
|
+
"kind": "bounded_grant",
|
|
2418
|
+
"status": (
|
|
2419
|
+
"authorized" if source_request_matches else
|
|
2420
|
+
(
|
|
2421
|
+
"missing_source_request" if source_request is None
|
|
2422
|
+
else "invalid_provenance"
|
|
2423
|
+
)
|
|
2424
|
+
),
|
|
2425
|
+
"grant_id": grant_use["grant_id"],
|
|
2426
|
+
"source_request_id": grant_use["source_request_id"],
|
|
2427
|
+
"decision_id": grant_use["decision_id"],
|
|
2428
|
+
"granted_by": grant_use["granted_by"],
|
|
2429
|
+
"scope_digest": grant_use["scope_digest"],
|
|
2430
|
+
"operation_id": grant_use["operation_id"],
|
|
2431
|
+
"consumed_at": grant_use["consumed_at"],
|
|
2432
|
+
"grant_created_at": grant_use["grant_created_at"],
|
|
2433
|
+
"expires_at": grant_use["grant_expires_at"],
|
|
2434
|
+
"ttl_unbounded": grant_use["grant_ttl_unbounded"],
|
|
2435
|
+
"ttl_state": (
|
|
2436
|
+
"unbounded" if grant_use["grant_ttl_unbounded"] else "bounded"
|
|
2437
|
+
),
|
|
2438
|
+
"grant_status": grant_use["grant_status"],
|
|
2439
|
+
"uses": grant_use["uses"],
|
|
2440
|
+
"max_uses": grant_use["max_uses"],
|
|
2441
|
+
"uses_unbounded": grant_use["grant_uses_unbounded"],
|
|
2442
|
+
"uses_state": (
|
|
2443
|
+
"unbounded" if grant_use["grant_uses_unbounded"] else "bounded"
|
|
2444
|
+
),
|
|
2445
|
+
"source_request_matches": source_request_matches,
|
|
2446
|
+
}
|
|
2447
|
+
expires_at = grant_use["grant_expires_at"]
|
|
2448
|
+
else:
|
|
2449
|
+
authorization = {"kind": "none", "status": "missing"}
|
|
2450
|
+
|
|
2451
|
+
request_value = source_request or direct_request
|
|
2452
|
+
queue_authority_state = await _queue_approval_state_projection(
|
|
2453
|
+
queue, job, authorization,
|
|
2454
|
+
)
|
|
2455
|
+
projection = {
|
|
2456
|
+
"schema": AUTHORIZATION_PROJECTION_SCHEMA,
|
|
2457
|
+
"version": 1,
|
|
2458
|
+
"authority_mode": authority_mode(),
|
|
2459
|
+
"job_id": job.job_id,
|
|
2460
|
+
"job_type": job.job_type.value,
|
|
2461
|
+
"job_status": job.status.value,
|
|
2462
|
+
"queue_authority_state": queue_authority_state,
|
|
2463
|
+
"action_digest": binding.action_digest,
|
|
2464
|
+
"scope_digest": binding.scope_digest,
|
|
2465
|
+
"binding_digest": binding_digest,
|
|
2466
|
+
"presentation": presentation,
|
|
2467
|
+
"presentation_digest": presentation_digest,
|
|
2468
|
+
"request": _request_projection(request_value),
|
|
2469
|
+
"request_digest": (
|
|
2470
|
+
request_value.get("request_digest") if request_value else None
|
|
2471
|
+
),
|
|
2472
|
+
"expires_at": expires_at,
|
|
2473
|
+
"authorization": authorization,
|
|
2474
|
+
"observed_at": datetime.now(timezone.utc).isoformat(),
|
|
2475
|
+
}
|
|
2476
|
+
projection["projection_digest"] = hashlib.sha256(json.dumps(
|
|
2477
|
+
projection,
|
|
2478
|
+
sort_keys=True,
|
|
2479
|
+
separators=(",", ":"),
|
|
2480
|
+
ensure_ascii=False,
|
|
2481
|
+
).encode("utf-8")).hexdigest()
|
|
2482
|
+
return projection
|
|
2483
|
+
|
|
2484
|
+
|
|
2485
|
+
@router.post("/approvals/requests/{request_id}/decision")
|
|
2486
|
+
async def decide_approval_request(
|
|
2487
|
+
request_id: str,
|
|
2488
|
+
body: ApprovalDecisionRequest,
|
|
2489
|
+
request: Request = None,
|
|
2490
|
+
) -> Dict[str, Any]:
|
|
2491
|
+
stored = _approval_store().get_request(request_id)
|
|
2492
|
+
if stored is None:
|
|
2493
|
+
raise HTTPException(status_code=404, detail="Approval request not found")
|
|
2494
|
+
queue = _get_queue()
|
|
2495
|
+
job = await queue.queue.get_job(stored["job_id"])
|
|
2496
|
+
if job is None:
|
|
2497
|
+
raise HTTPException(status_code=409, detail={
|
|
2498
|
+
"code": "approval_job_missing",
|
|
2499
|
+
"message": "the approval request's job no longer exists",
|
|
2500
|
+
})
|
|
2501
|
+
return await _decide_job(
|
|
2502
|
+
job=job,
|
|
2503
|
+
decision=body.decision,
|
|
2504
|
+
decision_id=body.decision_id,
|
|
2505
|
+
approval_request_id=request_id,
|
|
2506
|
+
expected_action_digest=body.expected_action_digest,
|
|
2507
|
+
grant=body.grant,
|
|
2508
|
+
request=request,
|
|
2509
|
+
)
|
|
2510
|
+
|
|
2511
|
+
|
|
2512
|
+
@router.get("/approvals/grants")
|
|
2513
|
+
async def list_bounded_grants(
|
|
2514
|
+
status: Optional[str] = None,
|
|
2515
|
+
) -> List[Dict[str, Any]]:
|
|
2516
|
+
return _approval_store().list_grants(status=status)
|
|
2517
|
+
|
|
2518
|
+
|
|
2519
|
+
@router.delete("/approvals/grants/{grant_id}")
|
|
2520
|
+
async def revoke_bounded_grant(grant_id: str) -> Dict[str, Any]:
|
|
2521
|
+
if not _approval_store().revoke_grant(grant_id):
|
|
2522
|
+
raise HTTPException(status_code=404, detail="Bounded grant not found or inactive")
|
|
2523
|
+
logger.info("Bounded approval grant revoked: %s", grant_id)
|
|
2524
|
+
return {"success": True, "grant_id": grant_id}
|
|
2525
|
+
|
|
2526
|
+
|
|
2527
|
+
# Legacy read/revoke paths remain during migration. They return exact-scope
|
|
2528
|
+
# grants and never mint action-name-only authority.
|
|
2529
|
+
@router.get("/approvals/standing")
|
|
2530
|
+
async def list_standing_approvals() -> List[Dict[str, Any]]:
|
|
2531
|
+
return _approval_store().list_grants()
|
|
2532
|
+
|
|
2533
|
+
|
|
2534
|
+
@router.delete("/approvals/standing/{action_name}")
|
|
2535
|
+
async def revoke_standing_approval(action_name: str) -> Dict[str, Any]:
|
|
2536
|
+
"""Revoke all active grants for an exact legacy action name."""
|
|
2537
|
+
store = _approval_store()
|
|
2538
|
+
matching = [
|
|
2539
|
+
item for item in store.list_grants(status="active")
|
|
2540
|
+
if item.get("scope", {}).get("action_name") == action_name
|
|
2541
|
+
]
|
|
2542
|
+
changed = False
|
|
2543
|
+
for item in matching:
|
|
2544
|
+
changed = store.revoke_grant(item["grant_id"]) or changed
|
|
2545
|
+
if not changed:
|
|
2546
|
+
raise HTTPException(
|
|
2547
|
+
status_code=404,
|
|
2548
|
+
detail=f"No active bounded approval for {action_name}",
|
|
2549
|
+
)
|
|
2550
|
+
logger.info("Bounded approvals revoked for %s", action_name)
|
|
2551
|
+
return {"success": True, "action_name": action_name}
|
|
2552
|
+
|
|
2553
|
+
|
|
2554
|
+
@router.get("/jobs/pending")
|
|
2555
|
+
async def list_pending_jobs(
|
|
2556
|
+
limit: int = Query(50, ge=1, le=200),
|
|
2557
|
+
task_type: Optional[str] = None,
|
|
2558
|
+
) -> List[Dict[str, Any]]:
|
|
2559
|
+
"""List pending (queued + claimed + running + blocked) jobs."""
|
|
2560
|
+
queue = _get_queue()
|
|
2561
|
+
jobs: List[Job] = []
|
|
2562
|
+
jobs.extend(await queue.queue.get_jobs_by_status(JobStatus.QUEUED))
|
|
2563
|
+
jobs.extend(await queue.queue.get_jobs_by_status(JobStatus.CLAIMED))
|
|
2564
|
+
jobs.extend(await queue.queue.get_jobs_by_status(JobStatus.RUNNING))
|
|
2565
|
+
jobs.extend(await queue.queue.get_jobs_by_status(JobStatus.BLOCKED))
|
|
2566
|
+
|
|
2567
|
+
items = []
|
|
2568
|
+
for job in jobs:
|
|
2569
|
+
if task_type and job.job_type.value != task_type:
|
|
2570
|
+
continue
|
|
2571
|
+
items.append(_job_to_dict(job))
|
|
2572
|
+
return items[:limit]
|
|
2573
|
+
|
|
2574
|
+
|
|
2575
|
+
@router.get("/jobs/completed")
|
|
2576
|
+
async def list_completed_jobs(
|
|
2577
|
+
since: Optional[str] = None,
|
|
2578
|
+
task_type: Optional[str] = None,
|
|
2579
|
+
limit: int = Query(20, ge=1, le=200),
|
|
2580
|
+
) -> List[Dict[str, Any]]:
|
|
2581
|
+
"""List completed jobs since a timestamp, optionally filtered by type."""
|
|
2582
|
+
queue = _get_queue()
|
|
2583
|
+
since_dt = _parse_dt(since) or datetime.min.replace(tzinfo=timezone.utc)
|
|
2584
|
+
completed = await queue.queue.get_completed_jobs_since(
|
|
2585
|
+
since_dt, limit=limit, job_type=task_type)
|
|
2586
|
+
return completed
|
|
2587
|
+
|
|
2588
|
+
|
|
2589
|
+
@router.get("/jobs/neutral")
|
|
2590
|
+
async def list_neutral_jobs(
|
|
2591
|
+
task_type: Optional[str] = None,
|
|
2592
|
+
limit: int = Query(20, ge=1, le=200),
|
|
2593
|
+
) -> List[Dict[str, Any]]:
|
|
2594
|
+
"""List terminal jobs that still need independent verification."""
|
|
2595
|
+
|
|
2596
|
+
queue = _get_queue()
|
|
2597
|
+
jobs = await queue.queue.get_jobs_by_status(JobStatus.NEUTRAL)
|
|
2598
|
+
return [
|
|
2599
|
+
_job_to_dict(job) for job in jobs
|
|
2600
|
+
if not task_type or job.job_type.value == task_type
|
|
2601
|
+
][:limit]
|
|
2602
|
+
|
|
2603
|
+
|
|
2604
|
+
@router.get("/inspection/jobs/{job_id}")
|
|
2605
|
+
async def inspect_job(job_id: str) -> Dict[str, Any]:
|
|
2606
|
+
"""Read one exact job for authenticated executor canary attestation."""
|
|
2607
|
+
|
|
2608
|
+
job = await _get_queue().queue.get_job(job_id)
|
|
2609
|
+
if job is None:
|
|
2610
|
+
raise HTTPException(status_code=404, detail="Job not found")
|
|
2611
|
+
return _job_inspection_dict(job)
|
|
2612
|
+
|
|
2613
|
+
|
|
2614
|
+
# ---------------------------------------------------------------------------
|
|
2615
|
+
# Stats
|
|
2616
|
+
# ---------------------------------------------------------------------------
|
|
2617
|
+
|
|
2618
|
+
@router.get("/governor")
|
|
2619
|
+
async def governor_status() -> Dict[str, Any]:
|
|
2620
|
+
"""Worker-governor status (item 5): enforcement mode + per-job-type
|
|
2621
|
+
earned-trust stages."""
|
|
2622
|
+
gov = _governor()
|
|
2623
|
+
queue = _get_queue()
|
|
2624
|
+
central = await queue.queue.governance_status()
|
|
2625
|
+
if gov is None:
|
|
2626
|
+
return {"available": False, "central": central}
|
|
2627
|
+
try:
|
|
2628
|
+
return {"available": True, **gov.status(), "central": central}
|
|
2629
|
+
except Exception as exc:
|
|
2630
|
+
return {"available": True, "error": str(exc), "central": central}
|
|
2631
|
+
|
|
2632
|
+
|
|
2633
|
+
@router.get("/stats")
|
|
2634
|
+
async def queue_stats(request: Request = None) -> Dict[str, Any]:
|
|
2635
|
+
"""Return queue statistics."""
|
|
2636
|
+
queue = _get_queue()
|
|
2637
|
+
stats = await queue.queue.get_queue_stats()
|
|
2638
|
+
scheduler = (
|
|
2639
|
+
getattr(request.app.state, "queue_scheduler", None)
|
|
2640
|
+
if request is not None else None
|
|
2641
|
+
)
|
|
2642
|
+
worker_truth = {
|
|
2643
|
+
"registered_workers": stats.registered_workers,
|
|
2644
|
+
"active_workers": stats.active_workers,
|
|
2645
|
+
"stale_workers": stats.stale_workers,
|
|
2646
|
+
"available_workers": stats.available_workers,
|
|
2647
|
+
"worker_heartbeat_ttl_secs": stats.worker_heartbeat_ttl_secs,
|
|
2648
|
+
}
|
|
2649
|
+
return {
|
|
2650
|
+
"by_status": stats.by_status,
|
|
2651
|
+
"by_type": stats.by_type,
|
|
2652
|
+
"total_workers": stats.total_workers,
|
|
2653
|
+
"available_workers": stats.available_workers,
|
|
2654
|
+
"registered_workers": stats.registered_workers,
|
|
2655
|
+
"active_workers": stats.active_workers,
|
|
2656
|
+
"stale_workers": stats.stale_workers,
|
|
2657
|
+
"worker_heartbeat_ttl_secs": stats.worker_heartbeat_ttl_secs,
|
|
2658
|
+
"last_user_message_at": load_last_user_message_at(),
|
|
2659
|
+
# Reuse the same worker snapshot so a heartbeat on the TTL boundary
|
|
2660
|
+
# cannot make top-level and nested readiness disagree in one response.
|
|
2661
|
+
"governance": await queue.queue.governance_status(worker_truth),
|
|
2662
|
+
"scheduler": (
|
|
2663
|
+
scheduler.health
|
|
2664
|
+
if scheduler is not None else {
|
|
2665
|
+
"running": False,
|
|
2666
|
+
"healthy": False,
|
|
2667
|
+
"last_tick_at": None,
|
|
2668
|
+
"last_error": "scheduler_unavailable",
|
|
2669
|
+
"tick_count": 0,
|
|
2670
|
+
}
|
|
2671
|
+
),
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
|
|
2675
|
+
@router.get("/digest")
|
|
2676
|
+
async def get_digest(
|
|
2677
|
+
hours: int = Query(6, ge=1, le=48),
|
|
2678
|
+
) -> Dict[str, Any]:
|
|
2679
|
+
"""Return a digest of completed and failed jobs in the last N hours."""
|
|
2680
|
+
queue = _get_queue()
|
|
2681
|
+
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
2682
|
+
digest = await queue.queue.get_digest_jobs(since)
|
|
2683
|
+
|
|
2684
|
+
completed_lines = []
|
|
2685
|
+
for job in digest.get("completed", []):
|
|
2686
|
+
payload = job.get("payload", {})
|
|
2687
|
+
desc = payload.get("description", job["job_id"])
|
|
2688
|
+
completed_lines.append(f"✓ {desc}")
|
|
2689
|
+
|
|
2690
|
+
failed_lines = []
|
|
2691
|
+
for job in digest.get("failed", []):
|
|
2692
|
+
payload = job.get("payload", {})
|
|
2693
|
+
desc = payload.get("description", job["job_id"])
|
|
2694
|
+
err = job.get("error", "unknown error")
|
|
2695
|
+
failed_lines.append(f"⚠ {desc} — {err}")
|
|
2696
|
+
|
|
2697
|
+
neutral_lines = []
|
|
2698
|
+
for job in digest.get("neutral", []):
|
|
2699
|
+
payload = job.get("payload", {})
|
|
2700
|
+
desc = payload.get("description", job["job_id"])
|
|
2701
|
+
reason = (job.get("tags") or {}).get(
|
|
2702
|
+
"governor_outcome_reason", "needs independent verification"
|
|
2703
|
+
)
|
|
2704
|
+
neutral_lines.append(f"? {desc} — {reason}")
|
|
2705
|
+
|
|
2706
|
+
return {
|
|
2707
|
+
"period_hours": hours,
|
|
2708
|
+
"since": since.isoformat(),
|
|
2709
|
+
"completed_count": len(digest.get("completed", [])),
|
|
2710
|
+
"failed_count": len(digest.get("failed", [])),
|
|
2711
|
+
"needs_verification_count": len(digest.get("neutral", [])),
|
|
2712
|
+
"completed": completed_lines,
|
|
2713
|
+
"failed": failed_lines,
|
|
2714
|
+
"needs_verification": neutral_lines,
|
|
2715
|
+
}
|