pulse-engine 0.2.53.dev20261105153545__tar.gz → 0.2.54__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/PKG-INFO +1 -1
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/pyproject.toml +1 -1
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/config.py +12 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/middleware/tenant.py +17 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/router_pipelines.py +61 -2
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/schemas.py +9 -2
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/service.py +6 -2
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/runners/prefect_pipeline_flow.py +194 -9
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/README.md +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/audio_transcription.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/batcher.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/digital_news.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/digital_news_metadata.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/document_event_campaign_mapper.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/exceptions.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/google_sheets.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/mart_export.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/opensearch_sql.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/opensearch_storage.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/prompts.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/report_generator.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/speech_content.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/speech_metadata.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/base.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/config.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/factory.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/providers/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/providers/deepgram.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/providers/openai_whisper.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/providers/selfhosted_job.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/providers/supadata.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/providers/youtube_captions.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/service.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/transcription/source.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/twitter.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/youtube_downloader.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/youtube_egress.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/youtube_metadata.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/auth.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/health.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/router.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/main.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/cookiecutter.json +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/.gitignore +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/Dockerfile +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/pipeline.yaml +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/pyproject.toml +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/src/pulse_{{cookiecutter.product_slug}}/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/tests/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/tests/unit/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/tests/unit/test_manifest.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/client.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/error_handlers.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/exceptions.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/job_token.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/logging.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/prefect_http.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/scope.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/security.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/pipeline_sync.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/repository.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/router.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/runner.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/schemas.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/service.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/tasks.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/translator.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/database.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/dependencies.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/deployment/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/deployment/infra_provisioner.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/evolution/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/evolution/adapter.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/evolution/client.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/evolution/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/evolution/router.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/evolution/service.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/extractor/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/extractor/adapters/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/extractor/base.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/main.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/__main__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/server.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/tools_kb.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/tools_modules.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/tools_pipelines.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/mcp/tools_processor.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/middleware/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/middleware/rate_limit.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/middleware/request_id.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/middleware/security_headers.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/notifications/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/notifications/apprise_notifier.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/config_parser.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/expression.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/repositories.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/router_modules.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/translators/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/translators/airflow_status.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/translators/airflow_translator.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/translators/base.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/translators/prefect_status.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/translators/prefect_translator.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/base.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/core/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/core/analysis.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/core/chunking.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/core/prompts.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/core/topic_splitter.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/defaults/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/defaults/core_processor.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/defaults/postprocessor.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/defaults/preprocessor.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/llm/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/llm/provider.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/ocr/gemini.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/pipeline.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/postprocessor/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/postprocessor/embeddings.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/postprocessor/tasks.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/preprocessor/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/preprocessor/tasks.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/router.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/processor/schemas.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/registry.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/agent.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/fixtures.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/prompts.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/service.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/staging.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/tools.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/validation.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/reporting/wiring.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/runners/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/runners/lambda_runner.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/runners/pipeline_runner.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/runners/prefect_runner.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/s3.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/backend.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/lambda_microvm.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/microvm_agent/Dockerfile +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/microvm_agent/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/microvm_agent/server.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sandbox/session.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/secrets.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/services/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/services/bootstrap.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/services/opensearch.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/catalog.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/repo.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/seed.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/seed_data/skill_files.json +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/seed_data/skills.json +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/sync.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/skills/tools.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/glue.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/models.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/prefect.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/repository.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/router.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/schemas.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/seed.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/seed_data/sm_pipelines.json +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/sm_pipelines/service.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/connectors/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/connectors/athena.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/connectors/base.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/connectors/opensearch.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/knowledge_base.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/router.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/storage/schemas.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/testing/__init__.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/testing/fixtures.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/testing/mocks.py +0 -0
- {pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/worker.py +0 -0
|
@@ -216,6 +216,18 @@ class Settings(BaseSettings):
|
|
|
216
216
|
glue_kb_processor_job_name: str = "Pulse KB Processor"
|
|
217
217
|
glue_region: str = ""
|
|
218
218
|
|
|
219
|
+
# Docker network joined by containers launched by compute=local, so a
|
|
220
|
+
# module can reach a local results backend (MinIO) running as a sibling.
|
|
221
|
+
# Empty = the daemon's default bridge.
|
|
222
|
+
pipeline_local_docker_network: str = ""
|
|
223
|
+
|
|
224
|
+
# TTL for the run-scoped token minted for *scheduled* pipelines, at both the
|
|
225
|
+
# trigger-time bake (``service.trigger``) and the per-firing mint
|
|
226
|
+
# (``service.init_run``). 90 days is the historical DD-058 value; it is
|
|
227
|
+
# settable so the expiry can be reproduced locally in minutes instead of
|
|
228
|
+
# months (PIPELINE_CRON_TOKEN_TTL_SECONDS=1200 for a 20-minute token).
|
|
229
|
+
pipeline_cron_token_ttl_seconds: int = 7_776_000
|
|
230
|
+
|
|
219
231
|
prefect_cron_work_pool: str = "default"
|
|
220
232
|
# Colon-separated dirs scanned by Celery Beat for pipeline YAML schedules
|
|
221
233
|
pulse_pipeline_dirs: str = ""
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/middleware/tenant.py
RENAMED
|
@@ -47,6 +47,22 @@ def _is_cron_internal_path(path: str) -> bool:
|
|
|
47
47
|
return parts[6] == "init" or (len(parts) == 8 and parts[7] == "status")
|
|
48
48
|
|
|
49
49
|
|
|
50
|
+
PIPELINE_INIT_PATH = "/api/v1/pipelines/runs/init"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _is_pipeline_init_internal(path: str, request: Request) -> bool:
|
|
54
|
+
"""True for a /runs/init call presenting the internal token.
|
|
55
|
+
|
|
56
|
+
Conditional on the header rather than a flat INTERNAL_TOKEN_PATHS entry: a
|
|
57
|
+
flat entry would skip tenant resolution for Bearer callers too and break
|
|
58
|
+
them. The token value itself is verified in the endpoint.
|
|
59
|
+
"""
|
|
60
|
+
return (
|
|
61
|
+
path == PIPELINE_INIT_PATH
|
|
62
|
+
and request.headers.get("x-cron-internal-token") is not None
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
50
66
|
class TenantMiddleware(BaseHTTPMiddleware):
|
|
51
67
|
def __init__(
|
|
52
68
|
self,
|
|
@@ -66,6 +82,7 @@ class TenantMiddleware(BaseHTTPMiddleware):
|
|
|
66
82
|
path in PUBLIC_PATHS
|
|
67
83
|
or path in INTERNAL_TOKEN_PATHS
|
|
68
84
|
or _is_cron_internal_path(path)
|
|
85
|
+
or _is_pipeline_init_internal(path, request)
|
|
69
86
|
):
|
|
70
87
|
return await call_next(request)
|
|
71
88
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import hmac
|
|
3
4
|
import json
|
|
4
5
|
from typing import Annotated, Any
|
|
5
6
|
|
|
@@ -172,6 +173,61 @@ async def trigger_pipeline_internal(
|
|
|
172
173
|
return TriggerResponse(run_id=run_id)
|
|
173
174
|
|
|
174
175
|
|
|
176
|
+
def _resolve_init_run_tenant(request: Request, body: InitRunRequest) -> str:
|
|
177
|
+
"""Tenant for /runs/init, from whichever credential the caller presented.
|
|
178
|
+
|
|
179
|
+
A scheduled firing presents the shared X-Cron-Internal-Token: the flow has no
|
|
180
|
+
tenant Bearer to offer, and baking one into the deployment's parameters is
|
|
181
|
+
exactly what expires and 401s (DD-058). Every other caller keeps the Bearer
|
|
182
|
+
path unchanged.
|
|
183
|
+
|
|
184
|
+
On the internal-token path the tenant is **carried by the credential**:
|
|
185
|
+
``PULSE_CRON_INTERNAL_TOKEN`` is ``"<tenant>:<secret>"``, and the tenant is
|
|
186
|
+
read off *our own* copy after ``compare_digest`` has proven the presented
|
|
187
|
+
string equal to it. A caller cannot name a tenant — presenting
|
|
188
|
+
``victim-tenant:<secret>`` simply fails the comparison. Trusting a
|
|
189
|
+
caller-supplied tenant would let any holder of the secret mint a job token —
|
|
190
|
+
scoped ``kb:write`` (``job_token.py:75``) and enforced at
|
|
191
|
+
``storage/router.py:26`` — against an arbitrary tenant's knowledge base.
|
|
192
|
+
|
|
193
|
+
Carrying the tenant *in* the credential also means the two cannot drift: one
|
|
194
|
+
value to set, one to rotate, and a second worker gets a second tenant by
|
|
195
|
+
holding a different token rather than by any code change.
|
|
196
|
+
|
|
197
|
+
The tenant is deliberately *not* read from the pipeline config. That needed
|
|
198
|
+
``pulse_pipeline_dirs``, which is unset in every deployed environment, so the
|
|
199
|
+
lookup returned ``None`` and 404'd every scheduled firing; and no pipeline
|
|
200
|
+
YAML sets ``config.tenant_id``, so even a populated dir would have resolved
|
|
201
|
+
every pipeline to ``"default"`` rather than its real owner.
|
|
202
|
+
"""
|
|
203
|
+
presented = request.headers.get("x-cron-internal-token")
|
|
204
|
+
if presented is None:
|
|
205
|
+
return get_tenant_id(request)
|
|
206
|
+
|
|
207
|
+
settings = request.app.state.settings
|
|
208
|
+
expected = getattr(settings, "pulse_cron_internal_token", "")
|
|
209
|
+
if not expected or not hmac.compare_digest(presented, expected):
|
|
210
|
+
raise HTTPException(
|
|
211
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
212
|
+
detail="Invalid or missing internal token",
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# Parsed from the value we hold, never from `presented` — which at this
|
|
216
|
+
# point is only known to be byte-equal to it. Fail loud on a malformed
|
|
217
|
+
# token: a wrong tenant is a silent cross-tenant write, which is far
|
|
218
|
+
# harder to notice than a failed run.
|
|
219
|
+
tenant, sep, secret = expected.partition(":")
|
|
220
|
+
if not sep or not tenant or not secret:
|
|
221
|
+
raise HTTPException(
|
|
222
|
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
223
|
+
detail=(
|
|
224
|
+
"PULSE_CRON_INTERNAL_TOKEN must be '<tenant_id>:<secret>'; "
|
|
225
|
+
"the internal-token path cannot resolve a tenant"
|
|
226
|
+
),
|
|
227
|
+
)
|
|
228
|
+
return tenant
|
|
229
|
+
|
|
230
|
+
|
|
175
231
|
@router.post(
|
|
176
232
|
"/runs/init",
|
|
177
233
|
status_code=status.HTTP_201_CREATED,
|
|
@@ -179,7 +235,7 @@ async def trigger_pipeline_internal(
|
|
|
179
235
|
)
|
|
180
236
|
async def init_pipeline_run(
|
|
181
237
|
body: InitRunRequest,
|
|
182
|
-
|
|
238
|
+
request: Request,
|
|
183
239
|
service: PipelineService = Depends(get_pipeline_service),
|
|
184
240
|
) -> InitRunResponse:
|
|
185
241
|
"""Register a fresh pipeline run for a cron-triggered execution.
|
|
@@ -187,6 +243,7 @@ async def init_pipeline_run(
|
|
|
187
243
|
Called by the Prefect flow when it detects a cron-placeholder run_id.
|
|
188
244
|
Returns a new run_id and a scoped auth token.
|
|
189
245
|
"""
|
|
246
|
+
tenant_id = _resolve_init_run_tenant(request, body)
|
|
190
247
|
run_id, token = await service.init_run(
|
|
191
248
|
tenant_id=tenant_id,
|
|
192
249
|
product=body.product,
|
|
@@ -195,7 +252,9 @@ async def init_pipeline_run(
|
|
|
195
252
|
orchestrator=body.orchestrator,
|
|
196
253
|
orchestrator_run_id=body.orchestrator_run_id,
|
|
197
254
|
)
|
|
198
|
-
|
|
255
|
+
# tenant_id is echoed so the flow adopts the tenant the engine resolved
|
|
256
|
+
# rather than keeping the one baked into its deployment parameters.
|
|
257
|
+
return InitRunResponse(run_id=run_id, token=token, tenant_id=tenant_id)
|
|
199
258
|
|
|
200
259
|
|
|
201
260
|
@router.post("/{run_id}/report-status", status_code=204, response_model=None)
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/schemas.py
RENAMED
|
@@ -39,7 +39,7 @@ _VCPU_TO_ECS_UNITS: dict[float, int] = {
|
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class ResourceConfig(BaseModel):
|
|
42
|
-
compute: Literal["ecs", "lambda", "vm"] | None = Field(
|
|
42
|
+
compute: Literal["ecs", "lambda", "vm", "local"] | None = Field(
|
|
43
43
|
default=None,
|
|
44
44
|
description="Compute backend. None = inherit infra.default_compute.",
|
|
45
45
|
)
|
|
@@ -118,7 +118,7 @@ class StepConfig(BaseModel):
|
|
|
118
118
|
|
|
119
119
|
class InfraConfig(BaseModel):
|
|
120
120
|
orchestrator: Literal["prefect"] = "prefect"
|
|
121
|
-
default_compute: Literal["ecs", "lambda", "vm"] = "ecs"
|
|
121
|
+
default_compute: Literal["ecs", "lambda", "vm", "local"] = "ecs"
|
|
122
122
|
schedule: str | None = None # cron string — passed to Prefect deployment schedules
|
|
123
123
|
results_backend: ResultsBackendConfig = Field(default_factory=ResultsBackendConfig)
|
|
124
124
|
# AWS Secrets Manager secret name/ARN containing VM SSH credentials
|
|
@@ -252,11 +252,18 @@ class InitRunRequest(BaseModel):
|
|
|
252
252
|
global_config: dict[str, Any] = Field(default_factory=dict)
|
|
253
253
|
orchestrator: str = "prefect"
|
|
254
254
|
orchestrator_run_id: str | None = None
|
|
255
|
+
# Advisory only — echoed in logs, never trusted for authorization. On the
|
|
256
|
+
# X-Cron-Internal-Token path the tenant is resolved server-side from the
|
|
257
|
+
# pipeline config; see router_pipelines._resolve_init_run_tenant.
|
|
258
|
+
tenant_id: str | None = None
|
|
255
259
|
|
|
256
260
|
|
|
257
261
|
class InitRunResponse(BaseModel):
|
|
258
262
|
run_id: str
|
|
259
263
|
token: str
|
|
264
|
+
# The tenant the engine resolved. The flow adopts this over whatever its
|
|
265
|
+
# deployment parameters claimed.
|
|
266
|
+
tenant_id: str | None = None
|
|
260
267
|
|
|
261
268
|
|
|
262
269
|
class ReportPipelineStatusRequest(BaseModel):
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/pipeline/service.py
RENAMED
|
@@ -112,7 +112,11 @@ class PipelineService:
|
|
|
112
112
|
|
|
113
113
|
# Issue run-scoped token — long-lived for cron deployments so scheduled
|
|
114
114
|
# runs don't expire within 24h of the initial trigger.
|
|
115
|
-
token_ttl =
|
|
115
|
+
token_ttl = (
|
|
116
|
+
self._settings.pipeline_cron_token_ttl_seconds
|
|
117
|
+
if config.infra.schedule
|
|
118
|
+
else 86400 # 24h for a one-off trigger
|
|
119
|
+
)
|
|
116
120
|
token = self._token_issuer.issue_token(
|
|
117
121
|
pipeline_run_id=run.id,
|
|
118
122
|
tenant_id=tenant_id,
|
|
@@ -357,7 +361,7 @@ class PipelineService:
|
|
|
357
361
|
token = self._token_issuer.issue_token(
|
|
358
362
|
pipeline_run_id=run.id,
|
|
359
363
|
tenant_id=tenant_id,
|
|
360
|
-
ttl_seconds=
|
|
364
|
+
ttl_seconds=self._settings.pipeline_cron_token_ttl_seconds,
|
|
361
365
|
)
|
|
362
366
|
return run.id, token
|
|
363
367
|
|
|
@@ -789,6 +789,137 @@ async def _dispatch_vm(
|
|
|
789
789
|
return {"_s3_ref": True, "bucket": s3_bucket, "key": output_s3_key}
|
|
790
790
|
|
|
791
791
|
|
|
792
|
+
async def _dispatch_local(
|
|
793
|
+
image: str,
|
|
794
|
+
env: dict[str, str],
|
|
795
|
+
timeout: int,
|
|
796
|
+
memory_mb: int = 512,
|
|
797
|
+
cpu_units: int = 256,
|
|
798
|
+
results_backend: dict[str, Any] | None = None,
|
|
799
|
+
) -> dict[str, Any]:
|
|
800
|
+
"""Run a module container on the host's own Docker daemon.
|
|
801
|
+
|
|
802
|
+
Local development and testing only. The contract is identical to the ECS
|
|
803
|
+
and VM backends — same ``_VM_RUNNER_SCRIPT``, inputs read from
|
|
804
|
+
``INPUTS_S3_KEY``, result written to ``OUTPUT_S3_KEY`` — so a pipeline runs
|
|
805
|
+
here exactly as it would in the cloud. What differs is only *where*: the
|
|
806
|
+
container is a sibling process on this machine, and "S3" is whatever
|
|
807
|
+
``AWS_ENDPOINT_URL_S3`` resolves to (MinIO, locally).
|
|
808
|
+
|
|
809
|
+
Deliberately not production: no task isolation, no autoscaling, no task
|
|
810
|
+
role, and the daemon socket must be reachable from the caller.
|
|
811
|
+
"""
|
|
812
|
+
import asyncio
|
|
813
|
+
import shutil
|
|
814
|
+
|
|
815
|
+
if shutil.which("docker") is None:
|
|
816
|
+
raise RuntimeError(
|
|
817
|
+
"compute=local needs the docker CLI on the worker, and a daemon "
|
|
818
|
+
"socket it can reach. Mount /var/run/docker.sock into the worker."
|
|
819
|
+
)
|
|
820
|
+
|
|
821
|
+
step_name = env.get("STEP_NAME", "unknown")
|
|
822
|
+
pipeline_run_id_val = env.get("PIPELINE_RUN_ID", "unknown")
|
|
823
|
+
|
|
824
|
+
try:
|
|
825
|
+
from pulse_engine.config import get_settings
|
|
826
|
+
|
|
827
|
+
settings = get_settings()
|
|
828
|
+
default_bucket = settings.pulse_s3_bucket
|
|
829
|
+
network = settings.pipeline_local_docker_network
|
|
830
|
+
except Exception:
|
|
831
|
+
default_bucket = env.get("PULSE_S3_BUCKET", "pulse-data")
|
|
832
|
+
network = ""
|
|
833
|
+
|
|
834
|
+
rb = results_backend or {}
|
|
835
|
+
s3_bucket = rb.get("bucket") or default_bucket
|
|
836
|
+
s3_prefix = rb.get("prefix", "pipeline-runs/").rstrip("/")
|
|
837
|
+
output_s3_key = f"{s3_prefix}/{pipeline_run_id_val}/output/{step_name}.json"
|
|
838
|
+
|
|
839
|
+
env["OUTPUT_S3_BUCKET"] = s3_bucket
|
|
840
|
+
env["OUTPUT_S3_KEY"] = output_s3_key
|
|
841
|
+
|
|
842
|
+
# Same offload as ECS/VM: the module always reads its inputs from the
|
|
843
|
+
# results backend, so a large payload never rides on the command line.
|
|
844
|
+
inputs_raw = env.pop("INPUTS", None)
|
|
845
|
+
if inputs_raw is not None and "INPUTS_S3_KEY" not in env:
|
|
846
|
+
import boto3 as _boto3
|
|
847
|
+
|
|
848
|
+
inputs_s3_key = f"{s3_prefix}/{pipeline_run_id_val}/input/{step_name}.json"
|
|
849
|
+
_boto3.client("s3").put_object(
|
|
850
|
+
Bucket=s3_bucket,
|
|
851
|
+
Key=inputs_s3_key,
|
|
852
|
+
Body=inputs_raw.encode(),
|
|
853
|
+
ContentType="application/json",
|
|
854
|
+
)
|
|
855
|
+
env["INPUTS_S3_KEY"] = inputs_s3_key
|
|
856
|
+
|
|
857
|
+
_name_safe = re.sub(r"[^a-zA-Z0-9_.-]", "-", f"{step_name}-{pipeline_run_id_val}")
|
|
858
|
+
container_name = f"pulse-local-{_name_safe}-{uuid.uuid4().hex[:8]}"[:128]
|
|
859
|
+
|
|
860
|
+
cmd = [
|
|
861
|
+
"docker",
|
|
862
|
+
"run",
|
|
863
|
+
"--rm",
|
|
864
|
+
"--name",
|
|
865
|
+
container_name,
|
|
866
|
+
"--memory",
|
|
867
|
+
f"{memory_mb}m",
|
|
868
|
+
"--cpus",
|
|
869
|
+
str(cpu_units / 1024),
|
|
870
|
+
]
|
|
871
|
+
if network:
|
|
872
|
+
cmd += ["--network", network]
|
|
873
|
+
for k, v in env.items():
|
|
874
|
+
cmd += ["-e", f"{k}={v}"]
|
|
875
|
+
# The container talks to the same results backend the engine does, so it
|
|
876
|
+
# needs the engine's own endpoint/credential view of it.
|
|
877
|
+
for passthrough in (
|
|
878
|
+
"AWS_ENDPOINT_URL_S3",
|
|
879
|
+
"AWS_ENDPOINT_URL",
|
|
880
|
+
"AWS_ACCESS_KEY_ID",
|
|
881
|
+
"AWS_SECRET_ACCESS_KEY",
|
|
882
|
+
"AWS_SESSION_TOKEN",
|
|
883
|
+
"AWS_REGION",
|
|
884
|
+
"AWS_DEFAULT_REGION",
|
|
885
|
+
):
|
|
886
|
+
val = os.environ.get(passthrough)
|
|
887
|
+
if val:
|
|
888
|
+
cmd += ["-e", f"{passthrough}={val}"]
|
|
889
|
+
cmd.append(image)
|
|
890
|
+
cmd += ["python", "-c", _VM_RUNNER_SCRIPT]
|
|
891
|
+
|
|
892
|
+
logger.info("Local step '%s' starting: %s", step_name, image)
|
|
893
|
+
proc = await asyncio.create_subprocess_exec(
|
|
894
|
+
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
|
|
895
|
+
)
|
|
896
|
+
try:
|
|
897
|
+
stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
|
898
|
+
except TimeoutError:
|
|
899
|
+
proc.kill()
|
|
900
|
+
await asyncio.create_subprocess_exec("docker", "rm", "-f", container_name)
|
|
901
|
+
raise RuntimeError(f"Local step '{step_name}' timed out after {timeout}s")
|
|
902
|
+
|
|
903
|
+
if proc.returncode != 0:
|
|
904
|
+
stderr = stderr_b.decode("utf-8", errors="replace")[:_STDERR_LIMIT]
|
|
905
|
+
logger.error(
|
|
906
|
+
"Local step '%s' failed rc=%s\nSTDOUT:\n%s\nSTDERR:\n%s",
|
|
907
|
+
step_name,
|
|
908
|
+
proc.returncode,
|
|
909
|
+
stdout_b.decode("utf-8", errors="replace")[:_STDERR_LIMIT],
|
|
910
|
+
stderr,
|
|
911
|
+
)
|
|
912
|
+
raise RuntimeError(f"Local step failed (exit {proc.returncode}): {stderr}")
|
|
913
|
+
|
|
914
|
+
logger.info(
|
|
915
|
+
"Local step '%s' completed, output at s3://%s/%s",
|
|
916
|
+
step_name,
|
|
917
|
+
s3_bucket,
|
|
918
|
+
output_s3_key,
|
|
919
|
+
)
|
|
920
|
+
return {"_s3_ref": True, "bucket": s3_bucket, "key": output_s3_key}
|
|
921
|
+
|
|
922
|
+
|
|
792
923
|
async def _dispatch_container(
|
|
793
924
|
compute: str,
|
|
794
925
|
image: str,
|
|
@@ -819,6 +950,15 @@ async def _dispatch_container(
|
|
|
819
950
|
results_backend=results_backend,
|
|
820
951
|
use_vpc=use_vpc,
|
|
821
952
|
)
|
|
953
|
+
elif compute == "local":
|
|
954
|
+
return await _dispatch_local(
|
|
955
|
+
image,
|
|
956
|
+
step_env,
|
|
957
|
+
timeout,
|
|
958
|
+
memory_mb=memory_mb,
|
|
959
|
+
cpu_units=cpu_units,
|
|
960
|
+
results_backend=results_backend,
|
|
961
|
+
)
|
|
822
962
|
elif compute == "vm":
|
|
823
963
|
if not vm_secret_id:
|
|
824
964
|
raise ValueError(
|
|
@@ -852,17 +992,53 @@ def _current_flow_run_id() -> str | None:
|
|
|
852
992
|
return None
|
|
853
993
|
|
|
854
994
|
|
|
995
|
+
def _cron_internal_token() -> str:
|
|
996
|
+
"""The shared, non-expiring internal token from this worker's own settings.
|
|
997
|
+
|
|
998
|
+
Raises rather than returning empty: a worker with no token configured must
|
|
999
|
+
fail at its first firing, loudly. The previous fallback to the deployment's
|
|
1000
|
+
baked token meant a misconfigured worker looked healthy for 90 days and then
|
|
1001
|
+
401'd — the exact way the 2026-10-15 outage stayed invisible.
|
|
1002
|
+
"""
|
|
1003
|
+
from pulse_engine.config import get_settings
|
|
1004
|
+
|
|
1005
|
+
token = get_settings().pulse_cron_internal_token or ""
|
|
1006
|
+
if not token:
|
|
1007
|
+
raise RuntimeError(
|
|
1008
|
+
"PULSE_CRON_INTERNAL_TOKEN is not set on this worker. A cron-triggered "
|
|
1009
|
+
"pipeline cannot authenticate its /runs/init bootstrap without it. "
|
|
1010
|
+
"Check the Prefect worker task definition (main.tf)."
|
|
1011
|
+
)
|
|
1012
|
+
return token
|
|
1013
|
+
|
|
1014
|
+
|
|
855
1015
|
def _init_cron_run(
|
|
856
1016
|
pulse_engine_url: str,
|
|
857
|
-
pulse_api_token: str,
|
|
858
1017
|
product: str,
|
|
859
1018
|
config_snapshot: dict[str, Any],
|
|
860
1019
|
global_config: dict[str, Any],
|
|
861
|
-
|
|
1020
|
+
tenant_id: str = "",
|
|
1021
|
+
) -> tuple[str, str, str]:
|
|
862
1022
|
"""Register a fresh pipeline run for this cron execution.
|
|
863
1023
|
|
|
864
|
-
|
|
865
|
-
|
|
1024
|
+
Authenticates with the static cron-internal token read from this worker's
|
|
1025
|
+
environment. It deliberately does NOT accept the deployment's
|
|
1026
|
+
``pulse_api_token``: that is a run-scoped token minted once when the
|
|
1027
|
+
deployment was registered and never refreshed (``service.py:106`` ->
|
|
1028
|
+
``prefect_translator.py:198``), so it eventually expires and every scheduled
|
|
1029
|
+
firing 401s right here, before any step runs. That is what took out 80 tweet
|
|
1030
|
+
pipelines on 2026-10-15, and the same class of failure DD-058 fixed for the
|
|
1031
|
+
cron-job path in July. The internal token has no expiry, and because it is
|
|
1032
|
+
read from the environment rather than from deployment parameters, existing
|
|
1033
|
+
deployments pick this up with no re-registration.
|
|
1034
|
+
|
|
1035
|
+
Returns ``(run_id, token, tenant_id)``. The engine resolves the tenant from
|
|
1036
|
+
the pipeline config and returns it; the flow adopts that answer over whatever
|
|
1037
|
+
its own deployment parameters claimed. An engine that predates that field
|
|
1038
|
+
returns none, in which case the flow keeps the tenant it was given.
|
|
1039
|
+
|
|
1040
|
+
Uses sync httpx since this is called from the synchronous pipeline_flow body
|
|
1041
|
+
before the async task layer starts.
|
|
866
1042
|
"""
|
|
867
1043
|
resp = httpx.post(
|
|
868
1044
|
f"{pulse_engine_url}/api/v1/pipelines/runs/init",
|
|
@@ -871,13 +1047,20 @@ def _init_cron_run(
|
|
|
871
1047
|
"config_snapshot": config_snapshot,
|
|
872
1048
|
"global_config": global_config,
|
|
873
1049
|
"orchestrator_run_id": _current_flow_run_id(),
|
|
1050
|
+
# Advisory only: the engine resolves the tenant server-side from the
|
|
1051
|
+
# pipeline config and ignores this for authorization.
|
|
1052
|
+
"tenant_id": tenant_id or None,
|
|
874
1053
|
},
|
|
875
|
-
headers={"
|
|
1054
|
+
headers={"X-Cron-Internal-Token": _cron_internal_token()},
|
|
876
1055
|
timeout=30.0,
|
|
877
1056
|
)
|
|
878
1057
|
resp.raise_for_status()
|
|
879
1058
|
data = resp.json()
|
|
880
|
-
return
|
|
1059
|
+
return (
|
|
1060
|
+
str(data["run_id"]),
|
|
1061
|
+
str(data["token"]),
|
|
1062
|
+
str(data.get("tenant_id") or tenant_id),
|
|
1063
|
+
)
|
|
881
1064
|
|
|
882
1065
|
|
|
883
1066
|
def _fetch_live_module_images(
|
|
@@ -1632,14 +1815,16 @@ def pipeline_flow(
|
|
|
1632
1815
|
"Cron-triggered run detected — registering fresh pipeline run for '%s'",
|
|
1633
1816
|
product,
|
|
1634
1817
|
)
|
|
1635
|
-
active_run_id, active_token = _init_cron_run(
|
|
1818
|
+
active_run_id, active_token, tenant_id = _init_cron_run(
|
|
1636
1819
|
pulse_engine_url,
|
|
1637
|
-
pulse_api_token,
|
|
1638
1820
|
product,
|
|
1639
1821
|
config_snapshot or {},
|
|
1640
1822
|
global_config,
|
|
1823
|
+
tenant_id,
|
|
1824
|
+
)
|
|
1825
|
+
logger.info(
|
|
1826
|
+
"Cron run registered: run_id=%s tenant=%s", active_run_id, tenant_id
|
|
1641
1827
|
)
|
|
1642
|
-
logger.info("Cron run registered: run_id=%s", active_run_id)
|
|
1643
1828
|
|
|
1644
1829
|
logger.info(
|
|
1645
1830
|
"Starting v2 pipeline flow: run_id=%s, steps=%d", active_run_id, len(dag)
|
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/batcher.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/models.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/prompts.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/adapters/twitter.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/__init__.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/__init__.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/auth.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/health.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/api/v1/router.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cli/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/exceptions.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/job_token.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/logging.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/prefect_http.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/scope.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/core/security.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/__init__.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/models.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/pipeline_sync.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/repository.py
RENAMED
|
File without changes
|
{pulse_engine-0.2.53.dev20261105153545 → pulse_engine-0.2.54}/src/pulse_engine/cron/router.py
RENAMED
|
File without changes
|