streamwright 0.1.11__tar.gz → 0.1.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamwright-0.1.11/src/streamwright.egg-info → streamwright-0.1.13}/PKG-INFO +1 -1
- {streamwright-0.1.11 → streamwright-0.1.13}/pyproject.toml +1 -1
- streamwright-0.1.13/src/streamwright/core/cli/commands/jobs.py +31 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/source.py +5 -14
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/jobs.py +9 -5
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/main.py +9 -2
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/parser.py +26 -7
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/cli.py +62 -9
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/plugin.py +2 -2
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/engine.py +4 -1
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/__init__.py +4 -1
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/summary.py +20 -0
- {streamwright-0.1.11 → streamwright-0.1.13/src/streamwright.egg-info}/PKG-INFO +1 -1
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_cli_subcommands.py +39 -2
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_logging.py +2 -0
- streamwright-0.1.11/src/streamwright/core/cli/commands/jobs.py +0 -27
- {streamwright-0.1.11 → streamwright-0.1.13}/LICENSE +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/README.md +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/setup.cfg +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/setup.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/catalog.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/__main__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/connectors.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/log_options.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/inputs.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/loader.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/reader.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/bookmarks.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/builders.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/errors.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/execution.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/factories.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/partitions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/planning.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/queries.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/readers.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/responses.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/runner.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/session.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/compilation.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/list_parameters.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/ordering.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/plans.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/sandbox.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/stream_plan.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/tables.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/types.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/values.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/writers.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/graph.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/api_calls.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/capabilities.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/database_connections.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/database_rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/databases.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/file_formats.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/file_rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/files.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/http.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/local.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/locations.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/queries.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/sandbox.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/store_reader.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/calls.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/connections.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/dependencies.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/findings.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/names.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/outputs.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/refs.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/runtime.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/secrets.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/sections.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/spec.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/steps.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/structure.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/adapter.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/definition_adapter.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/engine.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/executor.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/guide.md +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/loading.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/reads.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/session.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/steps.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/writers/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/writers/tables.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/fields.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/column_rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/formats.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/headers.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/mapped.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/model.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/plain.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/inputs.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/manifest.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/naming.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/plan.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/policy.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/refs.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/runner.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/schema.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/dialects.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/migrations.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/protocol.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/store.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/types.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/validate.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/write_rules.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/auth.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/downloads.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/encoding.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/errors.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/http.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/paginators.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/policies.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/base.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/dlt_columns.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/dlt_output.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/ducklake.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/file_names.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/files.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/loading.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/output.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/staging.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/warehouse.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/py.typed +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/components.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/connector_types.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/constants.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_declarations.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_guide.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_terms.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_validation.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/definitions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/formatting.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/handlers.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/lines.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/metrics.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/network.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/record_factory.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/redaction.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/setup.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/plugins.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/redact.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/source_connectors.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/templates.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/testing.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/constants.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/contract.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/engines/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/engines/duckdb.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/engines/source.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/messaging.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/network.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/databases.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/files.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/http.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/messages.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/tables.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/py.typed +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/contract.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/helpers.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/storage.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/analysis.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/conditions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/quote.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/auth.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/checker.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/engine.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/issues.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/job_manifests.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/json_schemas.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/marked_yaml.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/partitions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/requests.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/schema.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/__init__.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/async_jobs.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/document.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/partitions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/references.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/request_partitions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/requests.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/spec.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/stream_context.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/streams.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/tables.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/templates.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/source.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/streams.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/vocabulary.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/SOURCES.txt +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/dependency_links.txt +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/entry_points.txt +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/requires.txt +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/top_level.txt +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_catalog.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_conditions.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_connector_api_family.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_connector_contract.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_connectors.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_dlt_output.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_engine_reach.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_exact_decimals.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_graph.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_http_engine.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_output_contract.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_output_formats.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_plugin_contract_gate.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_runner.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_sdk_plugins.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_sql.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_sql_quote.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_transform.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_validation.py +0 -0
- {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_warehouse_output.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamwright
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.13
|
|
4
4
|
Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
|
|
5
5
|
Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamwright"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.13"
|
|
8
8
|
description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "Apache-2.0"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
from streamwright.core.jobs.engine import load_job_engine
|
|
7
|
+
from streamwright.core.runtime import plugins
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def register(subparsers):
|
|
11
|
+
parser = subparsers.add_parser("duckdb", help="validate, plan and run DuckDBJob manifests and manage their "
|
|
12
|
+
"batches (the explicit form of run|validate|plan JOB)",
|
|
13
|
+
add_help=False)
|
|
14
|
+
parser.add_argument("-h", "--help", action="store_true", help="show the streamwright duckdb help")
|
|
15
|
+
parser.add_argument("arguments", nargs=argparse.REMAINDER)
|
|
16
|
+
parser.set_defaults(command_handler=lambda args: execute(["--help"] if args.help else args.arguments))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def execute(arguments):
|
|
20
|
+
return run_job_kind("DuckDBJob", arguments, prog="streamwright duckdb")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def run_job_kind(kind, argv, prog=None):
|
|
24
|
+
"""Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
|
|
25
|
+
try:
|
|
26
|
+
job_engine = load_job_engine(kind)
|
|
27
|
+
except plugins.PluginError as exc:
|
|
28
|
+
sys.stderr.write("streamwright: %s\n" % exc)
|
|
29
|
+
return 2
|
|
30
|
+
# `prog`: the command's name in usage lines (`streamwright duckdb`), for engines whose main takes it
|
|
31
|
+
return (job_engine.main(argv, prog=prog) if prog else job_engine.main(argv)) or 0
|
|
@@ -102,18 +102,6 @@ def _load_state(path):
|
|
|
102
102
|
return state.get("value", state) if state.get("type") == "STATE" else state
|
|
103
103
|
|
|
104
104
|
|
|
105
|
-
def _summary_problem(path):
|
|
106
|
-
"""Why --summary FILE cannot be written (checked before the run starts), or None."""
|
|
107
|
-
if not path:
|
|
108
|
-
return None
|
|
109
|
-
if os.path.isdir(path):
|
|
110
|
-
return "--summary %s is a folder" % path
|
|
111
|
-
folder = os.path.dirname(os.path.abspath(path))
|
|
112
|
-
if not os.path.isdir(folder):
|
|
113
|
-
return "--summary %s: the folder %s does not exist" % (path, folder)
|
|
114
|
-
return None
|
|
115
|
-
|
|
116
|
-
|
|
117
105
|
def run(args, logging_setup=None):
|
|
118
106
|
"""
|
|
119
107
|
streamwright run. Whatever the outcome, a run that started logs its end (RunMetrics.finish), and --summary FILE is
|
|
@@ -121,7 +109,7 @@ def run(args, logging_setup=None):
|
|
|
121
109
|
the loggers' levels and handlers can drop - redacted and cut as log lines are. `logging_setup`: the command's
|
|
122
110
|
logs.LogSetup, which gets the run's Redactor and the network loggers of the source's connector.
|
|
123
111
|
"""
|
|
124
|
-
problem =
|
|
112
|
+
problem = logs.summary_problem(getattr(args, "summary", None))
|
|
125
113
|
if problem:
|
|
126
114
|
LOG.error("%s", problem)
|
|
127
115
|
return 2
|
|
@@ -150,7 +138,10 @@ def run(args, logging_setup=None):
|
|
|
150
138
|
messages.append(message)
|
|
151
139
|
metrics.error = "; ".join(messages) or None
|
|
152
140
|
try:
|
|
153
|
-
|
|
141
|
+
if args.summary == logs.LOG_SUMMARY: # a bare --summary: a log line (stdout carries the records)
|
|
142
|
+
logs.log_summary(LOG, metrics.summary())
|
|
143
|
+
else:
|
|
144
|
+
logs.write_summary(args.summary, metrics.summary())
|
|
154
145
|
except (OSError, TypeError, ValueError) as exc:
|
|
155
146
|
LOG.error("could not write the summary %s: %s", args.summary, exc)
|
|
156
147
|
code = 1 if code == 0 else code
|
|
@@ -38,10 +38,14 @@ def add_job_arguments(parser):
|
|
|
38
38
|
"""Add common job manifest and execution options to an argparse parser."""
|
|
39
39
|
parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
|
|
40
40
|
parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
|
|
41
|
-
parser.add_argument("--overlay", action="append", default=[], metavar="FILE"
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
parser.add_argument("--
|
|
45
|
-
parser.add_argument("--
|
|
41
|
+
parser.add_argument("--overlay", action="append", default=[], metavar="FILE",
|
|
42
|
+
help="a YAML overlay merged onto the job, in order (repeatable; a null deletes a key, a list "
|
|
43
|
+
"replaces the job's)")
|
|
44
|
+
parser.add_argument("--config", metavar="FILE", help="a YAML/JSON file of spec.config values")
|
|
45
|
+
parser.add_argument("--set", action="append", default=[], type=parse_set, metavar="NAME=VALUE",
|
|
46
|
+
help="a spec.config value (repeatable; lists as a,b,c); overrides --config")
|
|
47
|
+
parser.add_argument("--allow-connector", action="append", default=[], metavar="NAME",
|
|
48
|
+
help="allow only this connector (repeatable); by default any installed connector")
|
|
49
|
+
parser.add_argument("--strict", action="store_true", help="treat warnings as errors")
|
|
46
50
|
add_log_options(parser)
|
|
47
51
|
return parser
|
|
@@ -27,7 +27,7 @@ from streamwright.core.cli.commands.connectors import run_connectors_command
|
|
|
27
27
|
from streamwright.core.cli.commands.source import run
|
|
28
28
|
from streamwright.core.cli.jobs import JOB_OPTION
|
|
29
29
|
from streamwright.core.cli.log_options import log_options, log_setup
|
|
30
|
-
from streamwright.core.cli.parser import build_parser, build_validate_parser, command_parsers
|
|
30
|
+
from streamwright.core.cli.parser import build_parser, build_plan_parser, build_validate_parser, command_parsers
|
|
31
31
|
from streamwright.core.validation import engine
|
|
32
32
|
from streamwright.core.validation.engine import NON_STRICT_CODES, check_components
|
|
33
33
|
|
|
@@ -52,6 +52,9 @@ def _extract_all_paths(args, command="run"):
|
|
|
52
52
|
value = args[index + 1]
|
|
53
53
|
if value:
|
|
54
54
|
paths.append(value)
|
|
55
|
+
if action.nargs == "?" and not equals and (index + 1 >= len(args) or args[index + 1].startswith("-")):
|
|
56
|
+
index += 1 # an optional value that is not given (a bare --summary)
|
|
57
|
+
continue
|
|
55
58
|
if not equals and action.nargs != 0:
|
|
56
59
|
index += 1
|
|
57
60
|
if action.nargs in ("+", "*"):
|
|
@@ -144,7 +147,11 @@ def main(argv=None):
|
|
|
144
147
|
if command == "validate":
|
|
145
148
|
return _validate_command(argv)
|
|
146
149
|
if command == "plan":
|
|
147
|
-
|
|
150
|
+
if any(argument in ("-h", "--help") for argument in argv[1:]):
|
|
151
|
+
build_plan_parser().print_help()
|
|
152
|
+
return 0
|
|
153
|
+
sys.stderr.write("streamwright: 'plan' needs a job manifest (JOB or --job JOB) of an installed engine's "
|
|
154
|
+
"kind; see streamwright plan -h\n")
|
|
148
155
|
return 2
|
|
149
156
|
args = build_parser().parse_args(argv)
|
|
150
157
|
if args.command == "connectors":
|
|
@@ -23,26 +23,43 @@ from streamwright.core.cli.commands import jobs
|
|
|
23
23
|
from streamwright.core.cli.jobs import JOB_OPTION, add_job_arguments
|
|
24
24
|
from streamwright.core.cli.log_options import add_log_options, key_value
|
|
25
25
|
from streamwright.core.config.inputs import SECRET_ENV_PREFIX
|
|
26
|
-
from streamwright.core.runtime import plugins
|
|
26
|
+
from streamwright.core.runtime import logs, plugins
|
|
27
27
|
from streamwright.core.validation import engine
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
def build_validate_parser():
|
|
31
31
|
"""`streamwright validate --help`: validation options, then the logging options (as streamwright run's)."""
|
|
32
32
|
parser = engine._parser("streamwright validate")
|
|
33
|
+
parser.epilog = ("A job manifest (kind: DuckDBJob, SparkJob) is checked by its engine, with that engine's options: "
|
|
34
|
+
"streamwright duckdb validate -h, streamwright spark validate -h.")
|
|
33
35
|
add_log_options(parser.add_argument_group("logging options"))
|
|
34
36
|
return parser
|
|
35
37
|
|
|
36
38
|
|
|
39
|
+
def build_plan_parser():
|
|
40
|
+
"""`streamwright plan --help`: the command takes a job manifest; its engine's plan command takes the options."""
|
|
41
|
+
parser = argparse.ArgumentParser(
|
|
42
|
+
prog="streamwright plan",
|
|
43
|
+
description="Show a job's execution plan - its steps, outputs and the order they run in - without running it "
|
|
44
|
+
"or needing secrets. The manifest's kind picks the engine, and its options are that engine's: "
|
|
45
|
+
"streamwright duckdb plan -h (kind: DuckDBJob), streamwright spark plan -h (kind: SparkJob).")
|
|
46
|
+
parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
|
|
47
|
+
parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
|
|
48
|
+
return parser
|
|
49
|
+
|
|
50
|
+
|
|
37
51
|
def build_parser():
|
|
38
52
|
parser = argparse.ArgumentParser(prog="streamwright",
|
|
39
|
-
description="Run and check StreamWright sources (`kind: source`)
|
|
53
|
+
description="Run and check StreamWright sources (`kind: source`) and jobs "
|
|
54
|
+
"(`kind: DuckDBJob`, `kind: SparkJob`).")
|
|
40
55
|
commands = parser.add_subparsers(dest="command", metavar="COMMAND")
|
|
41
56
|
commands.required = True
|
|
42
57
|
|
|
43
|
-
run = commands.add_parser("run", help="run a source and write its records",
|
|
58
|
+
run = commands.add_parser("run", help="run a source and write its records, or a job (JOB or --job JOB)",
|
|
44
59
|
description="Run a source. Records go to stdout as Singer messages unless --output "
|
|
45
|
-
"names a folder for files or a warehouse; logs go to stderr."
|
|
60
|
+
"names a folder for files or a warehouse; logs go to stderr. A job "
|
|
61
|
+
"manifest (kind: DuckDBJob, SparkJob) runs on its engine, with that "
|
|
62
|
+
"engine's options: streamwright duckdb run -h, streamwright spark run -h.")
|
|
46
63
|
run.add_argument("source", nargs="?", metavar="SOURCE",
|
|
47
64
|
help="a source folder (source.yaml and one file per stream in streams/) or a source file; or a "
|
|
48
65
|
"job manifest (kind: DuckDBJob, SparkJob), which runs on its engine")
|
|
@@ -83,10 +100,11 @@ def build_parser():
|
|
|
83
100
|
"EXPORT.TIMESTAMP.UNIQUE.jsonl, or .csv, .tsv, .parquet)")
|
|
84
101
|
run.add_argument("--allow-connector", action="append", dest="allowed_connectors", metavar="NAME",
|
|
85
102
|
help="allow only this connector (repeatable); by default any installed connector can be used")
|
|
86
|
-
run.add_argument("--summary", metavar="FILE",
|
|
103
|
+
run.add_argument("--summary", nargs="?", const=logs.LOG_SUMMARY, metavar="FILE",
|
|
87
104
|
help="also write the run's summary as JSON to FILE, atomically, whatever the outcome: status "
|
|
88
105
|
"(ok or failed), times, the streams' counts (partitions, windows, pages, requests, retries, "
|
|
89
|
-
"records read and written), the outputs written, the state's bookmarks and the error"
|
|
106
|
+
"records read and written), the outputs written, the state's bookmarks and the error; a "
|
|
107
|
+
"bare --summary logs it instead, as one INFO line of compact JSON (event run_summary)")
|
|
90
108
|
add_log_options(run)
|
|
91
109
|
|
|
92
110
|
conn = commands.add_parser("connectors",
|
|
@@ -113,7 +131,8 @@ def build_parser():
|
|
|
113
131
|
add_help=False)
|
|
114
132
|
validate.add_argument("arguments", nargs=argparse.REMAINDER)
|
|
115
133
|
|
|
116
|
-
plan_cmd = commands.add_parser("plan", help="show execution plan
|
|
134
|
+
plan_cmd = commands.add_parser("plan", help="show a job's execution plan: its steps, outputs and their order "
|
|
135
|
+
"(no secrets needed)", add_help=False)
|
|
117
136
|
plan_cmd.add_argument("arguments", nargs=argparse.REMAINDER)
|
|
118
137
|
installed_commands = {item["name"] for item in plugins.discover(plugins.COMMANDS)}
|
|
119
138
|
if "duckdb" not in installed_commands: # editable installs have stale entry-point metadata
|
|
@@ -20,7 +20,7 @@ The DuckDBJob command line (registered for `kind: DuckDBJob` in the `streamwrigh
|
|
|
20
20
|
|
|
21
21
|
validate JOB [--graph]
|
|
22
22
|
plan JOB
|
|
23
|
-
run JOB [--secrets FILE] [--dry-run]
|
|
23
|
+
run JOB [--secrets FILE] [--dry-run] [--summary FILE]
|
|
24
24
|
state list|retry|abandon JOB [--batch ID] [--secrets FILE]
|
|
25
25
|
(every command: --config FILE, --set NAME=VALUE, --overlay FILE, --allow-connector NAME, --strict, and the logging
|
|
26
26
|
options of `streamwright run`: --log-level, --log NAME=LEVEL, --log-format, --log-config, --log-max-chars)
|
|
@@ -30,6 +30,7 @@ job or failed run, 2 usage or input error. Validation and planning never need se
|
|
|
30
30
|
"""
|
|
31
31
|
|
|
32
32
|
import argparse
|
|
33
|
+
import logging
|
|
33
34
|
import sys
|
|
34
35
|
|
|
35
36
|
from streamwright.core.cli.jobs import FAILURE, INVALID_INPUT, SUCCESS, add_job_arguments, resolve_job
|
|
@@ -40,9 +41,13 @@ from streamwright.core.jobs.inputs import read_config_values
|
|
|
40
41
|
from streamwright.core.jobs.manifest import JobError
|
|
41
42
|
from streamwright.core.jobs.runner import owner_id
|
|
42
43
|
from streamwright.core.jobs.state import StateError
|
|
44
|
+
from streamwright.core.runtime import logs
|
|
45
|
+
from streamwright.core.runtime.logs.lines import iso_time, utc_now
|
|
43
46
|
|
|
44
47
|
__all__ = ["main"]
|
|
45
48
|
|
|
49
|
+
LOG = logging.getLogger("streamwright.duckdb")
|
|
50
|
+
|
|
46
51
|
|
|
47
52
|
def _err(text):
|
|
48
53
|
sys.stderr.write(text + "\n")
|
|
@@ -91,13 +96,54 @@ def _cmd_plan(args):
|
|
|
91
96
|
return SUCCESS
|
|
92
97
|
|
|
93
98
|
|
|
99
|
+
def _run_summary(args, result, started_at, failure):
|
|
100
|
+
"""The --summary of a job run: status, times, job, batches, each output's rows and files, and the error."""
|
|
101
|
+
finished_at = utc_now()
|
|
102
|
+
outputs = [{"name": name, "rows": rows, "files": [location for location in result.locations.get(name, ())
|
|
103
|
+
if location],
|
|
104
|
+
"already_committed": name in result.skipped}
|
|
105
|
+
for name, rows in sorted(result.outputs.items())] if result is not None else []
|
|
106
|
+
summary = {"status": "ok" if failure is None and result is not None and not result.exit_code else "failed",
|
|
107
|
+
"started_at": iso_time(started_at), "finished_at": iso_time(finished_at),
|
|
108
|
+
"duration_s": round((finished_at - started_at).total_seconds(), 3),
|
|
109
|
+
"job": result.job_name if result else None, "manifest": args.job, "dry_run": args.dry_run,
|
|
110
|
+
"batches": list(result.batches) if result else [], "outputs": outputs}
|
|
111
|
+
if summary["status"] != "ok":
|
|
112
|
+
if failure is not None and not isinstance(failure, Exception):
|
|
113
|
+
error = "interrupted"
|
|
114
|
+
elif failure is not None:
|
|
115
|
+
error = "%s: %s" % (type(failure).__name__, failure)
|
|
116
|
+
else:
|
|
117
|
+
error = result.error or "the run failed"
|
|
118
|
+
summary["error"] = args.log_setup.message(error)
|
|
119
|
+
return summary
|
|
120
|
+
|
|
121
|
+
|
|
94
122
|
def _cmd_run(args):
|
|
95
|
-
|
|
96
|
-
|
|
123
|
+
problem = logs.summary_problem(args.summary)
|
|
124
|
+
if problem:
|
|
125
|
+
_err("Error: %s" % problem)
|
|
126
|
+
return INVALID_INPUT
|
|
127
|
+
started_at, result, failure, summary_written = utc_now(), None, None, True
|
|
128
|
+
try:
|
|
129
|
+
result = run_job(args.job, config=_config(args), secrets=_secrets(args), overlays=args.overlay,
|
|
130
|
+
allowed_connectors=_allowed(args), dry_run=args.dry_run, strict=args.strict)
|
|
131
|
+
except BaseException as exc:
|
|
132
|
+
failure = exc
|
|
133
|
+
raise
|
|
134
|
+
finally:
|
|
135
|
+
if args.summary == logs.LOG_SUMMARY: # a bare --summary: a log line, whatever the outcome
|
|
136
|
+
logs.log_summary(LOG, _run_summary(args, result, started_at, failure))
|
|
137
|
+
elif args.summary:
|
|
138
|
+
try:
|
|
139
|
+
logs.write_summary(args.summary, _run_summary(args, result, started_at, failure))
|
|
140
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
141
|
+
_err("Error: could not write the summary %s: %s" % (args.summary, exc))
|
|
142
|
+
summary_written = False
|
|
97
143
|
sys.stdout.write("Job: %s\n" % result.job_name)
|
|
98
144
|
if args.dry_run:
|
|
99
145
|
sys.stdout.write(result.plan + "\n")
|
|
100
|
-
return SUCCESS
|
|
146
|
+
return SUCCESS if summary_written else FAILURE
|
|
101
147
|
for name, rows in sorted(result.outputs.items()):
|
|
102
148
|
if name in result.skipped:
|
|
103
149
|
sys.stdout.write(" %-28s already committed in an earlier attempt\n" % name)
|
|
@@ -113,7 +159,7 @@ def _cmd_run(args):
|
|
|
113
159
|
sys.stdout.writelines(" %s\n" % location for location in locations)
|
|
114
160
|
if result.exit_code:
|
|
115
161
|
_err("Run failed: %s" % (result.error or "see the log"))
|
|
116
|
-
return result.exit_code
|
|
162
|
+
return result.exit_code or (SUCCESS if summary_written else FAILURE)
|
|
117
163
|
|
|
118
164
|
|
|
119
165
|
def _cmd_state(args):
|
|
@@ -151,7 +197,8 @@ def _cmd_state(args):
|
|
|
151
197
|
def _inputs(parser, secrets=False):
|
|
152
198
|
add_job_arguments(parser)
|
|
153
199
|
if secrets:
|
|
154
|
-
parser.add_argument("--secrets",
|
|
200
|
+
parser.add_argument("--secrets", metavar="FILE",
|
|
201
|
+
help="a YAML file of spec.secrets values (or STREAMWRIGHT_SECRET_<NAME> variables)")
|
|
155
202
|
|
|
156
203
|
|
|
157
204
|
def _build_parser(prog):
|
|
@@ -167,12 +214,18 @@ def _build_parser(prog):
|
|
|
167
214
|
plan.set_defaults(func=_cmd_plan, parser=plan)
|
|
168
215
|
run = sub.add_parser("run", allow_abbrev=False, help="run a DuckDBJob (unfinished batches resume first)")
|
|
169
216
|
run.add_argument("--dry-run", action="store_true", help="check and plan, write nothing")
|
|
217
|
+
run.add_argument("--summary", nargs="?", const=logs.LOG_SUMMARY, metavar="FILE",
|
|
218
|
+
help="also write the run's summary as JSON to FILE, atomically, whatever the outcome: status (ok "
|
|
219
|
+
"or failed), times, the batches, each output's rows and files, and the error; a bare "
|
|
220
|
+
"--summary logs it instead, as one INFO line of compact JSON (event run_summary)")
|
|
170
221
|
_inputs(run, secrets=True)
|
|
171
222
|
run.set_defaults(func=_cmd_run, parser=run)
|
|
172
223
|
state = sub.add_parser("state", allow_abbrev=False, help="list, retry or abandon a job's batches (runtime.state)")
|
|
173
|
-
state.add_argument("action", choices=["list", "retry", "abandon"]
|
|
174
|
-
|
|
175
|
-
|
|
224
|
+
state.add_argument("action", choices=["list", "retry", "abandon"],
|
|
225
|
+
help="list: the job's recent batches; retry: run a failed batch again from where it failed; "
|
|
226
|
+
"abandon: give a failed batch up so the job can run again")
|
|
227
|
+
state.add_argument("--batch", metavar="ID", help="the batch id (retry, abandon)")
|
|
228
|
+
state.add_argument("--limit", type=int, default=20, metavar="N", help="list: the number of batches (default 20)")
|
|
176
229
|
_inputs(state, secrets=True)
|
|
177
230
|
state.set_defaults(func=_cmd_state, parser=state)
|
|
178
231
|
return parser
|
|
@@ -31,8 +31,8 @@ class DuckDBJobEngine:
|
|
|
31
31
|
|
|
32
32
|
kind = KIND
|
|
33
33
|
|
|
34
|
-
def main(self, argv=None):
|
|
35
|
-
return cli.main(argv)
|
|
34
|
+
def main(self, argv=None, prog="streamwright"):
|
|
35
|
+
return cli.main(argv, prog=prog)
|
|
36
36
|
|
|
37
37
|
def validate_job(self, job, *, overlays=(), config=None, strict=False, allowed=None):
|
|
38
38
|
return engine.validate_job(job, overlays=overlays, config=config, strict=strict, allowed=allowed)
|
|
@@ -41,7 +41,10 @@ class JobEngine(Protocol):
|
|
|
41
41
|
kind: str # the manifest `kind` it runs, e.g. "DuckDBJob" (= the entry-point name)
|
|
42
42
|
|
|
43
43
|
def main(self, argv: Sequence[str] | None = None) -> int:
|
|
44
|
-
"""
|
|
44
|
+
"""
|
|
45
|
+
Runs the engine's command line on the original arguments (`validate|plan|run JOB ...`); the exit code. An engine
|
|
46
|
+
may also take `prog`, the command's name in its usage lines (`streamwright duckdb` for the explicit form).
|
|
47
|
+
"""
|
|
45
48
|
|
|
46
49
|
def validate_job(self, job: str | Mapping[str, Any], *, overlays: Sequence[str] = (),
|
|
47
50
|
config: Mapping[str, Any] | None = None, strict: bool = False) -> JobReport:
|
|
@@ -57,7 +57,7 @@ from streamwright.core.runtime.logs.metrics import RunMetrics, StreamMetrics
|
|
|
57
57
|
from streamwright.core.runtime.logs.network import http_details, http_summary, request_details, sdk_summary
|
|
58
58
|
from streamwright.core.runtime.logs.redaction import mask_form, mask_headers, mask_text, mask_url, sensitive
|
|
59
59
|
from streamwright.core.runtime.logs.setup import LogSetup
|
|
60
|
-
from streamwright.core.runtime.logs.summary import write_summary
|
|
60
|
+
from streamwright.core.runtime.logs.summary import LOG_SUMMARY, log_summary, summary_problem, write_summary
|
|
61
61
|
|
|
62
62
|
__all__ = [
|
|
63
63
|
"DEFAULT_MAX_CHARS",
|
|
@@ -65,6 +65,7 @@ __all__ = [
|
|
|
65
65
|
"FORMATS",
|
|
66
66
|
"LEVELS",
|
|
67
67
|
"LOG",
|
|
68
|
+
"LOG_SUMMARY",
|
|
68
69
|
"NETWORK",
|
|
69
70
|
"PROGRESS_PAGES",
|
|
70
71
|
"PROGRESS_SECONDS",
|
|
@@ -78,6 +79,7 @@ __all__ = [
|
|
|
78
79
|
"http_details",
|
|
79
80
|
"http_summary",
|
|
80
81
|
"level_number",
|
|
82
|
+
"log_summary",
|
|
81
83
|
"mask_form",
|
|
82
84
|
"mask_headers",
|
|
83
85
|
"mask_text",
|
|
@@ -85,6 +87,7 @@ __all__ = [
|
|
|
85
87
|
"request_details",
|
|
86
88
|
"sdk_summary",
|
|
87
89
|
"sensitive",
|
|
90
|
+
"summary_problem",
|
|
88
91
|
"write_summary",
|
|
89
92
|
]
|
|
90
93
|
|
|
@@ -22,6 +22,26 @@ import json
|
|
|
22
22
|
import os
|
|
23
23
|
import tempfile
|
|
24
24
|
|
|
25
|
+
LOG_SUMMARY = "-" # a bare --summary: the summary as a log line
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def log_summary(logger, summary):
|
|
29
|
+
"""The run summary as one INFO log line of compact JSON (event `run_summary`), on the command's log handlers."""
|
|
30
|
+
logger.info("run summary: %s", json.dumps(summary, separators=(",", ":"), default=str),
|
|
31
|
+
extra={"event": "run_summary"})
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def summary_problem(path):
|
|
35
|
+
"""Why --summary FILE cannot be written (checked before a run starts), or None."""
|
|
36
|
+
if not path or path == LOG_SUMMARY:
|
|
37
|
+
return None
|
|
38
|
+
if os.path.isdir(path):
|
|
39
|
+
return "--summary %s is a folder" % path
|
|
40
|
+
folder = os.path.dirname(os.path.abspath(path))
|
|
41
|
+
if not os.path.isdir(folder):
|
|
42
|
+
return "--summary %s: the folder %s does not exist" % (path, folder)
|
|
43
|
+
return None
|
|
44
|
+
|
|
25
45
|
|
|
26
46
|
def write_summary(path, summary):
|
|
27
47
|
"""Writes the run summary as JSON to `path`, atomically (a temporary file in its folder, then renamed)."""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamwright
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.13
|
|
4
4
|
Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
|
|
5
5
|
Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -6,8 +6,12 @@ import subprocess
|
|
|
6
6
|
import sys
|
|
7
7
|
from types import SimpleNamespace
|
|
8
8
|
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
9
11
|
from streamwright.core import cli
|
|
10
12
|
from streamwright.core.cli.jobs import parse_set
|
|
13
|
+
from streamwright.core.cli.parser import build_parser, build_validate_parser
|
|
14
|
+
from streamwright.core.jobs.duckdb import cli as duckdb_cli
|
|
11
15
|
from streamwright.core.runtime import plugins
|
|
12
16
|
|
|
13
17
|
|
|
@@ -82,6 +86,8 @@ def test_job_option_routes_like_the_job_path(tmp_path, capsys, monkeypatch):
|
|
|
82
86
|
for argv in (["run", "--job", str(manifest)], ["validate", "-j", str(manifest)],
|
|
83
87
|
["plan", "--job=%s" % manifest], ["run", str(manifest), "--job", str(manifest)]):
|
|
84
88
|
assert cli.main(argv) == 0 and received[-1] == argv
|
|
89
|
+
for argv in (["run", "--summary", "--job", str(manifest)], ["run", "--job", str(manifest), "--summary"]):
|
|
90
|
+
assert cli.main(argv) == 0 and received[-1] == argv # a bare --summary takes no path
|
|
85
91
|
source = tmp_path / "source.yaml"
|
|
86
92
|
source.write_text("kind: source\n")
|
|
87
93
|
assert cli.main(["run", "--job", str(source)]) == 2
|
|
@@ -124,9 +130,10 @@ def test_command_entry_is_registered_from_metadata(monkeypatch, capsys):
|
|
|
124
130
|
def test_builtin_job_shorthand_uses_plugin_route(monkeypatch):
|
|
125
131
|
jobs_module = importlib.import_module("streamwright.core.cli.commands.jobs")
|
|
126
132
|
calls = []
|
|
127
|
-
monkeypatch.setattr(jobs_module, "run_job_kind",
|
|
133
|
+
monkeypatch.setattr(jobs_module, "run_job_kind",
|
|
134
|
+
lambda kind, argv, prog=None: calls.append((kind, argv, prog)) or 0)
|
|
128
135
|
assert cli.main(["duckdb", "run", "job.yaml"]) == 0
|
|
129
|
-
assert calls == [("DuckDBJob", ["run", "job.yaml"])]
|
|
136
|
+
assert calls == [("DuckDBJob", ["run", "job.yaml"], "streamwright duckdb")] # its usage lines name the command
|
|
130
137
|
|
|
131
138
|
|
|
132
139
|
def test_shared_job_set_parser_uses_jobs_input_rules():
|
|
@@ -137,3 +144,33 @@ def test_shared_job_set_parser_uses_jobs_input_rules():
|
|
|
137
144
|
assert "NAME=VALUE" in str(exc)
|
|
138
145
|
else:
|
|
139
146
|
raise AssertionError("an invalid --set must be rejected")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _without_help(parser, path):
|
|
150
|
+
missing = []
|
|
151
|
+
for action in parser._actions:
|
|
152
|
+
if isinstance(action, argparse._HelpAction) or action.help == argparse.SUPPRESS:
|
|
153
|
+
continue
|
|
154
|
+
if isinstance(action, argparse._SubParsersAction):
|
|
155
|
+
helps = {choice.dest: choice.help for choice in action._choices_actions}
|
|
156
|
+
for name, sub in action.choices.items():
|
|
157
|
+
missing += [] if helps.get(name) else ["%s %s" % (path, name)]
|
|
158
|
+
missing += _without_help(sub, "%s %s" % (path, name))
|
|
159
|
+
elif not action.help and action.nargs != argparse.REMAINDER: # (REMAINDER: another parser's arguments)
|
|
160
|
+
missing.append("%s %s" % (path, "/".join(action.option_strings) or action.dest))
|
|
161
|
+
return missing
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def test_every_command_and_option_has_help(capsys):
|
|
165
|
+
plugin_commands = {item["name"] for item in plugins.discover(plugins.COMMANDS)} - {"duckdb"}
|
|
166
|
+
missing = [item for item in _without_help(build_parser(), "streamwright")
|
|
167
|
+
if item.split()[1] not in plugin_commands] # (installed plug-ins check their own)
|
|
168
|
+
assert missing == []
|
|
169
|
+
assert _without_help(build_validate_parser(), "streamwright validate") == []
|
|
170
|
+
assert _without_help(duckdb_cli._build_parser("streamwright duckdb"), "streamwright duckdb") == []
|
|
171
|
+
assert cli.main(["plan", "-h"]) == 0
|
|
172
|
+
with pytest.raises(SystemExit) as shown:
|
|
173
|
+
cli.main(["duckdb", "-h"]) # forwarded to the engine's own parser
|
|
174
|
+
assert shown.value.code == 0
|
|
175
|
+
out = capsys.readouterr().out
|
|
176
|
+
assert "usage: streamwright plan" in out and "usage: streamwright duckdb" in out
|
|
@@ -931,6 +931,8 @@ def test_the_summary_of_a_run_that_cannot_start(api, tmp_path, monkeypatch, caps
|
|
|
931
931
|
|
|
932
932
|
code, _, err = run_cli(capsys, path, "--summary", tmp_path / "nowhere" / "run.json")
|
|
933
933
|
assert code == 2 and "the folder %s does not exist" % (tmp_path / "nowhere") in err and api.requests == []
|
|
934
|
+
code, _, err = run_cli(capsys, path, "--summary") # bare: a log line (a source's stdout carries its records)
|
|
935
|
+
assert code == 2 and 'run summary: {"status":"failed"' in err and api.requests == []
|
|
934
936
|
|
|
935
937
|
|
|
936
938
|
# * ----------------------------------
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
|
|
2
|
-
|
|
3
|
-
import argparse
|
|
4
|
-
import sys
|
|
5
|
-
|
|
6
|
-
from streamwright.core.jobs.engine import load_job_engine
|
|
7
|
-
from streamwright.core.runtime import plugins
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def register(subparsers):
|
|
11
|
-
parser = subparsers.add_parser("duckdb", help="run an installed DuckDBJob engine", add_help=False)
|
|
12
|
-
parser.add_argument("arguments", nargs=argparse.REMAINDER)
|
|
13
|
-
parser.set_defaults(command_handler=lambda args: execute(args.arguments))
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def execute(arguments):
|
|
17
|
-
return run_job_kind("DuckDBJob", arguments)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def run_job_kind(kind, argv):
|
|
21
|
-
"""Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
|
|
22
|
-
try:
|
|
23
|
-
job_engine = load_job_engine(kind)
|
|
24
|
-
except plugins.PluginError as exc:
|
|
25
|
-
sys.stderr.write("streamwright: %s\n" % exc)
|
|
26
|
-
return 2
|
|
27
|
-
return job_engine.main(argv) or 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/connectors.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/list_parameters.py
RENAMED
|
File without changes
|