streamwright 0.1.12__tar.gz → 0.1.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streamwright-0.1.12/src/streamwright.egg-info → streamwright-0.1.14}/PKG-INFO +2 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/pyproject.toml +3 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/catalog.py +16 -0
- streamwright-0.1.14/src/streamwright/core/cli/__init__.py +22 -0
- streamwright-0.1.14/src/streamwright/core/cli/__main__.py +22 -0
- streamwright-0.1.14/src/streamwright/core/cli/commands/__init__.py +18 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/cli/commands/connectors.py +14 -0
- streamwright-0.1.14/src/streamwright/core/cli/commands/jobs.py +54 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/cli/commands/source.py +20 -1
- streamwright-0.1.14/src/streamwright/core/cli/commands/state.py +168 -0
- streamwright-0.1.14/src/streamwright/core/cli/help_format.py +47 -0
- streamwright-0.1.14/src/streamwright/core/cli/job_state.py +286 -0
- streamwright-0.1.14/src/streamwright/core/cli/jobs.py +227 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/cli/log_options.py +6 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/cli/main.py +52 -7
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/cli/parser.py +70 -18
- streamwright-0.1.14/src/streamwright/core/config/__init__.py +16 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/config/inputs.py +8 -4
- streamwright-0.1.14/src/streamwright/core/engine/__init__.py +16 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/bookmarks.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/builders.py +17 -0
- streamwright-0.1.14/src/streamwright/core/engine/errors.py +24 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/execution.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/factories.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/partitions.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/planning.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/queries.py +0 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/readers.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/responses.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/rules.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/runner.py +2 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/session.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/__init__.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/sandbox.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/tables.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/types.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/values.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/writers.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/graph.py +17 -0
- streamwright-0.1.14/src/streamwright/core/io/__init__.py +26 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/api_calls.py +17 -0
- streamwright-0.1.14/src/streamwright/core/io/capabilities.py +35 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/database_connections.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/database_rules.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/databases.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/file_formats.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/file_rules.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/files.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/http.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/local.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/locations.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/queries.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/sandbox.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/io/store_reader.py +17 -0
- streamwright-0.1.14/src/streamwright/core/jobs/check/__init__.py +70 -0
- streamwright-0.1.14/src/streamwright/core/jobs/check/connections.py +43 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/outputs.py +5 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/refs.py +4 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/rules.py +18 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/runtime.py +4 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/adapter.py +27 -6
- streamwright-0.1.14/src/streamwright/core/jobs/duckdb/cli.py +367 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/definition_adapter.py +34 -0
- streamwright-0.1.14/src/streamwright/core/jobs/duckdb/engine.py +400 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/executor.py +81 -18
- streamwright-0.1.14/src/streamwright/core/jobs/duckdb/guide.md +385 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/loading.py +20 -2
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/plugin.py +5 -2
- streamwright-0.1.14/src/streamwright/core/jobs/duckdb/results.py +68 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/rules.py +66 -12
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/session.py +9 -2
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/steps.py +45 -3
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/engine.py +6 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/fields.py +5 -1
- streamwright-0.1.14/src/streamwright/core/jobs/heartbeat.py +71 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/inputs.py +30 -2
- streamwright-0.1.14/src/streamwright/core/jobs/locking.py +301 -0
- streamwright-0.1.14/src/streamwright/core/jobs/log_labels.py +76 -0
- streamwright-0.1.14/src/streamwright/core/jobs/lookups/__init__.py +33 -0
- streamwright-0.1.14/src/streamwright/core/jobs/lookups/extract.py +104 -0
- streamwright-0.1.14/src/streamwright/core/jobs/lookups/passes.py +93 -0
- streamwright-0.1.14/src/streamwright/core/jobs/lookups/providers.py +269 -0
- streamwright-0.1.14/src/streamwright/core/jobs/lookups/resolver.py +269 -0
- streamwright-0.1.14/src/streamwright/core/jobs/lookups/rules.py +138 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/manifest.py +23 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/policy.py +2 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/refs.py +10 -1
- streamwright-0.1.14/src/streamwright/core/jobs/runner.py +416 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/state/__init__.py +31 -6
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/state/dialects.py +77 -5
- streamwright-0.1.14/src/streamwright/core/jobs/state/leases.py +147 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/memory.py +58 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/migrations/0001_initial_schema.py +154 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/migrations/__init__.py +22 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/migrations/runner.py +190 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/state/protocol.py +42 -18
- streamwright-0.1.14/src/streamwright/core/jobs/state/sql.py +146 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/store.py +350 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/__init__.py +64 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/base.py +86 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/events.py +50 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/jobs.py +80 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/leases.py +44 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/migrations.py +39 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/receipts.py +45 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/sequences.py +36 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/tables/signatures.py +39 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/target.py +66 -0
- streamwright-0.1.14/src/streamwright/core/jobs/state/url.py +201 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/types.py +9 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/validate.py +28 -1
- streamwright-0.1.14/src/streamwright/core/net/__init__.py +16 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/downloads.py +0 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/errors.py +17 -0
- streamwright-0.1.14/src/streamwright/core/outputs/__init__.py +16 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/base.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/dlt_columns.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/dlt_output.py +0 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/loading.py +17 -0
- streamwright-0.1.14/src/streamwright/core/runtime/__init__.py +16 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/components.py +0 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/connector_types.py +17 -0
- streamwright-0.1.14/src/streamwright/core/runtime/constants.py +21 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/contract.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/contract_declarations.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/contract_guide.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/contract_terms.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/contract_validation.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/definitions.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/__init__.py +7 -1
- streamwright-0.1.14/src/streamwright/core/runtime/logs/context.py +118 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/formatting.py +18 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/lines.py +4 -2
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/record_factory.py +3 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/setup.py +3 -3
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/plugins.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/redact.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/source_connectors.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/testing.py +0 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/__init__.py +17 -0
- streamwright-0.1.14/src/streamwright/core/sdk/constants.py +22 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/contract.py +17 -0
- streamwright-0.1.14/src/streamwright/core/sdk/engines/__init__.py +18 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/engines/duckdb.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/engines/source.py +17 -0
- streamwright-0.1.14/src/streamwright/core/sdk/messaging.py +77 -0
- streamwright-0.1.14/src/streamwright/core/sdk/network.py +26 -0
- streamwright-0.1.14/src/streamwright/core/sdk/operations/__init__.py +27 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/operations/databases.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/operations/files.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/operations/http.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/operations/messages.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/operations/tables.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/testing/__init__.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/testing/contract.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/testing/helpers.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/testing/storage.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sql/analysis.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sql/conditions.py +17 -0
- streamwright-0.1.14/src/streamwright/core/validation/__init__.py +20 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/auth.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/checker.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/issues.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/partitions.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/requests.py +17 -0
- streamwright-0.1.14/src/streamwright/core/validation/sections/__init__.py +34 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/streams.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14/src/streamwright.egg-info}/PKG-INFO +2 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright.egg-info/SOURCES.txt +31 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright.egg-info/entry_points.txt +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright.egg-info/requires.txt +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_catalog.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_cli_subcommands.py +74 -2
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_conditions.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_connector_api_family.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_connector_contract.py +19 -1
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_connectors.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_dlt_output.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_engine_reach.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_exact_decimals.py +17 -0
- streamwright-0.1.12/src/streamwright/core/jobs/check/connections.py → streamwright-0.1.14/tests/test_graph.py +8 -7
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_http_engine.py +1 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_logging.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_output_contract.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_output_formats.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_plugin_contract_gate.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_runner.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_sdk_plugins.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_sql.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_sql_quote.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_transform.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_validation.py +17 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/tests/test_warehouse_output.py +17 -0
- streamwright-0.1.12/src/streamwright/core/cli/__init__.py +0 -5
- streamwright-0.1.12/src/streamwright/core/cli/__main__.py +0 -5
- streamwright-0.1.12/src/streamwright/core/cli/commands/__init__.py +0 -1
- streamwright-0.1.12/src/streamwright/core/cli/commands/jobs.py +0 -27
- streamwright-0.1.12/src/streamwright/core/cli/jobs.py +0 -47
- streamwright-0.1.12/src/streamwright/core/config/__init__.py +0 -0
- streamwright-0.1.12/src/streamwright/core/engine/__init__.py +0 -0
- streamwright-0.1.12/src/streamwright/core/engine/errors.py +0 -7
- streamwright-0.1.12/src/streamwright/core/io/__init__.py +0 -9
- streamwright-0.1.12/src/streamwright/core/io/capabilities.py +0 -17
- streamwright-0.1.12/src/streamwright/core/jobs/check/__init__.py +0 -41
- streamwright-0.1.12/src/streamwright/core/jobs/duckdb/cli.py +0 -259
- streamwright-0.1.12/src/streamwright/core/jobs/duckdb/engine.py +0 -276
- streamwright-0.1.12/src/streamwright/core/jobs/duckdb/guide.md +0 -133
- streamwright-0.1.12/src/streamwright/core/jobs/runner.py +0 -308
- streamwright-0.1.12/src/streamwright/core/jobs/state/migrations.py +0 -85
- streamwright-0.1.12/src/streamwright/core/jobs/state/store.py +0 -397
- streamwright-0.1.12/src/streamwright/core/net/__init__.py +0 -0
- streamwright-0.1.12/src/streamwright/core/outputs/__init__.py +0 -0
- streamwright-0.1.12/src/streamwright/core/runtime/__init__.py +0 -0
- streamwright-0.1.12/src/streamwright/core/runtime/constants.py +0 -4
- streamwright-0.1.12/src/streamwright/core/sdk/constants.py +0 -5
- streamwright-0.1.12/src/streamwright/core/sdk/engines/__init__.py +0 -1
- streamwright-0.1.12/src/streamwright/core/sdk/messaging.py +0 -43
- streamwright-0.1.12/src/streamwright/core/sdk/network.py +0 -9
- streamwright-0.1.12/src/streamwright/core/sdk/operations/__init__.py +0 -10
- streamwright-0.1.12/src/streamwright/core/validation/__init__.py +0 -3
- streamwright-0.1.12/src/streamwright/core/validation/sections/__init__.py +0 -17
- streamwright-0.1.12/tests/test_graph.py +0 -12
- {streamwright-0.1.12 → streamwright-0.1.14}/LICENSE +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/README.md +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/setup.cfg +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/setup.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/__init__.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/config/loader.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/config/reader.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/compilation.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/list_parameters.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/ordering.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/plans.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/engine/sql/stream_plan.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/__init__.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/calls.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/dependencies.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/findings.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/names.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/secrets.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/sections.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/spec.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/steps.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/check/structure.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/__init__.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/reads.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/writers/__init__.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/duckdb/writers/tables.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/__init__.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/column_rules.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/formats.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/headers.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/mapped.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/model.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/ingest/plain.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/naming.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/plan.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/schema.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/jobs/write_rules.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/auth.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/encoding.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/http.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/paginators.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/net/policies.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/ducklake.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/file_names.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/files.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/output.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/staging.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/outputs/warehouse.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/py.typed +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/handlers.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/metrics.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/network.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/redaction.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/logs/summary.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/runtime/templates.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sdk/py.typed +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sql/__init__.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/sql/quote.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/engine.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/job_manifests.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/json_schemas.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/marked_yaml.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/schema.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/async_jobs.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/document.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/partitions.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/references.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/request_partitions.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/requests.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/spec.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/stream_context.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/streams.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/tables.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/sections/templates.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/source.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/validation/vocabulary.py +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright.egg-info/dependency_links.txt +0 -0
- {streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streamwright
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.14
|
|
4
4
|
Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
|
|
5
5
|
Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -21,6 +21,7 @@ License-File: LICENSE
|
|
|
21
21
|
Requires-Dist: pyyaml==6.0.2
|
|
22
22
|
Requires-Dist: requests<3,>=2.25
|
|
23
23
|
Requires-Dist: duckdb<2.1,>=2.0
|
|
24
|
+
Requires-Dist: SQLAlchemy<3,>=2.0
|
|
24
25
|
Requires-Dist: pytz
|
|
25
26
|
Provides-Extra: dlt
|
|
26
27
|
Requires-Dist: dlt>=1.0; extra == "dlt"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamwright"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.14"
|
|
8
8
|
description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "Apache-2.0"}
|
|
@@ -27,6 +27,7 @@ dependencies = [
|
|
|
27
27
|
# `select`: the SQL that shapes each stream's records runs in an embedded DuckDB (which needs pytz for
|
|
28
28
|
# timestamps with time zones)
|
|
29
29
|
"duckdb>=2.0,<2.1",
|
|
30
|
+
"SQLAlchemy>=2.0,<3",
|
|
30
31
|
"pytz",
|
|
31
32
|
]
|
|
32
33
|
|
|
@@ -47,6 +48,7 @@ sw = "streamwright.core.cli:main"
|
|
|
47
48
|
|
|
48
49
|
[project.entry-points."streamwright.commands"]
|
|
49
50
|
duckdb = "streamwright.core.cli.commands.jobs:register"
|
|
51
|
+
state = "streamwright.core.cli.commands.state:register"
|
|
50
52
|
|
|
51
53
|
[project.entry-points."streamwright.job_kinds"]
|
|
52
54
|
DuckDBJob = "streamwright.core.jobs.duckdb.plugin:JOB_ENGINE"
|
|
@@ -1,4 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
2
18
|
"""The connector catalog: discover and install connectors from the StreamWright hub.
|
|
3
19
|
|
|
4
20
|
A catalog is a static index that maps a connector key to how it's installed - a
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
18
|
+
"""Public StreamWright command-line interface."""
|
|
19
|
+
|
|
20
|
+
from streamwright.core.cli.main import command_parsers, detect_kind, main, run
|
|
21
|
+
|
|
22
|
+
__all__ = ["command_parsers", "detect_kind", "main", "run"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
18
|
+
"""Run the CLI as a module."""
|
|
19
|
+
|
|
20
|
+
from streamwright.core.cli import main
|
|
21
|
+
|
|
22
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
18
|
+
"""Built-in commands."""
|
{streamwright-0.1.12 → streamwright-0.1.14}/src/streamwright/core/cli/commands/connectors.py
RENAMED
|
@@ -122,6 +122,10 @@ def _connector_lines(names=None):
|
|
|
122
122
|
return lines
|
|
123
123
|
|
|
124
124
|
|
|
125
|
+
_CATALOG_ACTIONS = ("ls", "list", "-ls", "--list", "i", "install", "-i", "--install", "find", "search", "s", "-s",
|
|
126
|
+
"-f", "--find", "--search")
|
|
127
|
+
|
|
128
|
+
|
|
125
129
|
def run_connectors_command(args):
|
|
126
130
|
"""`streamwright connectors`: bare or installed names = installed connectors; ls/i/find use the catalog (hub)."""
|
|
127
131
|
if getattr(args, "flag_list", False):
|
|
@@ -137,6 +141,16 @@ def run_connectors_command(args):
|
|
|
137
141
|
action = getattr(args, "action", None)
|
|
138
142
|
query = getattr(args, "query", [])
|
|
139
143
|
|
|
144
|
+
install = (action or "").lower() in ("i", "install", "-i", "--install")
|
|
145
|
+
if args.yes and not install:
|
|
146
|
+
sys.stderr.write("streamwright: --yes only applies to installing (connectors -i KEY --yes)\n")
|
|
147
|
+
return 2
|
|
148
|
+
installed_only = action is None or (not install and (action or "").lower() not in _CATALOG_ACTIONS
|
|
149
|
+
and {action, *query} <= set(components.available()))
|
|
150
|
+
if args.hub_url and installed_only:
|
|
151
|
+
sys.stderr.write("streamwright: --hub-url applies to the catalog (list, find, install), not to the installed "
|
|
152
|
+
"connectors\n")
|
|
153
|
+
return 2
|
|
140
154
|
if action is None:
|
|
141
155
|
lines = _connector_lines()
|
|
142
156
|
print("\n".join(lines) if lines else "no connectors installed (use streamwright connectors list to find one)")
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
18
|
+
"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import sys
|
|
22
|
+
|
|
23
|
+
from streamwright.core.jobs.engine import load_job_engine
|
|
24
|
+
from streamwright.core.runtime import plugins
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def register(subparsers):
|
|
28
|
+
parser = subparsers.add_parser("duckdb", help="validate, plan and run DuckDBJob manifests and manage their "
|
|
29
|
+
"batches (the explicit form of run|validate|plan JOB)",
|
|
30
|
+
add_help=False)
|
|
31
|
+
parser.add_argument("-h", "--help", action="store_true", help="show the streamwright duckdb help")
|
|
32
|
+
parser.add_argument("arguments", nargs=argparse.REMAINDER)
|
|
33
|
+
parser.set_defaults(command_handler=lambda args: execute(["--help"] if args.help else args.arguments),
|
|
34
|
+
full_parser=lambda: engine_parser("DuckDBJob", "streamwright duckdb"))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def engine_parser(kind, prog):
|
|
38
|
+
"""The engine's own argument parser (its subcommands and options), for help and option inspection."""
|
|
39
|
+
return load_job_engine(kind).parser(prog)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def execute(arguments):
|
|
43
|
+
return run_job_kind("DuckDBJob", arguments, prog="streamwright duckdb")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_job_kind(kind, argv, prog=None):
|
|
47
|
+
"""Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
|
|
48
|
+
try:
|
|
49
|
+
job_engine = load_job_engine(kind)
|
|
50
|
+
except plugins.PluginError as exc:
|
|
51
|
+
sys.stderr.write("streamwright: %s\n" % exc)
|
|
52
|
+
return 2
|
|
53
|
+
# `prog`: the command's name in usage lines (`streamwright duckdb`), for engines whose main takes it
|
|
54
|
+
return (job_engine.main(argv, prog=prog) if prog else job_engine.main(argv)) or 0
|
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
1
18
|
"""
|
|
2
19
|
`streamwright run`: preparation, execution and reporting of a source run (RunCommand), and the run with its
|
|
3
20
|
metrics, summary and output handling (run).
|
|
@@ -166,11 +183,13 @@ def _today_in_zone(started, timezone):
|
|
|
166
183
|
def _resolve_config_and_secrets(spec, provided_config, provided_secrets, today):
|
|
167
184
|
"""(config, secrets) resolved from the spec's declared inputs, reporting every section's problems together."""
|
|
168
185
|
problems, values = [], {}
|
|
186
|
+
config_names = tuple(spec.get("config") or ())
|
|
169
187
|
for section, label, provided in (("config", "config", provided_config),
|
|
170
188
|
("secrets", "secret", provided_secrets)):
|
|
171
189
|
try:
|
|
172
190
|
values[section] = resolve_inputs(spec.get(section), provided, label, today=today,
|
|
173
|
-
hide_values=section == "secrets"
|
|
191
|
+
hide_values=section == "secrets",
|
|
192
|
+
config_names=config_names if section == "secrets" else ())
|
|
174
193
|
except InputError as exc:
|
|
175
194
|
problems.append(str(exc))
|
|
176
195
|
if problems:
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
18
|
+
"""State database CLI commands: migrate, show, check (StreamWright state management)."""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import sys
|
|
22
|
+
|
|
23
|
+
from streamwright.core.cli.help_format import DescriptionFormatter
|
|
24
|
+
from streamwright.core.jobs.state.url import ENV_STATE_URL, migration_runner_for_url
|
|
25
|
+
|
|
26
|
+
__all__ = ["check", "cmd_check", "cmd_migrate", "cmd_show", "migrate", "register", "show"]
|
|
27
|
+
|
|
28
|
+
SUCCESS = 0
|
|
29
|
+
FAILURE = 1
|
|
30
|
+
INVALID_INPUT = 2
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
_DESCRIPTION = """\
|
|
34
|
+
The bookkeeping tables of runtime.state (sw_jobs, sw_leases, sw_sequences, sw_receipts, sw_events, sw_signatures,
|
|
35
|
+
sw_migrations), in a database named by --url or $%s. A job run checks them and fails while a migration is
|
|
36
|
+
pending; it applies them itself only with --apply-state-migrations or runtime.state.apply_migrations: true. These
|
|
37
|
+
commands let a deployment apply them ahead of time and check them. A database that holds the tables of an earlier
|
|
38
|
+
StreamWright release is imported by migration 0001 (batches, job keys, sequences and the store identity carry over;
|
|
39
|
+
the old tables are kept). A job's batches are listed, retried and abandoned with `streamwright duckdb state` and
|
|
40
|
+
`sw spark state`.""" % ENV_STATE_URL
|
|
41
|
+
|
|
42
|
+
_URL_HELP = ("the state database: sqlite:///state.db (relative to the working folder), sqlite:////abs/state.db, "
|
|
43
|
+
"duckdb:///state.duckdb (?catalog=NAME), postgresql://user:pass@host:5432/db?schema=streamwright "
|
|
44
|
+
"(other parameters, e.g. sslmode, go to the driver), mysql://user:pass@host:3306/db (the tables live in "
|
|
45
|
+
"database db); percent-encode special characters in the user and password (default: $%s)"
|
|
46
|
+
% ENV_STATE_URL)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def register(subparsers):
|
|
50
|
+
"""Register the `state` command and its subcommands."""
|
|
51
|
+
parser = subparsers.add_parser(
|
|
52
|
+
"state",
|
|
53
|
+
help="create, upgrade and check the state tables of runtime.state (migrate, show, check)",
|
|
54
|
+
description=_DESCRIPTION,
|
|
55
|
+
formatter_class=DescriptionFormatter,
|
|
56
|
+
)
|
|
57
|
+
sub = parser.add_subparsers(dest="state_action", required=True, metavar="ACTION")
|
|
58
|
+
|
|
59
|
+
raw = DescriptionFormatter
|
|
60
|
+
p_migrate = sub.add_parser("migrate", help="apply the pending migrations (idempotent; concurrent runs wait for "
|
|
61
|
+
"one another)", formatter_class=raw,
|
|
62
|
+
description="Create the state tables, or upgrade them: apply the pending migrations "
|
|
63
|
+
"in order, each in its own transaction. It is idempotent, and concurrent "
|
|
64
|
+
"runs wait for one another.\n\n"
|
|
65
|
+
" streamwright state migrate --url postgresql://sw@db:5432/etl\n"
|
|
66
|
+
" streamwright state migrate --url sqlite:///state.db --dry-run")
|
|
67
|
+
_add_url_options(p_migrate)
|
|
68
|
+
p_migrate.add_argument("--dry-run", action="store_true",
|
|
69
|
+
help="print the SQL of the pending migrations instead of applying them")
|
|
70
|
+
p_migrate.set_defaults(command_handler=cmd_migrate)
|
|
71
|
+
|
|
72
|
+
p_show = sub.add_parser("show", help="list every migration: applied or not, when, and the store identity",
|
|
73
|
+
formatter_class=raw,
|
|
74
|
+
description="List every migration, applied or pending, with when it was applied, and the "
|
|
75
|
+
"store id it was applied to.\n\n"
|
|
76
|
+
" streamwright state show --url mysql://sw@db:3306/etl")
|
|
77
|
+
_add_url_options(p_show)
|
|
78
|
+
p_show.set_defaults(command_handler=cmd_show)
|
|
79
|
+
|
|
80
|
+
p_check = sub.add_parser("check", help="exit 0 when every migration is applied, 1 when one is pending (for "
|
|
81
|
+
"deployment checks)", formatter_class=raw,
|
|
82
|
+
description="Check that the state tables are current, for deployment pipelines: exit 0 "
|
|
83
|
+
"when every migration is applied, 1 when one is pending (run migrate), 2 "
|
|
84
|
+
"when the URL is missing or invalid.\n\n"
|
|
85
|
+
" streamwright state check --url \"$STREAMWRIGHT_STATE_URL\" || exit 1")
|
|
86
|
+
_add_url_options(p_check)
|
|
87
|
+
p_check.set_defaults(command_handler=cmd_check)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _add_url_options(parser: argparse.ArgumentParser):
|
|
91
|
+
parser.add_argument("--url", metavar="URL", help=_URL_HELP)
|
|
92
|
+
parser.add_argument(
|
|
93
|
+
"--schema",
|
|
94
|
+
metavar="SCHEMA",
|
|
95
|
+
help="the schema of the tables, overriding the URL's ?schema= (default: streamwright; MySQL: the URL's "
|
|
96
|
+
"database; SQLite keeps the tables unprefixed)",
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _runner(args):
|
|
101
|
+
try:
|
|
102
|
+
return migration_runner_for_url(args.url, schema=args.schema)
|
|
103
|
+
except (ValueError, ImportError) as exc:
|
|
104
|
+
sys.stderr.write("streamwright state: %s\n" % exc)
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def migrate(runner, dry_run=False) -> int:
|
|
109
|
+
"""Applies a store's pending migrations (or prints their SQL); the exit code."""
|
|
110
|
+
if dry_run:
|
|
111
|
+
plan = runner.apply(dry_run=True)
|
|
112
|
+
if not plan:
|
|
113
|
+
sys.stdout.write("The state tables in %s are up to date: no pending migrations.\n" % runner.target)
|
|
114
|
+
else:
|
|
115
|
+
sys.stdout.write("-- State database: %s\n" % runner.target)
|
|
116
|
+
for item in plan:
|
|
117
|
+
sys.stdout.write("-- Migration %s: %s\n" % (item["name"], item.get("description") or ""))
|
|
118
|
+
sys.stdout.writelines("%s;\n\n" % statement for statement in item["statements"])
|
|
119
|
+
return SUCCESS
|
|
120
|
+
try:
|
|
121
|
+
applied = runner.apply()
|
|
122
|
+
except Exception as exc: # the database's own error (permissions, connection, ...)
|
|
123
|
+
sys.stderr.write("streamwright state migrate: %s\n" % exc)
|
|
124
|
+
return FAILURE
|
|
125
|
+
if not applied:
|
|
126
|
+
sys.stdout.write("The state tables in %s are up to date: no migrations to apply.\n" % runner.target)
|
|
127
|
+
for record in applied:
|
|
128
|
+
sys.stdout.write("Applied migration %s to %s (store id %s)\n" % (record["name"], runner.target,
|
|
129
|
+
record["store_id"]))
|
|
130
|
+
return SUCCESS
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def show(runner) -> int:
|
|
134
|
+
"""Lists every migration: applied or not, when, and the store identity."""
|
|
135
|
+
records = runner.show()
|
|
136
|
+
sys.stdout.write("State database: %s\n" % runner.target)
|
|
137
|
+
sys.stdout.write("%-8s %-25s %-9s %-26s %s\n" % ("Version", "Name", "Applied", "Applied at", "Store id"))
|
|
138
|
+
for record in records:
|
|
139
|
+
sys.stdout.write("%-8s %-25s %-9s %-26s %s\n" % (
|
|
140
|
+
record["version"], record["name"], "yes" if record["applied"] else "no", record["applied_at"] or "",
|
|
141
|
+
record["store_id"] or ""))
|
|
142
|
+
return SUCCESS
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def check(runner) -> int:
|
|
146
|
+
"""Exit 0 when every migration is applied, 1 (listing the pending ones) otherwise."""
|
|
147
|
+
pending = [record["name"] for record in runner.show() if not record["applied"]]
|
|
148
|
+
if pending:
|
|
149
|
+
sys.stdout.write("The state tables in %s need migrations: %s (run migrate)\n" % (runner.target,
|
|
150
|
+
", ".join(pending)))
|
|
151
|
+
return FAILURE
|
|
152
|
+
sys.stdout.write("The state tables in %s are up to date.\n" % runner.target)
|
|
153
|
+
return SUCCESS
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def cmd_migrate(args) -> int:
|
|
157
|
+
runner = _runner(args)
|
|
158
|
+
return INVALID_INPUT if runner is None else migrate(runner, args.dry_run)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def cmd_show(args) -> int:
|
|
162
|
+
runner = _runner(args)
|
|
163
|
+
return INVALID_INPUT if runner is None else show(runner)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def cmd_check(args) -> int:
|
|
167
|
+
runner = _runner(args)
|
|
168
|
+
return INVALID_INPUT if runner is None else check(runner)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# /*************************************************************************
|
|
3
|
+
# * Copyright 2026 Karthick Jaganathan
|
|
4
|
+
# *
|
|
5
|
+
# * Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# * you may not use this file except in compliance with the License.
|
|
7
|
+
# * You may obtain a copy of the License at
|
|
8
|
+
# *
|
|
9
|
+
# * https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
# *
|
|
11
|
+
# * Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# * See the License for the specific language governing permissions and
|
|
15
|
+
# * limitations under the License.
|
|
16
|
+
# **************************************************************************/
|
|
17
|
+
|
|
18
|
+
"""The help formatter of StreamWright's commands: descriptions wrap to the terminal, examples keep their lines."""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import textwrap
|
|
22
|
+
|
|
23
|
+
__all__ = ["DescriptionFormatter", "sentence"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DescriptionFormatter(argparse.HelpFormatter):
|
|
27
|
+
"""
|
|
28
|
+
Wraps each paragraph of a description or epilog (blank-line separated) to the terminal's width; a paragraph
|
|
29
|
+
with an indented line (examples, tables) keeps its lines as written.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def _fill_text(self, text, width, indent):
|
|
33
|
+
paragraphs = []
|
|
34
|
+
for paragraph in text.strip("\n").split("\n\n"):
|
|
35
|
+
lines = paragraph.splitlines()
|
|
36
|
+
if any(line[:1].isspace() for line in lines):
|
|
37
|
+
paragraphs.append("\n".join(indent + line for line in lines))
|
|
38
|
+
else:
|
|
39
|
+
paragraphs.append(textwrap.fill(" ".join(line.strip() for line in lines), width,
|
|
40
|
+
initial_indent=indent, subsequent_indent=indent))
|
|
41
|
+
return "\n\n".join(paragraphs)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def sentence(text):
|
|
45
|
+
"""A help line (`list the batches`) as a description's sentence (`List the batches.`)."""
|
|
46
|
+
text = text.strip()
|
|
47
|
+
return text[:1].upper() + text[1:] + ("" if text.endswith(".") else ".")
|