streamwright 0.1.11__tar.gz → 0.1.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. {streamwright-0.1.11/src/streamwright.egg-info → streamwright-0.1.13}/PKG-INFO +1 -1
  2. {streamwright-0.1.11 → streamwright-0.1.13}/pyproject.toml +1 -1
  3. streamwright-0.1.13/src/streamwright/core/cli/commands/jobs.py +31 -0
  4. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/source.py +5 -14
  5. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/jobs.py +9 -5
  6. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/main.py +9 -2
  7. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/parser.py +26 -7
  8. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/cli.py +62 -9
  9. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/plugin.py +2 -2
  10. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/engine.py +4 -1
  11. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/__init__.py +4 -1
  12. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/summary.py +20 -0
  13. {streamwright-0.1.11 → streamwright-0.1.13/src/streamwright.egg-info}/PKG-INFO +1 -1
  14. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_cli_subcommands.py +39 -2
  15. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_logging.py +2 -0
  16. streamwright-0.1.11/src/streamwright/core/cli/commands/jobs.py +0 -27
  17. {streamwright-0.1.11 → streamwright-0.1.13}/LICENSE +0 -0
  18. {streamwright-0.1.11 → streamwright-0.1.13}/README.md +0 -0
  19. {streamwright-0.1.11 → streamwright-0.1.13}/setup.cfg +0 -0
  20. {streamwright-0.1.11 → streamwright-0.1.13}/setup.py +0 -0
  21. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/__init__.py +0 -0
  22. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/catalog.py +0 -0
  23. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/__init__.py +0 -0
  24. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/__main__.py +0 -0
  25. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/__init__.py +0 -0
  26. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/commands/connectors.py +0 -0
  27. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/cli/log_options.py +0 -0
  28. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/__init__.py +0 -0
  29. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/inputs.py +0 -0
  30. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/loader.py +0 -0
  31. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/config/reader.py +0 -0
  32. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/__init__.py +0 -0
  33. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/bookmarks.py +0 -0
  34. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/builders.py +0 -0
  35. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/errors.py +0 -0
  36. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/execution.py +0 -0
  37. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/factories.py +0 -0
  38. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/partitions.py +0 -0
  39. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/planning.py +0 -0
  40. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/queries.py +0 -0
  41. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/readers.py +0 -0
  42. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/responses.py +0 -0
  43. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/rules.py +0 -0
  44. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/runner.py +0 -0
  45. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/session.py +0 -0
  46. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/__init__.py +0 -0
  47. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/compilation.py +0 -0
  48. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/list_parameters.py +0 -0
  49. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/ordering.py +0 -0
  50. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/plans.py +0 -0
  51. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/sandbox.py +0 -0
  52. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/stream_plan.py +0 -0
  53. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/tables.py +0 -0
  54. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/sql/types.py +0 -0
  55. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/values.py +0 -0
  56. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/engine/writers.py +0 -0
  57. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/graph.py +0 -0
  58. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/__init__.py +0 -0
  59. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/api_calls.py +0 -0
  60. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/capabilities.py +0 -0
  61. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/database_connections.py +0 -0
  62. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/database_rules.py +0 -0
  63. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/databases.py +0 -0
  64. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/file_formats.py +0 -0
  65. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/file_rules.py +0 -0
  66. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/files.py +0 -0
  67. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/http.py +0 -0
  68. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/local.py +0 -0
  69. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/locations.py +0 -0
  70. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/queries.py +0 -0
  71. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/sandbox.py +0 -0
  72. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/io/store_reader.py +0 -0
  73. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/__init__.py +0 -0
  74. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/calls.py +0 -0
  75. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/__init__.py +0 -0
  76. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/connections.py +0 -0
  77. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/dependencies.py +0 -0
  78. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/findings.py +0 -0
  79. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/names.py +0 -0
  80. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/outputs.py +0 -0
  81. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/refs.py +0 -0
  82. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/rules.py +0 -0
  83. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/runtime.py +0 -0
  84. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/secrets.py +0 -0
  85. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/sections.py +0 -0
  86. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/spec.py +0 -0
  87. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/steps.py +0 -0
  88. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/check/structure.py +0 -0
  89. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/__init__.py +0 -0
  90. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/adapter.py +0 -0
  91. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/definition_adapter.py +0 -0
  92. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/engine.py +0 -0
  93. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/executor.py +0 -0
  94. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/guide.md +0 -0
  95. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/loading.py +0 -0
  96. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/reads.py +0 -0
  97. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/rules.py +0 -0
  98. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/session.py +0 -0
  99. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/steps.py +0 -0
  100. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/writers/__init__.py +0 -0
  101. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/writers/tables.py +0 -0
  102. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/fields.py +0 -0
  103. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/__init__.py +0 -0
  104. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/column_rules.py +0 -0
  105. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/formats.py +0 -0
  106. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/headers.py +0 -0
  107. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/mapped.py +0 -0
  108. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/model.py +0 -0
  109. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/ingest/plain.py +0 -0
  110. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/inputs.py +0 -0
  111. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/manifest.py +0 -0
  112. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/naming.py +0 -0
  113. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/plan.py +0 -0
  114. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/policy.py +0 -0
  115. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/refs.py +0 -0
  116. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/runner.py +0 -0
  117. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/schema.py +0 -0
  118. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/__init__.py +0 -0
  119. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/dialects.py +0 -0
  120. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/migrations.py +0 -0
  121. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/protocol.py +0 -0
  122. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/state/store.py +0 -0
  123. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/types.py +0 -0
  124. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/validate.py +0 -0
  125. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/jobs/write_rules.py +0 -0
  126. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/__init__.py +0 -0
  127. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/auth.py +0 -0
  128. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/downloads.py +0 -0
  129. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/encoding.py +0 -0
  130. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/errors.py +0 -0
  131. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/http.py +0 -0
  132. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/paginators.py +0 -0
  133. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/net/policies.py +0 -0
  134. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/__init__.py +0 -0
  135. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/base.py +0 -0
  136. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/dlt_columns.py +0 -0
  137. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/dlt_output.py +0 -0
  138. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/ducklake.py +0 -0
  139. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/file_names.py +0 -0
  140. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/files.py +0 -0
  141. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/loading.py +0 -0
  142. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/output.py +0 -0
  143. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/staging.py +0 -0
  144. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/outputs/warehouse.py +0 -0
  145. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/py.typed +0 -0
  146. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/__init__.py +0 -0
  147. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/components.py +0 -0
  148. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/connector_types.py +0 -0
  149. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/constants.py +0 -0
  150. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract.py +0 -0
  151. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_declarations.py +0 -0
  152. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_guide.py +0 -0
  153. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_terms.py +0 -0
  154. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/contract_validation.py +0 -0
  155. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/definitions.py +0 -0
  156. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/formatting.py +0 -0
  157. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/handlers.py +0 -0
  158. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/lines.py +0 -0
  159. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/metrics.py +0 -0
  160. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/network.py +0 -0
  161. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/record_factory.py +0 -0
  162. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/redaction.py +0 -0
  163. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/logs/setup.py +0 -0
  164. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/plugins.py +0 -0
  165. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/redact.py +0 -0
  166. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/source_connectors.py +0 -0
  167. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/templates.py +0 -0
  168. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/runtime/testing.py +0 -0
  169. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/__init__.py +0 -0
  170. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/constants.py +0 -0
  171. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/contract.py +0 -0
  172. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/engines/__init__.py +0 -0
  173. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/engines/duckdb.py +0 -0
  174. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/engines/source.py +0 -0
  175. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/messaging.py +0 -0
  176. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/network.py +0 -0
  177. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/__init__.py +0 -0
  178. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/databases.py +0 -0
  179. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/files.py +0 -0
  180. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/http.py +0 -0
  181. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/messages.py +0 -0
  182. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/operations/tables.py +0 -0
  183. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/py.typed +0 -0
  184. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/__init__.py +0 -0
  185. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/contract.py +0 -0
  186. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/helpers.py +0 -0
  187. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sdk/testing/storage.py +0 -0
  188. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/__init__.py +0 -0
  189. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/analysis.py +0 -0
  190. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/conditions.py +0 -0
  191. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/sql/quote.py +0 -0
  192. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/__init__.py +0 -0
  193. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/auth.py +0 -0
  194. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/checker.py +0 -0
  195. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/engine.py +0 -0
  196. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/issues.py +0 -0
  197. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/job_manifests.py +0 -0
  198. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/json_schemas.py +0 -0
  199. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/marked_yaml.py +0 -0
  200. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/partitions.py +0 -0
  201. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/requests.py +0 -0
  202. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/schema.py +0 -0
  203. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/__init__.py +0 -0
  204. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/async_jobs.py +0 -0
  205. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/document.py +0 -0
  206. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/partitions.py +0 -0
  207. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/references.py +0 -0
  208. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/request_partitions.py +0 -0
  209. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/requests.py +0 -0
  210. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/spec.py +0 -0
  211. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/stream_context.py +0 -0
  212. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/streams.py +0 -0
  213. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/tables.py +0 -0
  214. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/sections/templates.py +0 -0
  215. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/source.py +0 -0
  216. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/streams.py +0 -0
  217. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright/core/validation/vocabulary.py +0 -0
  218. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/SOURCES.txt +0 -0
  219. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/dependency_links.txt +0 -0
  220. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/entry_points.txt +0 -0
  221. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/requires.txt +0 -0
  222. {streamwright-0.1.11 → streamwright-0.1.13}/src/streamwright.egg-info/top_level.txt +0 -0
  223. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_catalog.py +0 -0
  224. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_conditions.py +0 -0
  225. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_connector_api_family.py +0 -0
  226. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_connector_contract.py +0 -0
  227. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_connectors.py +0 -0
  228. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_dlt_output.py +0 -0
  229. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_engine_reach.py +0 -0
  230. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_exact_decimals.py +0 -0
  231. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_graph.py +0 -0
  232. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_http_engine.py +0 -0
  233. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_output_contract.py +0 -0
  234. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_output_formats.py +0 -0
  235. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_plugin_contract_gate.py +0 -0
  236. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_runner.py +0 -0
  237. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_sdk_plugins.py +0 -0
  238. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_sql.py +0 -0
  239. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_sql_quote.py +0 -0
  240. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_transform.py +0 -0
  241. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_validation.py +0 -0
  242. {streamwright-0.1.11 → streamwright-0.1.13}/tests/test_warehouse_output.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamwright
3
- Version: 0.1.11
3
+ Version: 0.1.13
4
4
  Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
5
5
  Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
6
6
  License: Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "streamwright"
7
- version = "0.1.11"
7
+ version = "0.1.13"
8
8
  description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
9
9
  readme = "README.md"
10
10
  license = {text = "Apache-2.0"}
@@ -0,0 +1,31 @@
1
+ """Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
2
+
3
+ import argparse
4
+ import sys
5
+
6
+ from streamwright.core.jobs.engine import load_job_engine
7
+ from streamwright.core.runtime import plugins
8
+
9
+
10
+ def register(subparsers):
11
+ parser = subparsers.add_parser("duckdb", help="validate, plan and run DuckDBJob manifests and manage their "
12
+ "batches (the explicit form of run|validate|plan JOB)",
13
+ add_help=False)
14
+ parser.add_argument("-h", "--help", action="store_true", help="show the streamwright duckdb help")
15
+ parser.add_argument("arguments", nargs=argparse.REMAINDER)
16
+ parser.set_defaults(command_handler=lambda args: execute(["--help"] if args.help else args.arguments))
17
+
18
+
19
+ def execute(arguments):
20
+ return run_job_kind("DuckDBJob", arguments, prog="streamwright duckdb")
21
+
22
+
23
+ def run_job_kind(kind, argv, prog=None):
24
+ """Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
25
+ try:
26
+ job_engine = load_job_engine(kind)
27
+ except plugins.PluginError as exc:
28
+ sys.stderr.write("streamwright: %s\n" % exc)
29
+ return 2
30
+ # `prog`: the command's name in usage lines (`streamwright duckdb`), for engines whose main takes it
31
+ return (job_engine.main(argv, prog=prog) if prog else job_engine.main(argv)) or 0
@@ -102,18 +102,6 @@ def _load_state(path):
102
102
  return state.get("value", state) if state.get("type") == "STATE" else state
103
103
 
104
104
 
105
- def _summary_problem(path):
106
- """Why --summary FILE cannot be written (checked before the run starts), or None."""
107
- if not path:
108
- return None
109
- if os.path.isdir(path):
110
- return "--summary %s is a folder" % path
111
- folder = os.path.dirname(os.path.abspath(path))
112
- if not os.path.isdir(folder):
113
- return "--summary %s: the folder %s does not exist" % (path, folder)
114
- return None
115
-
116
-
117
105
  def run(args, logging_setup=None):
118
106
  """
119
107
  streamwright run. Whatever the outcome, a run that started logs its end (RunMetrics.finish), and --summary FILE is
@@ -121,7 +109,7 @@ def run(args, logging_setup=None):
121
109
  the loggers' levels and handlers can drop - redacted and cut as log lines are. `logging_setup`: the command's
122
110
  logs.LogSetup, which gets the run's Redactor and the network loggers of the source's connector.
123
111
  """
124
- problem = _summary_problem(getattr(args, "summary", None))
112
+ problem = logs.summary_problem(getattr(args, "summary", None))
125
113
  if problem:
126
114
  LOG.error("%s", problem)
127
115
  return 2
@@ -150,7 +138,10 @@ def run(args, logging_setup=None):
150
138
  messages.append(message)
151
139
  metrics.error = "; ".join(messages) or None
152
140
  try:
153
- logs.write_summary(args.summary, metrics.summary())
141
+ if args.summary == logs.LOG_SUMMARY: # a bare --summary: a log line (stdout carries the records)
142
+ logs.log_summary(LOG, metrics.summary())
143
+ else:
144
+ logs.write_summary(args.summary, metrics.summary())
154
145
  except (OSError, TypeError, ValueError) as exc:
155
146
  LOG.error("could not write the summary %s: %s", args.summary, exc)
156
147
  code = 1 if code == 0 else code
@@ -38,10 +38,14 @@ def add_job_arguments(parser):
38
38
  """Add common job manifest and execution options to an argparse parser."""
39
39
  parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
40
40
  parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
41
- parser.add_argument("--overlay", action="append", default=[], metavar="FILE")
42
- parser.add_argument("--config", metavar="FILE")
43
- parser.add_argument("--set", action="append", default=[], type=parse_set, metavar="NAME=VALUE")
44
- parser.add_argument("--allow-connector", action="append", default=[], metavar="NAME")
45
- parser.add_argument("--strict", action="store_true")
41
+ parser.add_argument("--overlay", action="append", default=[], metavar="FILE",
42
+ help="a YAML overlay merged onto the job, in order (repeatable; a null deletes a key, a list "
43
+ "replaces the job's)")
44
+ parser.add_argument("--config", metavar="FILE", help="a YAML/JSON file of spec.config values")
45
+ parser.add_argument("--set", action="append", default=[], type=parse_set, metavar="NAME=VALUE",
46
+ help="a spec.config value (repeatable; lists as a,b,c); overrides --config")
47
+ parser.add_argument("--allow-connector", action="append", default=[], metavar="NAME",
48
+ help="allow only this connector (repeatable); by default any installed connector")
49
+ parser.add_argument("--strict", action="store_true", help="treat warnings as errors")
46
50
  add_log_options(parser)
47
51
  return parser
@@ -27,7 +27,7 @@ from streamwright.core.cli.commands.connectors import run_connectors_command
27
27
  from streamwright.core.cli.commands.source import run
28
28
  from streamwright.core.cli.jobs import JOB_OPTION
29
29
  from streamwright.core.cli.log_options import log_options, log_setup
30
- from streamwright.core.cli.parser import build_parser, build_validate_parser, command_parsers
30
+ from streamwright.core.cli.parser import build_parser, build_plan_parser, build_validate_parser, command_parsers
31
31
  from streamwright.core.validation import engine
32
32
  from streamwright.core.validation.engine import NON_STRICT_CODES, check_components
33
33
 
@@ -52,6 +52,9 @@ def _extract_all_paths(args, command="run"):
52
52
  value = args[index + 1]
53
53
  if value:
54
54
  paths.append(value)
55
+ if action.nargs == "?" and not equals and (index + 1 >= len(args) or args[index + 1].startswith("-")):
56
+ index += 1 # an optional value that is not given (a bare --summary)
57
+ continue
55
58
  if not equals and action.nargs != 0:
56
59
  index += 1
57
60
  if action.nargs in ("+", "*"):
@@ -144,7 +147,11 @@ def main(argv=None):
144
147
  if command == "validate":
145
148
  return _validate_command(argv)
146
149
  if command == "plan":
147
- sys.stderr.write("streamwright: 'plan' needs an installed job manifest kind and path\n")
150
+ if any(argument in ("-h", "--help") for argument in argv[1:]):
151
+ build_plan_parser().print_help()
152
+ return 0
153
+ sys.stderr.write("streamwright: 'plan' needs a job manifest (JOB or --job JOB) of an installed engine's "
154
+ "kind; see streamwright plan -h\n")
148
155
  return 2
149
156
  args = build_parser().parse_args(argv)
150
157
  if args.command == "connectors":
@@ -23,26 +23,43 @@ from streamwright.core.cli.commands import jobs
23
23
  from streamwright.core.cli.jobs import JOB_OPTION, add_job_arguments
24
24
  from streamwright.core.cli.log_options import add_log_options, key_value
25
25
  from streamwright.core.config.inputs import SECRET_ENV_PREFIX
26
- from streamwright.core.runtime import plugins
26
+ from streamwright.core.runtime import logs, plugins
27
27
  from streamwright.core.validation import engine
28
28
 
29
29
 
30
30
  def build_validate_parser():
31
31
  """`streamwright validate --help`: validation options, then the logging options (as streamwright run's)."""
32
32
  parser = engine._parser("streamwright validate")
33
+ parser.epilog = ("A job manifest (kind: DuckDBJob, SparkJob) is checked by its engine, with that engine's options: "
34
+ "streamwright duckdb validate -h, streamwright spark validate -h.")
33
35
  add_log_options(parser.add_argument_group("logging options"))
34
36
  return parser
35
37
 
36
38
 
39
+ def build_plan_parser():
40
+ """`streamwright plan --help`: the command takes a job manifest; its engine's plan command takes the options."""
41
+ parser = argparse.ArgumentParser(
42
+ prog="streamwright plan",
43
+ description="Show a job's execution plan - its steps, outputs and the order they run in - without running it "
44
+ "or needing secrets. The manifest's kind picks the engine, and its options are that engine's: "
45
+ "streamwright duckdb plan -h (kind: DuckDBJob), streamwright spark plan -h (kind: SparkJob).")
46
+ parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
47
+ parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
48
+ return parser
49
+
50
+
37
51
  def build_parser():
38
52
  parser = argparse.ArgumentParser(prog="streamwright",
39
- description="Run and check StreamWright sources (`kind: source`).")
53
+ description="Run and check StreamWright sources (`kind: source`) and jobs "
54
+ "(`kind: DuckDBJob`, `kind: SparkJob`).")
40
55
  commands = parser.add_subparsers(dest="command", metavar="COMMAND")
41
56
  commands.required = True
42
57
 
43
- run = commands.add_parser("run", help="run a source and write its records",
58
+ run = commands.add_parser("run", help="run a source and write its records, or a job (JOB or --job JOB)",
44
59
  description="Run a source. Records go to stdout as Singer messages unless --output "
45
- "names a folder for files or a warehouse; logs go to stderr.")
60
+ "names a folder for files or a warehouse; logs go to stderr. A job "
61
+ "manifest (kind: DuckDBJob, SparkJob) runs on its engine, with that "
62
+ "engine's options: streamwright duckdb run -h, streamwright spark run -h.")
46
63
  run.add_argument("source", nargs="?", metavar="SOURCE",
47
64
  help="a source folder (source.yaml and one file per stream in streams/) or a source file; or a "
48
65
  "job manifest (kind: DuckDBJob, SparkJob), which runs on its engine")
@@ -83,10 +100,11 @@ def build_parser():
83
100
  "EXPORT.TIMESTAMP.UNIQUE.jsonl, or .csv, .tsv, .parquet)")
84
101
  run.add_argument("--allow-connector", action="append", dest="allowed_connectors", metavar="NAME",
85
102
  help="allow only this connector (repeatable); by default any installed connector can be used")
86
- run.add_argument("--summary", metavar="FILE",
103
+ run.add_argument("--summary", nargs="?", const=logs.LOG_SUMMARY, metavar="FILE",
87
104
  help="also write the run's summary as JSON to FILE, atomically, whatever the outcome: status "
88
105
  "(ok or failed), times, the streams' counts (partitions, windows, pages, requests, retries, "
89
- "records read and written), the outputs written, the state's bookmarks and the error")
106
+ "records read and written), the outputs written, the state's bookmarks and the error; a "
107
+ "bare --summary logs it instead, as one INFO line of compact JSON (event run_summary)")
90
108
  add_log_options(run)
91
109
 
92
110
  conn = commands.add_parser("connectors",
@@ -113,7 +131,8 @@ def build_parser():
113
131
  add_help=False)
114
132
  validate.add_argument("arguments", nargs=argparse.REMAINDER)
115
133
 
116
- plan_cmd = commands.add_parser("plan", help="show execution plan DAG for a DuckDB job", add_help=False)
134
+ plan_cmd = commands.add_parser("plan", help="show a job's execution plan: its steps, outputs and their order "
135
+ "(no secrets needed)", add_help=False)
117
136
  plan_cmd.add_argument("arguments", nargs=argparse.REMAINDER)
118
137
  installed_commands = {item["name"] for item in plugins.discover(plugins.COMMANDS)}
119
138
  if "duckdb" not in installed_commands: # editable installs have stale entry-point metadata
@@ -20,7 +20,7 @@ The DuckDBJob command line (registered for `kind: DuckDBJob` in the `streamwrigh
20
20
 
21
21
  validate JOB [--graph]
22
22
  plan JOB
23
- run JOB [--secrets FILE] [--dry-run]
23
+ run JOB [--secrets FILE] [--dry-run] [--summary FILE]
24
24
  state list|retry|abandon JOB [--batch ID] [--secrets FILE]
25
25
  (every command: --config FILE, --set NAME=VALUE, --overlay FILE, --allow-connector NAME, --strict, and the logging
26
26
  options of `streamwright run`: --log-level, --log NAME=LEVEL, --log-format, --log-config, --log-max-chars)
@@ -30,6 +30,7 @@ job or failed run, 2 usage or input error. Validation and planning never need se
30
30
  """
31
31
 
32
32
  import argparse
33
+ import logging
33
34
  import sys
34
35
 
35
36
  from streamwright.core.cli.jobs import FAILURE, INVALID_INPUT, SUCCESS, add_job_arguments, resolve_job
@@ -40,9 +41,13 @@ from streamwright.core.jobs.inputs import read_config_values
40
41
  from streamwright.core.jobs.manifest import JobError
41
42
  from streamwright.core.jobs.runner import owner_id
42
43
  from streamwright.core.jobs.state import StateError
44
+ from streamwright.core.runtime import logs
45
+ from streamwright.core.runtime.logs.lines import iso_time, utc_now
43
46
 
44
47
  __all__ = ["main"]
45
48
 
49
+ LOG = logging.getLogger("streamwright.duckdb")
50
+
46
51
 
47
52
  def _err(text):
48
53
  sys.stderr.write(text + "\n")
@@ -91,13 +96,54 @@ def _cmd_plan(args):
91
96
  return SUCCESS
92
97
 
93
98
 
99
+ def _run_summary(args, result, started_at, failure):
100
+ """The --summary of a job run: status, times, job, batches, each output's rows and files, and the error."""
101
+ finished_at = utc_now()
102
+ outputs = [{"name": name, "rows": rows, "files": [location for location in result.locations.get(name, ())
103
+ if location],
104
+ "already_committed": name in result.skipped}
105
+ for name, rows in sorted(result.outputs.items())] if result is not None else []
106
+ summary = {"status": "ok" if failure is None and result is not None and not result.exit_code else "failed",
107
+ "started_at": iso_time(started_at), "finished_at": iso_time(finished_at),
108
+ "duration_s": round((finished_at - started_at).total_seconds(), 3),
109
+ "job": result.job_name if result else None, "manifest": args.job, "dry_run": args.dry_run,
110
+ "batches": list(result.batches) if result else [], "outputs": outputs}
111
+ if summary["status"] != "ok":
112
+ if failure is not None and not isinstance(failure, Exception):
113
+ error = "interrupted"
114
+ elif failure is not None:
115
+ error = "%s: %s" % (type(failure).__name__, failure)
116
+ else:
117
+ error = result.error or "the run failed"
118
+ summary["error"] = args.log_setup.message(error)
119
+ return summary
120
+
121
+
94
122
  def _cmd_run(args):
95
- result = run_job(args.job, config=_config(args), secrets=_secrets(args), overlays=args.overlay,
96
- allowed_connectors=_allowed(args), dry_run=args.dry_run, strict=args.strict)
123
+ problem = logs.summary_problem(args.summary)
124
+ if problem:
125
+ _err("Error: %s" % problem)
126
+ return INVALID_INPUT
127
+ started_at, result, failure, summary_written = utc_now(), None, None, True
128
+ try:
129
+ result = run_job(args.job, config=_config(args), secrets=_secrets(args), overlays=args.overlay,
130
+ allowed_connectors=_allowed(args), dry_run=args.dry_run, strict=args.strict)
131
+ except BaseException as exc:
132
+ failure = exc
133
+ raise
134
+ finally:
135
+ if args.summary == logs.LOG_SUMMARY: # a bare --summary: a log line, whatever the outcome
136
+ logs.log_summary(LOG, _run_summary(args, result, started_at, failure))
137
+ elif args.summary:
138
+ try:
139
+ logs.write_summary(args.summary, _run_summary(args, result, started_at, failure))
140
+ except (OSError, TypeError, ValueError) as exc:
141
+ _err("Error: could not write the summary %s: %s" % (args.summary, exc))
142
+ summary_written = False
97
143
  sys.stdout.write("Job: %s\n" % result.job_name)
98
144
  if args.dry_run:
99
145
  sys.stdout.write(result.plan + "\n")
100
- return SUCCESS
146
+ return SUCCESS if summary_written else FAILURE
101
147
  for name, rows in sorted(result.outputs.items()):
102
148
  if name in result.skipped:
103
149
  sys.stdout.write(" %-28s already committed in an earlier attempt\n" % name)
@@ -113,7 +159,7 @@ def _cmd_run(args):
113
159
  sys.stdout.writelines(" %s\n" % location for location in locations)
114
160
  if result.exit_code:
115
161
  _err("Run failed: %s" % (result.error or "see the log"))
116
- return result.exit_code
162
+ return result.exit_code or (SUCCESS if summary_written else FAILURE)
117
163
 
118
164
 
119
165
  def _cmd_state(args):
@@ -151,7 +197,8 @@ def _cmd_state(args):
151
197
  def _inputs(parser, secrets=False):
152
198
  add_job_arguments(parser)
153
199
  if secrets:
154
- parser.add_argument("--secrets", help="a YAML file of secret values (or STREAMWRIGHT_SECRET_<NAME>)")
200
+ parser.add_argument("--secrets", metavar="FILE",
201
+ help="a YAML file of spec.secrets values (or STREAMWRIGHT_SECRET_<NAME> variables)")
155
202
 
156
203
 
157
204
  def _build_parser(prog):
@@ -167,12 +214,18 @@ def _build_parser(prog):
167
214
  plan.set_defaults(func=_cmd_plan, parser=plan)
168
215
  run = sub.add_parser("run", allow_abbrev=False, help="run a DuckDBJob (unfinished batches resume first)")
169
216
  run.add_argument("--dry-run", action="store_true", help="check and plan, write nothing")
217
+ run.add_argument("--summary", nargs="?", const=logs.LOG_SUMMARY, metavar="FILE",
218
+ help="also write the run's summary as JSON to FILE, atomically, whatever the outcome: status (ok "
219
+ "or failed), times, the batches, each output's rows and files, and the error; a bare "
220
+ "--summary logs it instead, as one INFO line of compact JSON (event run_summary)")
170
221
  _inputs(run, secrets=True)
171
222
  run.set_defaults(func=_cmd_run, parser=run)
172
223
  state = sub.add_parser("state", allow_abbrev=False, help="list, retry or abandon a job's batches (runtime.state)")
173
- state.add_argument("action", choices=["list", "retry", "abandon"])
174
- state.add_argument("--batch", help="the batch id (retry, abandon)")
175
- state.add_argument("--limit", type=int, default=20, help="batches to list")
224
+ state.add_argument("action", choices=["list", "retry", "abandon"],
225
+ help="list: the job's recent batches; retry: run a failed batch again from where it failed; "
226
+ "abandon: give a failed batch up so the job can run again")
227
+ state.add_argument("--batch", metavar="ID", help="the batch id (retry, abandon)")
228
+ state.add_argument("--limit", type=int, default=20, metavar="N", help="list: the number of batches (default 20)")
176
229
  _inputs(state, secrets=True)
177
230
  state.set_defaults(func=_cmd_state, parser=state)
178
231
  return parser
@@ -31,8 +31,8 @@ class DuckDBJobEngine:
31
31
 
32
32
  kind = KIND
33
33
 
34
- def main(self, argv=None):
35
- return cli.main(argv)
34
+ def main(self, argv=None, prog="streamwright"):
35
+ return cli.main(argv, prog=prog)
36
36
 
37
37
  def validate_job(self, job, *, overlays=(), config=None, strict=False, allowed=None):
38
38
  return engine.validate_job(job, overlays=overlays, config=config, strict=strict, allowed=allowed)
@@ -41,7 +41,10 @@ class JobEngine(Protocol):
41
41
  kind: str # the manifest `kind` it runs, e.g. "DuckDBJob" (= the entry-point name)
42
42
 
43
43
  def main(self, argv: Sequence[str] | None = None) -> int:
44
- """Runs the engine's command line on the original arguments (`validate|plan|run JOB ...`); the exit code."""
44
+ """
45
+ Runs the engine's command line on the original arguments (`validate|plan|run JOB ...`); the exit code. An engine
46
+ may also take `prog`, the command's name in its usage lines (`streamwright duckdb` for the explicit form).
47
+ """
45
48
 
46
49
  def validate_job(self, job: str | Mapping[str, Any], *, overlays: Sequence[str] = (),
47
50
  config: Mapping[str, Any] | None = None, strict: bool = False) -> JobReport:
@@ -57,7 +57,7 @@ from streamwright.core.runtime.logs.metrics import RunMetrics, StreamMetrics
57
57
  from streamwright.core.runtime.logs.network import http_details, http_summary, request_details, sdk_summary
58
58
  from streamwright.core.runtime.logs.redaction import mask_form, mask_headers, mask_text, mask_url, sensitive
59
59
  from streamwright.core.runtime.logs.setup import LogSetup
60
- from streamwright.core.runtime.logs.summary import write_summary
60
+ from streamwright.core.runtime.logs.summary import LOG_SUMMARY, log_summary, summary_problem, write_summary
61
61
 
62
62
  __all__ = [
63
63
  "DEFAULT_MAX_CHARS",
@@ -65,6 +65,7 @@ __all__ = [
65
65
  "FORMATS",
66
66
  "LEVELS",
67
67
  "LOG",
68
+ "LOG_SUMMARY",
68
69
  "NETWORK",
69
70
  "PROGRESS_PAGES",
70
71
  "PROGRESS_SECONDS",
@@ -78,6 +79,7 @@ __all__ = [
78
79
  "http_details",
79
80
  "http_summary",
80
81
  "level_number",
82
+ "log_summary",
81
83
  "mask_form",
82
84
  "mask_headers",
83
85
  "mask_text",
@@ -85,6 +87,7 @@ __all__ = [
85
87
  "request_details",
86
88
  "sdk_summary",
87
89
  "sensitive",
90
+ "summary_problem",
88
91
  "write_summary",
89
92
  ]
90
93
 
@@ -22,6 +22,26 @@ import json
22
22
  import os
23
23
  import tempfile
24
24
 
25
+ LOG_SUMMARY = "-" # a bare --summary: the summary as a log line
26
+
27
+
28
+ def log_summary(logger, summary):
29
+ """The run summary as one INFO log line of compact JSON (event `run_summary`), on the command's log handlers."""
30
+ logger.info("run summary: %s", json.dumps(summary, separators=(",", ":"), default=str),
31
+ extra={"event": "run_summary"})
32
+
33
+
34
+ def summary_problem(path):
35
+ """Why --summary FILE cannot be written (checked before a run starts), or None."""
36
+ if not path or path == LOG_SUMMARY:
37
+ return None
38
+ if os.path.isdir(path):
39
+ return "--summary %s is a folder" % path
40
+ folder = os.path.dirname(os.path.abspath(path))
41
+ if not os.path.isdir(folder):
42
+ return "--summary %s: the folder %s does not exist" % (path, folder)
43
+ return None
44
+
25
45
 
26
46
  def write_summary(path, summary):
27
47
  """Writes the run summary as JSON to `path`, atomically (a temporary file in its folder, then renamed)."""
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streamwright
3
- Version: 0.1.11
3
+ Version: 0.1.13
4
4
  Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
5
5
  Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
6
6
  License: Apache-2.0
@@ -6,8 +6,12 @@ import subprocess
6
6
  import sys
7
7
  from types import SimpleNamespace
8
8
 
9
+ import pytest
10
+
9
11
  from streamwright.core import cli
10
12
  from streamwright.core.cli.jobs import parse_set
13
+ from streamwright.core.cli.parser import build_parser, build_validate_parser
14
+ from streamwright.core.jobs.duckdb import cli as duckdb_cli
11
15
  from streamwright.core.runtime import plugins
12
16
 
13
17
 
@@ -82,6 +86,8 @@ def test_job_option_routes_like_the_job_path(tmp_path, capsys, monkeypatch):
82
86
  for argv in (["run", "--job", str(manifest)], ["validate", "-j", str(manifest)],
83
87
  ["plan", "--job=%s" % manifest], ["run", str(manifest), "--job", str(manifest)]):
84
88
  assert cli.main(argv) == 0 and received[-1] == argv
89
+ for argv in (["run", "--summary", "--job", str(manifest)], ["run", "--job", str(manifest), "--summary"]):
90
+ assert cli.main(argv) == 0 and received[-1] == argv # a bare --summary takes no path
85
91
  source = tmp_path / "source.yaml"
86
92
  source.write_text("kind: source\n")
87
93
  assert cli.main(["run", "--job", str(source)]) == 2
@@ -124,9 +130,10 @@ def test_command_entry_is_registered_from_metadata(monkeypatch, capsys):
124
130
  def test_builtin_job_shorthand_uses_plugin_route(monkeypatch):
125
131
  jobs_module = importlib.import_module("streamwright.core.cli.commands.jobs")
126
132
  calls = []
127
- monkeypatch.setattr(jobs_module, "run_job_kind", lambda kind, argv: calls.append((kind, argv)) or 0)
133
+ monkeypatch.setattr(jobs_module, "run_job_kind",
134
+ lambda kind, argv, prog=None: calls.append((kind, argv, prog)) or 0)
128
135
  assert cli.main(["duckdb", "run", "job.yaml"]) == 0
129
- assert calls == [("DuckDBJob", ["run", "job.yaml"])]
136
+ assert calls == [("DuckDBJob", ["run", "job.yaml"], "streamwright duckdb")] # its usage lines name the command
130
137
 
131
138
 
132
139
  def test_shared_job_set_parser_uses_jobs_input_rules():
@@ -137,3 +144,33 @@ def test_shared_job_set_parser_uses_jobs_input_rules():
137
144
  assert "NAME=VALUE" in str(exc)
138
145
  else:
139
146
  raise AssertionError("an invalid --set must be rejected")
147
+
148
+
149
+ def _without_help(parser, path):
150
+ missing = []
151
+ for action in parser._actions:
152
+ if isinstance(action, argparse._HelpAction) or action.help == argparse.SUPPRESS:
153
+ continue
154
+ if isinstance(action, argparse._SubParsersAction):
155
+ helps = {choice.dest: choice.help for choice in action._choices_actions}
156
+ for name, sub in action.choices.items():
157
+ missing += [] if helps.get(name) else ["%s %s" % (path, name)]
158
+ missing += _without_help(sub, "%s %s" % (path, name))
159
+ elif not action.help and action.nargs != argparse.REMAINDER: # (REMAINDER: another parser's arguments)
160
+ missing.append("%s %s" % (path, "/".join(action.option_strings) or action.dest))
161
+ return missing
162
+
163
+
164
+ def test_every_command_and_option_has_help(capsys):
165
+ plugin_commands = {item["name"] for item in plugins.discover(plugins.COMMANDS)} - {"duckdb"}
166
+ missing = [item for item in _without_help(build_parser(), "streamwright")
167
+ if item.split()[1] not in plugin_commands] # (installed plug-ins check their own)
168
+ assert missing == []
169
+ assert _without_help(build_validate_parser(), "streamwright validate") == []
170
+ assert _without_help(duckdb_cli._build_parser("streamwright duckdb"), "streamwright duckdb") == []
171
+ assert cli.main(["plan", "-h"]) == 0
172
+ with pytest.raises(SystemExit) as shown:
173
+ cli.main(["duckdb", "-h"]) # forwarded to the engine's own parser
174
+ assert shown.value.code == 0
175
+ out = capsys.readouterr().out
176
+ assert "usage: streamwright plan" in out and "usage: streamwright duckdb" in out
@@ -931,6 +931,8 @@ def test_the_summary_of_a_run_that_cannot_start(api, tmp_path, monkeypatch, caps
931
931
 
932
932
  code, _, err = run_cli(capsys, path, "--summary", tmp_path / "nowhere" / "run.json")
933
933
  assert code == 2 and "the folder %s does not exist" % (tmp_path / "nowhere") in err and api.requests == []
934
+ code, _, err = run_cli(capsys, path, "--summary") # bare: a log line (a source's stdout carries its records)
935
+ assert code == 2 and 'run summary: {"status":"failed"' in err and api.requests == []
934
936
 
935
937
 
936
938
  # * ----------------------------------
@@ -1,27 +0,0 @@
1
- """Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
2
-
3
- import argparse
4
- import sys
5
-
6
- from streamwright.core.jobs.engine import load_job_engine
7
- from streamwright.core.runtime import plugins
8
-
9
-
10
- def register(subparsers):
11
- parser = subparsers.add_parser("duckdb", help="run an installed DuckDBJob engine", add_help=False)
12
- parser.add_argument("arguments", nargs=argparse.REMAINDER)
13
- parser.set_defaults(command_handler=lambda args: execute(args.arguments))
14
-
15
-
16
- def execute(arguments):
17
- return run_job_kind("DuckDBJob", arguments)
18
-
19
-
20
- def run_job_kind(kind, argv):
21
- """Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
22
- try:
23
- job_engine = load_job_engine(kind)
24
- except plugins.PluginError as exc:
25
- sys.stderr.write("streamwright: %s\n" % exc)
26
- return 2
27
- return job_engine.main(argv) or 0
File without changes
File without changes
File without changes
File without changes