streamwright-spark 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. streamwright/spark/__init__.py +1 -0
  2. streamwright/spark/cli/__init__.py +14 -0
  3. streamwright/spark/cli/__main__.py +7 -0
  4. streamwright/spark/cli/aws_secrets.py +58 -0
  5. streamwright/spark/cli/common.py +73 -0
  6. streamwright/spark/cli/inputs.py +40 -0
  7. streamwright/spark/cli/lock.py +65 -0
  8. streamwright/spark/cli/main.py +94 -0
  9. streamwright/spark/cli/run.py +339 -0
  10. streamwright/spark/cli/state.py +133 -0
  11. streamwright/spark/cli/submit_format.py +43 -0
  12. streamwright/spark/cli/validate.py +60 -0
  13. streamwright/spark/contracts/__init__.py +36 -0
  14. streamwright/spark/contracts/adapter.py +7 -0
  15. streamwright/spark/contracts/events.py +5 -0
  16. streamwright/spark/contracts/launcher.py +71 -0
  17. streamwright/spark/contracts/state.py +5 -0
  18. streamwright/spark/contracts/types.py +29 -0
  19. streamwright/spark/docs.py +19 -0
  20. streamwright/spark/engine/__init__.py +1 -0
  21. streamwright/spark/engine/executor.py +276 -0
  22. streamwright/spark/engine/runtime.py +154 -0
  23. streamwright/spark/engine/session.py +95 -0
  24. streamwright/spark/entrypoint.py +33 -0
  25. streamwright/spark/guide.md +105 -0
  26. streamwright/spark/io/__init__.py +1 -0
  27. streamwright/spark/io/capabilities.py +13 -0
  28. streamwright/spark/io/database_rules.py +79 -0
  29. streamwright/spark/io/delta.py +314 -0
  30. streamwright/spark/io/file_rules.py +201 -0
  31. streamwright/spark/io/files.py +222 -0
  32. streamwright/spark/io/jdbc.py +320 -0
  33. streamwright/spark/io/messaging.py +40 -0
  34. streamwright/spark/io/sql.py +56 -0
  35. streamwright/spark/job/__init__.py +20 -0
  36. streamwright/spark/job/check.py +20 -0
  37. streamwright/spark/job/reader.py +10 -0
  38. streamwright/spark/job/rules.py +325 -0
  39. streamwright/spark/job/runtime_rules.py +99 -0
  40. streamwright/spark/job/validate.py +80 -0
  41. streamwright/spark/launchers/__init__.py +1 -0
  42. streamwright/spark/launchers/common.py +83 -0
  43. streamwright/spark/launchers/emr_ec2.py +200 -0
  44. streamwright/spark/launchers/local.py +22 -0
  45. streamwright/spark/launchers/s3_jobs.py +25 -0
  46. streamwright/spark/launchers/standalone.py +58 -0
  47. streamwright/spark/plan/__init__.py +5 -0
  48. streamwright/spark/plan/analysis.py +166 -0
  49. streamwright/spark/plugin.py +37 -0
  50. streamwright/spark/runtime/__init__.py +1 -0
  51. streamwright/spark/runtime/definition.py +104 -0
  52. streamwright/spark/runtime/errors.py +11 -0
  53. streamwright/spark/runtime/lock.py +126 -0
  54. streamwright/spark/runtime/registry.py +101 -0
  55. streamwright/spark/schema.py +20 -0
  56. streamwright/spark/sdk/__init__.py +55 -0
  57. streamwright/spark/sdk/py.typed +0 -0
  58. streamwright/spark/storage/__init__.py +1 -0
  59. streamwright/spark/storage/exports.py +155 -0
  60. streamwright/spark/storage/ingest.py +175 -0
  61. streamwright/spark/storage/paths.py +60 -0
  62. streamwright_spark-0.1.0.dist-info/METADATA +107 -0
  63. streamwright_spark-0.1.0.dist-info/RECORD +67 -0
  64. streamwright_spark-0.1.0.dist-info/WHEEL +5 -0
  65. streamwright_spark-0.1.0.dist-info/entry_points.txt +14 -0
  66. streamwright_spark-0.1.0.dist-info/licenses/LICENSE +201 -0
  67. streamwright_spark-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,339 @@
1
+ """sw spark run: submit a job to its platform, or run the core batch loop on the driver."""
2
+
3
+ import datetime
4
+ import json
5
+ import os
6
+ import re
7
+ import sys
8
+ from importlib import import_module, metadata
9
+
10
+ from streamwright.core.jobs import BrokerCall, Runner
11
+ from streamwright.core.jobs.manifest import manifest_directory
12
+ from streamwright.core.jobs.refs import references
13
+ from streamwright.core.runtime.redact import Redactor
14
+ from streamwright.core.sdk.contract import InvalidConnectionError
15
+ from streamwright.spark.cli.common import connections_of, err, load_plugins
16
+ from streamwright.spark.cli.inputs import config_values, secret_values
17
+ from streamwright.spark.cli.state import open_store
18
+ from streamwright.spark.cli.submit_format import format_spark_submit
19
+ from streamwright.spark.contracts import LaunchError, LaunchFailureError
20
+ from streamwright.spark.io.messaging import open_broker
21
+ from streamwright.spark.job import check_job, load_job, resolve
22
+ from streamwright.spark.job.reader import resolve_job_location
23
+ from streamwright.spark.plan import build_plan, role_of
24
+ from streamwright.spark.runtime import lock as lockmod
25
+
26
+ __all__ = ["add_arguments", "run"]
27
+
28
+
29
+ def add_arguments(parser):
30
+ parser.add_argument("--secrets", help="path to a secrets YAML file or JSON string of secret values")
31
+ mode = parser.add_mutually_exclusive_group()
32
+ mode.add_argument("--once", action="store_true", help="one batch (default)")
33
+ mode.add_argument("--drain", action="store_true", help="batches until the source is empty")
34
+ mode.add_argument(
35
+ "--available-now",
36
+ action="store_true",
37
+ help="batches until messages available at launch are processed (Trigger.AvailableNow)",
38
+ )
39
+ mode.add_argument("--follow", action="store_true", help="keep polling until interrupted")
40
+ parser.add_argument("--interval", default="10s", help="polling interval when running with --follow (default: 10s)")
41
+ parser.add_argument(
42
+ "--max-messages", type=int, default=500, help="maximum messages per batch (0: recover and relay only)"
43
+ )
44
+ parser.add_argument("--run-key", help="idempotency key for jobs without a broker")
45
+ parser.add_argument(
46
+ "--conf",
47
+ action="append",
48
+ default=[],
49
+ metavar="K=V",
50
+ help="Spark configuration override (e.g. --conf spark.executor.memory=4g)",
51
+ )
52
+ parser.add_argument(
53
+ "--log-level",
54
+ choices=["ALL", "DEBUG", "ERROR", "FATAL", "INFO", "OFF", "TRACE", "WARN"],
55
+ type=str.upper,
56
+ help="Spark logging level (e.g. --log-level ERROR or WARN)",
57
+ )
58
+ parser.add_argument("--lock", help="path to the lock file to verify dependencies and versions against")
59
+ parser.add_argument("--lock-required", action="store_true", help="require a valid lock file before running")
60
+ parser.add_argument(
61
+ "--no-init-state",
62
+ action="store_true",
63
+ help="do not create the bookkeeping tables (they must exist: `sw spark state init JOB`)",
64
+ )
65
+ parser.add_argument(
66
+ "--dry-run", action="store_true", help="resolve the job and print the launch submission without executing"
67
+ )
68
+ parser.add_argument(
69
+ "--detach", action="store_true", help="submit the job to the cluster launcher and exit without waiting"
70
+ )
71
+ parser.add_argument("--summary", action="store_true", help="print a summary of the executed batch upon completion")
72
+ parser.add_argument("--driver", action="store_true", help="internal: run on the cluster driver")
73
+
74
+
75
+ def _seconds(text):
76
+ match = re.match(r"^([0-9]+)([smh])$", str(text))
77
+ if not match:
78
+ raise ValueError("invalid duration %r (expected e.g. 10s, 5m)" % (text,))
79
+ return int(match.group(1)) * {"s": 1, "m": 60, "h": 3600}[match.group(2)]
80
+
81
+
82
+ def run(args):
83
+ adapters, launchers, plugin_problems = load_plugins(args.allow_connector)
84
+ job = load_job(args.job, args.overlay)
85
+ problems, warnings = check_job(job, adapters, launchers)
86
+ problems += plugin_problems
87
+ for text in problems:
88
+ err("error: " + text)
89
+ if problems:
90
+ return 2
91
+ for text in warnings:
92
+ err("warning: " + text)
93
+ config = config_values(job, args)
94
+ platform = job["runtime"]["platform"]
95
+ lock = lockmod.load(args.lock) if args.lock else None
96
+ if args.lock_required and lock is None and platform["type"] != "local":
97
+ err("error: --lock-required: no lock given")
98
+ return 2
99
+ if args.dry_run:
100
+ try:
101
+ _preview_dry_run(job, adapters, args, lock=lock, platform=platform, launcher=launchers[platform["type"]])
102
+ except LaunchError as exc:
103
+ err("error: %s" % exc)
104
+ return 2
105
+ return 0
106
+ if platform["type"] != "local" and not args.driver:
107
+ placeholders = {name: "{{ secrets.%s }}" % name for name in ((job.get("spec") or {}).get("secrets") or {})}
108
+ submission, problems = resolve(job, config, placeholders)
109
+ if problems:
110
+ for text in problems:
111
+ err("error: " + text)
112
+ return 2
113
+ return _submit(submission, args, lock, submission["runtime"]["platform"], launchers[platform["type"]])
114
+ return _drive(job, adapters, args, config, lock)
115
+
116
+
117
+ def _spark_conf(job, extra_conf):
118
+ conf = dict((job.get("runtime") or {}).get("spark", {}).get("conf") or {})
119
+ env = (job.get("runtime") or {}).get("env") or {}
120
+ secret_values_in_env = [
121
+ str(v) for k, v in env.items() if k.startswith("STREAMWRIGHT_SECRET_") and v and not references(v)
122
+ ]
123
+ for k, v in env.items():
124
+ if k.startswith("STREAMWRIGHT_SECRET_") or (
125
+ isinstance(v, str) and any(ref.startswith("secrets.") for ref in references(v))
126
+ ):
127
+ continue
128
+ conf.setdefault("spark.yarn.appMasterEnv.%s" % k, str(v))
129
+ conf.setdefault("spark.executorEnv.%s" % k, str(v))
130
+ conf.update(item.split("=", 1) for item in extra_conf if "=" in item)
131
+ redact = Redactor(secret_values_in_env) # the one secret-matching rule (the forms logs mask)
132
+ if any(redact(str(value)) != str(value) for value in conf.values()):
133
+ raise LaunchError("resolved secrets cannot be copied into Spark conf")
134
+ if any(
135
+ key.startswith(("spark.yarn.appMasterEnv.STREAMWRIGHT_SECRET_", "spark.executorEnv.STREAMWRIGHT_SECRET_"))
136
+ for key in conf
137
+ ):
138
+ raise LaunchError("secret environment variables cannot be put in Spark conf; use runtime.secrets")
139
+ return conf
140
+
141
+
142
+ def _driver_flags(args):
143
+ flags = []
144
+ for path in args.overlay:
145
+ flags += ["--overlay", os.path.basename(path)]
146
+ for item in args.set:
147
+ flags += ["--set", item]
148
+ if args.config:
149
+ flags += ["--config", os.path.basename(args.config)]
150
+ flags += (
151
+ ["--follow"]
152
+ if args.follow
153
+ else ["--available-now"]
154
+ if getattr(args, "available_now", False)
155
+ else ["--drain"]
156
+ if args.drain
157
+ else ["--once"]
158
+ )
159
+ flags += ["--interval", args.interval, "--max-messages", str(args.max_messages)]
160
+ if args.run_key:
161
+ flags += ["--run-key", args.run_key]
162
+ if args.no_init_state:
163
+ flags += ["--no-init-state"]
164
+ if getattr(args, "log_level", None):
165
+ flags += ["--log-level", args.log_level]
166
+ return flags
167
+
168
+
169
+ def _preview_dry_run(job, adapters, args, *, lock, platform, launcher):
170
+ print("Execution Plan:")
171
+ print(build_plan(job, adapters).describe())
172
+ print("\nLaunch Submission (platform: %s):" % platform["type"])
173
+ conf = _spark_conf(job, args.conf)
174
+ files = list(args.overlay) + ([args.config] if args.config else [])
175
+ preview = launcher.preview(
176
+ args.job, _driver_flags(args), conf, lock or {}, platform=platform, lock_path=args.lock, files=files
177
+ )
178
+ if preview.get("target"):
179
+ print(" Target Cluster: %s" % preview["target"])
180
+ if preview.get("command"):
181
+ print(" spark-submit command:\n " + format_spark_submit(preview["command"], indent=4))
182
+ if preview.get("step"):
183
+ print(" EMR Step JSON:\n " + json.dumps(preview["step"], indent=2).replace("\n", "\n "))
184
+
185
+
186
+ def _submit(job, args, lock, platform, launcher):
187
+ try:
188
+ conf = _spark_conf(job, args.conf)
189
+ except LaunchError as exc:
190
+ err("error: %s" % exc)
191
+ return 2
192
+ if getattr(args, "secrets", None):
193
+ err("error: --secrets is not delivered to cluster drivers; use runtime.secrets")
194
+ return 2
195
+ flags = _driver_flags(args)
196
+ env = (lock or {}).get("python_env") or {}
197
+ archive = env.get("archive") if env.get("archive") and os.path.exists(env["archive"]) else None
198
+ job_path = os.path.join(args.job, "job.yaml") if os.path.isdir(args.job) else resolve_job_location(args.job)
199
+ try:
200
+ lock_problems = launcher.check_lock(lock or {}, platform)
201
+ for text in lock_problems:
202
+ err("error: " + text)
203
+ if lock_problems:
204
+ return 2
205
+ files = list(args.overlay) + ([args.config] if args.config else [])
206
+ names = [os.path.basename(path.rstrip("/")) for path in [job_path, args.lock, *files] if path]
207
+ if len(names) != len(set(names)):
208
+ raise LaunchError("job, lock, overlays and config must have distinct file names for cluster staging")
209
+ return launcher.submit(
210
+ job_path,
211
+ flags,
212
+ conf,
213
+ lock or {},
214
+ detach=args.detach,
215
+ lock_path=args.lock,
216
+ env_archive=archive,
217
+ platform=platform,
218
+ files=files,
219
+ )
220
+ except LaunchFailureError as exc:
221
+ err("error: %s" % exc)
222
+ return 1
223
+ except LaunchError as exc:
224
+ err("error: %s" % exc)
225
+ return 2
226
+
227
+
228
+ def _verify_lock(lock):
229
+ if lock is None:
230
+ return []
231
+ mismatches = lockmod.check_lock(lock, metadata.version("pyspark"))
232
+ if not mismatches:
233
+ os.environ["STREAMWRIGHT_LOCK_SHA"] = lockmod.fingerprint(lock)
234
+ return mismatches
235
+
236
+
237
+ def _configure_driver_conf(resolved, args, secrets):
238
+ conf = _spark_conf(resolved, getattr(args, "conf", None) or [])
239
+ redact = Redactor(secrets.values()) # the one secret-matching rule (the forms logs mask)
240
+ if any(redact(str(value)) != str(value) for value in conf.values()):
241
+ raise LaunchError("resolved secrets cannot be copied into Spark conf")
242
+ resolved.setdefault("runtime", {}).setdefault("spark", {})["conf"] = conf
243
+
244
+
245
+ def _broker_calls(resolved, connections, adapters):
246
+ brokers = {}
247
+
248
+ def broker_for(name):
249
+ if name not in brokers:
250
+ conn = connections[name]
251
+ adapter = adapters[conn.provider]
252
+ brokers[name] = open_broker(adapter, conn.auth)
253
+ return brokers[name]
254
+
255
+ consumer = None
256
+ publishers = {}
257
+ for step in resolved.get("steps") or []:
258
+ if role_of(resolved, step, adapters) == "consume":
259
+ consumer = BrokerCall(broker_for(step["connection"]), step["arguments"])
260
+ for output in resolved.get("outputs") or []:
261
+ if role_of(resolved, output, adapters) == "publish":
262
+ publishers[output["name"]] = BrokerCall(broker_for(output["connection"]), output["arguments"])
263
+ return consumer, publishers
264
+
265
+
266
+ def _drive(job, adapters, args, config, lock):
267
+ """On the driver (local, or inside the cluster): verify, connect, run the batch loop. Needs Spark."""
268
+ executor_type = import_module("streamwright.spark.engine.executor").SparkExecutor
269
+ session_module = import_module("streamwright.spark.engine.session")
270
+
271
+ secrets = secret_values(job, args)
272
+ resolved, problems = resolve(job, config, secrets)
273
+ for text in problems:
274
+ err("error: " + text)
275
+ if problems:
276
+ return 2
277
+ for key, value in ((resolved.get("runtime") or {}).get("env") or {}).items():
278
+ os.environ[key] = str(value)
279
+ try:
280
+ connections = connections_of(resolved, adapters=adapters, declared=job, config=config, secrets=secrets)
281
+ except InvalidConnectionError as exc:
282
+ err("error: " + str(exc))
283
+ return 2
284
+ mismatches = _verify_lock(lock)
285
+ for text in mismatches:
286
+ err("error: lock: " + text)
287
+ if mismatches:
288
+ return 2
289
+ store = open_store(resolved, connections, adapters, manifest_directory(args.job))
290
+ if not args.no_init_state:
291
+ store.initialize()
292
+ try:
293
+ _configure_driver_conf(resolved, args, secrets)
294
+ except LaunchError as exc:
295
+ err("error: " + str(exc))
296
+ return 2
297
+ spark = session_module.build_session(
298
+ resolved,
299
+ connections,
300
+ adapters,
301
+ local=resolved["runtime"]["platform"]["type"] == "local",
302
+ log_level=getattr(args, "log_level", None),
303
+ )
304
+ app_id, driver_host = session_module.resolve_spark_metadata(spark)
305
+ executor = executor_type(
306
+ spark, resolved, connections, adapters, config=config, limits=resolved["runtime"].get("limits"),
307
+ state=store, declared_connections=(job.get("connections") or {}), secrets=secrets,
308
+ allowed_connectors=getattr(args, "allow_connector", None),
309
+ )
310
+ consumer, publishers = _broker_calls(resolved, connections, adapters)
311
+
312
+ def log(event, **fields):
313
+ print(
314
+ json.dumps(
315
+ dict(fields, event=event, ts=datetime.datetime.now(datetime.UTC).isoformat()), default=str
316
+ ),
317
+ file=sys.stderr,
318
+ )
319
+
320
+ mode = (
321
+ "follow"
322
+ if args.follow
323
+ else "available_now"
324
+ if getattr(args, "available_now", False)
325
+ else "drain"
326
+ if args.drain
327
+ else "once"
328
+ )
329
+ runner = Runner(
330
+ resolved["name"],
331
+ store,
332
+ executor,
333
+ consumer,
334
+ publishers,
335
+ redact=Redactor(secrets.values()),
336
+ log=log,
337
+ engine_metadata={"app_id": app_id, "driver_host": driver_host},
338
+ )
339
+ return runner.run(mode, _seconds(args.interval), args.max_messages, args.run_key)
@@ -0,0 +1,133 @@
1
+ """sw spark state: inspect and repair bookkeeping."""
2
+
3
+ import datetime
4
+ import json
5
+ import os
6
+
7
+ from streamwright.core.jobs.manifest import manifest_directory
8
+ from streamwright.core.jobs.state import open_sqlite
9
+ from streamwright.core.jobs.state import open_store as open_dialect
10
+ from streamwright.spark.cli.common import add_job_arguments, connections_of, err, load_plugins
11
+ from streamwright.spark.cli.inputs import config_values, secret_values
12
+ from streamwright.spark.job import check_job, load_job, resolve
13
+
14
+ __all__ = ["add_arguments", "open_store", "run"]
15
+
16
+ ACTIONS = ("init", "status", "batches", "show", "retry", "abandon", "purge", "lease")
17
+
18
+
19
+ def add_arguments(parser):
20
+ parser.add_argument("action", choices=ACTIONS, help="state action to perform: " + ", ".join(ACTIONS))
21
+ add_job_arguments(parser) # `state status JOB [ARG]`
22
+ parser.add_argument("arg", nargs="?", help="optional argument for action (e.g. batch_id for show/retry/abandon)")
23
+ parser.add_argument("--secrets", help="path to a secrets YAML file or JSON string of secret values")
24
+
25
+
26
+ def open_store(resolved, connections, adapters, base_dir):
27
+ """The bookkeeping store named by runtime.state: a sqlite file, or the connection's own adapter."""
28
+ state = resolved["runtime"]["state"]
29
+ if "sqlite" in state:
30
+ path = state["sqlite"]
31
+ if resolved["runtime"]["platform"]["type"] != "local" and not os.path.isabs(path):
32
+ raise ValueError("remote jobs require an absolute runtime.state.sqlite path")
33
+ if path != ":memory:" and not os.path.isabs(path):
34
+ path = os.path.join(base_dir or os.getcwd(), path)
35
+ return open_sqlite(path)
36
+ conn = connections[state["connection"]]
37
+ connect, dialect = adapters[conn.provider].state_connection(conn.auth)
38
+ return open_dialect(connect, dialect, schema=state.get("schema", "streamwright"))
39
+
40
+
41
+ def run(args):
42
+ adapters, launchers, _ = load_plugins(args.allow_connector)
43
+ job = load_job(args.job, args.overlay)
44
+ problems, _ = check_job(job, adapters, launchers)
45
+ if problems:
46
+ for text in problems:
47
+ err("error: " + text)
48
+ return 2
49
+ config = config_values(job, args)
50
+ secrets = secret_values(job, args)
51
+ resolved, missing = resolve(job, config, secrets)
52
+ if missing:
53
+ for text in missing:
54
+ err("error: " + text)
55
+ return 2
56
+ connections = connections_of(resolved, adapters=adapters, declared=job, config=config, secrets=secrets)
57
+ store = open_store(resolved, connections, adapters, manifest_directory(args.job))
58
+ return _run_action(store, job["name"], args)
59
+
60
+
61
+ def _list_batches(store, name, limit):
62
+ for batch in store.list_batches(name, limit):
63
+ metadata = batch.get("engine_metadata") or {}
64
+ meta = [
65
+ f"{label}={metadata[key]}"
66
+ for key, label in (("app_id", "app"), ("driver_host", "host"))
67
+ if metadata.get(key)
68
+ ]
69
+ meta_str = (" " + " ".join(meta)) if meta else ""
70
+ print(
71
+ "%-24s %-10s seq=%d updated=%s%s %s"
72
+ % (
73
+ batch["batch_id"],
74
+ batch["status"],
75
+ batch["batch_sequence"],
76
+ batch["updated_at"],
77
+ meta_str,
78
+ (batch["error"] or "")[:80],
79
+ )
80
+ )
81
+
82
+
83
+ def _show_batch(store, name, batch_id):
84
+ batch = store.get_batch(name, batch_id)
85
+ if batch is None:
86
+ err("no such batch")
87
+ return 2
88
+ manifest = dict(batch["manifest"], messages="%d message(s)" % len(batch["manifest"].get("messages", [])))
89
+ print(
90
+ json.dumps(
91
+ dict(batch, manifest=manifest, receipts=store.get_receipts(name, batch["batch_id"])), indent=2, default=str
92
+ )
93
+ )
94
+ return 0
95
+
96
+
97
+ def _change_batch(store, name, args):
98
+ lease = store.acquire_lease(name, "cli:%d" % os.getpid(), 60)
99
+ try:
100
+ batch = store.get_batch(name, args.arg or "")
101
+ if batch is None:
102
+ err("no such batch")
103
+ return 2
104
+ if args.action == "retry":
105
+ store.retry_batch(lease, batch["batch_id"])
106
+ else:
107
+ store.record_batch(lease, batch["batch_id"], "abandoned", batch["manifest"])
108
+ print("%s: %s" % (batch["batch_id"], "queued for retry" if args.action == "retry" else "abandoned"))
109
+ finally:
110
+ store.release_lease(lease)
111
+ return 0
112
+
113
+
114
+ def _run_action(store, name, args):
115
+ if args.action == "init":
116
+ store.initialize()
117
+ print("initialized")
118
+ elif args.action in ("status", "batches"):
119
+ _list_batches(store, name, 1 if args.action == "status" else 20)
120
+ elif args.action == "show":
121
+ return _show_batch(store, name, args.arg or "")
122
+ elif args.action == "lease":
123
+ lease = store.get_lease(name)
124
+ if lease is None:
125
+ print("no lease recorded for %s" % name)
126
+ else:
127
+ print(json.dumps(lease, indent=2))
128
+ elif args.action in ("retry", "abandon"):
129
+ return _change_batch(store, name, args)
130
+ elif args.action == "purge":
131
+ cutoff = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=int(args.arg or 30))
132
+ print("purged %d batch(es)" % store.purge_batches(name, cutoff))
133
+ return 0
@@ -0,0 +1,43 @@
1
+ """Render a spark-submit argument list for dry-run previews."""
2
+
3
+ _BOOL_FLAGS = frozenset(
4
+ {
5
+ "--once",
6
+ "--drain",
7
+ "--available-now",
8
+ "--follow",
9
+ "--no-init-state",
10
+ "--dry-run",
11
+ "--detach",
12
+ "--summary",
13
+ "--driver",
14
+ "--supervise",
15
+ "--verbose",
16
+ }
17
+ )
18
+
19
+
20
+ def _quoted(text):
21
+ value = str(text)
22
+ return f'"{value}"' if " " in value and not (value.startswith('"') and value.endswith('"')) else value
23
+
24
+
25
+ def format_spark_submit(arguments, indent=4):
26
+ """Group submit flags with their values in a human-readable multiline command."""
27
+ lines = []
28
+ index = 0
29
+ if arguments and arguments[0] == "spark-submit":
30
+ lines.append("spark-submit")
31
+ index = 1
32
+ prefix = " " * indent
33
+ while index < len(arguments):
34
+ argument = str(arguments[index])
35
+ if argument.startswith("-") and argument not in _BOOL_FLAGS and index + 1 < len(arguments):
36
+ value = str(arguments[index + 1])
37
+ if not value.startswith("-"):
38
+ lines.append(f"{prefix}{argument} {_quoted(value)}")
39
+ index += 2
40
+ continue
41
+ lines.append(f"{prefix}{_quoted(argument)}")
42
+ index += 1
43
+ return " \\\n".join(lines)
@@ -0,0 +1,60 @@
1
+ """sw spark validate: check jobs without Spark, brokers or credentials."""
2
+
3
+ import contextlib
4
+
5
+ from streamwright.core.jobs.inputs import InputError, parse_set, read_config_values
6
+ from streamwright.spark.cli.common import err
7
+ from streamwright.spark.job import validate as validate_job
8
+ from streamwright.spark.plan import build_plan
9
+ from streamwright.spark.runtime.registry import Registry, RegistryError
10
+
11
+ __all__ = ["add_arguments", "run"]
12
+
13
+
14
+ def add_arguments(parser):
15
+ parser.add_argument("--analyze", action="store_true", help="print the Spark actions and recomputation warnings")
16
+ parser.add_argument("--graph", action="store_true", help="print step dependencies")
17
+ parser.add_argument("--strict", action="store_true", help="warnings fail the check")
18
+ parser.add_argument("--lock", help="path to a lock file to verify compatibility against")
19
+
20
+
21
+ def run(args):
22
+ code = 0
23
+ for path in args.job:
24
+ try:
25
+ config = read_config_values(args.config) if args.config else {}
26
+ config.update(parse_set(args.set))
27
+ except InputError as exc:
28
+ err("%s: %s" % (path, exc))
29
+ code = 2
30
+ continue
31
+ report = validate_job(
32
+ path, overlays=args.overlay, allow=args.allow_connector, lock=args.lock, strict=args.strict, config=config
33
+ )
34
+ job = report.job or {}
35
+ registry = Registry()
36
+ adapters = {}
37
+ for connection in (job.get("connections") or {}).values():
38
+ if not isinstance(connection, dict):
39
+ continue
40
+ provider = connection.get("provider")
41
+ if provider and provider not in adapters:
42
+ with contextlib.suppress(RegistryError):
43
+ adapters[provider] = registry.load(provider, allow=args.allow_connector)
44
+ if report.valid and (args.analyze or args.graph):
45
+ plan = build_plan(job, adapters)
46
+ if args.analyze:
47
+ print(plan.describe())
48
+ if args.graph:
49
+ for node in plan.nodes:
50
+ print("%-8s %-22s <- %s" % (node.kind, node.name, ", ".join(node.uses) or "-"))
51
+ for issue in report.issues:
52
+ err("%s: %s: %s" % (path, issue.severity, issue.message))
53
+ if report.valid and args.analyze:
54
+ for warning in build_plan(job, adapters).warnings:
55
+ err("%s: warning: %s" % (path, warning))
56
+ if not report.valid:
57
+ code = 2
58
+ else:
59
+ print("%s: ok" % path)
60
+ return code
@@ -0,0 +1,36 @@
1
+ """Spark adapter errors and launcher contracts; job types come from core."""
2
+
3
+ from streamwright.spark.contracts.adapter import AdapterError
4
+ from streamwright.spark.contracts.events import Broker
5
+ from streamwright.spark.contracts.launcher import LAUNCHER_OWNED_CONF, Launcher, LaunchError, LaunchFailureError
6
+ from streamwright.spark.contracts.state import StateStore
7
+ from streamwright.spark.contracts.types import (
8
+ CONTRACT_VERSION,
9
+ BatchContext,
10
+ Connection,
11
+ DataFrame,
12
+ Lease,
13
+ Message,
14
+ Observation,
15
+ SparkSession,
16
+ WriteReceipt,
17
+ )
18
+
19
+ __all__ = [
20
+ "CONTRACT_VERSION",
21
+ "LAUNCHER_OWNED_CONF",
22
+ "AdapterError",
23
+ "BatchContext",
24
+ "Broker",
25
+ "Connection",
26
+ "DataFrame",
27
+ "LaunchError",
28
+ "LaunchFailureError",
29
+ "Launcher",
30
+ "Lease",
31
+ "Message",
32
+ "Observation",
33
+ "SparkSession",
34
+ "StateStore",
35
+ "WriteReceipt",
36
+ ]
@@ -0,0 +1,7 @@
1
+ """Spark adapter errors."""
2
+
3
+ __all__ = ["AdapterError"]
4
+
5
+
6
+ class AdapterError(Exception):
7
+ pass
@@ -0,0 +1,5 @@
1
+ """The shared broker contract used by Spark adapters."""
2
+
3
+ from streamwright.core.jobs.types import Broker
4
+
5
+ __all__ = ["Broker"]