streamwright-spark 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- streamwright/spark/__init__.py +1 -0
- streamwright/spark/cli/__init__.py +14 -0
- streamwright/spark/cli/__main__.py +7 -0
- streamwright/spark/cli/aws_secrets.py +58 -0
- streamwright/spark/cli/common.py +73 -0
- streamwright/spark/cli/inputs.py +40 -0
- streamwright/spark/cli/lock.py +65 -0
- streamwright/spark/cli/main.py +94 -0
- streamwright/spark/cli/run.py +339 -0
- streamwright/spark/cli/state.py +133 -0
- streamwright/spark/cli/submit_format.py +43 -0
- streamwright/spark/cli/validate.py +60 -0
- streamwright/spark/contracts/__init__.py +36 -0
- streamwright/spark/contracts/adapter.py +7 -0
- streamwright/spark/contracts/events.py +5 -0
- streamwright/spark/contracts/launcher.py +71 -0
- streamwright/spark/contracts/state.py +5 -0
- streamwright/spark/contracts/types.py +29 -0
- streamwright/spark/docs.py +19 -0
- streamwright/spark/engine/__init__.py +1 -0
- streamwright/spark/engine/executor.py +276 -0
- streamwright/spark/engine/runtime.py +154 -0
- streamwright/spark/engine/session.py +95 -0
- streamwright/spark/entrypoint.py +33 -0
- streamwright/spark/guide.md +105 -0
- streamwright/spark/io/__init__.py +1 -0
- streamwright/spark/io/capabilities.py +13 -0
- streamwright/spark/io/database_rules.py +79 -0
- streamwright/spark/io/delta.py +314 -0
- streamwright/spark/io/file_rules.py +201 -0
- streamwright/spark/io/files.py +222 -0
- streamwright/spark/io/jdbc.py +320 -0
- streamwright/spark/io/messaging.py +40 -0
- streamwright/spark/io/sql.py +56 -0
- streamwright/spark/job/__init__.py +20 -0
- streamwright/spark/job/check.py +20 -0
- streamwright/spark/job/reader.py +10 -0
- streamwright/spark/job/rules.py +325 -0
- streamwright/spark/job/runtime_rules.py +99 -0
- streamwright/spark/job/validate.py +80 -0
- streamwright/spark/launchers/__init__.py +1 -0
- streamwright/spark/launchers/common.py +83 -0
- streamwright/spark/launchers/emr_ec2.py +200 -0
- streamwright/spark/launchers/local.py +22 -0
- streamwright/spark/launchers/s3_jobs.py +25 -0
- streamwright/spark/launchers/standalone.py +58 -0
- streamwright/spark/plan/__init__.py +5 -0
- streamwright/spark/plan/analysis.py +166 -0
- streamwright/spark/plugin.py +37 -0
- streamwright/spark/runtime/__init__.py +1 -0
- streamwright/spark/runtime/definition.py +104 -0
- streamwright/spark/runtime/errors.py +11 -0
- streamwright/spark/runtime/lock.py +126 -0
- streamwright/spark/runtime/registry.py +101 -0
- streamwright/spark/schema.py +20 -0
- streamwright/spark/sdk/__init__.py +55 -0
- streamwright/spark/sdk/py.typed +0 -0
- streamwright/spark/storage/__init__.py +1 -0
- streamwright/spark/storage/exports.py +155 -0
- streamwright/spark/storage/ingest.py +175 -0
- streamwright/spark/storage/paths.py +60 -0
- streamwright_spark-0.1.0.dist-info/METADATA +107 -0
- streamwright_spark-0.1.0.dist-info/RECORD +67 -0
- streamwright_spark-0.1.0.dist-info/WHEEL +5 -0
- streamwright_spark-0.1.0.dist-info/entry_points.txt +14 -0
- streamwright_spark-0.1.0.dist-info/licenses/LICENSE +201 -0
- streamwright_spark-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""sw spark run: submit a job to its platform, or run the core batch loop on the driver."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
from importlib import import_module, metadata
|
|
9
|
+
|
|
10
|
+
from streamwright.core.jobs import BrokerCall, Runner
|
|
11
|
+
from streamwright.core.jobs.manifest import manifest_directory
|
|
12
|
+
from streamwright.core.jobs.refs import references
|
|
13
|
+
from streamwright.core.runtime.redact import Redactor
|
|
14
|
+
from streamwright.core.sdk.contract import InvalidConnectionError
|
|
15
|
+
from streamwright.spark.cli.common import connections_of, err, load_plugins
|
|
16
|
+
from streamwright.spark.cli.inputs import config_values, secret_values
|
|
17
|
+
from streamwright.spark.cli.state import open_store
|
|
18
|
+
from streamwright.spark.cli.submit_format import format_spark_submit
|
|
19
|
+
from streamwright.spark.contracts import LaunchError, LaunchFailureError
|
|
20
|
+
from streamwright.spark.io.messaging import open_broker
|
|
21
|
+
from streamwright.spark.job import check_job, load_job, resolve
|
|
22
|
+
from streamwright.spark.job.reader import resolve_job_location
|
|
23
|
+
from streamwright.spark.plan import build_plan, role_of
|
|
24
|
+
from streamwright.spark.runtime import lock as lockmod
|
|
25
|
+
|
|
26
|
+
__all__ = ["add_arguments", "run"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def add_arguments(parser):
|
|
30
|
+
parser.add_argument("--secrets", help="path to a secrets YAML file or JSON string of secret values")
|
|
31
|
+
mode = parser.add_mutually_exclusive_group()
|
|
32
|
+
mode.add_argument("--once", action="store_true", help="one batch (default)")
|
|
33
|
+
mode.add_argument("--drain", action="store_true", help="batches until the source is empty")
|
|
34
|
+
mode.add_argument(
|
|
35
|
+
"--available-now",
|
|
36
|
+
action="store_true",
|
|
37
|
+
help="batches until messages available at launch are processed (Trigger.AvailableNow)",
|
|
38
|
+
)
|
|
39
|
+
mode.add_argument("--follow", action="store_true", help="keep polling until interrupted")
|
|
40
|
+
parser.add_argument("--interval", default="10s", help="polling interval when running with --follow (default: 10s)")
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"--max-messages", type=int, default=500, help="maximum messages per batch (0: recover and relay only)"
|
|
43
|
+
)
|
|
44
|
+
parser.add_argument("--run-key", help="idempotency key for jobs without a broker")
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"--conf",
|
|
47
|
+
action="append",
|
|
48
|
+
default=[],
|
|
49
|
+
metavar="K=V",
|
|
50
|
+
help="Spark configuration override (e.g. --conf spark.executor.memory=4g)",
|
|
51
|
+
)
|
|
52
|
+
parser.add_argument(
|
|
53
|
+
"--log-level",
|
|
54
|
+
choices=["ALL", "DEBUG", "ERROR", "FATAL", "INFO", "OFF", "TRACE", "WARN"],
|
|
55
|
+
type=str.upper,
|
|
56
|
+
help="Spark logging level (e.g. --log-level ERROR or WARN)",
|
|
57
|
+
)
|
|
58
|
+
parser.add_argument("--lock", help="path to the lock file to verify dependencies and versions against")
|
|
59
|
+
parser.add_argument("--lock-required", action="store_true", help="require a valid lock file before running")
|
|
60
|
+
parser.add_argument(
|
|
61
|
+
"--no-init-state",
|
|
62
|
+
action="store_true",
|
|
63
|
+
help="do not create the bookkeeping tables (they must exist: `sw spark state init JOB`)",
|
|
64
|
+
)
|
|
65
|
+
parser.add_argument(
|
|
66
|
+
"--dry-run", action="store_true", help="resolve the job and print the launch submission without executing"
|
|
67
|
+
)
|
|
68
|
+
parser.add_argument(
|
|
69
|
+
"--detach", action="store_true", help="submit the job to the cluster launcher and exit without waiting"
|
|
70
|
+
)
|
|
71
|
+
parser.add_argument("--summary", action="store_true", help="print a summary of the executed batch upon completion")
|
|
72
|
+
parser.add_argument("--driver", action="store_true", help="internal: run on the cluster driver")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _seconds(text):
|
|
76
|
+
match = re.match(r"^([0-9]+)([smh])$", str(text))
|
|
77
|
+
if not match:
|
|
78
|
+
raise ValueError("invalid duration %r (expected e.g. 10s, 5m)" % (text,))
|
|
79
|
+
return int(match.group(1)) * {"s": 1, "m": 60, "h": 3600}[match.group(2)]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def run(args):
|
|
83
|
+
adapters, launchers, plugin_problems = load_plugins(args.allow_connector)
|
|
84
|
+
job = load_job(args.job, args.overlay)
|
|
85
|
+
problems, warnings = check_job(job, adapters, launchers)
|
|
86
|
+
problems += plugin_problems
|
|
87
|
+
for text in problems:
|
|
88
|
+
err("error: " + text)
|
|
89
|
+
if problems:
|
|
90
|
+
return 2
|
|
91
|
+
for text in warnings:
|
|
92
|
+
err("warning: " + text)
|
|
93
|
+
config = config_values(job, args)
|
|
94
|
+
platform = job["runtime"]["platform"]
|
|
95
|
+
lock = lockmod.load(args.lock) if args.lock else None
|
|
96
|
+
if args.lock_required and lock is None and platform["type"] != "local":
|
|
97
|
+
err("error: --lock-required: no lock given")
|
|
98
|
+
return 2
|
|
99
|
+
if args.dry_run:
|
|
100
|
+
try:
|
|
101
|
+
_preview_dry_run(job, adapters, args, lock=lock, platform=platform, launcher=launchers[platform["type"]])
|
|
102
|
+
except LaunchError as exc:
|
|
103
|
+
err("error: %s" % exc)
|
|
104
|
+
return 2
|
|
105
|
+
return 0
|
|
106
|
+
if platform["type"] != "local" and not args.driver:
|
|
107
|
+
placeholders = {name: "{{ secrets.%s }}" % name for name in ((job.get("spec") or {}).get("secrets") or {})}
|
|
108
|
+
submission, problems = resolve(job, config, placeholders)
|
|
109
|
+
if problems:
|
|
110
|
+
for text in problems:
|
|
111
|
+
err("error: " + text)
|
|
112
|
+
return 2
|
|
113
|
+
return _submit(submission, args, lock, submission["runtime"]["platform"], launchers[platform["type"]])
|
|
114
|
+
return _drive(job, adapters, args, config, lock)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _spark_conf(job, extra_conf):
|
|
118
|
+
conf = dict((job.get("runtime") or {}).get("spark", {}).get("conf") or {})
|
|
119
|
+
env = (job.get("runtime") or {}).get("env") or {}
|
|
120
|
+
secret_values_in_env = [
|
|
121
|
+
str(v) for k, v in env.items() if k.startswith("STREAMWRIGHT_SECRET_") and v and not references(v)
|
|
122
|
+
]
|
|
123
|
+
for k, v in env.items():
|
|
124
|
+
if k.startswith("STREAMWRIGHT_SECRET_") or (
|
|
125
|
+
isinstance(v, str) and any(ref.startswith("secrets.") for ref in references(v))
|
|
126
|
+
):
|
|
127
|
+
continue
|
|
128
|
+
conf.setdefault("spark.yarn.appMasterEnv.%s" % k, str(v))
|
|
129
|
+
conf.setdefault("spark.executorEnv.%s" % k, str(v))
|
|
130
|
+
conf.update(item.split("=", 1) for item in extra_conf if "=" in item)
|
|
131
|
+
redact = Redactor(secret_values_in_env) # the one secret-matching rule (the forms logs mask)
|
|
132
|
+
if any(redact(str(value)) != str(value) for value in conf.values()):
|
|
133
|
+
raise LaunchError("resolved secrets cannot be copied into Spark conf")
|
|
134
|
+
if any(
|
|
135
|
+
key.startswith(("spark.yarn.appMasterEnv.STREAMWRIGHT_SECRET_", "spark.executorEnv.STREAMWRIGHT_SECRET_"))
|
|
136
|
+
for key in conf
|
|
137
|
+
):
|
|
138
|
+
raise LaunchError("secret environment variables cannot be put in Spark conf; use runtime.secrets")
|
|
139
|
+
return conf
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _driver_flags(args):
|
|
143
|
+
flags = []
|
|
144
|
+
for path in args.overlay:
|
|
145
|
+
flags += ["--overlay", os.path.basename(path)]
|
|
146
|
+
for item in args.set:
|
|
147
|
+
flags += ["--set", item]
|
|
148
|
+
if args.config:
|
|
149
|
+
flags += ["--config", os.path.basename(args.config)]
|
|
150
|
+
flags += (
|
|
151
|
+
["--follow"]
|
|
152
|
+
if args.follow
|
|
153
|
+
else ["--available-now"]
|
|
154
|
+
if getattr(args, "available_now", False)
|
|
155
|
+
else ["--drain"]
|
|
156
|
+
if args.drain
|
|
157
|
+
else ["--once"]
|
|
158
|
+
)
|
|
159
|
+
flags += ["--interval", args.interval, "--max-messages", str(args.max_messages)]
|
|
160
|
+
if args.run_key:
|
|
161
|
+
flags += ["--run-key", args.run_key]
|
|
162
|
+
if args.no_init_state:
|
|
163
|
+
flags += ["--no-init-state"]
|
|
164
|
+
if getattr(args, "log_level", None):
|
|
165
|
+
flags += ["--log-level", args.log_level]
|
|
166
|
+
return flags
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _preview_dry_run(job, adapters, args, *, lock, platform, launcher):
|
|
170
|
+
print("Execution Plan:")
|
|
171
|
+
print(build_plan(job, adapters).describe())
|
|
172
|
+
print("\nLaunch Submission (platform: %s):" % platform["type"])
|
|
173
|
+
conf = _spark_conf(job, args.conf)
|
|
174
|
+
files = list(args.overlay) + ([args.config] if args.config else [])
|
|
175
|
+
preview = launcher.preview(
|
|
176
|
+
args.job, _driver_flags(args), conf, lock or {}, platform=platform, lock_path=args.lock, files=files
|
|
177
|
+
)
|
|
178
|
+
if preview.get("target"):
|
|
179
|
+
print(" Target Cluster: %s" % preview["target"])
|
|
180
|
+
if preview.get("command"):
|
|
181
|
+
print(" spark-submit command:\n " + format_spark_submit(preview["command"], indent=4))
|
|
182
|
+
if preview.get("step"):
|
|
183
|
+
print(" EMR Step JSON:\n " + json.dumps(preview["step"], indent=2).replace("\n", "\n "))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _submit(job, args, lock, platform, launcher):
|
|
187
|
+
try:
|
|
188
|
+
conf = _spark_conf(job, args.conf)
|
|
189
|
+
except LaunchError as exc:
|
|
190
|
+
err("error: %s" % exc)
|
|
191
|
+
return 2
|
|
192
|
+
if getattr(args, "secrets", None):
|
|
193
|
+
err("error: --secrets is not delivered to cluster drivers; use runtime.secrets")
|
|
194
|
+
return 2
|
|
195
|
+
flags = _driver_flags(args)
|
|
196
|
+
env = (lock or {}).get("python_env") or {}
|
|
197
|
+
archive = env.get("archive") if env.get("archive") and os.path.exists(env["archive"]) else None
|
|
198
|
+
job_path = os.path.join(args.job, "job.yaml") if os.path.isdir(args.job) else resolve_job_location(args.job)
|
|
199
|
+
try:
|
|
200
|
+
lock_problems = launcher.check_lock(lock or {}, platform)
|
|
201
|
+
for text in lock_problems:
|
|
202
|
+
err("error: " + text)
|
|
203
|
+
if lock_problems:
|
|
204
|
+
return 2
|
|
205
|
+
files = list(args.overlay) + ([args.config] if args.config else [])
|
|
206
|
+
names = [os.path.basename(path.rstrip("/")) for path in [job_path, args.lock, *files] if path]
|
|
207
|
+
if len(names) != len(set(names)):
|
|
208
|
+
raise LaunchError("job, lock, overlays and config must have distinct file names for cluster staging")
|
|
209
|
+
return launcher.submit(
|
|
210
|
+
job_path,
|
|
211
|
+
flags,
|
|
212
|
+
conf,
|
|
213
|
+
lock or {},
|
|
214
|
+
detach=args.detach,
|
|
215
|
+
lock_path=args.lock,
|
|
216
|
+
env_archive=archive,
|
|
217
|
+
platform=platform,
|
|
218
|
+
files=files,
|
|
219
|
+
)
|
|
220
|
+
except LaunchFailureError as exc:
|
|
221
|
+
err("error: %s" % exc)
|
|
222
|
+
return 1
|
|
223
|
+
except LaunchError as exc:
|
|
224
|
+
err("error: %s" % exc)
|
|
225
|
+
return 2
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _verify_lock(lock):
|
|
229
|
+
if lock is None:
|
|
230
|
+
return []
|
|
231
|
+
mismatches = lockmod.check_lock(lock, metadata.version("pyspark"))
|
|
232
|
+
if not mismatches:
|
|
233
|
+
os.environ["STREAMWRIGHT_LOCK_SHA"] = lockmod.fingerprint(lock)
|
|
234
|
+
return mismatches
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _configure_driver_conf(resolved, args, secrets):
|
|
238
|
+
conf = _spark_conf(resolved, getattr(args, "conf", None) or [])
|
|
239
|
+
redact = Redactor(secrets.values()) # the one secret-matching rule (the forms logs mask)
|
|
240
|
+
if any(redact(str(value)) != str(value) for value in conf.values()):
|
|
241
|
+
raise LaunchError("resolved secrets cannot be copied into Spark conf")
|
|
242
|
+
resolved.setdefault("runtime", {}).setdefault("spark", {})["conf"] = conf
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _broker_calls(resolved, connections, adapters):
|
|
246
|
+
brokers = {}
|
|
247
|
+
|
|
248
|
+
def broker_for(name):
|
|
249
|
+
if name not in brokers:
|
|
250
|
+
conn = connections[name]
|
|
251
|
+
adapter = adapters[conn.provider]
|
|
252
|
+
brokers[name] = open_broker(adapter, conn.auth)
|
|
253
|
+
return brokers[name]
|
|
254
|
+
|
|
255
|
+
consumer = None
|
|
256
|
+
publishers = {}
|
|
257
|
+
for step in resolved.get("steps") or []:
|
|
258
|
+
if role_of(resolved, step, adapters) == "consume":
|
|
259
|
+
consumer = BrokerCall(broker_for(step["connection"]), step["arguments"])
|
|
260
|
+
for output in resolved.get("outputs") or []:
|
|
261
|
+
if role_of(resolved, output, adapters) == "publish":
|
|
262
|
+
publishers[output["name"]] = BrokerCall(broker_for(output["connection"]), output["arguments"])
|
|
263
|
+
return consumer, publishers
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _drive(job, adapters, args, config, lock):
|
|
267
|
+
"""On the driver (local, or inside the cluster): verify, connect, run the batch loop. Needs Spark."""
|
|
268
|
+
executor_type = import_module("streamwright.spark.engine.executor").SparkExecutor
|
|
269
|
+
session_module = import_module("streamwright.spark.engine.session")
|
|
270
|
+
|
|
271
|
+
secrets = secret_values(job, args)
|
|
272
|
+
resolved, problems = resolve(job, config, secrets)
|
|
273
|
+
for text in problems:
|
|
274
|
+
err("error: " + text)
|
|
275
|
+
if problems:
|
|
276
|
+
return 2
|
|
277
|
+
for key, value in ((resolved.get("runtime") or {}).get("env") or {}).items():
|
|
278
|
+
os.environ[key] = str(value)
|
|
279
|
+
try:
|
|
280
|
+
connections = connections_of(resolved, adapters=adapters, declared=job, config=config, secrets=secrets)
|
|
281
|
+
except InvalidConnectionError as exc:
|
|
282
|
+
err("error: " + str(exc))
|
|
283
|
+
return 2
|
|
284
|
+
mismatches = _verify_lock(lock)
|
|
285
|
+
for text in mismatches:
|
|
286
|
+
err("error: lock: " + text)
|
|
287
|
+
if mismatches:
|
|
288
|
+
return 2
|
|
289
|
+
store = open_store(resolved, connections, adapters, manifest_directory(args.job))
|
|
290
|
+
if not args.no_init_state:
|
|
291
|
+
store.initialize()
|
|
292
|
+
try:
|
|
293
|
+
_configure_driver_conf(resolved, args, secrets)
|
|
294
|
+
except LaunchError as exc:
|
|
295
|
+
err("error: " + str(exc))
|
|
296
|
+
return 2
|
|
297
|
+
spark = session_module.build_session(
|
|
298
|
+
resolved,
|
|
299
|
+
connections,
|
|
300
|
+
adapters,
|
|
301
|
+
local=resolved["runtime"]["platform"]["type"] == "local",
|
|
302
|
+
log_level=getattr(args, "log_level", None),
|
|
303
|
+
)
|
|
304
|
+
app_id, driver_host = session_module.resolve_spark_metadata(spark)
|
|
305
|
+
executor = executor_type(
|
|
306
|
+
spark, resolved, connections, adapters, config=config, limits=resolved["runtime"].get("limits"),
|
|
307
|
+
state=store, declared_connections=(job.get("connections") or {}), secrets=secrets,
|
|
308
|
+
allowed_connectors=getattr(args, "allow_connector", None),
|
|
309
|
+
)
|
|
310
|
+
consumer, publishers = _broker_calls(resolved, connections, adapters)
|
|
311
|
+
|
|
312
|
+
def log(event, **fields):
|
|
313
|
+
print(
|
|
314
|
+
json.dumps(
|
|
315
|
+
dict(fields, event=event, ts=datetime.datetime.now(datetime.UTC).isoformat()), default=str
|
|
316
|
+
),
|
|
317
|
+
file=sys.stderr,
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
mode = (
|
|
321
|
+
"follow"
|
|
322
|
+
if args.follow
|
|
323
|
+
else "available_now"
|
|
324
|
+
if getattr(args, "available_now", False)
|
|
325
|
+
else "drain"
|
|
326
|
+
if args.drain
|
|
327
|
+
else "once"
|
|
328
|
+
)
|
|
329
|
+
runner = Runner(
|
|
330
|
+
resolved["name"],
|
|
331
|
+
store,
|
|
332
|
+
executor,
|
|
333
|
+
consumer,
|
|
334
|
+
publishers,
|
|
335
|
+
redact=Redactor(secrets.values()),
|
|
336
|
+
log=log,
|
|
337
|
+
engine_metadata={"app_id": app_id, "driver_host": driver_host},
|
|
338
|
+
)
|
|
339
|
+
return runner.run(mode, _seconds(args.interval), args.max_messages, args.run_key)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""sw spark state: inspect and repair bookkeeping."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
from streamwright.core.jobs.manifest import manifest_directory
|
|
8
|
+
from streamwright.core.jobs.state import open_sqlite
|
|
9
|
+
from streamwright.core.jobs.state import open_store as open_dialect
|
|
10
|
+
from streamwright.spark.cli.common import add_job_arguments, connections_of, err, load_plugins
|
|
11
|
+
from streamwright.spark.cli.inputs import config_values, secret_values
|
|
12
|
+
from streamwright.spark.job import check_job, load_job, resolve
|
|
13
|
+
|
|
14
|
+
__all__ = ["add_arguments", "open_store", "run"]
|
|
15
|
+
|
|
16
|
+
ACTIONS = ("init", "status", "batches", "show", "retry", "abandon", "purge", "lease")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def add_arguments(parser):
|
|
20
|
+
parser.add_argument("action", choices=ACTIONS, help="state action to perform: " + ", ".join(ACTIONS))
|
|
21
|
+
add_job_arguments(parser) # `state status JOB [ARG]`
|
|
22
|
+
parser.add_argument("arg", nargs="?", help="optional argument for action (e.g. batch_id for show/retry/abandon)")
|
|
23
|
+
parser.add_argument("--secrets", help="path to a secrets YAML file or JSON string of secret values")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def open_store(resolved, connections, adapters, base_dir):
|
|
27
|
+
"""The bookkeeping store named by runtime.state: a sqlite file, or the connection's own adapter."""
|
|
28
|
+
state = resolved["runtime"]["state"]
|
|
29
|
+
if "sqlite" in state:
|
|
30
|
+
path = state["sqlite"]
|
|
31
|
+
if resolved["runtime"]["platform"]["type"] != "local" and not os.path.isabs(path):
|
|
32
|
+
raise ValueError("remote jobs require an absolute runtime.state.sqlite path")
|
|
33
|
+
if path != ":memory:" and not os.path.isabs(path):
|
|
34
|
+
path = os.path.join(base_dir or os.getcwd(), path)
|
|
35
|
+
return open_sqlite(path)
|
|
36
|
+
conn = connections[state["connection"]]
|
|
37
|
+
connect, dialect = adapters[conn.provider].state_connection(conn.auth)
|
|
38
|
+
return open_dialect(connect, dialect, schema=state.get("schema", "streamwright"))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def run(args):
|
|
42
|
+
adapters, launchers, _ = load_plugins(args.allow_connector)
|
|
43
|
+
job = load_job(args.job, args.overlay)
|
|
44
|
+
problems, _ = check_job(job, adapters, launchers)
|
|
45
|
+
if problems:
|
|
46
|
+
for text in problems:
|
|
47
|
+
err("error: " + text)
|
|
48
|
+
return 2
|
|
49
|
+
config = config_values(job, args)
|
|
50
|
+
secrets = secret_values(job, args)
|
|
51
|
+
resolved, missing = resolve(job, config, secrets)
|
|
52
|
+
if missing:
|
|
53
|
+
for text in missing:
|
|
54
|
+
err("error: " + text)
|
|
55
|
+
return 2
|
|
56
|
+
connections = connections_of(resolved, adapters=adapters, declared=job, config=config, secrets=secrets)
|
|
57
|
+
store = open_store(resolved, connections, adapters, manifest_directory(args.job))
|
|
58
|
+
return _run_action(store, job["name"], args)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _list_batches(store, name, limit):
|
|
62
|
+
for batch in store.list_batches(name, limit):
|
|
63
|
+
metadata = batch.get("engine_metadata") or {}
|
|
64
|
+
meta = [
|
|
65
|
+
f"{label}={metadata[key]}"
|
|
66
|
+
for key, label in (("app_id", "app"), ("driver_host", "host"))
|
|
67
|
+
if metadata.get(key)
|
|
68
|
+
]
|
|
69
|
+
meta_str = (" " + " ".join(meta)) if meta else ""
|
|
70
|
+
print(
|
|
71
|
+
"%-24s %-10s seq=%d updated=%s%s %s"
|
|
72
|
+
% (
|
|
73
|
+
batch["batch_id"],
|
|
74
|
+
batch["status"],
|
|
75
|
+
batch["batch_sequence"],
|
|
76
|
+
batch["updated_at"],
|
|
77
|
+
meta_str,
|
|
78
|
+
(batch["error"] or "")[:80],
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _show_batch(store, name, batch_id):
|
|
84
|
+
batch = store.get_batch(name, batch_id)
|
|
85
|
+
if batch is None:
|
|
86
|
+
err("no such batch")
|
|
87
|
+
return 2
|
|
88
|
+
manifest = dict(batch["manifest"], messages="%d message(s)" % len(batch["manifest"].get("messages", [])))
|
|
89
|
+
print(
|
|
90
|
+
json.dumps(
|
|
91
|
+
dict(batch, manifest=manifest, receipts=store.get_receipts(name, batch["batch_id"])), indent=2, default=str
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
return 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _change_batch(store, name, args):
|
|
98
|
+
lease = store.acquire_lease(name, "cli:%d" % os.getpid(), 60)
|
|
99
|
+
try:
|
|
100
|
+
batch = store.get_batch(name, args.arg or "")
|
|
101
|
+
if batch is None:
|
|
102
|
+
err("no such batch")
|
|
103
|
+
return 2
|
|
104
|
+
if args.action == "retry":
|
|
105
|
+
store.retry_batch(lease, batch["batch_id"])
|
|
106
|
+
else:
|
|
107
|
+
store.record_batch(lease, batch["batch_id"], "abandoned", batch["manifest"])
|
|
108
|
+
print("%s: %s" % (batch["batch_id"], "queued for retry" if args.action == "retry" else "abandoned"))
|
|
109
|
+
finally:
|
|
110
|
+
store.release_lease(lease)
|
|
111
|
+
return 0
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _run_action(store, name, args):
|
|
115
|
+
if args.action == "init":
|
|
116
|
+
store.initialize()
|
|
117
|
+
print("initialized")
|
|
118
|
+
elif args.action in ("status", "batches"):
|
|
119
|
+
_list_batches(store, name, 1 if args.action == "status" else 20)
|
|
120
|
+
elif args.action == "show":
|
|
121
|
+
return _show_batch(store, name, args.arg or "")
|
|
122
|
+
elif args.action == "lease":
|
|
123
|
+
lease = store.get_lease(name)
|
|
124
|
+
if lease is None:
|
|
125
|
+
print("no lease recorded for %s" % name)
|
|
126
|
+
else:
|
|
127
|
+
print(json.dumps(lease, indent=2))
|
|
128
|
+
elif args.action in ("retry", "abandon"):
|
|
129
|
+
return _change_batch(store, name, args)
|
|
130
|
+
elif args.action == "purge":
|
|
131
|
+
cutoff = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=int(args.arg or 30))
|
|
132
|
+
print("purged %d batch(es)" % store.purge_batches(name, cutoff))
|
|
133
|
+
return 0
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Render a spark-submit argument list for dry-run previews."""
|
|
2
|
+
|
|
3
|
+
_BOOL_FLAGS = frozenset(
|
|
4
|
+
{
|
|
5
|
+
"--once",
|
|
6
|
+
"--drain",
|
|
7
|
+
"--available-now",
|
|
8
|
+
"--follow",
|
|
9
|
+
"--no-init-state",
|
|
10
|
+
"--dry-run",
|
|
11
|
+
"--detach",
|
|
12
|
+
"--summary",
|
|
13
|
+
"--driver",
|
|
14
|
+
"--supervise",
|
|
15
|
+
"--verbose",
|
|
16
|
+
}
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _quoted(text):
|
|
21
|
+
value = str(text)
|
|
22
|
+
return f'"{value}"' if " " in value and not (value.startswith('"') and value.endswith('"')) else value
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def format_spark_submit(arguments, indent=4):
|
|
26
|
+
"""Group submit flags with their values in a human-readable multiline command."""
|
|
27
|
+
lines = []
|
|
28
|
+
index = 0
|
|
29
|
+
if arguments and arguments[0] == "spark-submit":
|
|
30
|
+
lines.append("spark-submit")
|
|
31
|
+
index = 1
|
|
32
|
+
prefix = " " * indent
|
|
33
|
+
while index < len(arguments):
|
|
34
|
+
argument = str(arguments[index])
|
|
35
|
+
if argument.startswith("-") and argument not in _BOOL_FLAGS and index + 1 < len(arguments):
|
|
36
|
+
value = str(arguments[index + 1])
|
|
37
|
+
if not value.startswith("-"):
|
|
38
|
+
lines.append(f"{prefix}{argument} {_quoted(value)}")
|
|
39
|
+
index += 2
|
|
40
|
+
continue
|
|
41
|
+
lines.append(f"{prefix}{_quoted(argument)}")
|
|
42
|
+
index += 1
|
|
43
|
+
return " \\\n".join(lines)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""sw spark validate: check jobs without Spark, brokers or credentials."""
|
|
2
|
+
|
|
3
|
+
import contextlib
|
|
4
|
+
|
|
5
|
+
from streamwright.core.jobs.inputs import InputError, parse_set, read_config_values
|
|
6
|
+
from streamwright.spark.cli.common import err
|
|
7
|
+
from streamwright.spark.job import validate as validate_job
|
|
8
|
+
from streamwright.spark.plan import build_plan
|
|
9
|
+
from streamwright.spark.runtime.registry import Registry, RegistryError
|
|
10
|
+
|
|
11
|
+
__all__ = ["add_arguments", "run"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def add_arguments(parser):
|
|
15
|
+
parser.add_argument("--analyze", action="store_true", help="print the Spark actions and recomputation warnings")
|
|
16
|
+
parser.add_argument("--graph", action="store_true", help="print step dependencies")
|
|
17
|
+
parser.add_argument("--strict", action="store_true", help="warnings fail the check")
|
|
18
|
+
parser.add_argument("--lock", help="path to a lock file to verify compatibility against")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run(args):
|
|
22
|
+
code = 0
|
|
23
|
+
for path in args.job:
|
|
24
|
+
try:
|
|
25
|
+
config = read_config_values(args.config) if args.config else {}
|
|
26
|
+
config.update(parse_set(args.set))
|
|
27
|
+
except InputError as exc:
|
|
28
|
+
err("%s: %s" % (path, exc))
|
|
29
|
+
code = 2
|
|
30
|
+
continue
|
|
31
|
+
report = validate_job(
|
|
32
|
+
path, overlays=args.overlay, allow=args.allow_connector, lock=args.lock, strict=args.strict, config=config
|
|
33
|
+
)
|
|
34
|
+
job = report.job or {}
|
|
35
|
+
registry = Registry()
|
|
36
|
+
adapters = {}
|
|
37
|
+
for connection in (job.get("connections") or {}).values():
|
|
38
|
+
if not isinstance(connection, dict):
|
|
39
|
+
continue
|
|
40
|
+
provider = connection.get("provider")
|
|
41
|
+
if provider and provider not in adapters:
|
|
42
|
+
with contextlib.suppress(RegistryError):
|
|
43
|
+
adapters[provider] = registry.load(provider, allow=args.allow_connector)
|
|
44
|
+
if report.valid and (args.analyze or args.graph):
|
|
45
|
+
plan = build_plan(job, adapters)
|
|
46
|
+
if args.analyze:
|
|
47
|
+
print(plan.describe())
|
|
48
|
+
if args.graph:
|
|
49
|
+
for node in plan.nodes:
|
|
50
|
+
print("%-8s %-22s <- %s" % (node.kind, node.name, ", ".join(node.uses) or "-"))
|
|
51
|
+
for issue in report.issues:
|
|
52
|
+
err("%s: %s: %s" % (path, issue.severity, issue.message))
|
|
53
|
+
if report.valid and args.analyze:
|
|
54
|
+
for warning in build_plan(job, adapters).warnings:
|
|
55
|
+
err("%s: warning: %s" % (path, warning))
|
|
56
|
+
if not report.valid:
|
|
57
|
+
code = 2
|
|
58
|
+
else:
|
|
59
|
+
print("%s: ok" % path)
|
|
60
|
+
return code
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Spark adapter errors and launcher contracts; job types come from core."""
|
|
2
|
+
|
|
3
|
+
from streamwright.spark.contracts.adapter import AdapterError
|
|
4
|
+
from streamwright.spark.contracts.events import Broker
|
|
5
|
+
from streamwright.spark.contracts.launcher import LAUNCHER_OWNED_CONF, Launcher, LaunchError, LaunchFailureError
|
|
6
|
+
from streamwright.spark.contracts.state import StateStore
|
|
7
|
+
from streamwright.spark.contracts.types import (
|
|
8
|
+
CONTRACT_VERSION,
|
|
9
|
+
BatchContext,
|
|
10
|
+
Connection,
|
|
11
|
+
DataFrame,
|
|
12
|
+
Lease,
|
|
13
|
+
Message,
|
|
14
|
+
Observation,
|
|
15
|
+
SparkSession,
|
|
16
|
+
WriteReceipt,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"CONTRACT_VERSION",
|
|
21
|
+
"LAUNCHER_OWNED_CONF",
|
|
22
|
+
"AdapterError",
|
|
23
|
+
"BatchContext",
|
|
24
|
+
"Broker",
|
|
25
|
+
"Connection",
|
|
26
|
+
"DataFrame",
|
|
27
|
+
"LaunchError",
|
|
28
|
+
"LaunchFailureError",
|
|
29
|
+
"Launcher",
|
|
30
|
+
"Lease",
|
|
31
|
+
"Message",
|
|
32
|
+
"Observation",
|
|
33
|
+
"SparkSession",
|
|
34
|
+
"StateStore",
|
|
35
|
+
"WriteReceipt",
|
|
36
|
+
]
|