gluekit 1.0.1.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gluekit/__init__.py +7 -0
- gluekit/app.py +0 -0
- gluekit/cli.py +64 -0
- gluekit/commands/__init__.py +1 -0
- gluekit/commands/add.py +455 -0
- gluekit/commands/build.py +816 -0
- gluekit/commands/checkout.py +114 -0
- gluekit/commands/clone.py +516 -0
- gluekit/commands/config_commands.py +180 -0
- gluekit/commands/constants.py +47 -0
- gluekit/commands/convert.py +336 -0
- gluekit/commands/edit.py +1104 -0
- gluekit/commands/helpers.py +1068 -0
- gluekit/commands/init.py +798 -0
- gluekit/commands/list.py +16 -0
- gluekit/commands/local_commands.py +680 -0
- gluekit/commands/pull.py +374 -0
- gluekit/commands/push.py +251 -0
- gluekit/commands/remove.py +161 -0
- gluekit/commands/run.py +126 -0
- gluekit/commands/status.py +97 -0
- gluekit/commands/sync.py +97 -0
- gluekit/commands/update.py +104 -0
- gluekit/job_mgmt/__init__.py +0 -0
- gluekit/job_mgmt/glue_jobs.py +1323 -0
- gluekit/job_mgmt/magics.py +122 -0
- gluekit/job_mgmt/resources/__init__.py +0 -0
- gluekit/job_mgmt/resources/glue_job_schema.json +40341 -0
- gluekit/job_mgmt/resources/magic_map.json +83 -0
- gluekit/job_mgmt/schema.py +165 -0
- gluekit/local/__init__.py +6 -0
- gluekit/local/awsglue/__init__.py +1 -0
- gluekit/local/awsglue/context.py +30 -0
- gluekit/local/awsglue/job.py +9 -0
- gluekit/local/awsglue/utils.py +17 -0
- gluekit/local/local.py +434 -0
- gluekit/local/local_fixtures.py +337 -0
- gluekit/local/pyspark/__init__.py +7 -0
- gluekit/local/pyspark/context.py +31 -0
- gluekit/local/pyspark/sql/__init__.py +6 -0
- gluekit/local/pyspark/sql/session.py +29 -0
- gluekit-1.0.1.dev1.dist-info/METADATA +1176 -0
- gluekit-1.0.1.dev1.dist-info/RECORD +46 -0
- gluekit-1.0.1.dev1.dist-info/WHEEL +5 -0
- gluekit-1.0.1.dev1.dist-info/entry_points.txt +2 -0
- gluekit-1.0.1.dev1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,816 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import shutil
|
|
5
|
+
import tarfile
|
|
6
|
+
import tomllib
|
|
7
|
+
import zipfile
|
|
8
|
+
from email.parser import Parser
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from tempfile import TemporaryDirectory
|
|
11
|
+
from typing import Any, Optional
|
|
12
|
+
|
|
13
|
+
import typer
|
|
14
|
+
from slugify import slugify
|
|
15
|
+
|
|
16
|
+
from ..job_mgmt.glue_jobs import (
|
|
17
|
+
_resolve_notebook_path,
|
|
18
|
+
upload_glue_job_files_from_config,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
from .constants import PUSH_COMPONENT_ALIASES, PUSH_COMPONENTS
|
|
22
|
+
from .helpers import (
|
|
23
|
+
_apply_saved_params_to_config_path,
|
|
24
|
+
_examples_epilog,
|
|
25
|
+
_find_workspace_root,
|
|
26
|
+
_get_checked_out_jobs,
|
|
27
|
+
_get_checked_out_profile,
|
|
28
|
+
_load_config_index,
|
|
29
|
+
_load_glue_set_store,
|
|
30
|
+
_raise_missing_local_config,
|
|
31
|
+
_resolve_single_job_name,
|
|
32
|
+
_save_checkout_local_paths,
|
|
33
|
+
_set_saved_scope,
|
|
34
|
+
_write_config_changes,
|
|
35
|
+
run_command,
|
|
36
|
+
)
|
|
37
|
+
from ..cli import app
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def run_uv(
|
|
41
|
+
*args: str,
|
|
42
|
+
cwd: Optional[Path] = None,
|
|
43
|
+
dry_run: bool = False,
|
|
44
|
+
verbose: bool = False,
|
|
45
|
+
) -> None:
|
|
46
|
+
run_command(["uv", *args], cwd=cwd, dry_run=dry_run, verbose=verbose)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def run_python_build(
|
|
50
|
+
cwd: Optional[Path] = None,
|
|
51
|
+
out_dir: Optional[Path] = None,
|
|
52
|
+
dry_run: bool = False,
|
|
53
|
+
verbose: bool = False,
|
|
54
|
+
) -> None:
|
|
55
|
+
command = ["python", "-m", "build"]
|
|
56
|
+
if out_dir is not None:
|
|
57
|
+
command.extend(["--outdir", str(out_dir)])
|
|
58
|
+
run_command(command, cwd=cwd, dry_run=dry_run, verbose=verbose)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _zip_path_for_sdist(sdist_path: Path) -> Path:
|
|
62
|
+
return sdist_path.with_name(f"{sdist_path.name[:-7]}.zip")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _package_name_from_sdist_contents(temp_root: Path, fallback: str) -> str:
|
|
66
|
+
extracted_entries = [entry for entry in temp_root.iterdir() if entry.exists()]
|
|
67
|
+
package_root = extracted_entries[0] if len(extracted_entries) == 1 else temp_root
|
|
68
|
+
|
|
69
|
+
pkg_info_candidates = [package_root / "PKG-INFO"]
|
|
70
|
+
pkg_info_candidates.extend(package_root.glob("*.dist-info/PKG-INFO"))
|
|
71
|
+
|
|
72
|
+
for pkg_info_path in pkg_info_candidates:
|
|
73
|
+
if not pkg_info_path.exists() or not pkg_info_path.is_file():
|
|
74
|
+
continue
|
|
75
|
+
message = Parser().parsestr(
|
|
76
|
+
pkg_info_path.read_text(encoding="utf-8", errors="ignore")
|
|
77
|
+
)
|
|
78
|
+
package_name = (message.get("Name") or "").strip()
|
|
79
|
+
if package_name:
|
|
80
|
+
return package_name
|
|
81
|
+
|
|
82
|
+
for entry in extracted_entries:
|
|
83
|
+
if entry.is_dir() and entry.name != fallback:
|
|
84
|
+
return entry.name
|
|
85
|
+
|
|
86
|
+
return fallback
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _resolve_sdist_package_dir(temp_root: Path, package_name: str) -> Path | None:
|
|
90
|
+
extracted_entries = [entry for entry in temp_root.iterdir() if entry.exists()]
|
|
91
|
+
package_root = extracted_entries[0] if len(extracted_entries) == 1 else temp_root
|
|
92
|
+
src_dir = package_root / "src"
|
|
93
|
+
if not src_dir.is_dir():
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
package_dirs = sorted(
|
|
97
|
+
entry
|
|
98
|
+
for entry in src_dir.iterdir()
|
|
99
|
+
if entry.is_dir() and (entry / "__init__.py").is_file()
|
|
100
|
+
)
|
|
101
|
+
if not package_dirs:
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
normalized_package_name = package_name.replace("-", "_")
|
|
105
|
+
for package_dir in package_dirs:
|
|
106
|
+
if package_dir.name == normalized_package_name:
|
|
107
|
+
return package_dir
|
|
108
|
+
|
|
109
|
+
if len(package_dirs) == 1:
|
|
110
|
+
return package_dirs[0]
|
|
111
|
+
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _write_package_zip(package_dir: Path, zip_path: Path) -> None:
|
|
116
|
+
zip_path.unlink(missing_ok=True)
|
|
117
|
+
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
118
|
+
for file_path in sorted(
|
|
119
|
+
path for path in package_dir.rglob("*") if path.is_file()
|
|
120
|
+
):
|
|
121
|
+
archive.write(file_path, arcname=file_path.relative_to(package_dir.parent))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _build_zip_artifacts_from_sdists(
|
|
125
|
+
dist_dir: Path,
|
|
126
|
+
dry_run: bool = False,
|
|
127
|
+
no_version_suffix: bool = False,
|
|
128
|
+
) -> None:
|
|
129
|
+
sdist_paths = sorted(path for path in dist_dir.glob("*.tar.gz") if path.is_file())
|
|
130
|
+
if not sdist_paths:
|
|
131
|
+
if dry_run:
|
|
132
|
+
typer.echo(
|
|
133
|
+
f"Would create zip artifact(s) from source distributions in: {dist_dir}"
|
|
134
|
+
)
|
|
135
|
+
else:
|
|
136
|
+
typer.echo(f"No source distribution archives found in: {dist_dir}")
|
|
137
|
+
return
|
|
138
|
+
|
|
139
|
+
for sdist_path in sdist_paths:
|
|
140
|
+
versioned_zip_path = _zip_path_for_sdist(sdist_path)
|
|
141
|
+
fallback_name = sdist_path.name[:-7]
|
|
142
|
+
with TemporaryDirectory() as temp_dir:
|
|
143
|
+
temp_root = Path(temp_dir)
|
|
144
|
+
with tarfile.open(sdist_path, "r:gz") as archive:
|
|
145
|
+
try:
|
|
146
|
+
archive.extractall(temp_root, filter="data")
|
|
147
|
+
except TypeError:
|
|
148
|
+
archive.extractall(temp_root)
|
|
149
|
+
extracted_entries = list(temp_root.iterdir())
|
|
150
|
+
package_name = _package_name_from_sdist_contents(temp_root, fallback_name)
|
|
151
|
+
package_dir = _resolve_sdist_package_dir(temp_root, package_name)
|
|
152
|
+
zip_package_name = (
|
|
153
|
+
package_dir.name if package_dir is not None else package_name
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
if dry_run:
|
|
157
|
+
if no_version_suffix:
|
|
158
|
+
typer.echo(
|
|
159
|
+
f"Would create zip artifact: {dist_dir / f'{zip_package_name}.zip'} from {sdist_path}"
|
|
160
|
+
)
|
|
161
|
+
else:
|
|
162
|
+
typer.echo(
|
|
163
|
+
f"Would create zip artifact: {versioned_zip_path} from {sdist_path}"
|
|
164
|
+
)
|
|
165
|
+
typer.echo(
|
|
166
|
+
"Would create unversioned zip artifact: "
|
|
167
|
+
f"{dist_dir / f'{zip_package_name}.zip'} from {versioned_zip_path}"
|
|
168
|
+
)
|
|
169
|
+
continue
|
|
170
|
+
|
|
171
|
+
zip_path = (
|
|
172
|
+
dist_dir / f"{zip_package_name}.zip"
|
|
173
|
+
if no_version_suffix
|
|
174
|
+
else versioned_zip_path
|
|
175
|
+
)
|
|
176
|
+
if package_dir is not None:
|
|
177
|
+
_write_package_zip(package_dir, zip_path)
|
|
178
|
+
else:
|
|
179
|
+
zip_path.unlink(missing_ok=True)
|
|
180
|
+
archive_base_name = str(zip_path.with_suffix(""))
|
|
181
|
+
if len(extracted_entries) == 1:
|
|
182
|
+
shutil.make_archive(
|
|
183
|
+
archive_base_name,
|
|
184
|
+
"zip",
|
|
185
|
+
root_dir=temp_root,
|
|
186
|
+
base_dir=extracted_entries[0].name,
|
|
187
|
+
)
|
|
188
|
+
else:
|
|
189
|
+
shutil.make_archive(archive_base_name, "zip", root_dir=temp_root)
|
|
190
|
+
|
|
191
|
+
unversioned_zip_path = zip_path
|
|
192
|
+
if not no_version_suffix:
|
|
193
|
+
unversioned_zip_path = dist_dir / f"{zip_package_name}.zip"
|
|
194
|
+
if unversioned_zip_path != zip_path:
|
|
195
|
+
shutil.copy2(zip_path, unversioned_zip_path)
|
|
196
|
+
else:
|
|
197
|
+
unversioned_zip_path = zip_path
|
|
198
|
+
|
|
199
|
+
if dry_run:
|
|
200
|
+
continue
|
|
201
|
+
|
|
202
|
+
typer.echo(f"Created zip artifact: {zip_path}")
|
|
203
|
+
if not no_version_suffix and unversioned_zip_path != zip_path:
|
|
204
|
+
typer.echo(f"Created unversioned zip artifact: {unversioned_zip_path}")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _resolve_build_output_dir(cwd: Optional[Path], out_dir: Optional[Path]) -> Path:
|
|
208
|
+
base_dir = cwd or Path.cwd()
|
|
209
|
+
if out_dir is None:
|
|
210
|
+
return base_dir / "dist"
|
|
211
|
+
if out_dir.is_absolute():
|
|
212
|
+
return out_dir
|
|
213
|
+
return base_dir / out_dir
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _parse_version_parts(version: str) -> tuple[int, int, int]:
|
|
217
|
+
parts = version.split(".")
|
|
218
|
+
if len(parts) != 3 or not all(part.isdigit() for part in parts):
|
|
219
|
+
raise typer.BadParameter(
|
|
220
|
+
f"Cannot bump non-semver project version: {version}. "
|
|
221
|
+
"Use --bump-version with an explicit X.Y.Z value."
|
|
222
|
+
)
|
|
223
|
+
return int(parts[0]), int(parts[1]), int(parts[2])
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _next_project_version(current_version: str, bump: str) -> str:
|
|
227
|
+
normalized = bump.strip().lower()
|
|
228
|
+
if normalized in {"major", "minor", "patch"}:
|
|
229
|
+
major, minor, patch = _parse_version_parts(current_version)
|
|
230
|
+
if normalized == "major":
|
|
231
|
+
return f"{major + 1}.0.0"
|
|
232
|
+
if normalized == "minor":
|
|
233
|
+
return f"{major}.{minor + 1}.0"
|
|
234
|
+
return f"{major}.{minor}.{patch + 1}"
|
|
235
|
+
|
|
236
|
+
if re.match(r"^\d+\.\d+\.\d+(?:[a-zA-Z0-9.+_-]+)?$", bump.strip()):
|
|
237
|
+
return bump.strip()
|
|
238
|
+
raise typer.BadParameter(
|
|
239
|
+
"Invalid --bump-version. Use major, minor, patch, or an explicit X.Y.Z version."
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _bump_project_version(
|
|
244
|
+
pyproject_path: Path,
|
|
245
|
+
bump: str,
|
|
246
|
+
*,
|
|
247
|
+
dry_run: bool = False,
|
|
248
|
+
) -> tuple[str, str]:
|
|
249
|
+
if not pyproject_path.exists():
|
|
250
|
+
raise typer.BadParameter(f"pyproject.toml not found: {pyproject_path}")
|
|
251
|
+
text = pyproject_path.read_text(encoding="utf-8")
|
|
252
|
+
project_match = re.search(r"(?ms)^\[project\]\s*(?P<body>.*?)(?=^\[|\Z)", text)
|
|
253
|
+
if not project_match:
|
|
254
|
+
raise typer.BadParameter("pyproject.toml is missing a [project] section.")
|
|
255
|
+
version_match = re.search(
|
|
256
|
+
r'(?m)^(?P<prefix>\s*version\s*=\s*")(?P<version>[^"]+)(?P<suffix>"\s*)$',
|
|
257
|
+
project_match.group("body"),
|
|
258
|
+
)
|
|
259
|
+
if not version_match:
|
|
260
|
+
raise typer.BadParameter("pyproject.toml [project] section is missing version.")
|
|
261
|
+
|
|
262
|
+
current_version = version_match.group("version")
|
|
263
|
+
next_version = _next_project_version(current_version, bump)
|
|
264
|
+
if current_version == next_version:
|
|
265
|
+
typer.echo(f"Project version already {next_version}.")
|
|
266
|
+
return current_version, next_version
|
|
267
|
+
|
|
268
|
+
if dry_run:
|
|
269
|
+
typer.echo(f"Would bump project version: {current_version} -> {next_version}")
|
|
270
|
+
return current_version, next_version
|
|
271
|
+
|
|
272
|
+
section_start = project_match.start("body")
|
|
273
|
+
version_start = section_start + version_match.start("version")
|
|
274
|
+
version_end = section_start + version_match.end("version")
|
|
275
|
+
pyproject_path.write_text(
|
|
276
|
+
f"{text[:version_start]}{next_version}{text[version_end:]}",
|
|
277
|
+
encoding="utf-8",
|
|
278
|
+
)
|
|
279
|
+
typer.echo(f"Bumped project version: {current_version} -> {next_version}")
|
|
280
|
+
return current_version, next_version
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _read_project_name_version(pyproject_path: Path) -> tuple[str, str]:
|
|
284
|
+
if not pyproject_path.exists():
|
|
285
|
+
raise typer.BadParameter(f"pyproject.toml not found: {pyproject_path}")
|
|
286
|
+
with pyproject_path.open("rb") as handle:
|
|
287
|
+
pyproject = tomllib.load(handle)
|
|
288
|
+
project = pyproject.get("project")
|
|
289
|
+
if not isinstance(project, dict):
|
|
290
|
+
raise typer.BadParameter("pyproject.toml is missing a [project] section.")
|
|
291
|
+
name = project.get("name")
|
|
292
|
+
version = project.get("version")
|
|
293
|
+
if not isinstance(name, str) or not name.strip():
|
|
294
|
+
raise typer.BadParameter("pyproject.toml [project] section is missing name.")
|
|
295
|
+
if not isinstance(version, str) or not version.strip():
|
|
296
|
+
raise typer.BadParameter("pyproject.toml [project] section is missing version.")
|
|
297
|
+
return name.strip(), version.strip()
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _wheel_distribution_name(project_name: str) -> str:
|
|
301
|
+
return re.sub(r"[-_.]+", "_", project_name).lower()
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _workspace_relative_path(path: Path, workspace_root: Path) -> Path:
|
|
305
|
+
try:
|
|
306
|
+
return path.resolve().relative_to(workspace_root.resolve())
|
|
307
|
+
except ValueError:
|
|
308
|
+
return path
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _resolve_built_package_wheel_path(
|
|
312
|
+
*,
|
|
313
|
+
workspace_root: Path,
|
|
314
|
+
out_dir: Optional[Path],
|
|
315
|
+
) -> Path:
|
|
316
|
+
project_name, project_version = _read_project_name_version(
|
|
317
|
+
workspace_root / "pyproject.toml"
|
|
318
|
+
)
|
|
319
|
+
dist_dir = _resolve_build_output_dir(workspace_root, out_dir)
|
|
320
|
+
if not dist_dir.exists():
|
|
321
|
+
raise typer.BadParameter(f"Package wheel directory not found: {dist_dir}")
|
|
322
|
+
if not dist_dir.is_dir():
|
|
323
|
+
raise typer.BadParameter(f"Package wheel path is not a directory: {dist_dir}")
|
|
324
|
+
|
|
325
|
+
wheel_prefix = f"{_wheel_distribution_name(project_name)}-{project_version}-"
|
|
326
|
+
matching_wheels = sorted(
|
|
327
|
+
path
|
|
328
|
+
for path in dist_dir.glob("*.whl")
|
|
329
|
+
if path.name.lower().startswith(wheel_prefix.lower())
|
|
330
|
+
)
|
|
331
|
+
if len(matching_wheels) == 1:
|
|
332
|
+
return _workspace_relative_path(matching_wheels[0], workspace_root)
|
|
333
|
+
|
|
334
|
+
all_wheels = sorted(path.as_posix() for path in dist_dir.glob("*.whl"))
|
|
335
|
+
if not matching_wheels:
|
|
336
|
+
rendered = ", ".join(all_wheels) or "(none)"
|
|
337
|
+
raise typer.BadParameter(
|
|
338
|
+
f"No package wheel found for {project_name}=={project_version} in {dist_dir}. "
|
|
339
|
+
f"Available wheels: {rendered}"
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
rendered = ", ".join(path.as_posix() for path in matching_wheels)
|
|
343
|
+
raise typer.BadParameter(
|
|
344
|
+
f"Multiple package wheels found for {project_name}=={project_version}: {rendered}."
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _apply_package_wheel_to_job(
|
|
349
|
+
*,
|
|
350
|
+
job_name: Optional[str],
|
|
351
|
+
config_dir: Path,
|
|
352
|
+
profile: Optional[str],
|
|
353
|
+
dry_run: bool,
|
|
354
|
+
package_wheel_path: Optional[Path] = None,
|
|
355
|
+
) -> Optional[str]:
|
|
356
|
+
from .add import (
|
|
357
|
+
_resolve_package_wheel_path,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
resolved_job_name = _resolve_single_job_name(job_name, "glue build --package-whl")
|
|
361
|
+
config_index = _load_config_index(config_dir)
|
|
362
|
+
config_entry = config_index.get(resolved_job_name)
|
|
363
|
+
if not config_entry:
|
|
364
|
+
_raise_missing_local_config(resolved_job_name, config_dir, "glue build")
|
|
365
|
+
|
|
366
|
+
config_path: Path = config_entry["config_path"]
|
|
367
|
+
config_data = config_entry["config"]
|
|
368
|
+
_apply_saved_params_to_config_path(
|
|
369
|
+
config_path=config_path,
|
|
370
|
+
config_data=config_data,
|
|
371
|
+
job_name=resolved_job_name,
|
|
372
|
+
profile=profile,
|
|
373
|
+
dry_run=dry_run,
|
|
374
|
+
)
|
|
375
|
+
try:
|
|
376
|
+
wheel_path = package_wheel_path or _resolve_package_wheel_path()
|
|
377
|
+
except typer.BadParameter:
|
|
378
|
+
if not dry_run:
|
|
379
|
+
raise
|
|
380
|
+
typer.echo(f"Would update package wheel for {resolved_job_name}: dist/*.whl")
|
|
381
|
+
return resolved_job_name
|
|
382
|
+
|
|
383
|
+
_apply_package_wheel_to_config(
|
|
384
|
+
job_name=resolved_job_name,
|
|
385
|
+
config_path=config_path,
|
|
386
|
+
config_data=config_data,
|
|
387
|
+
package_wheel_path=wheel_path,
|
|
388
|
+
profile=profile,
|
|
389
|
+
dry_run=dry_run,
|
|
390
|
+
)
|
|
391
|
+
return resolved_job_name
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _apply_package_wheel_to_config(
|
|
395
|
+
*,
|
|
396
|
+
job_name: str,
|
|
397
|
+
config_path: Path,
|
|
398
|
+
config_data: dict[str, Any],
|
|
399
|
+
package_wheel_path: Path,
|
|
400
|
+
profile: Optional[str],
|
|
401
|
+
dry_run: bool,
|
|
402
|
+
) -> None:
|
|
403
|
+
from .add import (
|
|
404
|
+
_apply_add_mutations,
|
|
405
|
+
_remove_tracked_package_wheels,
|
|
406
|
+
_sync_existing_notebook_config,
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
removal_updates = _remove_tracked_package_wheels(config_data)
|
|
410
|
+
replacement_updates = _remove_existing_additional_python_modules(config_data)
|
|
411
|
+
updates = removal_updates + replacement_updates + _apply_add_mutations(
|
|
412
|
+
config_data=config_data,
|
|
413
|
+
job_name=job_name,
|
|
414
|
+
items=[package_wheel_path.as_posix()],
|
|
415
|
+
as_path=True,
|
|
416
|
+
as_pypi=False,
|
|
417
|
+
)
|
|
418
|
+
_write_config_changes(config_path, config_data, updates, dry_run=dry_run)
|
|
419
|
+
if updates:
|
|
420
|
+
_sync_existing_notebook_config(
|
|
421
|
+
config_data=config_data,
|
|
422
|
+
job_name=job_name,
|
|
423
|
+
dry_run=dry_run,
|
|
424
|
+
)
|
|
425
|
+
_save_package_wheel_mapping_param(
|
|
426
|
+
job_name=job_name,
|
|
427
|
+
config_data=config_data,
|
|
428
|
+
package_wheel_path=package_wheel_path,
|
|
429
|
+
profile=profile,
|
|
430
|
+
dry_run=dry_run,
|
|
431
|
+
)
|
|
432
|
+
_sync_checked_out_build_artifacts(
|
|
433
|
+
job_name=job_name,
|
|
434
|
+
config_path=config_path,
|
|
435
|
+
config_data=config_data,
|
|
436
|
+
dry_run=dry_run,
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _remove_existing_additional_python_modules(config_data: dict[str, Any]) -> list[str]:
|
|
441
|
+
default_args = config_data.setdefault("DefaultArguments", {})
|
|
442
|
+
if not isinstance(default_args, dict):
|
|
443
|
+
raise typer.BadParameter("DefaultArguments must be an object.")
|
|
444
|
+
|
|
445
|
+
existing_modules = default_args.pop("--additional-python-modules", None)
|
|
446
|
+
if existing_modules:
|
|
447
|
+
return [
|
|
448
|
+
"Replaced --additional-python-modules entries: "
|
|
449
|
+
f"{existing_modules}"
|
|
450
|
+
]
|
|
451
|
+
return []
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _save_package_wheel_mapping_param(
|
|
455
|
+
*,
|
|
456
|
+
job_name: str,
|
|
457
|
+
config_data: dict[str, Any],
|
|
458
|
+
package_wheel_path: Path,
|
|
459
|
+
profile: Optional[str],
|
|
460
|
+
dry_run: bool,
|
|
461
|
+
) -> None:
|
|
462
|
+
local_path = package_wheel_path.as_posix()
|
|
463
|
+
sc = config_data.get("SourceControlDetails", {})
|
|
464
|
+
additional_files = sc.get("AdditionalPythonFiles") if isinstance(sc, dict) else None
|
|
465
|
+
if not isinstance(additional_files, list):
|
|
466
|
+
return
|
|
467
|
+
|
|
468
|
+
remote_path = None
|
|
469
|
+
for entry in additional_files:
|
|
470
|
+
if not isinstance(entry, dict):
|
|
471
|
+
continue
|
|
472
|
+
if entry.get("LocalPath") == local_path and isinstance(entry.get("S3Path"), str):
|
|
473
|
+
remote_path = entry["S3Path"]
|
|
474
|
+
break
|
|
475
|
+
if not remote_path:
|
|
476
|
+
return
|
|
477
|
+
|
|
478
|
+
mapping = {"local": local_path, "remote": remote_path}
|
|
479
|
+
key = "DefaultArguments.--additional-python-modules"
|
|
480
|
+
if dry_run:
|
|
481
|
+
profile_label = f" for profile {profile}" if profile else ""
|
|
482
|
+
typer.echo(f"Would save wheel mapping{profile_label}: {job_name} {mapping}")
|
|
483
|
+
return
|
|
484
|
+
_set_saved_scope({key: mapping}, job_name, False, profile=profile)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _automation_build_enabled(key: str) -> bool:
|
|
488
|
+
store = _load_glue_set_store()
|
|
489
|
+
automation = store.get("automation", {})
|
|
490
|
+
if not isinstance(automation, dict):
|
|
491
|
+
return False
|
|
492
|
+
build = automation.get("build", {})
|
|
493
|
+
if not isinstance(build, dict):
|
|
494
|
+
return False
|
|
495
|
+
return build.get(key) is True
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _sync_checked_out_build_artifacts(
|
|
499
|
+
*,
|
|
500
|
+
job_name: str,
|
|
501
|
+
config_path: Path,
|
|
502
|
+
config_data: dict[str, Any],
|
|
503
|
+
dry_run: bool,
|
|
504
|
+
) -> None:
|
|
505
|
+
from .convert import _update_notebook_config_cell, _update_script_config_cell
|
|
506
|
+
|
|
507
|
+
_save_checkout_local_paths(
|
|
508
|
+
job_name=job_name,
|
|
509
|
+
config_path=config_path,
|
|
510
|
+
config_data=config_data,
|
|
511
|
+
dry_run=dry_run,
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
sc = config_data.get("SourceControlDetails", {})
|
|
515
|
+
if not isinstance(sc, dict):
|
|
516
|
+
sc = {}
|
|
517
|
+
script_path = Path(
|
|
518
|
+
sc.get("ScriptLocation")
|
|
519
|
+
or sc.get("LocalPath")
|
|
520
|
+
or f"glue/scripts/{slugify(job_name)}.py"
|
|
521
|
+
)
|
|
522
|
+
notebook_value = sc.get("NotebookLocation") or sc.get("NotebookPath")
|
|
523
|
+
notebook_path = (
|
|
524
|
+
Path(notebook_value)
|
|
525
|
+
if isinstance(notebook_value, str)
|
|
526
|
+
else Path(_resolve_notebook_path(script_path))
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
_update_script_config_cell(script_path, config_data, dry_run)
|
|
530
|
+
if notebook_path.exists():
|
|
531
|
+
_update_notebook_config_cell(
|
|
532
|
+
notebook_path,
|
|
533
|
+
config_data,
|
|
534
|
+
{"--additional-python-modules", "--extra-py-files"},
|
|
535
|
+
dry_run,
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _sync_checked_out_configs_after_build(
|
|
540
|
+
*,
|
|
541
|
+
job_name: Optional[str],
|
|
542
|
+
config_dir: Path,
|
|
543
|
+
profile: Optional[str],
|
|
544
|
+
dry_run: bool,
|
|
545
|
+
) -> Optional[str]:
|
|
546
|
+
job_names = [job_name] if job_name else _get_checked_out_jobs()
|
|
547
|
+
if not job_names:
|
|
548
|
+
return None
|
|
549
|
+
|
|
550
|
+
config_index = _load_config_index(config_dir)
|
|
551
|
+
for selected_job in job_names:
|
|
552
|
+
config_entry = config_index.get(selected_job)
|
|
553
|
+
if not config_entry:
|
|
554
|
+
_raise_missing_local_config(selected_job, config_dir, "glue build")
|
|
555
|
+
config_path: Path = config_entry["config_path"]
|
|
556
|
+
config_data = config_entry["config"]
|
|
557
|
+
_apply_saved_params_to_config_path(
|
|
558
|
+
config_path=config_path,
|
|
559
|
+
config_data=config_data,
|
|
560
|
+
job_name=selected_job,
|
|
561
|
+
profile=profile,
|
|
562
|
+
dry_run=dry_run,
|
|
563
|
+
)
|
|
564
|
+
_sync_checked_out_build_artifacts(
|
|
565
|
+
job_name=selected_job,
|
|
566
|
+
config_path=config_path,
|
|
567
|
+
config_data=config_data,
|
|
568
|
+
dry_run=dry_run,
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
if len(job_names) == 1:
|
|
572
|
+
return job_names[0]
|
|
573
|
+
return None
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _push_after_build(
|
|
577
|
+
*,
|
|
578
|
+
job_name: Optional[str],
|
|
579
|
+
config_dir: Path,
|
|
580
|
+
profile: Optional[str],
|
|
581
|
+
dry_run: bool,
|
|
582
|
+
include: Optional[list[str]],
|
|
583
|
+
exclude: Optional[list[str]],
|
|
584
|
+
update_config: bool,
|
|
585
|
+
auto_login: bool,
|
|
586
|
+
) -> None:
|
|
587
|
+
from .edit import _normalize_component_filters
|
|
588
|
+
|
|
589
|
+
components = _normalize_component_filters(
|
|
590
|
+
include,
|
|
591
|
+
exclude,
|
|
592
|
+
allowed=PUSH_COMPONENTS,
|
|
593
|
+
aliases=PUSH_COMPONENT_ALIASES,
|
|
594
|
+
context_label="glue build --push",
|
|
595
|
+
)
|
|
596
|
+
active_profile = profile or _get_checked_out_profile()
|
|
597
|
+
job_names = [job_name] if job_name else _get_checked_out_jobs()
|
|
598
|
+
if not job_names:
|
|
599
|
+
raise typer.BadParameter(
|
|
600
|
+
"glue build --push requires --job-name or an active checkout."
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
config_index = _load_config_index(config_dir)
|
|
604
|
+
for selected_job in job_names:
|
|
605
|
+
config_entry = config_index.get(selected_job)
|
|
606
|
+
if not config_entry:
|
|
607
|
+
_raise_missing_local_config(selected_job, config_dir, "glue build --push")
|
|
608
|
+
config_path: Path = config_entry["config_path"]
|
|
609
|
+
config_data = config_entry["config"]
|
|
610
|
+
_apply_saved_params_to_config_path(
|
|
611
|
+
config_path=config_path,
|
|
612
|
+
config_data=config_data,
|
|
613
|
+
job_name=selected_job,
|
|
614
|
+
profile=active_profile,
|
|
615
|
+
dry_run=dry_run,
|
|
616
|
+
)
|
|
617
|
+
typer.echo(f"Pushing after build: {selected_job}")
|
|
618
|
+
upload_glue_job_files_from_config(
|
|
619
|
+
config_data,
|
|
620
|
+
dry_run=dry_run,
|
|
621
|
+
update_job_config=update_config and "job-config" in components,
|
|
622
|
+
include_components=components,
|
|
623
|
+
profile_name=active_profile,
|
|
624
|
+
auto_login=auto_login,
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def run_project_build(
|
|
629
|
+
build_tool: str,
|
|
630
|
+
cwd: Optional[Path] = None,
|
|
631
|
+
out_dir: Optional[Path] = None,
|
|
632
|
+
no_version_suffix: bool = False,
|
|
633
|
+
dry_run: bool = False,
|
|
634
|
+
verbose: bool = False,
|
|
635
|
+
) -> None:
|
|
636
|
+
tool = build_tool.strip().lower()
|
|
637
|
+
if tool not in {"auto", "uv", "build"}:
|
|
638
|
+
raise typer.BadParameter("Invalid --build-tool. Use one of: auto, uv, build.")
|
|
639
|
+
|
|
640
|
+
if tool == "uv":
|
|
641
|
+
args = ["build"]
|
|
642
|
+
if out_dir is not None:
|
|
643
|
+
args.extend(["--out-dir", str(out_dir)])
|
|
644
|
+
run_uv(*args, cwd=cwd, dry_run=dry_run, verbose=verbose)
|
|
645
|
+
elif tool == "build":
|
|
646
|
+
run_python_build(cwd=cwd, out_dir=out_dir, dry_run=dry_run, verbose=verbose)
|
|
647
|
+
elif shutil.which("uv"):
|
|
648
|
+
args = ["build"]
|
|
649
|
+
if out_dir is not None:
|
|
650
|
+
args.extend(["--out-dir", str(out_dir)])
|
|
651
|
+
run_uv(*args, cwd=cwd, dry_run=dry_run, verbose=verbose)
|
|
652
|
+
else:
|
|
653
|
+
typer.echo("uv not found, falling back to: python -m build")
|
|
654
|
+
run_python_build(cwd=cwd, out_dir=out_dir, dry_run=dry_run, verbose=verbose)
|
|
655
|
+
|
|
656
|
+
dist_dir = _resolve_build_output_dir(cwd, out_dir)
|
|
657
|
+
_build_zip_artifacts_from_sdists(
|
|
658
|
+
dist_dir,
|
|
659
|
+
dry_run=dry_run,
|
|
660
|
+
no_version_suffix=no_version_suffix,
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
@app.command(
|
|
665
|
+
"build",
|
|
666
|
+
epilog=_examples_epilog(
|
|
667
|
+
"gluekit build",
|
|
668
|
+
"gluekit build --build-tool uv",
|
|
669
|
+
"gluekit build --out-dir build/artifacts --no-version-suffix",
|
|
670
|
+
"gluekit build --build-tool build --dry-run",
|
|
671
|
+
),
|
|
672
|
+
)
|
|
673
|
+
def glue_build(
|
|
674
|
+
job_name: Optional[str] = typer.Option(
|
|
675
|
+
None,
|
|
676
|
+
"--job-name",
|
|
677
|
+
help="Checked-out or explicit local Glue job to update when using --package-whl or --push.",
|
|
678
|
+
),
|
|
679
|
+
dry_run: bool = typer.Option(
|
|
680
|
+
False,
|
|
681
|
+
"--dry-run",
|
|
682
|
+
help="Show what would be built and zipped without running build commands.",
|
|
683
|
+
),
|
|
684
|
+
bump_version: Optional[str] = typer.Option(
|
|
685
|
+
None,
|
|
686
|
+
"--bump-version",
|
|
687
|
+
"--bump",
|
|
688
|
+
help="Bump pyproject.toml [project].version before building: major, minor, patch, or X.Y.Z.",
|
|
689
|
+
),
|
|
690
|
+
build_tool: str = typer.Option(
|
|
691
|
+
"auto",
|
|
692
|
+
"--build-tool",
|
|
693
|
+
help="Build frontend to use: auto, uv, or build.",
|
|
694
|
+
),
|
|
695
|
+
out_dir: Optional[Path] = typer.Option(
|
|
696
|
+
None,
|
|
697
|
+
"--out-dir",
|
|
698
|
+
help="The output directory to which distributions should be written.",
|
|
699
|
+
),
|
|
700
|
+
no_version_suffix: bool = typer.Option(
|
|
701
|
+
False,
|
|
702
|
+
"--no-version-suffix",
|
|
703
|
+
help="Remove the version suffix from generated zip artifact names.",
|
|
704
|
+
),
|
|
705
|
+
verbose: bool = typer.Option(
|
|
706
|
+
False,
|
|
707
|
+
"--verbose",
|
|
708
|
+
"-v",
|
|
709
|
+
help="Print build commands before execution.",
|
|
710
|
+
),
|
|
711
|
+
package_whl: bool = typer.Option(
|
|
712
|
+
False,
|
|
713
|
+
"--package-whl",
|
|
714
|
+
help="After building, replace the tracked dist/*.whl in the selected Glue config.",
|
|
715
|
+
),
|
|
716
|
+
push: bool = typer.Option(
|
|
717
|
+
False,
|
|
718
|
+
"--push",
|
|
719
|
+
help="Push the selected Glue config and artifacts after building.",
|
|
720
|
+
),
|
|
721
|
+
include: Optional[list[str]] = typer.Option(
|
|
722
|
+
None,
|
|
723
|
+
"--include",
|
|
724
|
+
"-i",
|
|
725
|
+
help=(
|
|
726
|
+
"For --push, include only specific components (script, notebook, "
|
|
727
|
+
"additional-python-modules, extra-files, job-config)."
|
|
728
|
+
),
|
|
729
|
+
),
|
|
730
|
+
exclude: Optional[list[str]] = typer.Option(
|
|
731
|
+
None,
|
|
732
|
+
"--exclude",
|
|
733
|
+
"-x",
|
|
734
|
+
help=(
|
|
735
|
+
"For --push, exclude specific components (script, notebook, "
|
|
736
|
+
"additional-python-modules, extra-files, job-config)."
|
|
737
|
+
),
|
|
738
|
+
),
|
|
739
|
+
update_config: bool = typer.Option(
|
|
740
|
+
True,
|
|
741
|
+
"--update-config/--no-update-config",
|
|
742
|
+
help="For --push, update Glue job configuration after uploading files.",
|
|
743
|
+
),
|
|
744
|
+
config_dir: Path = typer.Option(
|
|
745
|
+
Path("glue/configs"),
|
|
746
|
+
"--config-dir",
|
|
747
|
+
help="Directory containing Glue job config files.",
|
|
748
|
+
),
|
|
749
|
+
profile: Optional[str] = typer.Option(
|
|
750
|
+
None,
|
|
751
|
+
"--profile",
|
|
752
|
+
"-p",
|
|
753
|
+
help=(
|
|
754
|
+
"AWS CLI credential profile for profile-scoped config params and, with "
|
|
755
|
+
"--push, real AWS Glue/S3 API calls."
|
|
756
|
+
),
|
|
757
|
+
),
|
|
758
|
+
auto_login: bool = typer.Option(
|
|
759
|
+
True,
|
|
760
|
+
"--auto-login/--no-auto-login",
|
|
761
|
+
help="For --push with a real AWS profile, automatically run 'aws sso login' when credentials are missing or expired.",
|
|
762
|
+
),
|
|
763
|
+
) -> None:
|
|
764
|
+
"""Build local artifacts; only touches AWS when --push is used."""
|
|
765
|
+
workspace_root = _find_workspace_root()
|
|
766
|
+
if bump_version:
|
|
767
|
+
_bump_project_version(
|
|
768
|
+
workspace_root / "pyproject.toml",
|
|
769
|
+
bump_version,
|
|
770
|
+
dry_run=dry_run,
|
|
771
|
+
)
|
|
772
|
+
run_project_build(
|
|
773
|
+
build_tool,
|
|
774
|
+
cwd=workspace_root,
|
|
775
|
+
out_dir=out_dir,
|
|
776
|
+
no_version_suffix=no_version_suffix,
|
|
777
|
+
dry_run=dry_run,
|
|
778
|
+
verbose=verbose,
|
|
779
|
+
)
|
|
780
|
+
selected_job_name = job_name
|
|
781
|
+
active_profile = profile or _get_checked_out_profile()
|
|
782
|
+
effective_package_whl = package_whl or push or _automation_build_enabled(
|
|
783
|
+
"package_whl"
|
|
784
|
+
)
|
|
785
|
+
if effective_package_whl:
|
|
786
|
+
package_wheel_path = None
|
|
787
|
+
if not dry_run and (bump_version or _automation_build_enabled("package_whl")):
|
|
788
|
+
package_wheel_path = _resolve_built_package_wheel_path(
|
|
789
|
+
workspace_root=workspace_root,
|
|
790
|
+
out_dir=out_dir,
|
|
791
|
+
)
|
|
792
|
+
selected_job_name = _apply_package_wheel_to_job(
|
|
793
|
+
job_name=job_name,
|
|
794
|
+
config_dir=config_dir,
|
|
795
|
+
profile=active_profile,
|
|
796
|
+
dry_run=dry_run,
|
|
797
|
+
package_wheel_path=package_wheel_path,
|
|
798
|
+
)
|
|
799
|
+
else:
|
|
800
|
+
selected_job_name = _sync_checked_out_configs_after_build(
|
|
801
|
+
job_name=job_name,
|
|
802
|
+
config_dir=config_dir,
|
|
803
|
+
profile=active_profile,
|
|
804
|
+
dry_run=dry_run,
|
|
805
|
+
)
|
|
806
|
+
if push:
|
|
807
|
+
_push_after_build(
|
|
808
|
+
job_name=selected_job_name,
|
|
809
|
+
config_dir=config_dir,
|
|
810
|
+
profile=active_profile,
|
|
811
|
+
dry_run=dry_run,
|
|
812
|
+
include=include,
|
|
813
|
+
exclude=exclude,
|
|
814
|
+
update_config=update_config,
|
|
815
|
+
auto_login=auto_login,
|
|
816
|
+
)
|