dagster-cloud-cli 1.12.16__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. dagster_cloud_cli/__init__.py +5 -0
  2. dagster_cloud_cli/commands/__init__.py +0 -0
  3. dagster_cloud_cli/commands/branch_deployment/__init__.py +154 -0
  4. dagster_cloud_cli/commands/ci/__init__.py +1207 -0
  5. dagster_cloud_cli/commands/ci/checks.py +135 -0
  6. dagster_cloud_cli/commands/ci/report.py +49 -0
  7. dagster_cloud_cli/commands/ci/state.py +129 -0
  8. dagster_cloud_cli/commands/ci/utils.py +22 -0
  9. dagster_cloud_cli/commands/config.py +250 -0
  10. dagster_cloud_cli/commands/deployment/__init__.py +108 -0
  11. dagster_cloud_cli/commands/deployment/alert_policies/__init__.py +0 -0
  12. dagster_cloud_cli/commands/deployment/alert_policies/commands.py +52 -0
  13. dagster_cloud_cli/commands/integration/__init__.py +6 -0
  14. dagster_cloud_cli/commands/integration/atlan/__init__.py +94 -0
  15. dagster_cloud_cli/commands/job/__init__.py +117 -0
  16. dagster_cloud_cli/commands/metrics.py +114 -0
  17. dagster_cloud_cli/commands/organization/__init__.py +134 -0
  18. dagster_cloud_cli/commands/organization/saml/__init__.py +0 -0
  19. dagster_cloud_cli/commands/organization/saml/commands.py +85 -0
  20. dagster_cloud_cli/commands/run/__init__.py +27 -0
  21. dagster_cloud_cli/commands/serverless/Dockerfile +38 -0
  22. dagster_cloud_cli/commands/serverless/Dockerfile.dagster-cloud-cli +20 -0
  23. dagster_cloud_cli/commands/serverless/__init__.py +560 -0
  24. dagster_cloud_cli/commands/workspace/__init__.py +358 -0
  25. dagster_cloud_cli/config/__init__.py +6 -0
  26. dagster_cloud_cli/config/jinja_template_loader.py +45 -0
  27. dagster_cloud_cli/config/list_merge_strategies.py +77 -0
  28. dagster_cloud_cli/config/loader.py +23 -0
  29. dagster_cloud_cli/config/merger.py +92 -0
  30. dagster_cloud_cli/config/models.py +76 -0
  31. dagster_cloud_cli/config_utils.py +506 -0
  32. dagster_cloud_cli/core/__init__.py +0 -0
  33. dagster_cloud_cli/core/agent_queue.py +3 -0
  34. dagster_cloud_cli/core/alert_types.py +26 -0
  35. dagster_cloud_cli/core/artifacts.py +192 -0
  36. dagster_cloud_cli/core/docker_runner.py +57 -0
  37. dagster_cloud_cli/core/errors.py +32 -0
  38. dagster_cloud_cli/core/graphql_client.py +420 -0
  39. dagster_cloud_cli/core/headers/__init__.py +0 -0
  40. dagster_cloud_cli/core/headers/auth.py +16 -0
  41. dagster_cloud_cli/core/headers/impl.py +31 -0
  42. dagster_cloud_cli/core/headers/versioning/__init__.py +0 -0
  43. dagster_cloud_cli/core/headers/versioning/constants.py +2 -0
  44. dagster_cloud_cli/core/pex_builder/__init__.py +8 -0
  45. dagster_cloud_cli/core/pex_builder/code_location.py +126 -0
  46. dagster_cloud_cli/core/pex_builder/deploy.py +255 -0
  47. dagster_cloud_cli/core/pex_builder/deps.py +495 -0
  48. dagster_cloud_cli/core/pex_builder/git_context.py +29 -0
  49. dagster_cloud_cli/core/pex_builder/github_context.py +178 -0
  50. dagster_cloud_cli/core/pex_builder/gitlab_context.py +30 -0
  51. dagster_cloud_cli/core/pex_builder/parse_workspace.py +39 -0
  52. dagster_cloud_cli/core/pex_builder/pex_registry.py +148 -0
  53. dagster_cloud_cli/core/pex_builder/platforms.py +6105 -0
  54. dagster_cloud_cli/core/pex_builder/selftest.py +14 -0
  55. dagster_cloud_cli/core/pex_builder/source.py +166 -0
  56. dagster_cloud_cli/core/pex_builder/util.py +165 -0
  57. dagster_cloud_cli/core/workspace.py +145 -0
  58. dagster_cloud_cli/docker_utils.py +98 -0
  59. dagster_cloud_cli/entrypoint.py +209 -0
  60. dagster_cloud_cli/gql.py +1025 -0
  61. dagster_cloud_cli/pex_utils.py +180 -0
  62. dagster_cloud_cli/py.typed +0 -0
  63. dagster_cloud_cli/types.py +78 -0
  64. dagster_cloud_cli/ui.py +101 -0
  65. dagster_cloud_cli/utils.py +107 -0
  66. dagster_cloud_cli/version.py +1 -0
  67. dagster_cloud_cli-1.12.16.dist-info/METADATA +31 -0
  68. dagster_cloud_cli-1.12.16.dist-info/RECORD +71 -0
  69. dagster_cloud_cli-1.12.16.dist-info/WHEEL +5 -0
  70. dagster_cloud_cli-1.12.16.dist-info/entry_points.txt +3 -0
  71. dagster_cloud_cli-1.12.16.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,495 @@
1
+ # Build deps.pex, given a project root
2
+
3
+ import enum
4
+ import hashlib
5
+ import importlib.metadata
6
+ import json
7
+ import logging
8
+ import os
9
+ import os.path
10
+ import re
11
+ import subprocess
12
+ import sys
13
+ import tempfile
14
+ from dataclasses import dataclass
15
+
16
+ import click
17
+ from dagster_shared.utils import find_uv_workspace_root
18
+ from packaging import version
19
+
20
+ try:
21
+ import tomllib # pyright: ignore[reportMissingImports]
22
+ except ImportError:
23
+ # Python < 3.11 fallback
24
+ import tomli as tomllib
25
+
26
+ from dagster_cloud_cli import ui
27
+ from dagster_cloud_cli.core import docker_runner
28
+ from dagster_cloud_cli.core.pex_builder import util
29
+
30
+
31
+ class BuildMethod(enum.Enum):
32
+ # try current environment, if build fails, try in a docker builder
33
+ DOCKER_FALLBACK = "docker-fallback"
34
+ LOCAL = "local" # try local environment only
35
+ DOCKER = "docker" # try docker builder only
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class DepsRequirements:
40
+ requirements_txt: str
41
+ python_version: version.Version
42
+ pex_flags: list[str]
43
+
44
+ @property
45
+ def hash(self) -> str:
46
+ # The hash uniquely identifies the list of requirements used to build a deps.pex.
47
+ # This is used as part of the cache key to reuse a cached deps.pex.
48
+ # Note requirements_txt may have floating dependencies, so this is not perfect and may
49
+ # reuse deps.pex even if a new PyPI package is published for a dependency.
50
+ # An easy workaround is to pin the dependency in setup.py.
51
+ normalized_pex_flags = sorted(set(self.pex_flags) - {"--resolve-local-platforms"})
52
+ return hashlib.sha1(
53
+ (
54
+ repr(self.requirements_txt) + str(self.python_version) + repr(normalized_pex_flags)
55
+ ).encode("utf-8")
56
+ ).hexdigest()
57
+
58
+
59
+ @dataclass(frozen=True)
60
+ class LocalPackages:
61
+ local_package_paths: list[str]
62
+
63
+
64
+ def local_path_for(line: str, relative_to: str) -> str | None:
65
+ # Return the abspath for a local package, iff this line points to a local package,
66
+ # otherwise return None.
67
+ # This handles relative or absolute paths specified in requirements.txt,
68
+ # eg "../some/other/dir" or "./subdir/" or "/abs/dir". For these directories we
69
+ # include the local package as part of the source pex.
70
+ # The "file://" urls (direct references) are correctly handled by the underlying pex build
71
+ # anyway and do not need special treatment here.
72
+
73
+ # Use a very specific match here to avoid accidentally matching URLs or other lines with slashes
74
+ path = None
75
+ if line.startswith("./") or line.startswith("../") or line.startswith("/"):
76
+ path = os.path.abspath(os.path.join(relative_to, line.strip()))
77
+
78
+ if path:
79
+ if not os.path.exists(path):
80
+ raise ValueError(
81
+ f"Could not find local directory {path!r} referenced in requirement {line!r}"
82
+ )
83
+
84
+ return path
85
+
86
+ return None
87
+
88
+
89
+ def get_requirements_lines(local_dir, python_interpreter: str) -> list[str]:
90
+ # Combine dependencies specified in requirements.txt, setup.py, and pyproject.toml
91
+
92
+ lines = get_requirements_txt_deps(local_dir)
93
+ lines.extend(get_setup_py_deps(local_dir, python_interpreter))
94
+ lines.extend(get_pyproject_toml_deps(local_dir))
95
+ return lines
96
+
97
+
98
+ def collect_requirements(code_directory, python_interpreter: str) -> tuple[list[str], list[str]]:
99
+ if not os.path.exists(code_directory):
100
+ raise Exception(
101
+ f"Specified a build directory that does not exist: {os.path.abspath(code_directory)}."
102
+ )
103
+
104
+ required_files = [
105
+ "setup.py",
106
+ "requirements.txt",
107
+ "pyproject.toml",
108
+ ]
109
+
110
+ if not any(os.path.exists(os.path.join(code_directory, file)) for file in required_files):
111
+ raise Exception(
112
+ f"Could not find a setup.py, requirements.txt, or pyproject.toml in build directory {os.path.abspath(code_directory)}."
113
+ )
114
+
115
+ # traverse all local packages and return the list of local packages and other requirements
116
+ pending = [os.path.abspath(code_directory)] # local packages to be processed
117
+ seen = set()
118
+
119
+ local_package_paths = []
120
+ deps_lines = []
121
+
122
+ while pending:
123
+ local_dir = pending.pop()
124
+ if local_dir in seen:
125
+ continue
126
+ seen.add(local_dir)
127
+
128
+ lines = get_requirements_lines(local_dir, python_interpreter)
129
+ # Separate out the local packages from other requirements
130
+ for line in lines:
131
+ local_package_path = local_path_for(line, relative_to=local_dir)
132
+ if local_package_path:
133
+ if local_package_path not in local_package_paths:
134
+ local_package_paths.append(local_package_path)
135
+ pending.append(local_package_path)
136
+ else:
137
+ deps_lines.append(line)
138
+
139
+ return local_package_paths, deps_lines
140
+
141
+
142
+ def get_deps_requirements(
143
+ code_directory, python_version: version.Version
144
+ ) -> tuple[LocalPackages, DepsRequirements]:
145
+ python_interpreter = util.python_interpreter_for(python_version)
146
+
147
+ ui.print(f"Finding dependencies using build directory {os.path.abspath(code_directory)}")
148
+
149
+ local_package_paths, deps_lines = collect_requirements(code_directory, python_interpreter)
150
+
151
+ deps_requirements_text = "\n".join(
152
+ sorted(set(deps_lines)) + [""]
153
+ ) # empty string adds trailing newline
154
+
155
+ ui.print(f"List of local packages: {local_package_paths}")
156
+ ui.print(f"List of dependencies: {deps_requirements_text}")
157
+
158
+ local_packages = LocalPackages(local_package_paths=local_package_paths)
159
+ deps_requirements = DepsRequirements(
160
+ requirements_txt=deps_requirements_text,
161
+ python_version=python_version,
162
+ pex_flags=util.get_pex_flags(python_version, build_sdists=True),
163
+ )
164
+ ui.print(f"deps_requirements_hash: {deps_requirements.hash}")
165
+
166
+ return local_packages, deps_requirements
167
+
168
+
169
+ def build_deps_pex(code_directory, output_directory, python_version) -> tuple[str, str]:
170
+ _, requirements = get_deps_requirements(code_directory, python_version)
171
+ return build_deps_from_requirements(
172
+ requirements, output_directory, build_method=BuildMethod.DOCKER_FALLBACK
173
+ )
174
+
175
+
176
+ # Resolving dependencies can be flaky - depends on the version of pip and the resolver algorithm.
177
+ # These flags allow trying multiple ways of building the deps.
178
+ # This also allows us to try new flags safely, by having automatic fallback.
179
+ TRY_FLAGS = [
180
+ ["--resolver-version=pip-2020-resolver"], # new resolver as recommended by pex team
181
+ # disabled but left here for easy revert
182
+ # [], # default set of flags defined in util.py
183
+ ]
184
+
185
+
186
+ class DepsBuildFailure(Exception):
187
+ def __init__(self, proc: subprocess.CompletedProcess):
188
+ self.proc = proc
189
+ self.stdout = proc.stdout.decode("utf-8")
190
+ self.stderr = proc.stderr.decode("utf-8")
191
+ lines = self.stdout.splitlines() + self.stderr.splitlines()
192
+ self.dependency_failure_lines = [
193
+ line
194
+ for line in lines
195
+ if "No matching distribution" in line
196
+ or "ResolutionImpossible" in line
197
+ or "No pre-built wheel was available" in line
198
+ ]
199
+
200
+ def format_error(self) -> str:
201
+ lines = []
202
+ lines.append("Dependency build failure details:\n")
203
+ lines.append("Command:\n" + util.indent(" ".join(self.proc.args)))
204
+ if self.stdout:
205
+ lines.append("\nOutput:\n" + util.indent(self.stdout))
206
+ if self.stderr:
207
+ lines.append("\nError:\n" + util.indent(self.stderr))
208
+ return "".join(lines)
209
+
210
+
211
+ def build_deps_from_requirements(
212
+ requirements: DepsRequirements,
213
+ output_directory: str,
214
+ build_method: BuildMethod,
215
+ ) -> tuple[str, str]:
216
+ os.makedirs(output_directory, exist_ok=True)
217
+ deps_requirements_filename = f"deps-requirements-{requirements.hash}.txt"
218
+ deps_requirements_path = os.path.join(output_directory, deps_requirements_filename)
219
+ tmp_pex_filename = f"deps-from-{requirements.hash}.pex"
220
+ tmp_pex_path = os.path.join(output_directory, tmp_pex_filename)
221
+
222
+ with open(deps_requirements_path, "w", encoding="utf-8") as deps_requirements_file:
223
+ deps_requirements_file.write(requirements.requirements_txt)
224
+
225
+ ui.print(
226
+ f"Building project dependencies for Python {requirements.python_version}, "
227
+ f"writing to {output_directory}",
228
+ )
229
+
230
+ def build_in_docker() -> None:
231
+ proc = docker_runner.run_dagster_cloud(
232
+ map_folders={"/output": output_directory},
233
+ run_args=[
234
+ "serverless",
235
+ "build-python-deps",
236
+ f"/output/{deps_requirements_filename}",
237
+ f"/output/{tmp_pex_filename}",
238
+ json.dumps(requirements.pex_flags),
239
+ ],
240
+ env={"PEX_VERBOSE": None}, # pass through this env, if set
241
+ )
242
+ if proc.returncode:
243
+ ui.error("Failed to build dependencies using docker")
244
+ if proc.stdout:
245
+ ui.error(proc.stdout.decode("utf-8"))
246
+ if proc.stderr:
247
+ ui.error(proc.stderr.decode("utf-8"))
248
+ sys.exit(1)
249
+
250
+ if build_method in [BuildMethod.DOCKER_FALLBACK, BuildMethod.LOCAL]:
251
+ try:
252
+ build_deps_from_requirements_file(
253
+ deps_requirements_path,
254
+ output_pex_path=tmp_pex_path,
255
+ pex_flags=requirements.pex_flags,
256
+ )
257
+ except DepsBuildFailure as err:
258
+ if build_method == BuildMethod.DOCKER_FALLBACK and err.dependency_failure_lines:
259
+ ui.warn(
260
+ "Failed to build dependencies in current environment:"
261
+ f"{''.join(err.dependency_failure_lines)}"
262
+ )
263
+ ui.warn("Falling back to build in a docker environment")
264
+ build_in_docker()
265
+ else:
266
+ raise ui.error("Failed to build dependencies:\n" + err.format_error())
267
+ else:
268
+ ui.print("Building project dependencies in a docker build environment")
269
+ build_in_docker()
270
+
271
+ pex_info = util.get_pex_info(tmp_pex_path)
272
+ pex_hash = pex_info["pex_hash"]
273
+ final_pex_path = os.path.join(output_directory, f"deps-{pex_hash}.pex")
274
+ os.rename(tmp_pex_path, final_pex_path)
275
+ ui.print(f"Wrote deps pex: {final_pex_path}")
276
+
277
+ distribution_names = pex_info["distributions"].keys()
278
+ # the distributions are named something like 'dagster-1.0.14-py3-none-any.whl'
279
+ # and 'dagster_cloud-1.1.7-py3-none-any.whl'
280
+ dep_names = ["dagster", "dagster_cloud"]
281
+ dep_versions = {}
282
+ for name in distribution_names:
283
+ for dep_name in dep_names:
284
+ pattern = re.compile(f"{dep_name}-(.+?)-py")
285
+ match = pattern.match(name)
286
+ if match:
287
+ dep_versions[dep_name] = match.group(1)
288
+ break
289
+
290
+ for dep_name in dep_names:
291
+ if dep_name not in dep_versions:
292
+ raise ValueError(f"The {dep_name} package dependency was expected but not found.")
293
+
294
+ return final_pex_path, dep_versions["dagster"]
295
+
296
+
297
+ def build_deps_from_requirements_file(
298
+ deps_requirements_path: str,
299
+ output_pex_path: str,
300
+ pex_flags: list[str],
301
+ ) -> None:
302
+ """Attempts to build a pex file from a requirements file and raises DepsBuildFailure on failure."""
303
+ # We try different sets of build flags and use the first one that works
304
+ try_flags = TRY_FLAGS.copy()
305
+ while try_flags:
306
+ add_on_flags = try_flags.pop(0)
307
+ pex_flags = pex_flags + add_on_flags
308
+ logging.debug(f"Running pex with {' '.join(pex_flags)}")
309
+ proc = util.build_pex(
310
+ sources_directories=[],
311
+ requirements_filepaths=[deps_requirements_path],
312
+ pex_flags=pex_flags,
313
+ output_pex_path=output_pex_path,
314
+ )
315
+ if proc.returncode:
316
+ if try_flags:
317
+ ui.warn(proc.stderr.decode("utf-8"))
318
+ ui.warn("Will retry building deps with a different resolution mechanism")
319
+ else:
320
+ raise DepsBuildFailure(proc)
321
+ else:
322
+ break
323
+
324
+
325
+ def get_requirements_txt_deps(code_directory: str) -> list[str]:
326
+ requirements_path = os.path.join(code_directory, "requirements.txt")
327
+ if not os.path.exists(requirements_path):
328
+ return []
329
+
330
+ # combine multi-line strings into a single string
331
+ combined_lines = []
332
+ current_line = ""
333
+
334
+ with open(requirements_path, encoding="utf-8") as file:
335
+ for line in file:
336
+ stripped_line = line.rstrip()
337
+ if stripped_line.endswith("\\"):
338
+ current_line += stripped_line[:-1]
339
+ else:
340
+ current_line += stripped_line
341
+ combined_lines.append(current_line)
342
+ current_line = ""
343
+
344
+ # Add any remaining content if the last line ends with a backslash
345
+ if current_line:
346
+ combined_lines.append(current_line)
347
+
348
+ lines = []
349
+ for raw_line in combined_lines:
350
+ # https://pip.pypa.io/en/stable/reference/requirements-file-format/#comments
351
+ line = re.sub(r"(^#|\s#).*", "", raw_line)
352
+ line = line.strip()
353
+ # remove current dir from the deps
354
+ if line in {"", "."}:
355
+ continue
356
+ lines.append(line)
357
+
358
+ return lines
359
+
360
+
361
+ def get_setup_py_deps(code_directory: str, python_interpreter: str) -> list[str]:
362
+ setup_py_path = os.path.join(code_directory, "setup.py")
363
+ if not os.path.exists(setup_py_path):
364
+ return []
365
+
366
+ lines = []
367
+ # write out egg_info files and load as distribution
368
+ with tempfile.TemporaryDirectory() as temp_dir:
369
+ proc = subprocess.run(
370
+ [python_interpreter, setup_py_path, "egg_info", f"--egg-base={temp_dir}"],
371
+ capture_output=True,
372
+ check=False,
373
+ cwd=code_directory,
374
+ )
375
+ if proc.returncode:
376
+ raise ValueError(
377
+ "Error running setup.py egg_info: "
378
+ + proc.stdout.decode("utf-8")
379
+ + proc.stderr.decode("utf-8")
380
+ )
381
+ dists = list(importlib.metadata.distributions(path=[temp_dir]))
382
+ if len(dists) != 1:
383
+ raise ValueError(f"Could not find distribution for {setup_py_path}")
384
+ dist = dists[0]
385
+ for requirement in dist.requires or []:
386
+ lines.append(requirement)
387
+
388
+ return lines
389
+
390
+
391
+ def _resolve_uv_workspace_dep(dep_name: str, code_directory: str) -> str | None:
392
+ """Find the relative path to a uv workspace member by package name.
393
+
394
+ Returns a relative path like "../shared-lib" or None if not found.
395
+ Assumes the directory name matches the package name.
396
+ """
397
+ result = find_uv_workspace_root(code_directory)
398
+ if not result:
399
+ return None
400
+
401
+ workspace_root, _ = result
402
+ candidate = os.path.join(str(workspace_root), dep_name)
403
+ if os.path.isdir(candidate):
404
+ return os.path.relpath(candidate, code_directory)
405
+
406
+ return None
407
+
408
+
409
+ def get_pyproject_toml_deps(code_directory: str) -> list[str]:
410
+ pyproject_path = os.path.join(code_directory, "pyproject.toml")
411
+ if not os.path.exists(pyproject_path):
412
+ return []
413
+
414
+ try:
415
+ with open(pyproject_path, "rb") as file:
416
+ pyproject_data = tomllib.load(file)
417
+ except Exception as e:
418
+ raise ValueError(f"Error parsing pyproject.toml: {e}")
419
+
420
+ lines = []
421
+
422
+ # Handle dependencies in [project] section (PEP 621)
423
+ project_section = pyproject_data.get("project", {})
424
+ dependencies = project_section.get("dependencies", [])
425
+ for dep in dependencies:
426
+ lines.append(str(dep))
427
+
428
+ # Handle optional dependencies in [project.optional-dependencies]
429
+ optional_deps = project_section.get("optional-dependencies", {})
430
+ for group_deps in optional_deps.values():
431
+ for dep in group_deps:
432
+ lines.append(str(dep))
433
+
434
+ # Handle legacy [tool.poetry.dependencies] for Poetry projects
435
+ poetry_section = pyproject_data.get("tool", {}).get("poetry", {})
436
+ poetry_deps = poetry_section.get("dependencies", {})
437
+ for dep_name, dep_spec in poetry_deps.items():
438
+ if dep_name == "python":
439
+ continue # Skip python version constraint
440
+
441
+ if isinstance(dep_spec, str):
442
+ lines.append(f"{dep_name}{dep_spec}")
443
+ elif isinstance(dep_spec, dict):
444
+ version_spec = dep_spec.get("version", "")
445
+ if version_spec:
446
+ lines.append(f"{dep_name}{version_spec}")
447
+ else:
448
+ # Handle complex dependency specs (git, path, etc.)
449
+ lines.append(dep_name)
450
+
451
+ # Handle legacy [tool.poetry.dev-dependencies] (older Poetry format)
452
+ poetry_dev_deps = poetry_section.get("dev-dependencies", {})
453
+ for dep_name, dep_spec in poetry_dev_deps.items():
454
+ if isinstance(dep_spec, str):
455
+ lines.append(f"{dep_name}{dep_spec}")
456
+ elif isinstance(dep_spec, dict):
457
+ version_spec = dep_spec.get("version", "")
458
+ if version_spec:
459
+ lines.append(f"{dep_name}{version_spec}")
460
+ else:
461
+ lines.append(dep_name)
462
+
463
+ # Handle [tool.uv.sources] - resolve workspace and path dependencies to local paths
464
+ uv_sources = pyproject_data.get("tool", {}).get("uv", {}).get("sources", {})
465
+ if uv_sources:
466
+ resolved_lines = []
467
+ for line in lines:
468
+ source_config = uv_sources.get(line)
469
+ if source_config:
470
+ if source_config.get("workspace"):
471
+ resolved_path = _resolve_uv_workspace_dep(line, code_directory)
472
+ if resolved_path:
473
+ resolved_lines.append(resolved_path)
474
+ continue
475
+ elif "path" in source_config:
476
+ resolved_lines.append(source_config["path"])
477
+ continue
478
+ resolved_lines.append(line)
479
+ return resolved_lines
480
+
481
+ return lines
482
+
483
+
484
+ @click.command()
485
+ @click.argument("project_dir", type=click.Path(exists=True))
486
+ @click.argument("build_output_dir", type=click.Path(exists=False))
487
+ @util.python_version_option()
488
+ def deps_main(project_dir, build_output_dir, python_version):
489
+ _deps_pex_path, _dagster_version = build_deps_pex(
490
+ project_dir, build_output_dir, util.parse_python_version(python_version)
491
+ )
492
+
493
+
494
+ if __name__ == "__main__":
495
+ deps_main()
@@ -0,0 +1,29 @@
1
+ import logging
2
+ import subprocess
3
+ from dataclasses import dataclass
4
+
5
+
6
+ @dataclass
7
+ class GitCommitMetadata:
8
+ timestamp: str
9
+ message: str
10
+ email: str
11
+ name: str
12
+
13
+
14
+ def get_git_commit_metadata(project_dir: str) -> GitCommitMetadata:
15
+ commands = {
16
+ "timestamp": f"git -C {project_dir} log -1 --format=%cd --date=unix".split(),
17
+ "message": f"git -C {project_dir} log -1 --format=%s".split(),
18
+ "email": f"git -C {project_dir} log -1 --format=%ae".split(),
19
+ "name": f"git -C {project_dir} log -1 --format=%an".split(),
20
+ }
21
+ metadata = {}
22
+ for key, command in commands.items():
23
+ logging.debug("Running %r", command)
24
+ proc = subprocess.run(command, capture_output=True, check=False)
25
+ if proc.returncode:
26
+ logging.error(f"git command failed: {proc.stdout}\n{proc.stderr}")
27
+ metadata[key] = proc.stdout.decode("utf-8").strip()
28
+
29
+ return GitCommitMetadata(**metadata)
@@ -0,0 +1,178 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ import pathlib
5
+ import subprocess
6
+ from contextlib import contextmanager
7
+
8
+ from dagster_cloud_cli import ui
9
+ from dagster_cloud_cli.core.pex_builder import util
10
+
11
+ # Loads event details from within a github action
12
+
13
+
14
+ class GithubEvent:
15
+ def __init__(self, project_dir: str):
16
+ self.github_server_url = os.environ["GITHUB_SERVER_URL"]
17
+ self.github_sha = os.environ["GITHUB_SHA"]
18
+ self.github_repository = os.environ["GITHUB_REPOSITORY"]
19
+ self.github_run_id = os.environ["GITHUB_RUN_ID"]
20
+ self.github_run_url = (
21
+ f"{self.github_server_url}/{self.github_repository}/actions/runs/{self.github_run_id}"
22
+ )
23
+ action_path = os.getenv("GITHUB_ACTION_PATH")
24
+ self.github_action_path = pathlib.Path(action_path) if action_path else None
25
+ event_path = os.getenv("GITHUB_EVENT_PATH")
26
+ if not event_path:
27
+ raise ValueError("GITHUB_EVENT_PATH not set")
28
+
29
+ # json details: https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads
30
+ self.event = event = json.load(open(event_path, encoding="utf-8"))
31
+
32
+ # get some commonly used fields
33
+ # not all events have "action", eg https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push
34
+ self.action = event.get("action")
35
+ self.repo_name = event["repository"]["full_name"]
36
+
37
+ self.branch_name: str | None = None
38
+ self.branch_url: str | None = None
39
+ self.pull_request_url: str | None = None
40
+ self.pull_request_id: str | None = None
41
+ self.pull_request_status: str | None = None
42
+
43
+ if "pull_request" in self.event:
44
+ pull_request = self.event["pull_request"]
45
+ # For PRs GITHUB_SHA is not the last commit in the branch, but head sha is
46
+ self.github_sha = pull_request["head"]["sha"]
47
+ self.branch_name = pull_request["head"]["ref"]
48
+ self.branch_url = f"{self.github_server_url}/{self.repo_name}/tree/{self.branch_name}"
49
+ self.pull_request_url = pull_request["html_url"]
50
+ self.pull_request_id = str(pull_request["number"])
51
+ self.pull_request_status = (
52
+ "merged" if pull_request.get("merged") else pull_request["state"]
53
+ ).upper()
54
+
55
+ self.commit_url = f"{self.github_server_url}/{self.repo_name}/tree/{self.github_sha}"
56
+
57
+ git_metadata = get_git_commit_metadata(project_dir)
58
+ self.timestamp = float(git_metadata["timestamp"])
59
+ self.commit_msg = git_metadata["message"]
60
+ self.author_name = git_metadata["name"]
61
+ self.author_email = git_metadata["email"]
62
+
63
+ def get_github_repo(self):
64
+ import github3
65
+
66
+ token = os.getenv("GITHUB_TOKEN")
67
+ if not token:
68
+ return None
69
+ gh = github3.login(token=token)
70
+ repo_owner, repo_name = self.github_repository.split("/", 1)
71
+ return gh.repository(repo_owner, repo_name)
72
+
73
+ def get_github_avatar_url(self) -> str | None:
74
+ repo = self.get_github_repo()
75
+ if not repo:
76
+ return
77
+ commit = repo.commit(self.github_sha)
78
+ return commit.author.get("avatar_url") if commit.author else None
79
+
80
+ def update_pr_comment(
81
+ self, body: str, orig_author: str | None = None, orig_text: str | None = None
82
+ ):
83
+ # orig_author and orig_text are used to identify an existing comment which is updated.
84
+ # if not provided, or not found, a new comment is created
85
+ import github3.exceptions
86
+
87
+ if not self.pull_request_id:
88
+ return
89
+
90
+ try:
91
+ repo = self.get_github_repo()
92
+ if not repo:
93
+ return
94
+ pr = repo.pull_request(int(self.pull_request_id))
95
+
96
+ for comment in pr.issue_comments():
97
+ if comment.user.login == orig_author and orig_text in comment.body:
98
+ comment.edit(body)
99
+ return
100
+
101
+ pr.create_comment(body)
102
+
103
+ except github3.exceptions.GitHubException:
104
+ logging.exception("Ignoring error when updating PR comment")
105
+
106
+
107
+ def get_git_commit_metadata(project_dir: str) -> dict[str, str]:
108
+ commands = {
109
+ "timestamp": f"git -C {project_dir} log -1 --format=%cd --date=unix".split(),
110
+ "message": f"git -C {project_dir} log -1 --format=%s".split(),
111
+ "email": f"git -C {project_dir} log -1 --format=%ae".split(),
112
+ "name": f"git -C {project_dir} log -1 --format=%an".split(),
113
+ }
114
+ metadata = {}
115
+ for key, command in commands.items():
116
+ logging.debug("Running %r", command)
117
+ proc = subprocess.run(command, capture_output=True, check=False)
118
+ if proc.returncode:
119
+ logging.error(f"git command failed: {proc.stdout}\n{proc.stderr}")
120
+ metadata[key] = proc.stdout.decode("utf-8").strip()
121
+
122
+ return metadata
123
+
124
+
125
+ def get_github_event(project_dir) -> GithubEvent:
126
+ return GithubEvent(project_dir)
127
+
128
+
129
+ def update_pr_comment(github_event: GithubEvent, action, deployment_name, location_name):
130
+ """Add or update the status comment on a github PR."""
131
+ # This reuses the src/create_or_update_comment.py script.
132
+ # We can't reuse actions/utils/notify here because we need to run this once for every location.
133
+ # To repeat an action github provides a matrix strategy but we don't use matrix due to latency
134
+ # concerns (it would launch a new container for every location)
135
+ if not github_event.pull_request_id:
136
+ ui.print("Not within a PR, not adding PR comment.")
137
+ return
138
+
139
+ if not github_event.github_action_path:
140
+ ui.warn("Unable to locate notification script, not adding PR comment.")
141
+ return
142
+
143
+ script_path = github_event.github_action_path.parent.parent / "src/create_or_update_comment.py"
144
+ if not script_path.exists():
145
+ ui.warn(f"Did not find {script_path}, not adding PR comment.")
146
+
147
+ env = {name: value for name, value in os.environ.items() if not name.startswith("PEX_")}
148
+ pr_id = str(github_event.pull_request_id)
149
+
150
+ env.update(
151
+ {
152
+ "INPUT_PR": pr_id,
153
+ "INPUT_ACTION": action,
154
+ "INPUT_DEPLOYMENT": deployment_name,
155
+ "INPUT_LOCATION_NAME": location_name,
156
+ "GITHUB_RUN_URL": github_event.github_run_url,
157
+ }
158
+ )
159
+ env = {name: value for name, value in env.items() if value is not None}
160
+ proc = util.run_python_subprocess([str(script_path)], env=env)
161
+ if proc.returncode:
162
+ ui.error(f"Could not update PR comment: {proc.stdout}\n{proc.stderr}")
163
+
164
+
165
+ @contextmanager
166
+ def log_group(title: str):
167
+ try:
168
+ yield
169
+ finally:
170
+ pass
171
+
172
+
173
+ def log_error(title: str, msg: str):
174
+ pass
175
+
176
+
177
+ if __name__ == "__main__":
178
+ pass