dagster-cloud-cli 1.12.16__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_cloud_cli/__init__.py +5 -0
- dagster_cloud_cli/commands/__init__.py +0 -0
- dagster_cloud_cli/commands/branch_deployment/__init__.py +154 -0
- dagster_cloud_cli/commands/ci/__init__.py +1207 -0
- dagster_cloud_cli/commands/ci/checks.py +135 -0
- dagster_cloud_cli/commands/ci/report.py +49 -0
- dagster_cloud_cli/commands/ci/state.py +129 -0
- dagster_cloud_cli/commands/ci/utils.py +22 -0
- dagster_cloud_cli/commands/config.py +250 -0
- dagster_cloud_cli/commands/deployment/__init__.py +108 -0
- dagster_cloud_cli/commands/deployment/alert_policies/__init__.py +0 -0
- dagster_cloud_cli/commands/deployment/alert_policies/commands.py +52 -0
- dagster_cloud_cli/commands/integration/__init__.py +6 -0
- dagster_cloud_cli/commands/integration/atlan/__init__.py +94 -0
- dagster_cloud_cli/commands/job/__init__.py +117 -0
- dagster_cloud_cli/commands/metrics.py +114 -0
- dagster_cloud_cli/commands/organization/__init__.py +134 -0
- dagster_cloud_cli/commands/organization/saml/__init__.py +0 -0
- dagster_cloud_cli/commands/organization/saml/commands.py +85 -0
- dagster_cloud_cli/commands/run/__init__.py +27 -0
- dagster_cloud_cli/commands/serverless/Dockerfile +38 -0
- dagster_cloud_cli/commands/serverless/Dockerfile.dagster-cloud-cli +20 -0
- dagster_cloud_cli/commands/serverless/__init__.py +560 -0
- dagster_cloud_cli/commands/workspace/__init__.py +358 -0
- dagster_cloud_cli/config/__init__.py +6 -0
- dagster_cloud_cli/config/jinja_template_loader.py +45 -0
- dagster_cloud_cli/config/list_merge_strategies.py +77 -0
- dagster_cloud_cli/config/loader.py +23 -0
- dagster_cloud_cli/config/merger.py +92 -0
- dagster_cloud_cli/config/models.py +76 -0
- dagster_cloud_cli/config_utils.py +506 -0
- dagster_cloud_cli/core/__init__.py +0 -0
- dagster_cloud_cli/core/agent_queue.py +3 -0
- dagster_cloud_cli/core/alert_types.py +26 -0
- dagster_cloud_cli/core/artifacts.py +192 -0
- dagster_cloud_cli/core/docker_runner.py +57 -0
- dagster_cloud_cli/core/errors.py +32 -0
- dagster_cloud_cli/core/graphql_client.py +420 -0
- dagster_cloud_cli/core/headers/__init__.py +0 -0
- dagster_cloud_cli/core/headers/auth.py +16 -0
- dagster_cloud_cli/core/headers/impl.py +31 -0
- dagster_cloud_cli/core/headers/versioning/__init__.py +0 -0
- dagster_cloud_cli/core/headers/versioning/constants.py +2 -0
- dagster_cloud_cli/core/pex_builder/__init__.py +8 -0
- dagster_cloud_cli/core/pex_builder/code_location.py +126 -0
- dagster_cloud_cli/core/pex_builder/deploy.py +255 -0
- dagster_cloud_cli/core/pex_builder/deps.py +495 -0
- dagster_cloud_cli/core/pex_builder/git_context.py +29 -0
- dagster_cloud_cli/core/pex_builder/github_context.py +178 -0
- dagster_cloud_cli/core/pex_builder/gitlab_context.py +30 -0
- dagster_cloud_cli/core/pex_builder/parse_workspace.py +39 -0
- dagster_cloud_cli/core/pex_builder/pex_registry.py +148 -0
- dagster_cloud_cli/core/pex_builder/platforms.py +6105 -0
- dagster_cloud_cli/core/pex_builder/selftest.py +14 -0
- dagster_cloud_cli/core/pex_builder/source.py +166 -0
- dagster_cloud_cli/core/pex_builder/util.py +165 -0
- dagster_cloud_cli/core/workspace.py +145 -0
- dagster_cloud_cli/docker_utils.py +98 -0
- dagster_cloud_cli/entrypoint.py +209 -0
- dagster_cloud_cli/gql.py +1025 -0
- dagster_cloud_cli/pex_utils.py +180 -0
- dagster_cloud_cli/py.typed +0 -0
- dagster_cloud_cli/types.py +78 -0
- dagster_cloud_cli/ui.py +101 -0
- dagster_cloud_cli/utils.py +107 -0
- dagster_cloud_cli/version.py +1 -0
- dagster_cloud_cli-1.12.16.dist-info/METADATA +31 -0
- dagster_cloud_cli-1.12.16.dist-info/RECORD +71 -0
- dagster_cloud_cli-1.12.16.dist-info/WHEEL +5 -0
- dagster_cloud_cli-1.12.16.dist-info/entry_points.txt +3 -0
- dagster_cloud_cli-1.12.16.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
# Build deps.pex, given a project root
|
|
2
|
+
|
|
3
|
+
import enum
|
|
4
|
+
import hashlib
|
|
5
|
+
import importlib.metadata
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
import os.path
|
|
10
|
+
import re
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
import click
|
|
17
|
+
from dagster_shared.utils import find_uv_workspace_root
|
|
18
|
+
from packaging import version
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
import tomllib # pyright: ignore[reportMissingImports]
|
|
22
|
+
except ImportError:
|
|
23
|
+
# Python < 3.11 fallback
|
|
24
|
+
import tomli as tomllib
|
|
25
|
+
|
|
26
|
+
from dagster_cloud_cli import ui
|
|
27
|
+
from dagster_cloud_cli.core import docker_runner
|
|
28
|
+
from dagster_cloud_cli.core.pex_builder import util
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class BuildMethod(enum.Enum):
|
|
32
|
+
# try current environment, if build fails, try in a docker builder
|
|
33
|
+
DOCKER_FALLBACK = "docker-fallback"
|
|
34
|
+
LOCAL = "local" # try local environment only
|
|
35
|
+
DOCKER = "docker" # try docker builder only
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class DepsRequirements:
|
|
40
|
+
requirements_txt: str
|
|
41
|
+
python_version: version.Version
|
|
42
|
+
pex_flags: list[str]
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def hash(self) -> str:
|
|
46
|
+
# The hash uniquely identifies the list of requirements used to build a deps.pex.
|
|
47
|
+
# This is used as part of the cache key to reuse a cached deps.pex.
|
|
48
|
+
# Note requirements_txt may have floating dependencies, so this is not perfect and may
|
|
49
|
+
# reuse deps.pex even if a new PyPI package is published for a dependency.
|
|
50
|
+
# An easy workaround is to pin the dependency in setup.py.
|
|
51
|
+
normalized_pex_flags = sorted(set(self.pex_flags) - {"--resolve-local-platforms"})
|
|
52
|
+
return hashlib.sha1(
|
|
53
|
+
(
|
|
54
|
+
repr(self.requirements_txt) + str(self.python_version) + repr(normalized_pex_flags)
|
|
55
|
+
).encode("utf-8")
|
|
56
|
+
).hexdigest()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class LocalPackages:
|
|
61
|
+
local_package_paths: list[str]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def local_path_for(line: str, relative_to: str) -> str | None:
|
|
65
|
+
# Return the abspath for a local package, iff this line points to a local package,
|
|
66
|
+
# otherwise return None.
|
|
67
|
+
# This handles relative or absolute paths specified in requirements.txt,
|
|
68
|
+
# eg "../some/other/dir" or "./subdir/" or "/abs/dir". For these directories we
|
|
69
|
+
# include the local package as part of the source pex.
|
|
70
|
+
# The "file://" urls (direct references) are correctly handled by the underlying pex build
|
|
71
|
+
# anyway and do not need special treatment here.
|
|
72
|
+
|
|
73
|
+
# Use a very specific match here to avoid accidentally matching URLs or other lines with slashes
|
|
74
|
+
path = None
|
|
75
|
+
if line.startswith("./") or line.startswith("../") or line.startswith("/"):
|
|
76
|
+
path = os.path.abspath(os.path.join(relative_to, line.strip()))
|
|
77
|
+
|
|
78
|
+
if path:
|
|
79
|
+
if not os.path.exists(path):
|
|
80
|
+
raise ValueError(
|
|
81
|
+
f"Could not find local directory {path!r} referenced in requirement {line!r}"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
return path
|
|
85
|
+
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_requirements_lines(local_dir, python_interpreter: str) -> list[str]:
|
|
90
|
+
# Combine dependencies specified in requirements.txt, setup.py, and pyproject.toml
|
|
91
|
+
|
|
92
|
+
lines = get_requirements_txt_deps(local_dir)
|
|
93
|
+
lines.extend(get_setup_py_deps(local_dir, python_interpreter))
|
|
94
|
+
lines.extend(get_pyproject_toml_deps(local_dir))
|
|
95
|
+
return lines
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def collect_requirements(code_directory, python_interpreter: str) -> tuple[list[str], list[str]]:
|
|
99
|
+
if not os.path.exists(code_directory):
|
|
100
|
+
raise Exception(
|
|
101
|
+
f"Specified a build directory that does not exist: {os.path.abspath(code_directory)}."
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
required_files = [
|
|
105
|
+
"setup.py",
|
|
106
|
+
"requirements.txt",
|
|
107
|
+
"pyproject.toml",
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
if not any(os.path.exists(os.path.join(code_directory, file)) for file in required_files):
|
|
111
|
+
raise Exception(
|
|
112
|
+
f"Could not find a setup.py, requirements.txt, or pyproject.toml in build directory {os.path.abspath(code_directory)}."
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# traverse all local packages and return the list of local packages and other requirements
|
|
116
|
+
pending = [os.path.abspath(code_directory)] # local packages to be processed
|
|
117
|
+
seen = set()
|
|
118
|
+
|
|
119
|
+
local_package_paths = []
|
|
120
|
+
deps_lines = []
|
|
121
|
+
|
|
122
|
+
while pending:
|
|
123
|
+
local_dir = pending.pop()
|
|
124
|
+
if local_dir in seen:
|
|
125
|
+
continue
|
|
126
|
+
seen.add(local_dir)
|
|
127
|
+
|
|
128
|
+
lines = get_requirements_lines(local_dir, python_interpreter)
|
|
129
|
+
# Separate out the local packages from other requirements
|
|
130
|
+
for line in lines:
|
|
131
|
+
local_package_path = local_path_for(line, relative_to=local_dir)
|
|
132
|
+
if local_package_path:
|
|
133
|
+
if local_package_path not in local_package_paths:
|
|
134
|
+
local_package_paths.append(local_package_path)
|
|
135
|
+
pending.append(local_package_path)
|
|
136
|
+
else:
|
|
137
|
+
deps_lines.append(line)
|
|
138
|
+
|
|
139
|
+
return local_package_paths, deps_lines
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def get_deps_requirements(
|
|
143
|
+
code_directory, python_version: version.Version
|
|
144
|
+
) -> tuple[LocalPackages, DepsRequirements]:
|
|
145
|
+
python_interpreter = util.python_interpreter_for(python_version)
|
|
146
|
+
|
|
147
|
+
ui.print(f"Finding dependencies using build directory {os.path.abspath(code_directory)}")
|
|
148
|
+
|
|
149
|
+
local_package_paths, deps_lines = collect_requirements(code_directory, python_interpreter)
|
|
150
|
+
|
|
151
|
+
deps_requirements_text = "\n".join(
|
|
152
|
+
sorted(set(deps_lines)) + [""]
|
|
153
|
+
) # empty string adds trailing newline
|
|
154
|
+
|
|
155
|
+
ui.print(f"List of local packages: {local_package_paths}")
|
|
156
|
+
ui.print(f"List of dependencies: {deps_requirements_text}")
|
|
157
|
+
|
|
158
|
+
local_packages = LocalPackages(local_package_paths=local_package_paths)
|
|
159
|
+
deps_requirements = DepsRequirements(
|
|
160
|
+
requirements_txt=deps_requirements_text,
|
|
161
|
+
python_version=python_version,
|
|
162
|
+
pex_flags=util.get_pex_flags(python_version, build_sdists=True),
|
|
163
|
+
)
|
|
164
|
+
ui.print(f"deps_requirements_hash: {deps_requirements.hash}")
|
|
165
|
+
|
|
166
|
+
return local_packages, deps_requirements
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def build_deps_pex(code_directory, output_directory, python_version) -> tuple[str, str]:
|
|
170
|
+
_, requirements = get_deps_requirements(code_directory, python_version)
|
|
171
|
+
return build_deps_from_requirements(
|
|
172
|
+
requirements, output_directory, build_method=BuildMethod.DOCKER_FALLBACK
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
# Resolving dependencies can be flaky - depends on the version of pip and the resolver algorithm.
|
|
177
|
+
# These flags allow trying multiple ways of building the deps.
|
|
178
|
+
# This also allows us to try new flags safely, by having automatic fallback.
|
|
179
|
+
TRY_FLAGS = [
|
|
180
|
+
["--resolver-version=pip-2020-resolver"], # new resolver as recommended by pex team
|
|
181
|
+
# disabled but left here for easy revert
|
|
182
|
+
# [], # default set of flags defined in util.py
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class DepsBuildFailure(Exception):
|
|
187
|
+
def __init__(self, proc: subprocess.CompletedProcess):
|
|
188
|
+
self.proc = proc
|
|
189
|
+
self.stdout = proc.stdout.decode("utf-8")
|
|
190
|
+
self.stderr = proc.stderr.decode("utf-8")
|
|
191
|
+
lines = self.stdout.splitlines() + self.stderr.splitlines()
|
|
192
|
+
self.dependency_failure_lines = [
|
|
193
|
+
line
|
|
194
|
+
for line in lines
|
|
195
|
+
if "No matching distribution" in line
|
|
196
|
+
or "ResolutionImpossible" in line
|
|
197
|
+
or "No pre-built wheel was available" in line
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
def format_error(self) -> str:
|
|
201
|
+
lines = []
|
|
202
|
+
lines.append("Dependency build failure details:\n")
|
|
203
|
+
lines.append("Command:\n" + util.indent(" ".join(self.proc.args)))
|
|
204
|
+
if self.stdout:
|
|
205
|
+
lines.append("\nOutput:\n" + util.indent(self.stdout))
|
|
206
|
+
if self.stderr:
|
|
207
|
+
lines.append("\nError:\n" + util.indent(self.stderr))
|
|
208
|
+
return "".join(lines)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def build_deps_from_requirements(
|
|
212
|
+
requirements: DepsRequirements,
|
|
213
|
+
output_directory: str,
|
|
214
|
+
build_method: BuildMethod,
|
|
215
|
+
) -> tuple[str, str]:
|
|
216
|
+
os.makedirs(output_directory, exist_ok=True)
|
|
217
|
+
deps_requirements_filename = f"deps-requirements-{requirements.hash}.txt"
|
|
218
|
+
deps_requirements_path = os.path.join(output_directory, deps_requirements_filename)
|
|
219
|
+
tmp_pex_filename = f"deps-from-{requirements.hash}.pex"
|
|
220
|
+
tmp_pex_path = os.path.join(output_directory, tmp_pex_filename)
|
|
221
|
+
|
|
222
|
+
with open(deps_requirements_path, "w", encoding="utf-8") as deps_requirements_file:
|
|
223
|
+
deps_requirements_file.write(requirements.requirements_txt)
|
|
224
|
+
|
|
225
|
+
ui.print(
|
|
226
|
+
f"Building project dependencies for Python {requirements.python_version}, "
|
|
227
|
+
f"writing to {output_directory}",
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def build_in_docker() -> None:
|
|
231
|
+
proc = docker_runner.run_dagster_cloud(
|
|
232
|
+
map_folders={"/output": output_directory},
|
|
233
|
+
run_args=[
|
|
234
|
+
"serverless",
|
|
235
|
+
"build-python-deps",
|
|
236
|
+
f"/output/{deps_requirements_filename}",
|
|
237
|
+
f"/output/{tmp_pex_filename}",
|
|
238
|
+
json.dumps(requirements.pex_flags),
|
|
239
|
+
],
|
|
240
|
+
env={"PEX_VERBOSE": None}, # pass through this env, if set
|
|
241
|
+
)
|
|
242
|
+
if proc.returncode:
|
|
243
|
+
ui.error("Failed to build dependencies using docker")
|
|
244
|
+
if proc.stdout:
|
|
245
|
+
ui.error(proc.stdout.decode("utf-8"))
|
|
246
|
+
if proc.stderr:
|
|
247
|
+
ui.error(proc.stderr.decode("utf-8"))
|
|
248
|
+
sys.exit(1)
|
|
249
|
+
|
|
250
|
+
if build_method in [BuildMethod.DOCKER_FALLBACK, BuildMethod.LOCAL]:
|
|
251
|
+
try:
|
|
252
|
+
build_deps_from_requirements_file(
|
|
253
|
+
deps_requirements_path,
|
|
254
|
+
output_pex_path=tmp_pex_path,
|
|
255
|
+
pex_flags=requirements.pex_flags,
|
|
256
|
+
)
|
|
257
|
+
except DepsBuildFailure as err:
|
|
258
|
+
if build_method == BuildMethod.DOCKER_FALLBACK and err.dependency_failure_lines:
|
|
259
|
+
ui.warn(
|
|
260
|
+
"Failed to build dependencies in current environment:"
|
|
261
|
+
f"{''.join(err.dependency_failure_lines)}"
|
|
262
|
+
)
|
|
263
|
+
ui.warn("Falling back to build in a docker environment")
|
|
264
|
+
build_in_docker()
|
|
265
|
+
else:
|
|
266
|
+
raise ui.error("Failed to build dependencies:\n" + err.format_error())
|
|
267
|
+
else:
|
|
268
|
+
ui.print("Building project dependencies in a docker build environment")
|
|
269
|
+
build_in_docker()
|
|
270
|
+
|
|
271
|
+
pex_info = util.get_pex_info(tmp_pex_path)
|
|
272
|
+
pex_hash = pex_info["pex_hash"]
|
|
273
|
+
final_pex_path = os.path.join(output_directory, f"deps-{pex_hash}.pex")
|
|
274
|
+
os.rename(tmp_pex_path, final_pex_path)
|
|
275
|
+
ui.print(f"Wrote deps pex: {final_pex_path}")
|
|
276
|
+
|
|
277
|
+
distribution_names = pex_info["distributions"].keys()
|
|
278
|
+
# the distributions are named something like 'dagster-1.0.14-py3-none-any.whl'
|
|
279
|
+
# and 'dagster_cloud-1.1.7-py3-none-any.whl'
|
|
280
|
+
dep_names = ["dagster", "dagster_cloud"]
|
|
281
|
+
dep_versions = {}
|
|
282
|
+
for name in distribution_names:
|
|
283
|
+
for dep_name in dep_names:
|
|
284
|
+
pattern = re.compile(f"{dep_name}-(.+?)-py")
|
|
285
|
+
match = pattern.match(name)
|
|
286
|
+
if match:
|
|
287
|
+
dep_versions[dep_name] = match.group(1)
|
|
288
|
+
break
|
|
289
|
+
|
|
290
|
+
for dep_name in dep_names:
|
|
291
|
+
if dep_name not in dep_versions:
|
|
292
|
+
raise ValueError(f"The {dep_name} package dependency was expected but not found.")
|
|
293
|
+
|
|
294
|
+
return final_pex_path, dep_versions["dagster"]
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def build_deps_from_requirements_file(
|
|
298
|
+
deps_requirements_path: str,
|
|
299
|
+
output_pex_path: str,
|
|
300
|
+
pex_flags: list[str],
|
|
301
|
+
) -> None:
|
|
302
|
+
"""Attempts to build a pex file from a requirements file and raises DepsBuildFailure on failure."""
|
|
303
|
+
# We try different sets of build flags and use the first one that works
|
|
304
|
+
try_flags = TRY_FLAGS.copy()
|
|
305
|
+
while try_flags:
|
|
306
|
+
add_on_flags = try_flags.pop(0)
|
|
307
|
+
pex_flags = pex_flags + add_on_flags
|
|
308
|
+
logging.debug(f"Running pex with {' '.join(pex_flags)}")
|
|
309
|
+
proc = util.build_pex(
|
|
310
|
+
sources_directories=[],
|
|
311
|
+
requirements_filepaths=[deps_requirements_path],
|
|
312
|
+
pex_flags=pex_flags,
|
|
313
|
+
output_pex_path=output_pex_path,
|
|
314
|
+
)
|
|
315
|
+
if proc.returncode:
|
|
316
|
+
if try_flags:
|
|
317
|
+
ui.warn(proc.stderr.decode("utf-8"))
|
|
318
|
+
ui.warn("Will retry building deps with a different resolution mechanism")
|
|
319
|
+
else:
|
|
320
|
+
raise DepsBuildFailure(proc)
|
|
321
|
+
else:
|
|
322
|
+
break
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def get_requirements_txt_deps(code_directory: str) -> list[str]:
|
|
326
|
+
requirements_path = os.path.join(code_directory, "requirements.txt")
|
|
327
|
+
if not os.path.exists(requirements_path):
|
|
328
|
+
return []
|
|
329
|
+
|
|
330
|
+
# combine multi-line strings into a single string
|
|
331
|
+
combined_lines = []
|
|
332
|
+
current_line = ""
|
|
333
|
+
|
|
334
|
+
with open(requirements_path, encoding="utf-8") as file:
|
|
335
|
+
for line in file:
|
|
336
|
+
stripped_line = line.rstrip()
|
|
337
|
+
if stripped_line.endswith("\\"):
|
|
338
|
+
current_line += stripped_line[:-1]
|
|
339
|
+
else:
|
|
340
|
+
current_line += stripped_line
|
|
341
|
+
combined_lines.append(current_line)
|
|
342
|
+
current_line = ""
|
|
343
|
+
|
|
344
|
+
# Add any remaining content if the last line ends with a backslash
|
|
345
|
+
if current_line:
|
|
346
|
+
combined_lines.append(current_line)
|
|
347
|
+
|
|
348
|
+
lines = []
|
|
349
|
+
for raw_line in combined_lines:
|
|
350
|
+
# https://pip.pypa.io/en/stable/reference/requirements-file-format/#comments
|
|
351
|
+
line = re.sub(r"(^#|\s#).*", "", raw_line)
|
|
352
|
+
line = line.strip()
|
|
353
|
+
# remove current dir from the deps
|
|
354
|
+
if line in {"", "."}:
|
|
355
|
+
continue
|
|
356
|
+
lines.append(line)
|
|
357
|
+
|
|
358
|
+
return lines
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def get_setup_py_deps(code_directory: str, python_interpreter: str) -> list[str]:
|
|
362
|
+
setup_py_path = os.path.join(code_directory, "setup.py")
|
|
363
|
+
if not os.path.exists(setup_py_path):
|
|
364
|
+
return []
|
|
365
|
+
|
|
366
|
+
lines = []
|
|
367
|
+
# write out egg_info files and load as distribution
|
|
368
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
|
369
|
+
proc = subprocess.run(
|
|
370
|
+
[python_interpreter, setup_py_path, "egg_info", f"--egg-base={temp_dir}"],
|
|
371
|
+
capture_output=True,
|
|
372
|
+
check=False,
|
|
373
|
+
cwd=code_directory,
|
|
374
|
+
)
|
|
375
|
+
if proc.returncode:
|
|
376
|
+
raise ValueError(
|
|
377
|
+
"Error running setup.py egg_info: "
|
|
378
|
+
+ proc.stdout.decode("utf-8")
|
|
379
|
+
+ proc.stderr.decode("utf-8")
|
|
380
|
+
)
|
|
381
|
+
dists = list(importlib.metadata.distributions(path=[temp_dir]))
|
|
382
|
+
if len(dists) != 1:
|
|
383
|
+
raise ValueError(f"Could not find distribution for {setup_py_path}")
|
|
384
|
+
dist = dists[0]
|
|
385
|
+
for requirement in dist.requires or []:
|
|
386
|
+
lines.append(requirement)
|
|
387
|
+
|
|
388
|
+
return lines
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _resolve_uv_workspace_dep(dep_name: str, code_directory: str) -> str | None:
|
|
392
|
+
"""Find the relative path to a uv workspace member by package name.
|
|
393
|
+
|
|
394
|
+
Returns a relative path like "../shared-lib" or None if not found.
|
|
395
|
+
Assumes the directory name matches the package name.
|
|
396
|
+
"""
|
|
397
|
+
result = find_uv_workspace_root(code_directory)
|
|
398
|
+
if not result:
|
|
399
|
+
return None
|
|
400
|
+
|
|
401
|
+
workspace_root, _ = result
|
|
402
|
+
candidate = os.path.join(str(workspace_root), dep_name)
|
|
403
|
+
if os.path.isdir(candidate):
|
|
404
|
+
return os.path.relpath(candidate, code_directory)
|
|
405
|
+
|
|
406
|
+
return None
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def get_pyproject_toml_deps(code_directory: str) -> list[str]:
|
|
410
|
+
pyproject_path = os.path.join(code_directory, "pyproject.toml")
|
|
411
|
+
if not os.path.exists(pyproject_path):
|
|
412
|
+
return []
|
|
413
|
+
|
|
414
|
+
try:
|
|
415
|
+
with open(pyproject_path, "rb") as file:
|
|
416
|
+
pyproject_data = tomllib.load(file)
|
|
417
|
+
except Exception as e:
|
|
418
|
+
raise ValueError(f"Error parsing pyproject.toml: {e}")
|
|
419
|
+
|
|
420
|
+
lines = []
|
|
421
|
+
|
|
422
|
+
# Handle dependencies in [project] section (PEP 621)
|
|
423
|
+
project_section = pyproject_data.get("project", {})
|
|
424
|
+
dependencies = project_section.get("dependencies", [])
|
|
425
|
+
for dep in dependencies:
|
|
426
|
+
lines.append(str(dep))
|
|
427
|
+
|
|
428
|
+
# Handle optional dependencies in [project.optional-dependencies]
|
|
429
|
+
optional_deps = project_section.get("optional-dependencies", {})
|
|
430
|
+
for group_deps in optional_deps.values():
|
|
431
|
+
for dep in group_deps:
|
|
432
|
+
lines.append(str(dep))
|
|
433
|
+
|
|
434
|
+
# Handle legacy [tool.poetry.dependencies] for Poetry projects
|
|
435
|
+
poetry_section = pyproject_data.get("tool", {}).get("poetry", {})
|
|
436
|
+
poetry_deps = poetry_section.get("dependencies", {})
|
|
437
|
+
for dep_name, dep_spec in poetry_deps.items():
|
|
438
|
+
if dep_name == "python":
|
|
439
|
+
continue # Skip python version constraint
|
|
440
|
+
|
|
441
|
+
if isinstance(dep_spec, str):
|
|
442
|
+
lines.append(f"{dep_name}{dep_spec}")
|
|
443
|
+
elif isinstance(dep_spec, dict):
|
|
444
|
+
version_spec = dep_spec.get("version", "")
|
|
445
|
+
if version_spec:
|
|
446
|
+
lines.append(f"{dep_name}{version_spec}")
|
|
447
|
+
else:
|
|
448
|
+
# Handle complex dependency specs (git, path, etc.)
|
|
449
|
+
lines.append(dep_name)
|
|
450
|
+
|
|
451
|
+
# Handle legacy [tool.poetry.dev-dependencies] (older Poetry format)
|
|
452
|
+
poetry_dev_deps = poetry_section.get("dev-dependencies", {})
|
|
453
|
+
for dep_name, dep_spec in poetry_dev_deps.items():
|
|
454
|
+
if isinstance(dep_spec, str):
|
|
455
|
+
lines.append(f"{dep_name}{dep_spec}")
|
|
456
|
+
elif isinstance(dep_spec, dict):
|
|
457
|
+
version_spec = dep_spec.get("version", "")
|
|
458
|
+
if version_spec:
|
|
459
|
+
lines.append(f"{dep_name}{version_spec}")
|
|
460
|
+
else:
|
|
461
|
+
lines.append(dep_name)
|
|
462
|
+
|
|
463
|
+
# Handle [tool.uv.sources] - resolve workspace and path dependencies to local paths
|
|
464
|
+
uv_sources = pyproject_data.get("tool", {}).get("uv", {}).get("sources", {})
|
|
465
|
+
if uv_sources:
|
|
466
|
+
resolved_lines = []
|
|
467
|
+
for line in lines:
|
|
468
|
+
source_config = uv_sources.get(line)
|
|
469
|
+
if source_config:
|
|
470
|
+
if source_config.get("workspace"):
|
|
471
|
+
resolved_path = _resolve_uv_workspace_dep(line, code_directory)
|
|
472
|
+
if resolved_path:
|
|
473
|
+
resolved_lines.append(resolved_path)
|
|
474
|
+
continue
|
|
475
|
+
elif "path" in source_config:
|
|
476
|
+
resolved_lines.append(source_config["path"])
|
|
477
|
+
continue
|
|
478
|
+
resolved_lines.append(line)
|
|
479
|
+
return resolved_lines
|
|
480
|
+
|
|
481
|
+
return lines
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
@click.command()
|
|
485
|
+
@click.argument("project_dir", type=click.Path(exists=True))
|
|
486
|
+
@click.argument("build_output_dir", type=click.Path(exists=False))
|
|
487
|
+
@util.python_version_option()
|
|
488
|
+
def deps_main(project_dir, build_output_dir, python_version):
|
|
489
|
+
_deps_pex_path, _dagster_version = build_deps_pex(
|
|
490
|
+
project_dir, build_output_dir, util.parse_python_version(python_version)
|
|
491
|
+
)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
if __name__ == "__main__":
|
|
495
|
+
deps_main()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import subprocess
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class GitCommitMetadata:
|
|
8
|
+
timestamp: str
|
|
9
|
+
message: str
|
|
10
|
+
email: str
|
|
11
|
+
name: str
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_git_commit_metadata(project_dir: str) -> GitCommitMetadata:
|
|
15
|
+
commands = {
|
|
16
|
+
"timestamp": f"git -C {project_dir} log -1 --format=%cd --date=unix".split(),
|
|
17
|
+
"message": f"git -C {project_dir} log -1 --format=%s".split(),
|
|
18
|
+
"email": f"git -C {project_dir} log -1 --format=%ae".split(),
|
|
19
|
+
"name": f"git -C {project_dir} log -1 --format=%an".split(),
|
|
20
|
+
}
|
|
21
|
+
metadata = {}
|
|
22
|
+
for key, command in commands.items():
|
|
23
|
+
logging.debug("Running %r", command)
|
|
24
|
+
proc = subprocess.run(command, capture_output=True, check=False)
|
|
25
|
+
if proc.returncode:
|
|
26
|
+
logging.error(f"git command failed: {proc.stdout}\n{proc.stderr}")
|
|
27
|
+
metadata[key] = proc.stdout.decode("utf-8").strip()
|
|
28
|
+
|
|
29
|
+
return GitCommitMetadata(**metadata)
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import pathlib
|
|
5
|
+
import subprocess
|
|
6
|
+
from contextlib import contextmanager
|
|
7
|
+
|
|
8
|
+
from dagster_cloud_cli import ui
|
|
9
|
+
from dagster_cloud_cli.core.pex_builder import util
|
|
10
|
+
|
|
11
|
+
# Loads event details from within a github action
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class GithubEvent:
|
|
15
|
+
def __init__(self, project_dir: str):
|
|
16
|
+
self.github_server_url = os.environ["GITHUB_SERVER_URL"]
|
|
17
|
+
self.github_sha = os.environ["GITHUB_SHA"]
|
|
18
|
+
self.github_repository = os.environ["GITHUB_REPOSITORY"]
|
|
19
|
+
self.github_run_id = os.environ["GITHUB_RUN_ID"]
|
|
20
|
+
self.github_run_url = (
|
|
21
|
+
f"{self.github_server_url}/{self.github_repository}/actions/runs/{self.github_run_id}"
|
|
22
|
+
)
|
|
23
|
+
action_path = os.getenv("GITHUB_ACTION_PATH")
|
|
24
|
+
self.github_action_path = pathlib.Path(action_path) if action_path else None
|
|
25
|
+
event_path = os.getenv("GITHUB_EVENT_PATH")
|
|
26
|
+
if not event_path:
|
|
27
|
+
raise ValueError("GITHUB_EVENT_PATH not set")
|
|
28
|
+
|
|
29
|
+
# json details: https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads
|
|
30
|
+
self.event = event = json.load(open(event_path, encoding="utf-8"))
|
|
31
|
+
|
|
32
|
+
# get some commonly used fields
|
|
33
|
+
# not all events have "action", eg https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push
|
|
34
|
+
self.action = event.get("action")
|
|
35
|
+
self.repo_name = event["repository"]["full_name"]
|
|
36
|
+
|
|
37
|
+
self.branch_name: str | None = None
|
|
38
|
+
self.branch_url: str | None = None
|
|
39
|
+
self.pull_request_url: str | None = None
|
|
40
|
+
self.pull_request_id: str | None = None
|
|
41
|
+
self.pull_request_status: str | None = None
|
|
42
|
+
|
|
43
|
+
if "pull_request" in self.event:
|
|
44
|
+
pull_request = self.event["pull_request"]
|
|
45
|
+
# For PRs GITHUB_SHA is not the last commit in the branch, but head sha is
|
|
46
|
+
self.github_sha = pull_request["head"]["sha"]
|
|
47
|
+
self.branch_name = pull_request["head"]["ref"]
|
|
48
|
+
self.branch_url = f"{self.github_server_url}/{self.repo_name}/tree/{self.branch_name}"
|
|
49
|
+
self.pull_request_url = pull_request["html_url"]
|
|
50
|
+
self.pull_request_id = str(pull_request["number"])
|
|
51
|
+
self.pull_request_status = (
|
|
52
|
+
"merged" if pull_request.get("merged") else pull_request["state"]
|
|
53
|
+
).upper()
|
|
54
|
+
|
|
55
|
+
self.commit_url = f"{self.github_server_url}/{self.repo_name}/tree/{self.github_sha}"
|
|
56
|
+
|
|
57
|
+
git_metadata = get_git_commit_metadata(project_dir)
|
|
58
|
+
self.timestamp = float(git_metadata["timestamp"])
|
|
59
|
+
self.commit_msg = git_metadata["message"]
|
|
60
|
+
self.author_name = git_metadata["name"]
|
|
61
|
+
self.author_email = git_metadata["email"]
|
|
62
|
+
|
|
63
|
+
def get_github_repo(self):
|
|
64
|
+
import github3
|
|
65
|
+
|
|
66
|
+
token = os.getenv("GITHUB_TOKEN")
|
|
67
|
+
if not token:
|
|
68
|
+
return None
|
|
69
|
+
gh = github3.login(token=token)
|
|
70
|
+
repo_owner, repo_name = self.github_repository.split("/", 1)
|
|
71
|
+
return gh.repository(repo_owner, repo_name)
|
|
72
|
+
|
|
73
|
+
def get_github_avatar_url(self) -> str | None:
|
|
74
|
+
repo = self.get_github_repo()
|
|
75
|
+
if not repo:
|
|
76
|
+
return
|
|
77
|
+
commit = repo.commit(self.github_sha)
|
|
78
|
+
return commit.author.get("avatar_url") if commit.author else None
|
|
79
|
+
|
|
80
|
+
def update_pr_comment(
|
|
81
|
+
self, body: str, orig_author: str | None = None, orig_text: str | None = None
|
|
82
|
+
):
|
|
83
|
+
# orig_author and orig_text are used to identify an existing comment which is updated.
|
|
84
|
+
# if not provided, or not found, a new comment is created
|
|
85
|
+
import github3.exceptions
|
|
86
|
+
|
|
87
|
+
if not self.pull_request_id:
|
|
88
|
+
return
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
repo = self.get_github_repo()
|
|
92
|
+
if not repo:
|
|
93
|
+
return
|
|
94
|
+
pr = repo.pull_request(int(self.pull_request_id))
|
|
95
|
+
|
|
96
|
+
for comment in pr.issue_comments():
|
|
97
|
+
if comment.user.login == orig_author and orig_text in comment.body:
|
|
98
|
+
comment.edit(body)
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
pr.create_comment(body)
|
|
102
|
+
|
|
103
|
+
except github3.exceptions.GitHubException:
|
|
104
|
+
logging.exception("Ignoring error when updating PR comment")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def get_git_commit_metadata(project_dir: str) -> dict[str, str]:
|
|
108
|
+
commands = {
|
|
109
|
+
"timestamp": f"git -C {project_dir} log -1 --format=%cd --date=unix".split(),
|
|
110
|
+
"message": f"git -C {project_dir} log -1 --format=%s".split(),
|
|
111
|
+
"email": f"git -C {project_dir} log -1 --format=%ae".split(),
|
|
112
|
+
"name": f"git -C {project_dir} log -1 --format=%an".split(),
|
|
113
|
+
}
|
|
114
|
+
metadata = {}
|
|
115
|
+
for key, command in commands.items():
|
|
116
|
+
logging.debug("Running %r", command)
|
|
117
|
+
proc = subprocess.run(command, capture_output=True, check=False)
|
|
118
|
+
if proc.returncode:
|
|
119
|
+
logging.error(f"git command failed: {proc.stdout}\n{proc.stderr}")
|
|
120
|
+
metadata[key] = proc.stdout.decode("utf-8").strip()
|
|
121
|
+
|
|
122
|
+
return metadata
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def get_github_event(project_dir) -> GithubEvent:
|
|
126
|
+
return GithubEvent(project_dir)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def update_pr_comment(github_event: GithubEvent, action, deployment_name, location_name):
|
|
130
|
+
"""Add or update the status comment on a github PR."""
|
|
131
|
+
# This reuses the src/create_or_update_comment.py script.
|
|
132
|
+
# We can't reuse actions/utils/notify here because we need to run this once for every location.
|
|
133
|
+
# To repeat an action github provides a matrix strategy but we don't use matrix due to latency
|
|
134
|
+
# concerns (it would launch a new container for every location)
|
|
135
|
+
if not github_event.pull_request_id:
|
|
136
|
+
ui.print("Not within a PR, not adding PR comment.")
|
|
137
|
+
return
|
|
138
|
+
|
|
139
|
+
if not github_event.github_action_path:
|
|
140
|
+
ui.warn("Unable to locate notification script, not adding PR comment.")
|
|
141
|
+
return
|
|
142
|
+
|
|
143
|
+
script_path = github_event.github_action_path.parent.parent / "src/create_or_update_comment.py"
|
|
144
|
+
if not script_path.exists():
|
|
145
|
+
ui.warn(f"Did not find {script_path}, not adding PR comment.")
|
|
146
|
+
|
|
147
|
+
env = {name: value for name, value in os.environ.items() if not name.startswith("PEX_")}
|
|
148
|
+
pr_id = str(github_event.pull_request_id)
|
|
149
|
+
|
|
150
|
+
env.update(
|
|
151
|
+
{
|
|
152
|
+
"INPUT_PR": pr_id,
|
|
153
|
+
"INPUT_ACTION": action,
|
|
154
|
+
"INPUT_DEPLOYMENT": deployment_name,
|
|
155
|
+
"INPUT_LOCATION_NAME": location_name,
|
|
156
|
+
"GITHUB_RUN_URL": github_event.github_run_url,
|
|
157
|
+
}
|
|
158
|
+
)
|
|
159
|
+
env = {name: value for name, value in env.items() if value is not None}
|
|
160
|
+
proc = util.run_python_subprocess([str(script_path)], env=env)
|
|
161
|
+
if proc.returncode:
|
|
162
|
+
ui.error(f"Could not update PR comment: {proc.stdout}\n{proc.stderr}")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@contextmanager
|
|
166
|
+
def log_group(title: str):
|
|
167
|
+
try:
|
|
168
|
+
yield
|
|
169
|
+
finally:
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def log_error(title: str, msg: str):
|
|
174
|
+
pass
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
if __name__ == "__main__":
|
|
178
|
+
pass
|