UEPyScripts 1.4.2__tar.gz → 1.4.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/PKG-INFO +1 -1
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ugs/git_utils.py +110 -68
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ugs/pull.py +18 -12
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ugs/push.py +33 -25
- uepyscripts-1.4.4/src/uepyscripts/tools/ugs/s3_settings.py +81 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/.gitignore +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/LICENSE +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/README.md +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/pyproject.toml +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/__init__.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/context/__init__.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/internal/__init__.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/internal/config.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/internal/engine.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/internal/engine_resolver.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/internal/project.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/py.typed +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/run/__init__.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/run/buildgraph.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/run/uat.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/run/ubt.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/__init__.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ci/buildgraph.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ci/cleanup.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/helpers.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/subprocess.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/__init__.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/check_engine_installation.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/close_editor.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/compile_editor.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/engine_installation/engine_destination.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/engine_installation/engine_installer.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/engine_installation/engine_source.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/generate_solution.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/run_editor.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ugs/cleanup.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ugs/ugs_types.py +0 -0
- {uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/winreg.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: UEPyScripts
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.4
|
|
4
4
|
Summary: Collection of Python tools and helpers designed to automate common Unreal Engine project tasks (builds, packaging, editor control, CI tasks, etc.).
|
|
5
5
|
Project-URL: Changelog, https://github.com/TheEmidee/UEPyScripts/releases
|
|
6
6
|
Project-URL: Homepage, https://github.com/TheEmidee/UEPyScripts
|
|
@@ -1,68 +1,110 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import subprocess
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
from uepyscripts import logger
|
|
6
|
-
|
|
7
|
-
ANCESTRY_LOOKBACK =
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def get_current_sha(cwd: Path) -> str:
|
|
11
|
-
sha = os.environ.get("GIT_COMMIT") # Jenkins sets this automatically
|
|
12
|
-
if sha:
|
|
13
|
-
return sha
|
|
14
|
-
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd, text=True).strip()
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def get_current_branch(cwd: Path) -> str:
|
|
18
|
-
branch = os.environ.get("GIT_BRANCH") # Jenkins' Git plugin sets this, sometimes as "origin/<branch>"
|
|
19
|
-
if branch:
|
|
20
|
-
return branch.split("/", 1)[1] if "/" in branch else branch
|
|
21
|
-
|
|
22
|
-
branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, text=True).strip()
|
|
23
|
-
if branch != "HEAD":
|
|
24
|
-
return branch
|
|
25
|
-
|
|
26
|
-
# Detached HEAD (e.g. a CI checkout of an explicit ref rather than a
|
|
27
|
-
# tracked branch, with no GIT_BRANCH env var set either) — fall back to
|
|
28
|
-
# whichever local or remote-tracking ref currently points at this commit.
|
|
29
|
-
refs = subprocess.check_output(
|
|
30
|
-
["git", "for-each-ref", "--points-at=HEAD", "--format=%(refname:short)", "refs/heads/", "refs/remotes/"],
|
|
31
|
-
cwd=cwd,
|
|
32
|
-
text=True,
|
|
33
|
-
).splitlines()
|
|
34
|
-
for ref in refs:
|
|
35
|
-
return ref.split("/", 1)[1] if "/" in ref else ref
|
|
36
|
-
|
|
37
|
-
logger.warning("Could not resolve current branch: HEAD is detached and no ref points at it")
|
|
38
|
-
return branch
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
def
|
|
42
|
-
out = subprocess.check_output(["git", "
|
|
43
|
-
return out
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def
|
|
47
|
-
"""
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
1
|
+
import os
|
|
2
|
+
import subprocess
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from uepyscripts import logger
|
|
6
|
+
|
|
7
|
+
ANCESTRY_LOOKBACK = 500
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_current_sha(cwd: Path) -> str:
|
|
11
|
+
sha = os.environ.get("GIT_COMMIT") # Jenkins sets this automatically
|
|
12
|
+
if sha:
|
|
13
|
+
return sha
|
|
14
|
+
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd, text=True).strip()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_current_branch(cwd: Path) -> str:
|
|
18
|
+
branch = os.environ.get("GIT_BRANCH") # Jenkins' Git plugin sets this, sometimes as "origin/<branch>"
|
|
19
|
+
if branch:
|
|
20
|
+
return branch.split("/", 1)[1] if "/" in branch else branch
|
|
21
|
+
|
|
22
|
+
branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, text=True).strip()
|
|
23
|
+
if branch != "HEAD":
|
|
24
|
+
return branch
|
|
25
|
+
|
|
26
|
+
# Detached HEAD (e.g. a CI checkout of an explicit ref rather than a
|
|
27
|
+
# tracked branch, with no GIT_BRANCH env var set either) — fall back to
|
|
28
|
+
# whichever local or remote-tracking ref currently points at this commit.
|
|
29
|
+
refs = subprocess.check_output(
|
|
30
|
+
["git", "for-each-ref", "--points-at=HEAD", "--format=%(refname:short)", "refs/heads/", "refs/remotes/"],
|
|
31
|
+
cwd=cwd,
|
|
32
|
+
text=True,
|
|
33
|
+
).splitlines()
|
|
34
|
+
for ref in refs:
|
|
35
|
+
return ref.split("/", 1)[1] if "/" in ref else ref
|
|
36
|
+
|
|
37
|
+
logger.warning("Could not resolve current branch: HEAD is detached and no ref points at it")
|
|
38
|
+
return branch
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def is_shallow_repository(cwd: Path) -> bool:
|
|
42
|
+
out = subprocess.check_output(["git", "rev-parse", "--is-shallow-repository"], cwd=cwd, text=True).strip()
|
|
43
|
+
return out == "true"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def deepen_shallow_history(cwd: Path, limit: int, remote: str = "origin") -> bool:
|
|
47
|
+
"""Pulls down the commits missing from a shallow checkout, and only the
|
|
48
|
+
commits: `--filter=tree:0` keeps trees and blobs on the server, so this
|
|
49
|
+
stays a metadata-sized fetch even on a repository the size of an engine
|
|
50
|
+
tree. Returns False (after logging) when the remote refuses, so callers
|
|
51
|
+
can carry on with whatever history is available locally."""
|
|
52
|
+
try:
|
|
53
|
+
subprocess.run(
|
|
54
|
+
["git", "fetch", "--no-tags", "--filter=tree:0", f"--deepen={limit}", remote],
|
|
55
|
+
cwd=cwd,
|
|
56
|
+
check=True,
|
|
57
|
+
capture_output=True,
|
|
58
|
+
text=True,
|
|
59
|
+
)
|
|
60
|
+
return True
|
|
61
|
+
except subprocess.CalledProcessError as e:
|
|
62
|
+
reason = e.stderr.strip() or e
|
|
63
|
+
logger.warning(f"Could not deepen shallow history from '{remote}' ({reason}); ancestry walking is limited to the local history")
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_local_ancestry(cwd: Path, limit: int = ANCESTRY_LOOKBACK) -> list[str]:
|
|
68
|
+
ancestry = _log_ancestry(cwd, limit)
|
|
69
|
+
|
|
70
|
+
# CI checkouts are typically shallow clones (Jenkins' cloneOption(depth: 1)),
|
|
71
|
+
# where HEAD is a grafted boundary commit: `git log HEAD` stops at HEAD
|
|
72
|
+
# itself and every already-published parent looks unreachable, which would
|
|
73
|
+
# downgrade an incremental publish into a full fresh sync. Deepen first so
|
|
74
|
+
# the walk sees the real ancestry.
|
|
75
|
+
if len(ancestry) < limit and is_shallow_repository(cwd):
|
|
76
|
+
logger.info(f"Shallow checkout detected ({len(ancestry)} commit(s) available); deepening history to {limit} commits")
|
|
77
|
+
if deepen_shallow_history(cwd, limit):
|
|
78
|
+
ancestry = _log_ancestry(cwd, limit)
|
|
79
|
+
|
|
80
|
+
return ancestry
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _log_ancestry(cwd: Path, limit: int) -> list[str]:
|
|
84
|
+
out = subprocess.check_output(["git", "log", f"-{limit}", "--format=%H", "HEAD"], cwd=cwd, text=True)
|
|
85
|
+
return out.splitlines()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def get_remote_reachable_shas(cwd: Path, remote: str = "origin") -> set[str]:
|
|
89
|
+
"""All commit SHAs reachable from any current branch tip on `remote`, after
|
|
90
|
+
pruning local remote-tracking refs for branches deleted on the remote."""
|
|
91
|
+
subprocess.run(["git", "fetch", "--prune", remote], cwd=cwd, check=True, capture_output=True, text=True)
|
|
92
|
+
out = subprocess.check_output(["git", "rev-list", f"--remotes={remote}"], cwd=cwd, text=True)
|
|
93
|
+
return set(out.splitlines())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def resolve_nearest_published_ancestor(index: list[str], ancestry: list[str]) -> tuple[str | None, bool]:
|
|
97
|
+
"""Finds the nearest ancestor SHA (including HEAD itself) present in the
|
|
98
|
+
index. Returns None if nothing has ever been published reachable from HEAD
|
|
99
|
+
within the walked ancestry. The bool is True when the match was found by
|
|
100
|
+
walking the ancestry itself (an exact match), False when falling back to
|
|
101
|
+
the most recent published version because nothing in the ancestry matched."""
|
|
102
|
+
index_set = set(index)
|
|
103
|
+
for sha in ancestry:
|
|
104
|
+
if sha in index_set:
|
|
105
|
+
return sha, True
|
|
106
|
+
|
|
107
|
+
if index:
|
|
108
|
+
return index[-1], False
|
|
109
|
+
|
|
110
|
+
return None, True
|
|
@@ -12,6 +12,7 @@ from uepyscripts import logger
|
|
|
12
12
|
from uepyscripts.internal.engine import resolve_engine
|
|
13
13
|
from uepyscripts.internal.project import resolve_project
|
|
14
14
|
from uepyscripts.tools.ugs.git_utils import get_local_ancestry, resolve_nearest_published_ancestor
|
|
15
|
+
from uepyscripts.tools.ugs.s3_settings import S3_CONFIG_SECTION, S3Settings, add_s3_arguments, resolve_s3_settings
|
|
15
16
|
from uepyscripts.tools.ugs.ugs_types import Manifest, VersionManifest
|
|
16
17
|
|
|
17
18
|
"""
|
|
@@ -44,14 +45,14 @@ class LocalState(TypedDict):
|
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
class Context:
|
|
47
|
-
def __init__(self, root_folder: Path,
|
|
48
|
+
def __init__(self, root_folder: Path, s3_settings: S3Settings) -> None:
|
|
48
49
|
self.root_folder: Path = root_folder
|
|
49
|
-
self.s3_bucket_name: str =
|
|
50
|
-
self.s3_bucket_region: str =
|
|
50
|
+
self.s3_bucket_name: str = s3_settings.bucket_name
|
|
51
|
+
self.s3_bucket_region: str = s3_settings.bucket_region
|
|
51
52
|
self.s3_client: S3Client = S3Client(
|
|
52
|
-
access_key=
|
|
53
|
-
secret_key=
|
|
54
|
-
region=
|
|
53
|
+
access_key=s3_settings.access_key,
|
|
54
|
+
secret_key=s3_settings.secret_key,
|
|
55
|
+
region=s3_settings.bucket_region,
|
|
55
56
|
)
|
|
56
57
|
logger.info("Download index.json")
|
|
57
58
|
self.commit_index = cast(list[str], self.s3_client.download_json(self.s3_bucket_name, "index.json", default=[]))
|
|
@@ -207,12 +208,15 @@ def sync(context: Context, target_version: str, state: LocalState) -> LocalState
|
|
|
207
208
|
|
|
208
209
|
def parse_arguments() -> argparse.Namespace:
|
|
209
210
|
"""Parse command line arguments."""
|
|
210
|
-
parser = argparse.ArgumentParser(
|
|
211
|
+
parser = argparse.ArgumentParser(
|
|
212
|
+
description=(
|
|
213
|
+
"Sync local engine + game binaries to match the current git HEAD. "
|
|
214
|
+
f"The S3 info is read from the [{S3_CONFIG_SECTION}] section of Config/PyScripts/config.ini, "
|
|
215
|
+
"and each value can be overridden by its command line argument."
|
|
216
|
+
)
|
|
217
|
+
)
|
|
211
218
|
parser.add_argument("--uproject-path", type=Path, help=("Path to a native uproject file"))
|
|
212
|
-
parser
|
|
213
|
-
parser.add_argument("--s3-bucket-region", type=str, help=("AWS S3 Bucket Region"))
|
|
214
|
-
parser.add_argument("--s3-access-key", type=str, help=("AWS S3 Access Key"))
|
|
215
|
-
parser.add_argument("--s3-secret-key", type=str, help=("AWS S3 Secret Key"))
|
|
219
|
+
add_s3_arguments(parser)
|
|
216
220
|
|
|
217
221
|
return parser.parse_args()
|
|
218
222
|
|
|
@@ -234,9 +238,11 @@ def main() -> None:
|
|
|
234
238
|
|
|
235
239
|
engine = resolve_engine(project)
|
|
236
240
|
|
|
241
|
+
s3_settings = resolve_s3_settings(project, args)
|
|
242
|
+
|
|
237
243
|
local_state = LocalStateManager()
|
|
238
244
|
|
|
239
|
-
context = Context(engine.root_path,
|
|
245
|
+
context = Context(engine.root_path, s3_settings)
|
|
240
246
|
|
|
241
247
|
warn_if_source_dirty()
|
|
242
248
|
|
|
@@ -35,6 +35,7 @@ from uepyscripts.tools.ugs.git_utils import (
|
|
|
35
35
|
get_local_ancestry,
|
|
36
36
|
resolve_nearest_published_ancestor,
|
|
37
37
|
)
|
|
38
|
+
from uepyscripts.tools.ugs.s3_settings import S3_CONFIG_SECTION, S3Settings, add_s3_arguments, resolve_s3_settings
|
|
38
39
|
from uepyscripts.tools.ugs.ugs_types import HashCacheEntry, HashCacheManifest, Manifest, VersionManifest
|
|
39
40
|
|
|
40
41
|
# ---- CONFIG ----------------------------------------------------------
|
|
@@ -92,12 +93,14 @@ class HashCache:
|
|
|
92
93
|
|
|
93
94
|
|
|
94
95
|
class Context:
|
|
95
|
-
def __init__(self, args: argparse.Namespace, root_path: Path) -> None:
|
|
96
|
+
def __init__(self, args: argparse.Namespace, root_path: Path, s3_settings: S3Settings) -> None:
|
|
96
97
|
self.args = args
|
|
98
|
+
self.s3_bucket_name: str = s3_settings.bucket_name
|
|
99
|
+
self.s3_bucket_region: str = s3_settings.bucket_region
|
|
97
100
|
self.s3_client: S3Client = S3Client(
|
|
98
|
-
access_key=
|
|
99
|
-
secret_key=
|
|
100
|
-
region=
|
|
101
|
+
access_key=s3_settings.access_key,
|
|
102
|
+
secret_key=s3_settings.secret_key,
|
|
103
|
+
region=s3_settings.bucket_region,
|
|
101
104
|
)
|
|
102
105
|
self.symbol_store_path: str = args.symbol_store_path
|
|
103
106
|
self.tmp_root: Path = Path(tempfile.mkdtemp(prefix="publish-"))
|
|
@@ -113,7 +116,7 @@ class Context:
|
|
|
113
116
|
an inexact match is fine here: a fingerprint mismatch just falls back to
|
|
114
117
|
a full hash for that file, no worse than a cold-cache case, so
|
|
115
118
|
there's no correctness risk in seeding from a non-ancestor."""
|
|
116
|
-
bucket_name = self.
|
|
119
|
+
bucket_name = self.s3_bucket_name
|
|
117
120
|
hash_cache_index = cast(list[str], self.s3_client.download_json(bucket_name, "hash_caches/index.json", default=[]))
|
|
118
121
|
baseline_sha, _ = resolve_nearest_published_ancestor(hash_cache_index, self.ancestry)
|
|
119
122
|
|
|
@@ -346,7 +349,7 @@ def prune_old_versions(context: Context, cfg: FilesConfiguration, index: list[st
|
|
|
346
349
|
keys_to_delete.append(f"checkpoints/{v}.7z")
|
|
347
350
|
|
|
348
351
|
logger.info(f"Pruning {len(versions_to_delete)} version(s), {len(checkpoints_to_delete)} checkpoint(s)")
|
|
349
|
-
context.s3_client.delete_keys(context.
|
|
352
|
+
context.s3_client.delete_keys(context.s3_bucket_name, keys_to_delete)
|
|
350
353
|
|
|
351
354
|
return index[cutoff_idx:], checkpoints_to_keep
|
|
352
355
|
|
|
@@ -364,7 +367,7 @@ def prune_old_hash_caches(context: Context, hash_cache_index: list[str], keep: i
|
|
|
364
367
|
to_keep = hash_cache_index[-keep:]
|
|
365
368
|
|
|
366
369
|
logger.info(f"Pruning {len(to_delete)} old hash cache(s)")
|
|
367
|
-
context.s3_client.delete_keys(context.
|
|
370
|
+
context.s3_client.delete_keys(context.s3_bucket_name, [f"hash_caches/{sha}.json" for sha in to_delete])
|
|
368
371
|
|
|
369
372
|
return to_keep
|
|
370
373
|
|
|
@@ -413,7 +416,13 @@ def publish_to_symbol_store(context: Context, paths: list[str], cfg: FilesConfig
|
|
|
413
416
|
|
|
414
417
|
def parse_arguments() -> argparse.Namespace:
|
|
415
418
|
"""Parse command line arguments."""
|
|
416
|
-
parser = argparse.ArgumentParser(
|
|
419
|
+
parser = argparse.ArgumentParser(
|
|
420
|
+
description=(
|
|
421
|
+
"Publish the engine + game binaries for the current git HEAD. "
|
|
422
|
+
f"The S3 info is read from the [{S3_CONFIG_SECTION}] section of Config/PyScripts/config.ini, "
|
|
423
|
+
"and each value can be overridden by its command line argument."
|
|
424
|
+
)
|
|
425
|
+
)
|
|
417
426
|
parser.add_argument("--uproject-path", type=Path, help=("Path to a native uproject file"))
|
|
418
427
|
parser.add_argument("--skip-upload", action=argparse.BooleanOptionalAction, help=("Set this to not upload anything on S3"))
|
|
419
428
|
parser.add_argument(
|
|
@@ -421,10 +430,7 @@ def parse_arguments() -> argparse.Namespace:
|
|
|
421
430
|
action=argparse.BooleanOptionalAction,
|
|
422
431
|
help=("Set this to not delete the temporary directory when the script finishes"),
|
|
423
432
|
)
|
|
424
|
-
parser
|
|
425
|
-
parser.add_argument("--s3-bucket-region", type=str, help=("AWS S3 Bucket Region"))
|
|
426
|
-
parser.add_argument("--s3-access-key", type=str, help=("AWS S3 Access Key"))
|
|
427
|
-
parser.add_argument("--s3-secret-key", type=str, help=("AWS S3 Secret Key"))
|
|
433
|
+
add_s3_arguments(parser)
|
|
428
434
|
parser.add_argument("--disable-symbol-store-upload", action=argparse.BooleanOptionalAction)
|
|
429
435
|
parser.add_argument("--symbol-store-path", type=str, help=("Path of the shared symbol store folder"))
|
|
430
436
|
parser.add_argument(
|
|
@@ -472,6 +478,8 @@ def main() -> None:
|
|
|
472
478
|
|
|
473
479
|
engine = resolve_engine(project)
|
|
474
480
|
|
|
481
|
+
s3_settings = resolve_s3_settings(project, args)
|
|
482
|
+
|
|
475
483
|
cfg = FilesConfiguration(
|
|
476
484
|
root_folder=engine.root_path,
|
|
477
485
|
directories=[
|
|
@@ -487,14 +495,14 @@ def main() -> None:
|
|
|
487
495
|
symstore_product=project.project_name,
|
|
488
496
|
)
|
|
489
497
|
|
|
490
|
-
context = Context(args, engine.root_path)
|
|
498
|
+
context = Context(args, engine.root_path, s3_settings)
|
|
491
499
|
|
|
492
500
|
try:
|
|
493
501
|
logger.info("=== Publish binary files ===")
|
|
494
502
|
logger.info(f"Current branch: {context.current_branch}")
|
|
495
503
|
|
|
496
504
|
logger.info("Download index.json")
|
|
497
|
-
index = cast(list[str], context.s3_client.download_json(context.
|
|
505
|
+
index = cast(list[str], context.s3_client.download_json(context.s3_bucket_name, "index.json", default=[]))
|
|
498
506
|
|
|
499
507
|
if len(index) == 0:
|
|
500
508
|
logger.info("Empty index.json. Fresh sync")
|
|
@@ -523,7 +531,7 @@ def main() -> None:
|
|
|
523
531
|
old_state = {}
|
|
524
532
|
else:
|
|
525
533
|
logger.info(f"Using {baseline_sha} as diff baseline for branch '{context.current_branch}'")
|
|
526
|
-
old_manifest = cast(VersionManifest, context.s3_client.download_json(context.
|
|
534
|
+
old_manifest = cast(VersionManifest, context.s3_client.download_json(context.s3_bucket_name, f"manifests/{baseline_sha}.json"))
|
|
527
535
|
old_state = old_manifest["files"]
|
|
528
536
|
|
|
529
537
|
changed, removed = compute_diff(old_state, new_state)
|
|
@@ -544,7 +552,7 @@ def main() -> None:
|
|
|
544
552
|
return
|
|
545
553
|
|
|
546
554
|
logger.info("Upload zip")
|
|
547
|
-
context.s3_client.upload_file(context.
|
|
555
|
+
context.s3_client.upload_file(context.s3_bucket_name, f"deltas/{version}.7z", zip_file_path)
|
|
548
556
|
zip_file_path.unlink()
|
|
549
557
|
|
|
550
558
|
manifest_payload: VersionManifest = {
|
|
@@ -553,7 +561,7 @@ def main() -> None:
|
|
|
553
561
|
"removed": removed,
|
|
554
562
|
}
|
|
555
563
|
context.s3_client.upload_bytes(
|
|
556
|
-
context.
|
|
564
|
+
context.s3_bucket_name,
|
|
557
565
|
f"manifests/{version}.json",
|
|
558
566
|
json.dumps(manifest_payload).encode(),
|
|
559
567
|
content_type="application/json",
|
|
@@ -562,7 +570,7 @@ def main() -> None:
|
|
|
562
570
|
index.append(version)
|
|
563
571
|
|
|
564
572
|
logger.info("Download JSON with checkpoints")
|
|
565
|
-
checkpoints = cast(list[str], context.s3_client.download_json(context.
|
|
573
|
+
checkpoints = cast(list[str], context.s3_client.download_json(context.s3_bucket_name, "checkpoints.json", default=[]))
|
|
566
574
|
|
|
567
575
|
logger.info(f"Checkpoint interval: {cfg.checkpoint_interval} - Number of deltas : {len(index)}")
|
|
568
576
|
|
|
@@ -576,7 +584,7 @@ def main() -> None:
|
|
|
576
584
|
zip_file_path = build_zip(cfg.root_folder, list(new_state.keys()), context.tmp_root)
|
|
577
585
|
|
|
578
586
|
logger.info("Upload zip")
|
|
579
|
-
context.s3_client.upload_file(context.
|
|
587
|
+
context.s3_client.upload_file(context.s3_bucket_name, f"checkpoints/{version}.7z", zip_file_path)
|
|
580
588
|
zip_file_path.unlink()
|
|
581
589
|
checkpoints.append(version)
|
|
582
590
|
else:
|
|
@@ -585,11 +593,11 @@ def main() -> None:
|
|
|
585
593
|
index, checkpoints = prune_old_versions(context, cfg, index, checkpoints)
|
|
586
594
|
|
|
587
595
|
logger.info("Upload index.json")
|
|
588
|
-
context.s3_client.upload_bytes(context.
|
|
596
|
+
context.s3_client.upload_bytes(context.s3_bucket_name, "index.json", json.dumps(index).encode(), content_type="application/json")
|
|
589
597
|
|
|
590
598
|
logger.info("Upload checkpoints.json")
|
|
591
599
|
context.s3_client.upload_bytes(
|
|
592
|
-
context.
|
|
600
|
+
context.s3_bucket_name, "checkpoints.json", json.dumps(checkpoints).encode(), content_type="application/json"
|
|
593
601
|
)
|
|
594
602
|
else:
|
|
595
603
|
logger.info("No changes in synced binaries.")
|
|
@@ -603,7 +611,7 @@ def main() -> None:
|
|
|
603
611
|
logger.info(f"Finished scan. Found {len(symstore_state)} files.")
|
|
604
612
|
|
|
605
613
|
logger.info("Download symstore-manifest.json")
|
|
606
|
-
prev_symstore_state = cast(Manifest, context.s3_client.download_json(context.
|
|
614
|
+
prev_symstore_state = cast(Manifest, context.s3_client.download_json(context.s3_bucket_name, "symstore-manifest.json", default={}))
|
|
607
615
|
symstore_changed, _ = compute_diff(prev_symstore_state, symstore_state)
|
|
608
616
|
|
|
609
617
|
if symstore_changed:
|
|
@@ -611,7 +619,7 @@ def main() -> None:
|
|
|
611
619
|
|
|
612
620
|
logger.info("Upload symstore-manifest.json")
|
|
613
621
|
context.s3_client.upload_bytes(
|
|
614
|
-
context.
|
|
622
|
+
context.s3_bucket_name, "symstore-manifest.json", json.dumps(symstore_state).encode(), content_type="application/json"
|
|
615
623
|
)
|
|
616
624
|
else:
|
|
617
625
|
logger.info("No changes for symbol store")
|
|
@@ -623,7 +631,7 @@ def main() -> None:
|
|
|
623
631
|
if not context.args.skip_upload:
|
|
624
632
|
logger.info("Upload hash cache")
|
|
625
633
|
context.s3_client.upload_bytes(
|
|
626
|
-
context.
|
|
634
|
+
context.s3_bucket_name,
|
|
627
635
|
f"hash_caches/{context.current_sha}.json",
|
|
628
636
|
json.dumps(context.hash_cache.to_manifest()).encode(),
|
|
629
637
|
content_type="application/json",
|
|
@@ -636,7 +644,7 @@ def main() -> None:
|
|
|
636
644
|
|
|
637
645
|
logger.info("Upload hash_caches/index.json")
|
|
638
646
|
context.s3_client.upload_bytes(
|
|
639
|
-
context.
|
|
647
|
+
context.s3_bucket_name,
|
|
640
648
|
"hash_caches/index.json",
|
|
641
649
|
json.dumps(context.hash_cache_index).encode(),
|
|
642
650
|
content_type="application/json",
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""
|
|
2
|
+
S3 connection info for the binary publish/sync tools.
|
|
3
|
+
|
|
4
|
+
Values are read from the project's Config/PyScripts/config.ini (section
|
|
5
|
+
[UGS.AWS]), and each one can be overridden by its command line argument.
|
|
6
|
+
Having no config file at all is fine — the command line can provide
|
|
7
|
+
everything — but a value that is missing from both sources is reported
|
|
8
|
+
before any S3 call is attempted.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
|
|
14
|
+
from uepyscripts import logger
|
|
15
|
+
from uepyscripts.internal.config import Config, resolve_config
|
|
16
|
+
from uepyscripts.internal.project import Project
|
|
17
|
+
|
|
18
|
+
S3_CONFIG_SECTION = "UGS.AWS"
|
|
19
|
+
|
|
20
|
+
# (attribute, command line argument, config.ini key)
|
|
21
|
+
S3_SETTINGS_FIELDS = [
|
|
22
|
+
("bucket_name", "--s3-bucket-name", "AWS_BucketName"),
|
|
23
|
+
("bucket_region", "--s3-bucket-region", "AWS_Region"),
|
|
24
|
+
("access_key", "--s3-access-key", "AWS_AccessKey"),
|
|
25
|
+
("secret_key", "--s3-secret-key", "AWS_SecretKey"),
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class S3Settings:
|
|
31
|
+
bucket_name: str
|
|
32
|
+
bucket_region: str
|
|
33
|
+
access_key: str
|
|
34
|
+
secret_key: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def add_s3_arguments(parser: argparse.ArgumentParser) -> None:
|
|
38
|
+
"""Registers the S3 command line arguments, which override the config file."""
|
|
39
|
+
group = parser.add_argument_group(
|
|
40
|
+
"AWS S3",
|
|
41
|
+
f"Defaults to the [{S3_CONFIG_SECTION}] section of Config/PyScripts/config.ini",
|
|
42
|
+
)
|
|
43
|
+
group.add_argument("--s3-bucket-name", type=str, help="AWS S3 Bucket Name (config: AWS_BucketName)")
|
|
44
|
+
group.add_argument("--s3-bucket-region", type=str, help="AWS S3 Bucket Region (config: AWS_Region)")
|
|
45
|
+
group.add_argument("--s3-access-key", type=str, help="AWS S3 Access Key (config: AWS_AccessKey)")
|
|
46
|
+
group.add_argument("--s3-secret-key", type=str, help="AWS S3 Secret Key (config: AWS_SecretKey)")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def read_config_value(config: Config, key: str) -> str | None:
|
|
50
|
+
"""A missing config file, section or key are all treated the same way — no value."""
|
|
51
|
+
if not config.valid or not config.config.has_section(S3_CONFIG_SECTION):
|
|
52
|
+
return None
|
|
53
|
+
|
|
54
|
+
value = config[S3_CONFIG_SECTION].get(key)
|
|
55
|
+
|
|
56
|
+
return value if value else None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def resolve_s3_settings(project: Project, args: argparse.Namespace) -> S3Settings:
|
|
60
|
+
"""Takes the S3 info from the config file, then lets the matching command
|
|
61
|
+
line arguments override each value. Exits with an explicit report of what
|
|
62
|
+
is missing rather than letting the first S3 call fail."""
|
|
63
|
+
config = resolve_config(project)
|
|
64
|
+
|
|
65
|
+
values: dict[str, str] = {}
|
|
66
|
+
missing: list[str] = []
|
|
67
|
+
|
|
68
|
+
for attribute, argument, config_key in S3_SETTINGS_FIELDS:
|
|
69
|
+
argument_value = getattr(args, argument.lstrip("-").replace("-", "_"), None)
|
|
70
|
+
value = argument_value or read_config_value(config, config_key)
|
|
71
|
+
|
|
72
|
+
if value:
|
|
73
|
+
values[attribute] = value
|
|
74
|
+
else:
|
|
75
|
+
missing.append(f" - {argument} (or [{S3_CONFIG_SECTION}] {config_key} in Config/PyScripts/config.ini)")
|
|
76
|
+
|
|
77
|
+
if missing:
|
|
78
|
+
logger.fatal("Missing S3 information:\n" + "\n".join(missing))
|
|
79
|
+
exit(1)
|
|
80
|
+
|
|
81
|
+
return S3Settings(**values)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{uepyscripts-1.4.2 → uepyscripts-1.4.4}/src/uepyscripts/tools/ue/check_engine_installation.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|