UEPyScripts 1.2.7__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of UEPyScripts might be problematic. Click here for more details.

Files changed (38) hide show
  1. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/PKG-INFO +3 -2
  2. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/pyproject.toml +6 -2
  3. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/config.py +9 -8
  4. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/engine.py +3 -1
  5. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/engine_resolver.py +8 -0
  6. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/project.py +27 -2
  7. uepyscripts-1.4.0/src/uepyscripts/run/buildgraph.py +125 -0
  8. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ci/buildgraph.py +19 -10
  9. uepyscripts-1.4.0/src/uepyscripts/tools/subprocess.py +50 -0
  10. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/engine_installation/engine_installer.py +4 -1
  11. uepyscripts-1.4.0/src/uepyscripts/tools/ugs/cleanup.py +126 -0
  12. uepyscripts-1.4.0/src/uepyscripts/tools/ugs/git_utils.py +49 -0
  13. uepyscripts-1.4.0/src/uepyscripts/tools/ugs/pull.py +278 -0
  14. uepyscripts-1.4.0/src/uepyscripts/tools/ugs/push.py +583 -0
  15. uepyscripts-1.4.0/src/uepyscripts/tools/ugs/ugs_types.py +9 -0
  16. uepyscripts-1.2.7/src/uepyscripts/run/buildgraph.py +0 -103
  17. uepyscripts-1.2.7/src/uepyscripts/tools/subprocess.py +0 -23
  18. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/.gitignore +0 -0
  19. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/LICENSE +0 -0
  20. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/README.md +0 -0
  21. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/__init__.py +0 -0
  22. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/context/__init__.py +0 -0
  23. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/__init__.py +0 -0
  24. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/run/__init__.py +0 -0
  25. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/run/uat.py +0 -0
  26. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/run/ubt.py +0 -0
  27. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/__init__.py +0 -0
  28. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ci/cleanup.py +0 -0
  29. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/helpers.py +0 -0
  30. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/__init__.py +0 -0
  31. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/check_engine_installation.py +0 -0
  32. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/close_editor.py +0 -0
  33. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/compile_editor.py +0 -0
  34. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/engine_installation/engine_destination.py +0 -0
  35. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/engine_installation/engine_source.py +0 -0
  36. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/generate_solution.py +0 -0
  37. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/run_editor.py +0 -0
  38. {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/winreg.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: UEPyScripts
3
- Version: 1.2.7
3
+ Version: 1.4.0
4
4
  Summary: Collection of Python tools and helpers designed to automate common Unreal Engine project tasks (builds, packaging, editor control, CI tasks, etc.).
5
5
  Project-URL: Changelog, https://github.com/TheEmidee/UEPyScripts/releases
6
6
  Project-URL: Homepage, https://github.com/TheEmidee/UEPyScripts
@@ -39,6 +39,7 @@ Classifier: Topic :: Software Development :: Libraries
39
39
  Classifier: Topic :: Utilities
40
40
  Requires-Python: >=3.11
41
41
  Requires-Dist: annotated-types
42
+ Requires-Dist: blake3
42
43
  Requires-Dist: colorama
43
44
  Requires-Dist: gamedevtools
44
45
  Requires-Dist: invoke
@@ -44,7 +44,8 @@ dependencies = [
44
44
  "urllib3",
45
45
  "invoke",
46
46
  "types-invoke",
47
- "GameDevTools"
47
+ "GameDevTools",
48
+ "blake3"
48
49
  ]
49
50
 
50
51
  urls.Changelog = "https://github.com/TheEmidee/UEPyScripts/releases"
@@ -91,7 +92,7 @@ fixable = ["ALL"]
91
92
  convention = "google"
92
93
 
93
94
  [tool.mypy]
94
- python_version = "3.9"
95
+ python_version = "3.12"
95
96
  warn_return_any = true
96
97
  disallow_untyped_defs = true
97
98
 
@@ -131,6 +132,9 @@ ue-generate-solution = "uepyscripts.tools.ue.generate_solution:main"
131
132
  ue-close-editor = "uepyscripts.tools.ue.close_editor:main"
132
133
  ue-compile-editor = "uepyscripts.tools.ue.compile_editor:main"
133
134
  ue-check-engine-installation = "uepyscripts.tools.ue.check_engine_installation:main"
135
+ ue-ugs-push = "uepyscripts.tools.ugs.push:main"
136
+ ue-ugs-pull = "uepyscripts.tools.ugs.pull:main"
137
+ ue-ugs-cleanup = "uepyscripts.tools.ugs.cleanup:main"
134
138
  ue-ci-cleanup = "uepyscripts.tools.ci.cleanup:main"
135
139
  ue-ci-run-buildgraph = "uepyscripts.tools.ci.buildgraph:main"
136
140
 
@@ -13,12 +13,16 @@ class Config:
13
13
 
14
14
  logger.debug("Initializing config")
15
15
 
16
- ini_files = [f for f in os.listdir(self.folder) if f.endswith(".ini")]
16
+ if self.folder.exists():
17
+ self.valid = True
18
+ ini_files = [f for f in os.listdir(self.folder) if f.endswith(".ini")]
17
19
 
18
- for ini_file in ini_files:
19
- ini_path = os.path.join(self.folder, ini_file)
20
- logger.debug(f"Reading file: {ini_path}")
21
- self.config.read(ini_path)
20
+ for ini_file in ini_files:
21
+ ini_path = os.path.join(self.folder, ini_file)
22
+ logger.debug(f"Reading file: {ini_path}")
23
+ self.config.read(ini_path)
24
+ else:
25
+ self.valid = False
22
26
 
23
27
  def __getitem__(self, key: str) -> configparser.SectionProxy:
24
28
  return self.config[key]
@@ -27,7 +31,4 @@ class Config:
27
31
  def resolve_config(project: Project) -> Config:
28
32
  config_folder = project.project_folders.config.joinpath("PyScripts")
29
33
 
30
- if not config_folder.exists():
31
- raise Exception("Impossible to find the PyScripts folder in Config")
32
-
33
34
  return Config(config_folder)
@@ -14,7 +14,8 @@ class Engine:
14
14
  class Runner:
15
15
  def __init__(self, path: Path, capture_output: bool, wait_process: bool = True, extra_args: list[str] = []) -> None:
16
16
  if not path.exists():
17
- raise FileNotFoundError(f"The file {path} does not exist")
17
+ logger.warning(f"The file {path} does not exist")
18
+ return
18
19
 
19
20
  self.path = path.resolve()
20
21
  self.capture_output = capture_output
@@ -49,6 +50,7 @@ class Engine:
49
50
  self.path = self.root_path.joinpath("Engine").resolve()
50
51
  self.build_path = self.path.joinpath("Build").resolve()
51
52
  self.batchfiles_path = self.build_path.joinpath("BatchFiles").resolve()
53
+ self.saved_path = self.path.joinpath("Saved").resolve()
52
54
  self.version = self.get_version_number()
53
55
  self.uat_path = self.Runner(self.batchfiles_path.joinpath("RunUAT.bat"), True)
54
56
  self.ubt_path = self.Runner(
@@ -11,6 +11,13 @@ from ..tools.helpers import get_engine_root_folder_from_env_var, is_engine_from_
11
11
  from ..tools.winreg import get_registry_value
12
12
 
13
13
 
14
+ def resolve_engine_from_native_project(project: Project) -> Optional[Path]:
15
+ if project.is_native_project:
16
+ return project.root_folder.parent
17
+
18
+ return None
19
+
20
+
14
21
  def resolve_engine_from_env_var(project: Project) -> Optional[Path]:
15
22
  return get_engine_root_folder_from_env_var(project.engine_association)
16
23
 
@@ -86,6 +93,7 @@ def resolve_engine_from_path(project: Project) -> Optional[Path]:
86
93
 
87
94
  def resolve_engine_path(project: Project) -> Path:
88
95
  resolvers = [
96
+ resolve_engine_from_native_project,
89
97
  resolve_engine_from_registry,
90
98
  resolve_engine_from_egs,
91
99
  resolve_engine_from_path,
@@ -1,4 +1,3 @@
1
- import os
2
1
  from pathlib import Path
3
2
  from typing import Optional
4
3
 
@@ -44,6 +43,10 @@ class Project:
44
43
  def engine_association(self) -> str:
45
44
  return self.uproject.engine_association
46
45
 
46
+ @property
47
+ def is_native_project(self) -> bool:
48
+ return not self.engine_association
49
+
47
50
  def __str__(self) -> str:
48
51
  return f"""
49
52
  ----- Project infos -----
@@ -55,6 +58,24 @@ class Project:
55
58
  """
56
59
 
57
60
 
61
+ EXCLUDED_DIR_NAMES = {"Engine", "FeaturePacks", "Samples", "Scripts", "Templates", ".git"}
62
+
63
+
64
+ def find_uproject_in_subfolders(root: Path) -> Optional[Path]:
65
+ """Recursively search for a .uproject file under root, skipping excluded folders."""
66
+ for entry in root.iterdir():
67
+ if entry.is_dir():
68
+ if entry.name in EXCLUDED_DIR_NAMES:
69
+ continue
70
+ found = find_uproject_in_subfolders(entry)
71
+ if found:
72
+ return found
73
+ elif entry.is_file() and entry.suffix == ".uproject":
74
+ return entry.resolve()
75
+
76
+ return None
77
+
78
+
58
79
  def find_parent_with_project_file(starting_path: Path, max_parents: int = 10) -> Optional[Path]:
59
80
  current_path = Path(starting_path).resolve()
60
81
 
@@ -65,12 +86,16 @@ def find_parent_with_project_file(starting_path: Path, max_parents: int = 10) ->
65
86
  if file.is_file() and file.suffix == ".uproject":
66
87
  return file.resolve()
67
88
 
89
+ build_version_path = path / "Engine" / "Build" / "Build.version"
90
+ if build_version_path.is_file():
91
+ return find_uproject_in_subfolders(path)
92
+
68
93
  return None
69
94
 
70
95
 
71
96
  def resolve_project(uproject_path: Optional[Path] = None) -> Project:
72
97
  if not uproject_path:
73
- dir_path = Path(os.path.dirname(os.path.realpath(__file__)))
98
+ dir_path = Path.cwd()
74
99
  uproject_path = find_parent_with_project_file(dir_path)
75
100
  if not uproject_path:
76
101
  raise Exception(f"Could not find a uproject file from {dir_path}")
@@ -0,0 +1,125 @@
1
+ import argparse
2
+ import re
3
+ from pathlib import Path
4
+ from typing import Optional, Sequence
5
+
6
+ from .. import logger
7
+ from ..internal.config import resolve_config
8
+ from ..internal.engine import resolve_engine
9
+ from ..internal.project import resolve_project
10
+
11
+
12
+ def run(target: str, arguments: list[str], script: Optional[str] = None, uproject_path: Optional[str] = None) -> int:
13
+ logger.info(f"Run Buildgraph - Target : {target}")
14
+ logger.info(f"Arguments : {arguments}")
15
+
16
+ project = resolve_project(Path(uproject_path) if uproject_path else None)
17
+ engine = resolve_engine(project)
18
+ config = resolve_config(project)
19
+
20
+ if script:
21
+ buildgraph_path = Path(script)
22
+ else:
23
+ buildgraph_path = project.root_folder.joinpath(config["Project"]["BuildgraphPath"])
24
+ logger.info(f"Buildgraph XML path : {buildgraph_path}")
25
+
26
+ if not buildgraph_path.exists():
27
+ raise Exception(f"Impossible to get a valid path to the buildgraph XML file. Current path : {buildgraph_path}")
28
+
29
+ extension = buildgraph_path.suffix
30
+ if extension != ".xml":
31
+ raise Exception(f"The buildgraph file must be a XML file. Current path : {buildgraph_path}")
32
+
33
+ uat_arguments = ["BuildGraph"]
34
+ uat_arguments.append(f"-script={buildgraph_path}")
35
+
36
+ # We can execute buildgraph without a target if the SingleNode argument is set
37
+ if target != "":
38
+ uat_arguments.append(f"-target={target}")
39
+
40
+ uat_arguments.append(f"-Project={project.uproject_path}")
41
+
42
+ if config.valid:
43
+ automation_scripts_directories = config["Project"].get("AutomationScriptsDirectories")
44
+ if automation_scripts_directories == "" or automation_scripts_directories is None:
45
+ logger.info("No automation scripts directory is set")
46
+ else:
47
+ automation_scripts_paths = automation_scripts_directories.split("+")
48
+
49
+ for automation_scripts_path_str in automation_scripts_paths:
50
+ automation_scripts_path = Path(automation_scripts_path_str)
51
+ automation_scripts_path = project.root_folder.joinpath(automation_scripts_path)
52
+ if not automation_scripts_path.exists():
53
+ logger.warning(f"The automation scripts directory does not exist. Current value {automation_scripts_path}")
54
+ continue
55
+
56
+ logger.info(f"Automation Scripts directory set to {automation_scripts_path}")
57
+ uat_arguments.append(f"-ScriptDir={automation_scripts_path}")
58
+
59
+ shared_properties_str = config["Project"].get("BuildgraphSharedProperties", "")
60
+ if shared_properties_str == "" or shared_properties_str is None:
61
+ logger.info("No shared properties to set")
62
+ else:
63
+ shared_properties = dict(pair.split("=") for pair in shared_properties_str.split("+")) if shared_properties_str else {}
64
+
65
+ if shared_properties is not None:
66
+ for key, value in shared_properties.items():
67
+ uat_arguments.append(f"-set:{key}={value}")
68
+
69
+ for arg in arguments:
70
+ uat_arguments.append(arg)
71
+
72
+ return engine.uat(uat_arguments)
73
+
74
+
75
+ def parse_arguments(argv: Optional[Sequence[str]] = None) -> tuple[argparse.Namespace, list[str]]:
76
+ """Parse command line arguments."""
77
+ parser = argparse.ArgumentParser(description="Execute a buildgraph task based on a target and properties.")
78
+ parser.add_argument("--target", type=str, default="", help="The target to run in the buildgraph file")
79
+ parser.add_argument(
80
+ "--script", type=str, default=None, help="Path to the buildgraph XML file. If not set, resolved from the project configuration."
81
+ )
82
+ parser.add_argument(
83
+ "--uproject",
84
+ type=str,
85
+ default=None,
86
+ help="Path to the .uproject file. Required when --script is set; otherwise auto-discovered from the current directory.",
87
+ )
88
+ return parser.parse_known_args(argv)
89
+
90
+
91
+ def validate_config(target: str, arguments: list[str], script: Optional[str], uproject_path: Optional[str]) -> None:
92
+ """Validate the configuration values."""
93
+ pattern = r'-SingleNode=[^"]*'
94
+
95
+ if not target and not any(re.search(pattern, item) for item in arguments):
96
+ raise ValueError("Target is required")
97
+
98
+ if script and not uproject_path:
99
+ raise ValueError("--uproject is required when --script is set")
100
+
101
+
102
+ def main(argv: Optional[Sequence[str]] = None) -> int:
103
+ args, other_arguments = parse_arguments(argv)
104
+
105
+ target = ""
106
+
107
+ if args.target:
108
+ target = args.target
109
+
110
+ try:
111
+ logger.info("Validating configuration")
112
+ validate_config(target, other_arguments, args.script, args.uproject)
113
+ except (ValueError, TypeError) as e:
114
+ logger.error(f"Configuration validation failed: {e}")
115
+ raise e
116
+
117
+ try:
118
+ return run(target, other_arguments, args.script, args.uproject)
119
+ except Exception as e:
120
+ logger.error(f"Execution failed: {e}")
121
+ raise e
122
+
123
+
124
+ if __name__ == "__main__":
125
+ main()
@@ -53,7 +53,11 @@ def parse_arguments(argv: Optional[Sequence[str]] = None) -> tuple[argparse.Name
53
53
  parser = argparse.ArgumentParser(description="Execute a buildgraph task using a shared storage.")
54
54
  parser.add_argument("--target", type=str, help="The target to run in the buildgraph file")
55
55
  parser.add_argument("--build_tag", type=str, help="The tag that will be used to define a folder on a shared storage")
56
- # parser.add_argument("remainder", nargs=argparse.REMAINDER, help="Properties (-set:K:V) and extra arguments")
56
+ parser.add_argument(
57
+ "--no-single-node",
58
+ action="store_true",
59
+ help="Set this flag to not use SingleNode but use Target. This also disables using the shared storage dir.",
60
+ )
57
61
 
58
62
  args, unknown_args = parser.parse_known_args(argv)
59
63
  return args, unknown_args
@@ -115,17 +119,22 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
115
119
 
116
120
  logger.info(f"Running ci.buildgraph with build tag: {args.build_tag} on machine {socket.gethostname()}")
117
121
 
118
- shared_storage_dir = Path(config["Jenkins"]["BuildgraphSharedStoragePath"]) / args.build_tag
119
- logger.info(f"Shared storage directory: {shared_storage_dir}")
122
+ default_arguments: list[str] = ["-BuildMachine", "-NoP4"]
120
123
 
121
- remove_task_shared_storage_dir(shared_storage_dir, args.target)
122
- try_delete_local_buildgraph_folder(args.build_tag)
123
- cleanup_local_folder()
124
+ if args.no_single_node:
125
+ logger.info("The flag --no-single-node was passed. Run the task with Target and not with SingleNode")
126
+ default_arguments += [f"--target={args.target}"]
127
+ else:
128
+ shared_storage_dir = Path(config["Jenkins"]["BuildgraphSharedStoragePath"]) / args.build_tag
129
+ logger.info(f"Shared storage directory: {shared_storage_dir}")
124
130
 
125
- updated_args = update_or_add_argument(
126
- unknown_args,
127
- ["-BuildMachine", f"-SharedStorageDir={shared_storage_dir}", "-WriteToSharedStorage", f"-SingleNode={args.target}", "-NoP4"],
128
- )
131
+ remove_task_shared_storage_dir(shared_storage_dir, args.target)
132
+ try_delete_local_buildgraph_folder(args.build_tag)
133
+ cleanup_local_folder()
134
+
135
+ default_arguments += [f"-SharedStorageDir={shared_storage_dir}", "-WriteToSharedStorage", f"-SingleNode={args.target}"]
136
+
137
+ updated_args = update_or_add_argument(unknown_args, default_arguments)
129
138
 
130
139
  return buildgraph.main(updated_args)
131
140
 
@@ -0,0 +1,50 @@
1
+ import subprocess
2
+ import sys
3
+ from typing import Any
4
+
5
+ from uepyscripts import logger
6
+
7
+
8
+ def run_subprocess(cmd: list[str], log_output: bool = True) -> int:
9
+ """Run a subprocess with the given command and return its exit code (or 0 if fire-and-forget)."""
10
+ kwargs: dict[str, Any] = {}
11
+ creationflags = 0
12
+
13
+ if sys.platform == "win32":
14
+ creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
15
+ startupinfo = subprocess.STARTUPINFO()
16
+ startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
17
+ startupinfo.wShowWindow = subprocess.SW_HIDE
18
+ kwargs["startupinfo"] = startupinfo
19
+ else:
20
+ kwargs["start_new_session"] = True
21
+
22
+ if log_output:
23
+ process = subprocess.Popen(
24
+ cmd,
25
+ stdout=subprocess.PIPE,
26
+ stderr=subprocess.STDOUT,
27
+ text=True,
28
+ bufsize=1,
29
+ creationflags=creationflags,
30
+ **kwargs,
31
+ )
32
+
33
+ assert process.stdout is not None
34
+ for line in process.stdout:
35
+ line = line.strip()
36
+ if line:
37
+ logger.info(line)
38
+
39
+ process.wait()
40
+ return process.returncode
41
+ else:
42
+ subprocess.Popen(
43
+ cmd,
44
+ stdout=subprocess.DEVNULL,
45
+ stderr=subprocess.DEVNULL,
46
+ stdin=subprocess.DEVNULL,
47
+ creationflags=creationflags,
48
+ **kwargs,
49
+ )
50
+ return 0
@@ -63,9 +63,9 @@ class TaskList:
63
63
 
64
64
  class EngineInstaller:
65
65
  def __init__(self, project: Project, unattended: bool = False) -> None:
66
+ self.project = project
66
67
  self.check_requirements()
67
68
 
68
- self.project = project
69
69
  try:
70
70
  self.engine_destination_folder = resolve_engine_destination(project)
71
71
  except FileNotFoundError as e:
@@ -95,6 +95,9 @@ class EngineInstaller:
95
95
  if not is_7z_installed():
96
96
  raise RuntimeError("7-Zip is not installed or not found in PATH. Please install 7-Zip to proceed.")
97
97
 
98
+ if self.project.is_native_project:
99
+ raise RuntimeError("A native project should be in the same folder than its engine")
100
+
98
101
  def get_project_platforms(self) -> List[str]:
99
102
  platforms: str = self.config["EngineUpdate.TurnKey"]["Platforms"]
100
103
  return platforms.split("+")
@@ -0,0 +1,126 @@
1
+ """
2
+ Prunes S3 data for commits whose branch no longer exists on the remote.
3
+
4
+ - A commit is "orphaned" once it is no longer reachable from any current
5
+ remote branch tip (i.e. its branch was deleted). This needs no per-branch
6
+ bookkeeping in S3 — reachability is computed straight from git, the same
7
+ way push.py/pull.py resolve diff baselines and sync targets.
8
+ - Deletes manifests/{sha}.json, deltas/{sha}.7z and checkpoints/{sha}.7z for
9
+ every orphaned sha, then rewrites index.json/checkpoints.json to drop them.
10
+ - Intended to run on a schedule (e.g. a periodic Jenkins job).
11
+ """
12
+
13
+ import argparse
14
+ import json
15
+ from pathlib import Path
16
+ from typing import cast
17
+
18
+ from gamedevtools.s3 import S3Client
19
+
20
+ from uepyscripts import logger
21
+ from uepyscripts.internal.engine import resolve_engine
22
+ from uepyscripts.internal.project import resolve_project
23
+ from uepyscripts.tools.ugs.git_utils import get_remote_reachable_shas
24
+
25
+
26
+ class Context:
27
+ def __init__(self, args: argparse.Namespace) -> None:
28
+ self.args = args
29
+ self.s3_client: S3Client = S3Client(
30
+ access_key=args.s3_access_key,
31
+ secret_key=args.s3_secret_key,
32
+ region=args.s3_bucket_region,
33
+ )
34
+
35
+
36
+ def build_orphaned_keys(orphaned: list[str]) -> list[str]:
37
+ """S3 keys to delete for each orphaned sha. Including a key that was never
38
+ written is harmless — S3 batch-delete is a no-op for missing keys, the same
39
+ assumption prune_old_versions (push.py) already relies on."""
40
+ keys: list[str] = []
41
+ for sha in orphaned:
42
+ keys.append(f"manifests/{sha}.json")
43
+ keys.append(f"deltas/{sha}.7z")
44
+ keys.append(f"checkpoints/{sha}.7z")
45
+ return keys
46
+
47
+
48
+ def parse_arguments() -> argparse.Namespace:
49
+ """Parse command line arguments."""
50
+ parser = argparse.ArgumentParser(description="Delete S3 binary-publish data for branches that no longer exist on the remote.")
51
+ parser.add_argument("--uproject-path", type=Path, help=("Path to a native uproject file"))
52
+ parser.add_argument("--s3-bucket-name", type=str, help=("AWS S3 Bucket Name"))
53
+ parser.add_argument("--s3-bucket-region", type=str, help=("AWS S3 Bucket Region"))
54
+ parser.add_argument("--s3-access-key", type=str, help=("AWS S3 Access Key"))
55
+ parser.add_argument("--s3-secret-key", type=str, help=("AWS S3 Secret Key"))
56
+ parser.add_argument("--remote", type=str, default="origin", help="Git remote to check branches against (default: origin)")
57
+ parser.add_argument(
58
+ "--dry-run",
59
+ action=argparse.BooleanOptionalAction,
60
+ default=False,
61
+ help="Log what would be deleted without touching S3",
62
+ )
63
+
64
+ return parser.parse_args()
65
+
66
+
67
+ def main() -> None:
68
+ args = parse_arguments()
69
+
70
+ logger.info("Cleanup orphaned binary publishes...")
71
+
72
+ try:
73
+ project = resolve_project(args.uproject_path)
74
+ except Exception as e:
75
+ logger.fatal(f"Project resolution failed: {e}")
76
+ exit(1)
77
+
78
+ engine = resolve_engine(project)
79
+ context = Context(args)
80
+
81
+ logger.info(f"Fetching '{args.remote}' and computing reachable commits")
82
+ reachable = get_remote_reachable_shas(engine.root_path, args.remote)
83
+
84
+ if not reachable:
85
+ logger.fatal(f"No commit is reachable from any branch on remote '{args.remote}' — aborting without deleting anything")
86
+ exit(1)
87
+
88
+ logger.info("Download index.json")
89
+ index = cast(list[str], context.s3_client.download_json(context.args.s3_bucket_name, "index.json", default=[]))
90
+
91
+ logger.info("Download checkpoints.json")
92
+ checkpoints = cast(list[str], context.s3_client.download_json(context.args.s3_bucket_name, "checkpoints.json", default=[]))
93
+
94
+ orphaned = [sha for sha in index if sha not in reachable]
95
+
96
+ if not orphaned:
97
+ logger.info(f"Nothing to clean up. {len(index)} published commit(s), all reachable from '{args.remote}'.")
98
+ return
99
+
100
+ logger.info(f"Found {len(orphaned)} orphaned commit(s) out of {len(index)} published")
101
+
102
+ keys = build_orphaned_keys(orphaned)
103
+
104
+ if args.dry_run:
105
+ logger.info(f"[dry-run] Would delete {len(keys)} key(s):")
106
+ for key in keys:
107
+ logger.info(f"[dry-run] {key}")
108
+ return
109
+
110
+ context.s3_client.delete_keys(context.args.s3_bucket_name, keys)
111
+
112
+ orphaned_set = set(orphaned)
113
+ index = [sha for sha in index if sha not in orphaned_set]
114
+ checkpoints = [sha for sha in checkpoints if sha not in orphaned_set]
115
+
116
+ logger.info("Upload index.json")
117
+ context.s3_client.upload_bytes(context.args.s3_bucket_name, "index.json", json.dumps(index).encode(), content_type="application/json")
118
+
119
+ logger.info("Upload checkpoints.json")
120
+ context.s3_client.upload_bytes(context.args.s3_bucket_name, "checkpoints.json", json.dumps(checkpoints).encode(), content_type="application/json")
121
+
122
+ logger.info(f"Cleanup done. Deleted {len(keys)} key(s) for {len(orphaned)} orphaned commit(s).")
123
+
124
+
125
+ if __name__ == "__main__":
126
+ main()
@@ -0,0 +1,49 @@
1
+ import os
2
+ import subprocess
3
+ from pathlib import Path
4
+
5
+ ANCESTRY_LOOKBACK = 2000
6
+
7
+
8
+ def get_current_sha(cwd: Path) -> str:
9
+ sha = os.environ.get("GIT_COMMIT") # Jenkins sets this automatically
10
+ if sha:
11
+ return sha
12
+ return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd, text=True).strip()
13
+
14
+
15
+ def get_current_branch(cwd: Path) -> str:
16
+ branch = os.environ.get("GIT_BRANCH") # Jenkins' Git plugin sets this, sometimes as "origin/<branch>"
17
+ if branch:
18
+ return branch.split("/", 1)[1] if "/" in branch else branch
19
+ return subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, text=True).strip()
20
+
21
+
22
+ def get_local_ancestry(cwd: Path, limit: int = ANCESTRY_LOOKBACK) -> list[str]:
23
+ out = subprocess.check_output(["git", "log", f"-{limit}", "--format=%H", "HEAD"], cwd=cwd, text=True)
24
+ return out.splitlines()
25
+
26
+
27
+ def get_remote_reachable_shas(cwd: Path, remote: str = "origin") -> set[str]:
28
+ """All commit SHAs reachable from any current branch tip on `remote`, after
29
+ pruning local remote-tracking refs for branches deleted on the remote."""
30
+ subprocess.run(["git", "fetch", "--prune", remote], cwd=cwd, check=True, capture_output=True, text=True)
31
+ out = subprocess.check_output(["git", "rev-list", f"--remotes={remote}"], cwd=cwd, text=True)
32
+ return set(out.splitlines())
33
+
34
+
35
+ def resolve_nearest_published_ancestor(index: list[str], ancestry: list[str]) -> tuple[str | None, bool]:
36
+ """Finds the nearest ancestor SHA (including HEAD itself) present in the
37
+ index. Returns None if nothing has ever been published reachable from HEAD
38
+ within the walked ancestry. The bool is True when the match was found by
39
+ walking the ancestry itself (an exact match), False when falling back to
40
+ the most recent published version because nothing in the ancestry matched."""
41
+ index_set = set(index)
42
+ for sha in ancestry:
43
+ if sha in index_set:
44
+ return sha, True
45
+
46
+ if index:
47
+ return index[-1], False
48
+
49
+ return None, True