UEPyScripts 1.2.7__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of UEPyScripts might be problematic. Click here for more details.
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/PKG-INFO +3 -2
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/pyproject.toml +6 -2
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/config.py +9 -8
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/engine.py +3 -1
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/engine_resolver.py +8 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/project.py +27 -2
- uepyscripts-1.4.0/src/uepyscripts/run/buildgraph.py +125 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ci/buildgraph.py +19 -10
- uepyscripts-1.4.0/src/uepyscripts/tools/subprocess.py +50 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/engine_installation/engine_installer.py +4 -1
- uepyscripts-1.4.0/src/uepyscripts/tools/ugs/cleanup.py +126 -0
- uepyscripts-1.4.0/src/uepyscripts/tools/ugs/git_utils.py +49 -0
- uepyscripts-1.4.0/src/uepyscripts/tools/ugs/pull.py +278 -0
- uepyscripts-1.4.0/src/uepyscripts/tools/ugs/push.py +583 -0
- uepyscripts-1.4.0/src/uepyscripts/tools/ugs/ugs_types.py +9 -0
- uepyscripts-1.2.7/src/uepyscripts/run/buildgraph.py +0 -103
- uepyscripts-1.2.7/src/uepyscripts/tools/subprocess.py +0 -23
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/.gitignore +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/LICENSE +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/README.md +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/__init__.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/context/__init__.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/internal/__init__.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/run/__init__.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/run/uat.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/run/ubt.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/__init__.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ci/cleanup.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/helpers.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/__init__.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/check_engine_installation.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/close_editor.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/compile_editor.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/engine_installation/engine_destination.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/engine_installation/engine_source.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/generate_solution.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/ue/run_editor.py +0 -0
- {uepyscripts-1.2.7 → uepyscripts-1.4.0}/src/uepyscripts/tools/winreg.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: UEPyScripts
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Collection of Python tools and helpers designed to automate common Unreal Engine project tasks (builds, packaging, editor control, CI tasks, etc.).
|
|
5
5
|
Project-URL: Changelog, https://github.com/TheEmidee/UEPyScripts/releases
|
|
6
6
|
Project-URL: Homepage, https://github.com/TheEmidee/UEPyScripts
|
|
@@ -39,6 +39,7 @@ Classifier: Topic :: Software Development :: Libraries
|
|
|
39
39
|
Classifier: Topic :: Utilities
|
|
40
40
|
Requires-Python: >=3.11
|
|
41
41
|
Requires-Dist: annotated-types
|
|
42
|
+
Requires-Dist: blake3
|
|
42
43
|
Requires-Dist: colorama
|
|
43
44
|
Requires-Dist: gamedevtools
|
|
44
45
|
Requires-Dist: invoke
|
|
@@ -44,7 +44,8 @@ dependencies = [
|
|
|
44
44
|
"urllib3",
|
|
45
45
|
"invoke",
|
|
46
46
|
"types-invoke",
|
|
47
|
-
"GameDevTools"
|
|
47
|
+
"GameDevTools",
|
|
48
|
+
"blake3"
|
|
48
49
|
]
|
|
49
50
|
|
|
50
51
|
urls.Changelog = "https://github.com/TheEmidee/UEPyScripts/releases"
|
|
@@ -91,7 +92,7 @@ fixable = ["ALL"]
|
|
|
91
92
|
convention = "google"
|
|
92
93
|
|
|
93
94
|
[tool.mypy]
|
|
94
|
-
python_version = "3.
|
|
95
|
+
python_version = "3.12"
|
|
95
96
|
warn_return_any = true
|
|
96
97
|
disallow_untyped_defs = true
|
|
97
98
|
|
|
@@ -131,6 +132,9 @@ ue-generate-solution = "uepyscripts.tools.ue.generate_solution:main"
|
|
|
131
132
|
ue-close-editor = "uepyscripts.tools.ue.close_editor:main"
|
|
132
133
|
ue-compile-editor = "uepyscripts.tools.ue.compile_editor:main"
|
|
133
134
|
ue-check-engine-installation = "uepyscripts.tools.ue.check_engine_installation:main"
|
|
135
|
+
ue-ugs-push = "uepyscripts.tools.ugs.push:main"
|
|
136
|
+
ue-ugs-pull = "uepyscripts.tools.ugs.pull:main"
|
|
137
|
+
ue-ugs-cleanup = "uepyscripts.tools.ugs.cleanup:main"
|
|
134
138
|
ue-ci-cleanup = "uepyscripts.tools.ci.cleanup:main"
|
|
135
139
|
ue-ci-run-buildgraph = "uepyscripts.tools.ci.buildgraph:main"
|
|
136
140
|
|
|
@@ -13,12 +13,16 @@ class Config:
|
|
|
13
13
|
|
|
14
14
|
logger.debug("Initializing config")
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
if self.folder.exists():
|
|
17
|
+
self.valid = True
|
|
18
|
+
ini_files = [f for f in os.listdir(self.folder) if f.endswith(".ini")]
|
|
17
19
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
for ini_file in ini_files:
|
|
21
|
+
ini_path = os.path.join(self.folder, ini_file)
|
|
22
|
+
logger.debug(f"Reading file: {ini_path}")
|
|
23
|
+
self.config.read(ini_path)
|
|
24
|
+
else:
|
|
25
|
+
self.valid = False
|
|
22
26
|
|
|
23
27
|
def __getitem__(self, key: str) -> configparser.SectionProxy:
|
|
24
28
|
return self.config[key]
|
|
@@ -27,7 +31,4 @@ class Config:
|
|
|
27
31
|
def resolve_config(project: Project) -> Config:
|
|
28
32
|
config_folder = project.project_folders.config.joinpath("PyScripts")
|
|
29
33
|
|
|
30
|
-
if not config_folder.exists():
|
|
31
|
-
raise Exception("Impossible to find the PyScripts folder in Config")
|
|
32
|
-
|
|
33
34
|
return Config(config_folder)
|
|
@@ -14,7 +14,8 @@ class Engine:
|
|
|
14
14
|
class Runner:
|
|
15
15
|
def __init__(self, path: Path, capture_output: bool, wait_process: bool = True, extra_args: list[str] = []) -> None:
|
|
16
16
|
if not path.exists():
|
|
17
|
-
|
|
17
|
+
logger.warning(f"The file {path} does not exist")
|
|
18
|
+
return
|
|
18
19
|
|
|
19
20
|
self.path = path.resolve()
|
|
20
21
|
self.capture_output = capture_output
|
|
@@ -49,6 +50,7 @@ class Engine:
|
|
|
49
50
|
self.path = self.root_path.joinpath("Engine").resolve()
|
|
50
51
|
self.build_path = self.path.joinpath("Build").resolve()
|
|
51
52
|
self.batchfiles_path = self.build_path.joinpath("BatchFiles").resolve()
|
|
53
|
+
self.saved_path = self.path.joinpath("Saved").resolve()
|
|
52
54
|
self.version = self.get_version_number()
|
|
53
55
|
self.uat_path = self.Runner(self.batchfiles_path.joinpath("RunUAT.bat"), True)
|
|
54
56
|
self.ubt_path = self.Runner(
|
|
@@ -11,6 +11,13 @@ from ..tools.helpers import get_engine_root_folder_from_env_var, is_engine_from_
|
|
|
11
11
|
from ..tools.winreg import get_registry_value
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
def resolve_engine_from_native_project(project: Project) -> Optional[Path]:
|
|
15
|
+
if project.is_native_project:
|
|
16
|
+
return project.root_folder.parent
|
|
17
|
+
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
14
21
|
def resolve_engine_from_env_var(project: Project) -> Optional[Path]:
|
|
15
22
|
return get_engine_root_folder_from_env_var(project.engine_association)
|
|
16
23
|
|
|
@@ -86,6 +93,7 @@ def resolve_engine_from_path(project: Project) -> Optional[Path]:
|
|
|
86
93
|
|
|
87
94
|
def resolve_engine_path(project: Project) -> Path:
|
|
88
95
|
resolvers = [
|
|
96
|
+
resolve_engine_from_native_project,
|
|
89
97
|
resolve_engine_from_registry,
|
|
90
98
|
resolve_engine_from_egs,
|
|
91
99
|
resolve_engine_from_path,
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import os
|
|
2
1
|
from pathlib import Path
|
|
3
2
|
from typing import Optional
|
|
4
3
|
|
|
@@ -44,6 +43,10 @@ class Project:
|
|
|
44
43
|
def engine_association(self) -> str:
|
|
45
44
|
return self.uproject.engine_association
|
|
46
45
|
|
|
46
|
+
@property
|
|
47
|
+
def is_native_project(self) -> bool:
|
|
48
|
+
return not self.engine_association
|
|
49
|
+
|
|
47
50
|
def __str__(self) -> str:
|
|
48
51
|
return f"""
|
|
49
52
|
----- Project infos -----
|
|
@@ -55,6 +58,24 @@ class Project:
|
|
|
55
58
|
"""
|
|
56
59
|
|
|
57
60
|
|
|
61
|
+
EXCLUDED_DIR_NAMES = {"Engine", "FeaturePacks", "Samples", "Scripts", "Templates", ".git"}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def find_uproject_in_subfolders(root: Path) -> Optional[Path]:
|
|
65
|
+
"""Recursively search for a .uproject file under root, skipping excluded folders."""
|
|
66
|
+
for entry in root.iterdir():
|
|
67
|
+
if entry.is_dir():
|
|
68
|
+
if entry.name in EXCLUDED_DIR_NAMES:
|
|
69
|
+
continue
|
|
70
|
+
found = find_uproject_in_subfolders(entry)
|
|
71
|
+
if found:
|
|
72
|
+
return found
|
|
73
|
+
elif entry.is_file() and entry.suffix == ".uproject":
|
|
74
|
+
return entry.resolve()
|
|
75
|
+
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
58
79
|
def find_parent_with_project_file(starting_path: Path, max_parents: int = 10) -> Optional[Path]:
|
|
59
80
|
current_path = Path(starting_path).resolve()
|
|
60
81
|
|
|
@@ -65,12 +86,16 @@ def find_parent_with_project_file(starting_path: Path, max_parents: int = 10) ->
|
|
|
65
86
|
if file.is_file() and file.suffix == ".uproject":
|
|
66
87
|
return file.resolve()
|
|
67
88
|
|
|
89
|
+
build_version_path = path / "Engine" / "Build" / "Build.version"
|
|
90
|
+
if build_version_path.is_file():
|
|
91
|
+
return find_uproject_in_subfolders(path)
|
|
92
|
+
|
|
68
93
|
return None
|
|
69
94
|
|
|
70
95
|
|
|
71
96
|
def resolve_project(uproject_path: Optional[Path] = None) -> Project:
|
|
72
97
|
if not uproject_path:
|
|
73
|
-
dir_path = Path
|
|
98
|
+
dir_path = Path.cwd()
|
|
74
99
|
uproject_path = find_parent_with_project_file(dir_path)
|
|
75
100
|
if not uproject_path:
|
|
76
101
|
raise Exception(f"Could not find a uproject file from {dir_path}")
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Optional, Sequence
|
|
5
|
+
|
|
6
|
+
from .. import logger
|
|
7
|
+
from ..internal.config import resolve_config
|
|
8
|
+
from ..internal.engine import resolve_engine
|
|
9
|
+
from ..internal.project import resolve_project
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def run(target: str, arguments: list[str], script: Optional[str] = None, uproject_path: Optional[str] = None) -> int:
|
|
13
|
+
logger.info(f"Run Buildgraph - Target : {target}")
|
|
14
|
+
logger.info(f"Arguments : {arguments}")
|
|
15
|
+
|
|
16
|
+
project = resolve_project(Path(uproject_path) if uproject_path else None)
|
|
17
|
+
engine = resolve_engine(project)
|
|
18
|
+
config = resolve_config(project)
|
|
19
|
+
|
|
20
|
+
if script:
|
|
21
|
+
buildgraph_path = Path(script)
|
|
22
|
+
else:
|
|
23
|
+
buildgraph_path = project.root_folder.joinpath(config["Project"]["BuildgraphPath"])
|
|
24
|
+
logger.info(f"Buildgraph XML path : {buildgraph_path}")
|
|
25
|
+
|
|
26
|
+
if not buildgraph_path.exists():
|
|
27
|
+
raise Exception(f"Impossible to get a valid path to the buildgraph XML file. Current path : {buildgraph_path}")
|
|
28
|
+
|
|
29
|
+
extension = buildgraph_path.suffix
|
|
30
|
+
if extension != ".xml":
|
|
31
|
+
raise Exception(f"The buildgraph file must be a XML file. Current path : {buildgraph_path}")
|
|
32
|
+
|
|
33
|
+
uat_arguments = ["BuildGraph"]
|
|
34
|
+
uat_arguments.append(f"-script={buildgraph_path}")
|
|
35
|
+
|
|
36
|
+
# We can execute buildgraph without a target if the SingleNode argument is set
|
|
37
|
+
if target != "":
|
|
38
|
+
uat_arguments.append(f"-target={target}")
|
|
39
|
+
|
|
40
|
+
uat_arguments.append(f"-Project={project.uproject_path}")
|
|
41
|
+
|
|
42
|
+
if config.valid:
|
|
43
|
+
automation_scripts_directories = config["Project"].get("AutomationScriptsDirectories")
|
|
44
|
+
if automation_scripts_directories == "" or automation_scripts_directories is None:
|
|
45
|
+
logger.info("No automation scripts directory is set")
|
|
46
|
+
else:
|
|
47
|
+
automation_scripts_paths = automation_scripts_directories.split("+")
|
|
48
|
+
|
|
49
|
+
for automation_scripts_path_str in automation_scripts_paths:
|
|
50
|
+
automation_scripts_path = Path(automation_scripts_path_str)
|
|
51
|
+
automation_scripts_path = project.root_folder.joinpath(automation_scripts_path)
|
|
52
|
+
if not automation_scripts_path.exists():
|
|
53
|
+
logger.warning(f"The automation scripts directory does not exist. Current value {automation_scripts_path}")
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
logger.info(f"Automation Scripts directory set to {automation_scripts_path}")
|
|
57
|
+
uat_arguments.append(f"-ScriptDir={automation_scripts_path}")
|
|
58
|
+
|
|
59
|
+
shared_properties_str = config["Project"].get("BuildgraphSharedProperties", "")
|
|
60
|
+
if shared_properties_str == "" or shared_properties_str is None:
|
|
61
|
+
logger.info("No shared properties to set")
|
|
62
|
+
else:
|
|
63
|
+
shared_properties = dict(pair.split("=") for pair in shared_properties_str.split("+")) if shared_properties_str else {}
|
|
64
|
+
|
|
65
|
+
if shared_properties is not None:
|
|
66
|
+
for key, value in shared_properties.items():
|
|
67
|
+
uat_arguments.append(f"-set:{key}={value}")
|
|
68
|
+
|
|
69
|
+
for arg in arguments:
|
|
70
|
+
uat_arguments.append(arg)
|
|
71
|
+
|
|
72
|
+
return engine.uat(uat_arguments)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def parse_arguments(argv: Optional[Sequence[str]] = None) -> tuple[argparse.Namespace, list[str]]:
|
|
76
|
+
"""Parse command line arguments."""
|
|
77
|
+
parser = argparse.ArgumentParser(description="Execute a buildgraph task based on a target and properties.")
|
|
78
|
+
parser.add_argument("--target", type=str, default="", help="The target to run in the buildgraph file")
|
|
79
|
+
parser.add_argument(
|
|
80
|
+
"--script", type=str, default=None, help="Path to the buildgraph XML file. If not set, resolved from the project configuration."
|
|
81
|
+
)
|
|
82
|
+
parser.add_argument(
|
|
83
|
+
"--uproject",
|
|
84
|
+
type=str,
|
|
85
|
+
default=None,
|
|
86
|
+
help="Path to the .uproject file. Required when --script is set; otherwise auto-discovered from the current directory.",
|
|
87
|
+
)
|
|
88
|
+
return parser.parse_known_args(argv)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def validate_config(target: str, arguments: list[str], script: Optional[str], uproject_path: Optional[str]) -> None:
|
|
92
|
+
"""Validate the configuration values."""
|
|
93
|
+
pattern = r'-SingleNode=[^"]*'
|
|
94
|
+
|
|
95
|
+
if not target and not any(re.search(pattern, item) for item in arguments):
|
|
96
|
+
raise ValueError("Target is required")
|
|
97
|
+
|
|
98
|
+
if script and not uproject_path:
|
|
99
|
+
raise ValueError("--uproject is required when --script is set")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
103
|
+
args, other_arguments = parse_arguments(argv)
|
|
104
|
+
|
|
105
|
+
target = ""
|
|
106
|
+
|
|
107
|
+
if args.target:
|
|
108
|
+
target = args.target
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
logger.info("Validating configuration")
|
|
112
|
+
validate_config(target, other_arguments, args.script, args.uproject)
|
|
113
|
+
except (ValueError, TypeError) as e:
|
|
114
|
+
logger.error(f"Configuration validation failed: {e}")
|
|
115
|
+
raise e
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
return run(target, other_arguments, args.script, args.uproject)
|
|
119
|
+
except Exception as e:
|
|
120
|
+
logger.error(f"Execution failed: {e}")
|
|
121
|
+
raise e
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
if __name__ == "__main__":
|
|
125
|
+
main()
|
|
@@ -53,7 +53,11 @@ def parse_arguments(argv: Optional[Sequence[str]] = None) -> tuple[argparse.Name
|
|
|
53
53
|
parser = argparse.ArgumentParser(description="Execute a buildgraph task using a shared storage.")
|
|
54
54
|
parser.add_argument("--target", type=str, help="The target to run in the buildgraph file")
|
|
55
55
|
parser.add_argument("--build_tag", type=str, help="The tag that will be used to define a folder on a shared storage")
|
|
56
|
-
|
|
56
|
+
parser.add_argument(
|
|
57
|
+
"--no-single-node",
|
|
58
|
+
action="store_true",
|
|
59
|
+
help="Set this flag to not use SingleNode but use Target. This also disables using the shared storage dir.",
|
|
60
|
+
)
|
|
57
61
|
|
|
58
62
|
args, unknown_args = parser.parse_known_args(argv)
|
|
59
63
|
return args, unknown_args
|
|
@@ -115,17 +119,22 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
|
115
119
|
|
|
116
120
|
logger.info(f"Running ci.buildgraph with build tag: {args.build_tag} on machine {socket.gethostname()}")
|
|
117
121
|
|
|
118
|
-
|
|
119
|
-
logger.info(f"Shared storage directory: {shared_storage_dir}")
|
|
122
|
+
default_arguments: list[str] = ["-BuildMachine", "-NoP4"]
|
|
120
123
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
+
if args.no_single_node:
|
|
125
|
+
logger.info("The flag --no-single-node was passed. Run the task with Target and not with SingleNode")
|
|
126
|
+
default_arguments += [f"--target={args.target}"]
|
|
127
|
+
else:
|
|
128
|
+
shared_storage_dir = Path(config["Jenkins"]["BuildgraphSharedStoragePath"]) / args.build_tag
|
|
129
|
+
logger.info(f"Shared storage directory: {shared_storage_dir}")
|
|
124
130
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
131
|
+
remove_task_shared_storage_dir(shared_storage_dir, args.target)
|
|
132
|
+
try_delete_local_buildgraph_folder(args.build_tag)
|
|
133
|
+
cleanup_local_folder()
|
|
134
|
+
|
|
135
|
+
default_arguments += [f"-SharedStorageDir={shared_storage_dir}", "-WriteToSharedStorage", f"-SingleNode={args.target}"]
|
|
136
|
+
|
|
137
|
+
updated_args = update_or_add_argument(unknown_args, default_arguments)
|
|
129
138
|
|
|
130
139
|
return buildgraph.main(updated_args)
|
|
131
140
|
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import sys
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from uepyscripts import logger
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def run_subprocess(cmd: list[str], log_output: bool = True) -> int:
|
|
9
|
+
"""Run a subprocess with the given command and return its exit code (or 0 if fire-and-forget)."""
|
|
10
|
+
kwargs: dict[str, Any] = {}
|
|
11
|
+
creationflags = 0
|
|
12
|
+
|
|
13
|
+
if sys.platform == "win32":
|
|
14
|
+
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
|
|
15
|
+
startupinfo = subprocess.STARTUPINFO()
|
|
16
|
+
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
|
17
|
+
startupinfo.wShowWindow = subprocess.SW_HIDE
|
|
18
|
+
kwargs["startupinfo"] = startupinfo
|
|
19
|
+
else:
|
|
20
|
+
kwargs["start_new_session"] = True
|
|
21
|
+
|
|
22
|
+
if log_output:
|
|
23
|
+
process = subprocess.Popen(
|
|
24
|
+
cmd,
|
|
25
|
+
stdout=subprocess.PIPE,
|
|
26
|
+
stderr=subprocess.STDOUT,
|
|
27
|
+
text=True,
|
|
28
|
+
bufsize=1,
|
|
29
|
+
creationflags=creationflags,
|
|
30
|
+
**kwargs,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
assert process.stdout is not None
|
|
34
|
+
for line in process.stdout:
|
|
35
|
+
line = line.strip()
|
|
36
|
+
if line:
|
|
37
|
+
logger.info(line)
|
|
38
|
+
|
|
39
|
+
process.wait()
|
|
40
|
+
return process.returncode
|
|
41
|
+
else:
|
|
42
|
+
subprocess.Popen(
|
|
43
|
+
cmd,
|
|
44
|
+
stdout=subprocess.DEVNULL,
|
|
45
|
+
stderr=subprocess.DEVNULL,
|
|
46
|
+
stdin=subprocess.DEVNULL,
|
|
47
|
+
creationflags=creationflags,
|
|
48
|
+
**kwargs,
|
|
49
|
+
)
|
|
50
|
+
return 0
|
|
@@ -63,9 +63,9 @@ class TaskList:
|
|
|
63
63
|
|
|
64
64
|
class EngineInstaller:
|
|
65
65
|
def __init__(self, project: Project, unattended: bool = False) -> None:
|
|
66
|
+
self.project = project
|
|
66
67
|
self.check_requirements()
|
|
67
68
|
|
|
68
|
-
self.project = project
|
|
69
69
|
try:
|
|
70
70
|
self.engine_destination_folder = resolve_engine_destination(project)
|
|
71
71
|
except FileNotFoundError as e:
|
|
@@ -95,6 +95,9 @@ class EngineInstaller:
|
|
|
95
95
|
if not is_7z_installed():
|
|
96
96
|
raise RuntimeError("7-Zip is not installed or not found in PATH. Please install 7-Zip to proceed.")
|
|
97
97
|
|
|
98
|
+
if self.project.is_native_project:
|
|
99
|
+
raise RuntimeError("A native project should be in the same folder than its engine")
|
|
100
|
+
|
|
98
101
|
def get_project_platforms(self) -> List[str]:
|
|
99
102
|
platforms: str = self.config["EngineUpdate.TurnKey"]["Platforms"]
|
|
100
103
|
return platforms.split("+")
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Prunes S3 data for commits whose branch no longer exists on the remote.
|
|
3
|
+
|
|
4
|
+
- A commit is "orphaned" once it is no longer reachable from any current
|
|
5
|
+
remote branch tip (i.e. its branch was deleted). This needs no per-branch
|
|
6
|
+
bookkeeping in S3 — reachability is computed straight from git, the same
|
|
7
|
+
way push.py/pull.py resolve diff baselines and sync targets.
|
|
8
|
+
- Deletes manifests/{sha}.json, deltas/{sha}.7z and checkpoints/{sha}.7z for
|
|
9
|
+
every orphaned sha, then rewrites index.json/checkpoints.json to drop them.
|
|
10
|
+
- Intended to run on a schedule (e.g. a periodic Jenkins job).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import cast
|
|
17
|
+
|
|
18
|
+
from gamedevtools.s3 import S3Client
|
|
19
|
+
|
|
20
|
+
from uepyscripts import logger
|
|
21
|
+
from uepyscripts.internal.engine import resolve_engine
|
|
22
|
+
from uepyscripts.internal.project import resolve_project
|
|
23
|
+
from uepyscripts.tools.ugs.git_utils import get_remote_reachable_shas
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Context:
|
|
27
|
+
def __init__(self, args: argparse.Namespace) -> None:
|
|
28
|
+
self.args = args
|
|
29
|
+
self.s3_client: S3Client = S3Client(
|
|
30
|
+
access_key=args.s3_access_key,
|
|
31
|
+
secret_key=args.s3_secret_key,
|
|
32
|
+
region=args.s3_bucket_region,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def build_orphaned_keys(orphaned: list[str]) -> list[str]:
|
|
37
|
+
"""S3 keys to delete for each orphaned sha. Including a key that was never
|
|
38
|
+
written is harmless — S3 batch-delete is a no-op for missing keys, the same
|
|
39
|
+
assumption prune_old_versions (push.py) already relies on."""
|
|
40
|
+
keys: list[str] = []
|
|
41
|
+
for sha in orphaned:
|
|
42
|
+
keys.append(f"manifests/{sha}.json")
|
|
43
|
+
keys.append(f"deltas/{sha}.7z")
|
|
44
|
+
keys.append(f"checkpoints/{sha}.7z")
|
|
45
|
+
return keys
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def parse_arguments() -> argparse.Namespace:
|
|
49
|
+
"""Parse command line arguments."""
|
|
50
|
+
parser = argparse.ArgumentParser(description="Delete S3 binary-publish data for branches that no longer exist on the remote.")
|
|
51
|
+
parser.add_argument("--uproject-path", type=Path, help=("Path to a native uproject file"))
|
|
52
|
+
parser.add_argument("--s3-bucket-name", type=str, help=("AWS S3 Bucket Name"))
|
|
53
|
+
parser.add_argument("--s3-bucket-region", type=str, help=("AWS S3 Bucket Region"))
|
|
54
|
+
parser.add_argument("--s3-access-key", type=str, help=("AWS S3 Access Key"))
|
|
55
|
+
parser.add_argument("--s3-secret-key", type=str, help=("AWS S3 Secret Key"))
|
|
56
|
+
parser.add_argument("--remote", type=str, default="origin", help="Git remote to check branches against (default: origin)")
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--dry-run",
|
|
59
|
+
action=argparse.BooleanOptionalAction,
|
|
60
|
+
default=False,
|
|
61
|
+
help="Log what would be deleted without touching S3",
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
return parser.parse_args()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def main() -> None:
|
|
68
|
+
args = parse_arguments()
|
|
69
|
+
|
|
70
|
+
logger.info("Cleanup orphaned binary publishes...")
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
project = resolve_project(args.uproject_path)
|
|
74
|
+
except Exception as e:
|
|
75
|
+
logger.fatal(f"Project resolution failed: {e}")
|
|
76
|
+
exit(1)
|
|
77
|
+
|
|
78
|
+
engine = resolve_engine(project)
|
|
79
|
+
context = Context(args)
|
|
80
|
+
|
|
81
|
+
logger.info(f"Fetching '{args.remote}' and computing reachable commits")
|
|
82
|
+
reachable = get_remote_reachable_shas(engine.root_path, args.remote)
|
|
83
|
+
|
|
84
|
+
if not reachable:
|
|
85
|
+
logger.fatal(f"No commit is reachable from any branch on remote '{args.remote}' — aborting without deleting anything")
|
|
86
|
+
exit(1)
|
|
87
|
+
|
|
88
|
+
logger.info("Download index.json")
|
|
89
|
+
index = cast(list[str], context.s3_client.download_json(context.args.s3_bucket_name, "index.json", default=[]))
|
|
90
|
+
|
|
91
|
+
logger.info("Download checkpoints.json")
|
|
92
|
+
checkpoints = cast(list[str], context.s3_client.download_json(context.args.s3_bucket_name, "checkpoints.json", default=[]))
|
|
93
|
+
|
|
94
|
+
orphaned = [sha for sha in index if sha not in reachable]
|
|
95
|
+
|
|
96
|
+
if not orphaned:
|
|
97
|
+
logger.info(f"Nothing to clean up. {len(index)} published commit(s), all reachable from '{args.remote}'.")
|
|
98
|
+
return
|
|
99
|
+
|
|
100
|
+
logger.info(f"Found {len(orphaned)} orphaned commit(s) out of {len(index)} published")
|
|
101
|
+
|
|
102
|
+
keys = build_orphaned_keys(orphaned)
|
|
103
|
+
|
|
104
|
+
if args.dry_run:
|
|
105
|
+
logger.info(f"[dry-run] Would delete {len(keys)} key(s):")
|
|
106
|
+
for key in keys:
|
|
107
|
+
logger.info(f"[dry-run] {key}")
|
|
108
|
+
return
|
|
109
|
+
|
|
110
|
+
context.s3_client.delete_keys(context.args.s3_bucket_name, keys)
|
|
111
|
+
|
|
112
|
+
orphaned_set = set(orphaned)
|
|
113
|
+
index = [sha for sha in index if sha not in orphaned_set]
|
|
114
|
+
checkpoints = [sha for sha in checkpoints if sha not in orphaned_set]
|
|
115
|
+
|
|
116
|
+
logger.info("Upload index.json")
|
|
117
|
+
context.s3_client.upload_bytes(context.args.s3_bucket_name, "index.json", json.dumps(index).encode(), content_type="application/json")
|
|
118
|
+
|
|
119
|
+
logger.info("Upload checkpoints.json")
|
|
120
|
+
context.s3_client.upload_bytes(context.args.s3_bucket_name, "checkpoints.json", json.dumps(checkpoints).encode(), content_type="application/json")
|
|
121
|
+
|
|
122
|
+
logger.info(f"Cleanup done. Deleted {len(keys)} key(s) for {len(orphaned)} orphaned commit(s).")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
main()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import subprocess
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
ANCESTRY_LOOKBACK = 2000
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_current_sha(cwd: Path) -> str:
|
|
9
|
+
sha = os.environ.get("GIT_COMMIT") # Jenkins sets this automatically
|
|
10
|
+
if sha:
|
|
11
|
+
return sha
|
|
12
|
+
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd, text=True).strip()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_current_branch(cwd: Path) -> str:
|
|
16
|
+
branch = os.environ.get("GIT_BRANCH") # Jenkins' Git plugin sets this, sometimes as "origin/<branch>"
|
|
17
|
+
if branch:
|
|
18
|
+
return branch.split("/", 1)[1] if "/" in branch else branch
|
|
19
|
+
return subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, text=True).strip()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_local_ancestry(cwd: Path, limit: int = ANCESTRY_LOOKBACK) -> list[str]:
|
|
23
|
+
out = subprocess.check_output(["git", "log", f"-{limit}", "--format=%H", "HEAD"], cwd=cwd, text=True)
|
|
24
|
+
return out.splitlines()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_remote_reachable_shas(cwd: Path, remote: str = "origin") -> set[str]:
|
|
28
|
+
"""All commit SHAs reachable from any current branch tip on `remote`, after
|
|
29
|
+
pruning local remote-tracking refs for branches deleted on the remote."""
|
|
30
|
+
subprocess.run(["git", "fetch", "--prune", remote], cwd=cwd, check=True, capture_output=True, text=True)
|
|
31
|
+
out = subprocess.check_output(["git", "rev-list", f"--remotes={remote}"], cwd=cwd, text=True)
|
|
32
|
+
return set(out.splitlines())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def resolve_nearest_published_ancestor(index: list[str], ancestry: list[str]) -> tuple[str | None, bool]:
|
|
36
|
+
"""Finds the nearest ancestor SHA (including HEAD itself) present in the
|
|
37
|
+
index. Returns None if nothing has ever been published reachable from HEAD
|
|
38
|
+
within the walked ancestry. The bool is True when the match was found by
|
|
39
|
+
walking the ancestry itself (an exact match), False when falling back to
|
|
40
|
+
the most recent published version because nothing in the ancestry matched."""
|
|
41
|
+
index_set = set(index)
|
|
42
|
+
for sha in ancestry:
|
|
43
|
+
if sha in index_set:
|
|
44
|
+
return sha, True
|
|
45
|
+
|
|
46
|
+
if index:
|
|
47
|
+
return index[-1], False
|
|
48
|
+
|
|
49
|
+
return None, True
|