UEPyScripts 1.4.3__tar.gz → 1.4.5__tar.gz

{uepyscripts-1.4.3 → uepyscripts-1.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: UEPyScripts
-Version: 1.4.3
+Version: 1.4.5
 Summary: Collection of Python tools and helpers designed to automate common Unreal Engine project tasks (builds, packaging, editor control, CI tasks, etc.).
 Project-URL: Changelog, https://github.com/TheEmidee/UEPyScripts/releases
 Project-URL: Homepage, https://github.com/TheEmidee/UEPyScripts

{uepyscripts-1.4.3 → uepyscripts-1.4.5}/src/uepyscripts/tools/ugs/pull.py

@@ -12,6 +12,7 @@ from uepyscripts import logger
 from uepyscripts.internal.engine import resolve_engine
 from uepyscripts.internal.project import resolve_project
 from uepyscripts.tools.ugs.git_utils import get_local_ancestry, resolve_nearest_published_ancestor
+from uepyscripts.tools.ugs.s3_settings import S3_CONFIG_SECTION, S3Settings, add_s3_arguments, resolve_s3_settings
 from uepyscripts.tools.ugs.ugs_types import Manifest, VersionManifest
 
 """
@@ -44,14 +45,14 @@ class LocalState(TypedDict):
 
 
 class Context:
-    def __init__(self, root_folder: Path, args: argparse.Namespace) -> None:
+    def __init__(self, root_folder: Path, s3_settings: S3Settings) -> None:
         self.root_folder: Path = root_folder
-        self.s3_bucket_name: str = args.s3_bucket_name
-        self.s3_bucket_region: str = args.s3_bucket_region
+        self.s3_bucket_name: str = s3_settings.bucket_name
+        self.s3_bucket_region: str = s3_settings.bucket_region
         self.s3_client: S3Client = S3Client(
-            access_key=args.s3_access_key,
-            secret_key=args.s3_secret_key,
-            region=args.s3_bucket_region,
+            access_key=s3_settings.access_key,
+            secret_key=s3_settings.secret_key,
+            region=s3_settings.bucket_region,
         )
         logger.info("Download index.json")
         self.commit_index = cast(list[str], self.s3_client.download_json(self.s3_bucket_name, "index.json", default=[]))
@@ -207,12 +208,15 @@ def sync(context: Context, target_version: str, state: LocalState) -> LocalState
 
 def parse_arguments() -> argparse.Namespace:
     """Parse command line arguments."""
-    parser = argparse.ArgumentParser(description="Check and install Unreal Engine installation for the given project.")
+    parser = argparse.ArgumentParser(
+        description=(
+            "Sync local engine + game binaries to match the current git HEAD. "
+            f"The S3 info is read from the [{S3_CONFIG_SECTION}] section of Config/PyScripts/config.ini, "
+            "and each value can be overridden by its command line argument."
+        )
+    )
     parser.add_argument("--uproject-path", type=Path, help=("Path to a native uproject file"))
-    parser.add_argument("--s3-bucket-name", type=str, help=("AWS S3 Bucket Name"))
-    parser.add_argument("--s3-bucket-region", type=str, help=("AWS S3 Bucket Region"))
-    parser.add_argument("--s3-access-key", type=str, help=("AWS S3 Access Key"))
-    parser.add_argument("--s3-secret-key", type=str, help=("AWS S3 Secret Key"))
+    add_s3_arguments(parser)
 
     return parser.parse_args()
 
@@ -234,9 +238,11 @@ def main() -> None:
 
     engine = resolve_engine(project)
 
+    s3_settings = resolve_s3_settings(project, args)
+
     local_state = LocalStateManager()
 
-    context = Context(engine.root_path, args)
+    context = Context(engine.root_path, s3_settings)
 
     warn_if_source_dirty()
 

{uepyscripts-1.4.3 → uepyscripts-1.4.5}/src/uepyscripts/tools/ugs/push.py

@@ -35,6 +35,7 @@ from uepyscripts.tools.ugs.git_utils import (
     get_local_ancestry,
     resolve_nearest_published_ancestor,
 )
+from uepyscripts.tools.ugs.s3_settings import S3_CONFIG_SECTION, S3Settings, add_s3_arguments, resolve_s3_settings
 from uepyscripts.tools.ugs.ugs_types import HashCacheEntry, HashCacheManifest, Manifest, VersionManifest
 
 # ---- CONFIG ----------------------------------------------------------
@@ -92,12 +93,14 @@ class HashCache:
 
 
 class Context:
-    def __init__(self, args: argparse.Namespace, root_path: Path) -> None:
+    def __init__(self, args: argparse.Namespace, root_path: Path, s3_settings: S3Settings) -> None:
         self.args = args
+        self.s3_bucket_name: str = s3_settings.bucket_name
+        self.s3_bucket_region: str = s3_settings.bucket_region
         self.s3_client: S3Client = S3Client(
-            access_key=args.s3_access_key,
-            secret_key=args.s3_secret_key,
-            region=args.s3_bucket_region,
+            access_key=s3_settings.access_key,
+            secret_key=s3_settings.secret_key,
+            region=s3_settings.bucket_region,
         )
         self.symbol_store_path: str = args.symbol_store_path
         self.tmp_root: Path = Path(tempfile.mkdtemp(prefix="publish-"))
@@ -113,7 +116,7 @@ class Context:
         an inexact match is fine here: a fingerprint mismatch just falls back to
         a full hash for that file, no worse than a cold-cache case, so
         there's no correctness risk in seeding from a non-ancestor."""
-        bucket_name = self.args.s3_bucket_name
+        bucket_name = self.s3_bucket_name
         hash_cache_index = cast(list[str], self.s3_client.download_json(bucket_name, "hash_caches/index.json", default=[]))
         baseline_sha, _ = resolve_nearest_published_ancestor(hash_cache_index, self.ancestry)
 
@@ -346,7 +349,7 @@ def prune_old_versions(context: Context, cfg: FilesConfiguration, index: list[st
         keys_to_delete.append(f"checkpoints/{v}.7z")
 
     logger.info(f"Pruning {len(versions_to_delete)} version(s), {len(checkpoints_to_delete)} checkpoint(s)")
-    context.s3_client.delete_keys(context.args.s3_bucket_name, keys_to_delete)
+    context.s3_client.delete_keys(context.s3_bucket_name, keys_to_delete)
 
     return index[cutoff_idx:], checkpoints_to_keep
 
@@ -364,7 +367,7 @@ def prune_old_hash_caches(context: Context, hash_cache_index: list[str], keep: i
     to_keep = hash_cache_index[-keep:]
 
     logger.info(f"Pruning {len(to_delete)} old hash cache(s)")
-    context.s3_client.delete_keys(context.args.s3_bucket_name, [f"hash_caches/{sha}.json" for sha in to_delete])
+    context.s3_client.delete_keys(context.s3_bucket_name, [f"hash_caches/{sha}.json" for sha in to_delete])
 
     return to_keep
 
@@ -413,7 +416,13 @@ def publish_to_symbol_store(context: Context, paths: list[str], cfg: FilesConfig
 
 def parse_arguments() -> argparse.Namespace:
     """Parse command line arguments."""
-    parser = argparse.ArgumentParser(description="Check and install Unreal Engine installation for the given project.")
+    parser = argparse.ArgumentParser(
+        description=(
+            "Publish the engine + game binaries for the current git HEAD. "
+            f"The S3 info is read from the [{S3_CONFIG_SECTION}] section of Config/PyScripts/config.ini, "
+            "and each value can be overridden by its command line argument."
+        )
+    )
     parser.add_argument("--uproject-path", type=Path, help=("Path to a native uproject file"))
     parser.add_argument("--skip-upload", action=argparse.BooleanOptionalAction, help=("Set this to not upload anything on S3"))
     parser.add_argument(
@@ -421,10 +430,7 @@ def parse_arguments() -> argparse.Namespace:
         action=argparse.BooleanOptionalAction,
         help=("Set this to not delete the temporary directory when the script finishes"),
     )
-    parser.add_argument("--s3-bucket-name", type=str, help=("AWS S3 Bucket Name"))
-    parser.add_argument("--s3-bucket-region", type=str, help=("AWS S3 Bucket Region"))
-    parser.add_argument("--s3-access-key", type=str, help=("AWS S3 Access Key"))
-    parser.add_argument("--s3-secret-key", type=str, help=("AWS S3 Secret Key"))
+    add_s3_arguments(parser)
     parser.add_argument("--disable-symbol-store-upload", action=argparse.BooleanOptionalAction)
     parser.add_argument("--symbol-store-path", type=str, help=("Path of the shared symbol store folder"))
     parser.add_argument(
@@ -472,11 +478,14 @@ def main() -> None:
 
     engine = resolve_engine(project)
 
+    s3_settings = resolve_s3_settings(project, args)
+
     cfg = FilesConfiguration(
         root_folder=engine.root_path,
         directories=[
             "Engine/Binaries/Win64",
             "Engine/Plugins",
+            "Engine/Platforms",
             "Engine/Intermediate/Build/BuildRules",
             "Engine/Intermediate/Build/Win64/**/",
             "Engine/Plugins/**/Intermediate/**/",
@@ -487,14 +496,14 @@ def main() -> None:
         symstore_product=project.project_name,
     )
 
-    context = Context(args, engine.root_path)
+    context = Context(args, engine.root_path, s3_settings)
 
     try:
         logger.info("=== Publish binary files ===")
         logger.info(f"Current branch: {context.current_branch}")
 
         logger.info("Download index.json")
-        index = cast(list[str], context.s3_client.download_json(context.args.s3_bucket_name, "index.json", default=[]))
+        index = cast(list[str], context.s3_client.download_json(context.s3_bucket_name, "index.json", default=[]))
 
         if len(index) == 0:
             logger.info("Empty index.json. Fresh sync")
@@ -523,7 +532,7 @@ def main() -> None:
             old_state = {}
         else:
             logger.info(f"Using {baseline_sha} as diff baseline for branch '{context.current_branch}'")
-            old_manifest = cast(VersionManifest, context.s3_client.download_json(context.args.s3_bucket_name, f"manifests/{baseline_sha}.json"))
+            old_manifest = cast(VersionManifest, context.s3_client.download_json(context.s3_bucket_name, f"manifests/{baseline_sha}.json"))
             old_state = old_manifest["files"]
 
         changed, removed = compute_diff(old_state, new_state)
@@ -544,7 +553,7 @@ def main() -> None:
                     return
 
                 logger.info("Upload zip")
-                context.s3_client.upload_file(context.args.s3_bucket_name, f"deltas/{version}.7z", zip_file_path)
+                context.s3_client.upload_file(context.s3_bucket_name, f"deltas/{version}.7z", zip_file_path)
                 zip_file_path.unlink()
 
             manifest_payload: VersionManifest = {
@@ -553,7 +562,7 @@ def main() -> None:
                 "removed": removed,
             }
             context.s3_client.upload_bytes(
-                context.args.s3_bucket_name,
+                context.s3_bucket_name,
                 f"manifests/{version}.json",
                 json.dumps(manifest_payload).encode(),
                 content_type="application/json",
@@ -562,7 +571,7 @@ def main() -> None:
             index.append(version)
 
             logger.info("Download JSON with checkpoints")
-            checkpoints = cast(list[str], context.s3_client.download_json(context.args.s3_bucket_name, "checkpoints.json", default=[]))
+            checkpoints = cast(list[str], context.s3_client.download_json(context.s3_bucket_name, "checkpoints.json", default=[]))
 
             logger.info(f"Checkpoint interval: {cfg.checkpoint_interval} - Number of deltas : {len(index)}")
 
@@ -576,7 +585,7 @@ def main() -> None:
                 zip_file_path = build_zip(cfg.root_folder, list(new_state.keys()), context.tmp_root)
 
                 logger.info("Upload zip")
-                context.s3_client.upload_file(context.args.s3_bucket_name, f"checkpoints/{version}.7z", zip_file_path)
+                context.s3_client.upload_file(context.s3_bucket_name, f"checkpoints/{version}.7z", zip_file_path)
                 zip_file_path.unlink()
                 checkpoints.append(version)
             else:
@@ -585,11 +594,11 @@ def main() -> None:
             index, checkpoints = prune_old_versions(context, cfg, index, checkpoints)
 
             logger.info("Upload index.json")
-            context.s3_client.upload_bytes(context.args.s3_bucket_name, "index.json", json.dumps(index).encode(), content_type="application/json")
+            context.s3_client.upload_bytes(context.s3_bucket_name, "index.json", json.dumps(index).encode(), content_type="application/json")
 
             logger.info("Upload checkpoints.json")
             context.s3_client.upload_bytes(
-                context.args.s3_bucket_name, "checkpoints.json", json.dumps(checkpoints).encode(), content_type="application/json"
+                context.s3_bucket_name, "checkpoints.json", json.dumps(checkpoints).encode(), content_type="application/json"
             )
         else:
             logger.info("No changes in synced binaries.")
@@ -603,7 +612,7 @@ def main() -> None:
             logger.info(f"Finished scan. Found {len(symstore_state)} files.")
 
             logger.info("Download symstore-manifest.json")
-            prev_symstore_state = cast(Manifest, context.s3_client.download_json(context.args.s3_bucket_name, "symstore-manifest.json", default={}))
+            prev_symstore_state = cast(Manifest, context.s3_client.download_json(context.s3_bucket_name, "symstore-manifest.json", default={}))
             symstore_changed, _ = compute_diff(prev_symstore_state, symstore_state)
 
             if symstore_changed:
@@ -611,7 +620,7 @@ def main() -> None:
 
                 logger.info("Upload symstore-manifest.json")
                 context.s3_client.upload_bytes(
-                    context.args.s3_bucket_name, "symstore-manifest.json", json.dumps(symstore_state).encode(), content_type="application/json"
+                    context.s3_bucket_name, "symstore-manifest.json", json.dumps(symstore_state).encode(), content_type="application/json"
                 )
             else:
                 logger.info("No changes for symbol store")
@@ -623,7 +632,7 @@ def main() -> None:
         if not context.args.skip_upload:
             logger.info("Upload hash cache")
             context.s3_client.upload_bytes(
-                context.args.s3_bucket_name,
+                context.s3_bucket_name,
                 f"hash_caches/{context.current_sha}.json",
                 json.dumps(context.hash_cache.to_manifest()).encode(),
                 content_type="application/json",
@@ -636,7 +645,7 @@ def main() -> None:
 
             logger.info("Upload hash_caches/index.json")
             context.s3_client.upload_bytes(
-                context.args.s3_bucket_name,
+                context.s3_bucket_name,
                 "hash_caches/index.json",
                 json.dumps(context.hash_cache_index).encode(),
                 content_type="application/json",

uepyscripts-1.4.5/src/uepyscripts/tools/ugs/s3_settings.py

@@ -0,0 +1,81 @@
+"""
+S3 connection info for the binary publish/sync tools.
+
+Values are read from the project's Config/PyScripts/config.ini (section
+[UGS.AWS]), and each one can be overridden by its command line argument.
+Having no config file at all is fine — the command line can provide
+everything — but a value that is missing from both sources is reported
+before any S3 call is attempted.
+"""
+
+import argparse
+from dataclasses import dataclass
+
+from uepyscripts import logger
+from uepyscripts.internal.config import Config, resolve_config
+from uepyscripts.internal.project import Project
+
+S3_CONFIG_SECTION = "UGS.AWS"
+
+# (attribute, command line argument, config.ini key)
+S3_SETTINGS_FIELDS = [
+    ("bucket_name", "--s3-bucket-name", "AWS_BucketName"),
+    ("bucket_region", "--s3-bucket-region", "AWS_Region"),
+    ("access_key", "--s3-access-key", "AWS_AccessKey"),
+    ("secret_key", "--s3-secret-key", "AWS_SecretKey"),
+]
+
+
+@dataclass
+class S3Settings:
+    bucket_name: str
+    bucket_region: str
+    access_key: str
+    secret_key: str
+
+
+def add_s3_arguments(parser: argparse.ArgumentParser) -> None:
+    """Registers the S3 command line arguments, which override the config file."""
+    group = parser.add_argument_group(
+        "AWS S3",
+        f"Defaults to the [{S3_CONFIG_SECTION}] section of Config/PyScripts/config.ini",
+    )
+    group.add_argument("--s3-bucket-name", type=str, help="AWS S3 Bucket Name (config: AWS_BucketName)")
+    group.add_argument("--s3-bucket-region", type=str, help="AWS S3 Bucket Region (config: AWS_Region)")
+    group.add_argument("--s3-access-key", type=str, help="AWS S3 Access Key (config: AWS_AccessKey)")
+    group.add_argument("--s3-secret-key", type=str, help="AWS S3 Secret Key (config: AWS_SecretKey)")
+
+
+def read_config_value(config: Config, key: str) -> str | None:
+    """A missing config file, section or key are all treated the same way — no value."""
+    if not config.valid or not config.config.has_section(S3_CONFIG_SECTION):
+        return None
+
+    value = config[S3_CONFIG_SECTION].get(key)
+
+    return value if value else None
+
+
+def resolve_s3_settings(project: Project, args: argparse.Namespace) -> S3Settings:
+    """Takes the S3 info from the config file, then lets the matching command
+    line arguments override each value. Exits with an explicit report of what
+    is missing rather than letting the first S3 call fail."""
+    config = resolve_config(project)
+
+    values: dict[str, str] = {}
+    missing: list[str] = []
+
+    for attribute, argument, config_key in S3_SETTINGS_FIELDS:
+        argument_value = getattr(args, argument.lstrip("-").replace("-", "_"), None)
+        value = argument_value or read_config_value(config, config_key)
+
+        if value:
+            values[attribute] = value
+        else:
+            missing.append(f"  - {argument} (or [{S3_CONFIG_SECTION}] {config_key} in Config/PyScripts/config.ini)")
+
+    if missing:
+        logger.fatal("Missing S3 information:\n" + "\n".join(missing))
+        exit(1)
+
+    return S3Settings(**values)