UEPyScripts 1.4.1__tar.gz → 1.4.3__tar.gz

{uepyscripts-1.4.1 → uepyscripts-1.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: UEPyScripts
-Version: 1.4.1
+Version: 1.4.3
 Summary: Collection of Python tools and helpers designed to automate common Unreal Engine project tasks (builds, packaging, editor control, CI tasks, etc.).
 Project-URL: Changelog, https://github.com/TheEmidee/UEPyScripts/releases
 Project-URL: Homepage, https://github.com/TheEmidee/UEPyScripts

uepyscripts-1.4.3/src/uepyscripts/tools/ugs/git_utils.py

@@ -0,0 +1,110 @@
+import os
+import subprocess
+from pathlib import Path
+
+from uepyscripts import logger
+
+ANCESTRY_LOOKBACK = 500
+
+
+def get_current_sha(cwd: Path) -> str:
+    sha = os.environ.get("GIT_COMMIT")  # Jenkins sets this automatically
+    if sha:
+        return sha
+    return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd, text=True).strip()
+
+
+def get_current_branch(cwd: Path) -> str:
+    branch = os.environ.get("GIT_BRANCH")  # Jenkins' Git plugin sets this, sometimes as "origin/<branch>"
+    if branch:
+        return branch.split("/", 1)[1] if "/" in branch else branch
+
+    branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, text=True).strip()
+    if branch != "HEAD":
+        return branch
+
+    # Detached HEAD (e.g. a CI checkout of an explicit ref rather than a
+    # tracked branch, with no GIT_BRANCH env var set either) — fall back to
+    # whichever local or remote-tracking ref currently points at this commit.
+    refs = subprocess.check_output(
+        ["git", "for-each-ref", "--points-at=HEAD", "--format=%(refname:short)", "refs/heads/", "refs/remotes/"],
+        cwd=cwd,
+        text=True,
+    ).splitlines()
+    for ref in refs:
+        return ref.split("/", 1)[1] if "/" in ref else ref
+
+    logger.warning("Could not resolve current branch: HEAD is detached and no ref points at it")
+    return branch
+
+
+def is_shallow_repository(cwd: Path) -> bool:
+    out = subprocess.check_output(["git", "rev-parse", "--is-shallow-repository"], cwd=cwd, text=True).strip()
+    return out == "true"
+
+
+def deepen_shallow_history(cwd: Path, limit: int, remote: str = "origin") -> bool:
+    """Pulls down the commits missing from a shallow checkout, and only the
+    commits: `--filter=tree:0` keeps trees and blobs on the server, so this
+    stays a metadata-sized fetch even on a repository the size of an engine
+    tree. Returns False (after logging) when the remote refuses, so callers
+    can carry on with whatever history is available locally."""
+    try:
+        subprocess.run(
+            ["git", "fetch", "--no-tags", "--filter=tree:0", f"--deepen={limit}", remote],
+            cwd=cwd,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return True
+    except subprocess.CalledProcessError as e:
+        reason = e.stderr.strip() or e
+        logger.warning(f"Could not deepen shallow history from '{remote}' ({reason}); ancestry walking is limited to the local history")
+        return False
+
+
+def get_local_ancestry(cwd: Path, limit: int = ANCESTRY_LOOKBACK) -> list[str]:
+    ancestry = _log_ancestry(cwd, limit)
+
+    # CI checkouts are typically shallow clones (Jenkins' cloneOption(depth: 1)),
+    # where HEAD is a grafted boundary commit: `git log HEAD` stops at HEAD
+    # itself and every already-published parent looks unreachable, which would
+    # downgrade an incremental publish into a full fresh sync. Deepen first so
+    # the walk sees the real ancestry.
+    if len(ancestry) < limit and is_shallow_repository(cwd):
+        logger.info(f"Shallow checkout detected ({len(ancestry)} commit(s) available); deepening history to {limit} commits")
+        if deepen_shallow_history(cwd, limit):
+            ancestry = _log_ancestry(cwd, limit)
+
+    return ancestry
+
+
+def _log_ancestry(cwd: Path, limit: int) -> list[str]:
+    out = subprocess.check_output(["git", "log", f"-{limit}", "--format=%H", "HEAD"], cwd=cwd, text=True)
+    return out.splitlines()
+
+
+def get_remote_reachable_shas(cwd: Path, remote: str = "origin") -> set[str]:
+    """All commit SHAs reachable from any current branch tip on `remote`, after
+    pruning local remote-tracking refs for branches deleted on the remote."""
+    subprocess.run(["git", "fetch", "--prune", remote], cwd=cwd, check=True, capture_output=True, text=True)
+    out = subprocess.check_output(["git", "rev-list", f"--remotes={remote}"], cwd=cwd, text=True)
+    return set(out.splitlines())
+
+
+def resolve_nearest_published_ancestor(index: list[str], ancestry: list[str]) -> tuple[str | None, bool]:
+    """Finds the nearest ancestor SHA (including HEAD itself) present in the
+    index. Returns None if nothing has ever been published reachable from HEAD
+    within the walked ancestry. The bool is True when the match was found by
+    walking the ancestry itself (an exact match), False when falling back to
+    the most recent published version because nothing in the ancestry matched."""
+    index_set = set(index)
+    for sha in ancestry:
+        if sha in index_set:
+            return sha, True
+
+    if index:
+        return index[-1], False
+
+    return None, True

{uepyscripts-1.4.1 → uepyscripts-1.4.3}/src/uepyscripts/tools/ugs/push.py

@@ -35,10 +35,9 @@ from uepyscripts.tools.ugs.git_utils import (
     get_local_ancestry,
     resolve_nearest_published_ancestor,
 )
-from uepyscripts.tools.ugs.ugs_types import Manifest, VersionManifest
+from uepyscripts.tools.ugs.ugs_types import HashCacheEntry, HashCacheManifest, Manifest, VersionManifest
 
 # ---- CONFIG ----------------------------------------------------------
-LOCAL_HASH_CACHE = ".hash-cache.json"
 
 # Extensions that go through the delta/checkpoint sync pipeline (no PDBs — see symstore below)
 SYNC_EXT = {".dll", ".exe", ".lib", ".modules", ".target"}
@@ -72,18 +71,18 @@ class HashCacheInfos:
 
 
 class HashCache:
-    def __init__(self) -> None:
+    def __init__(self, commit_sha: str) -> None:
+        self.commit_sha = commit_sha
         self.cache: dict[str, HashCacheInfos] = dict()
-        self.local_hash_cache = Path(LOCAL_HASH_CACHE)
 
-        if self.local_hash_cache.exists():
-            raw = json.loads(self.local_hash_cache.read_text())
-            self.cache = {path: HashCacheInfos(**entry) for path, entry in raw.items()}
+    def seed_from(self, files: dict[str, HashCacheEntry]) -> None:
+        self.cache = {path: HashCacheInfos(**entry) for path, entry in files.items()}
 
-    def save(self) -> None:
-        self.local_hash_cache.parent.mkdir(parents=True, exist_ok=True)
-        serializable = {path: asdict(entry) for path, entry in self.cache.items()}
-        self.local_hash_cache.write_text(json.dumps(serializable, indent=2, sort_keys=True))
+    def to_manifest(self) -> HashCacheManifest:
+        return {
+            "commit_sha": self.commit_sha,
+            "files": {path: cast(HashCacheEntry, asdict(entry)) for path, entry in self.cache.items()},
+        }
 
     def get(self, path: str) -> HashCacheInfos | None:
         return self.cache.get(path)
@@ -102,10 +101,35 @@ class Context:
         )
         self.symbol_store_path: str = args.symbol_store_path
         self.tmp_root: Path = Path(tempfile.mkdtemp(prefix="publish-"))
-        self.hash_cache = HashCache()
         self.root_path = root_path
         self.current_sha = get_current_sha(root_path)
         self.current_branch = get_current_branch(root_path)
+        self.ancestry: list[str] = get_local_ancestry(root_path)
+        self.hash_cache, self.hash_cache_index = self._load_hash_cache()
+
+    def _load_hash_cache(self) -> tuple[HashCache, list[str]]:
+        """Resolves the nearest commit with an uploaded hash cache and seeds a
+        fresh HashCache from it. Unlike the manifest baseline used for diffing,
+        an inexact match is fine here: a fingerprint mismatch just falls back to
+        a full hash for that file, no worse than a cold-cache case, so
+        there's no correctness risk in seeding from a non-ancestor."""
+        bucket_name = self.args.s3_bucket_name
+        hash_cache_index = cast(list[str], self.s3_client.download_json(bucket_name, "hash_caches/index.json", default=[]))
+        baseline_sha, _ = resolve_nearest_published_ancestor(hash_cache_index, self.ancestry)
+
+        hash_cache = HashCache(self.current_sha)
+        if baseline_sha is None:
+            logger.info("No hash cache baseline found — starting with a cold hash cache")
+            return hash_cache, hash_cache_index
+
+        try:
+            manifest = cast(HashCacheManifest, self.s3_client.download_json(bucket_name, f"hash_caches/{baseline_sha}.json"))
+            hash_cache.seed_from(manifest["files"])
+            logger.info(f"Seeded hash cache from {baseline_sha} ({len(manifest['files'])} entries)")
+        except Exception as e:
+            logger.warning(f"Failed to load hash cache from hash_caches/{baseline_sha}.json ({e}); starting with a cold hash cache")
+
+        return hash_cache, hash_cache_index
 
     def finalize(self) -> None:
         if not self.args.keep_temp_directory:
@@ -327,6 +351,24 @@ def prune_old_versions(context: Context, cfg: FilesConfiguration, index: list[st
     return index[cutoff_idx:], checkpoints_to_keep
 
 
+def prune_old_hash_caches(context: Context, hash_cache_index: list[str], keep: int) -> list[str]:
+    """Simple count-based retention for the hash-cache side-channel: unlike
+    index.json (only appended on actual content changes), hash_cache_index
+    grows once per run, so it needs its own, faster-turnover retention
+    rather than piggybacking on prune_old_versions's checkpoint-aligned
+    logic."""
+    if len(hash_cache_index) <= keep:
+        return hash_cache_index
+
+    to_delete = hash_cache_index[:-keep]
+    to_keep = hash_cache_index[-keep:]
+
+    logger.info(f"Pruning {len(to_delete)} old hash cache(s)")
+    context.s3_client.delete_keys(context.args.s3_bucket_name, [f"hash_caches/{sha}.json" for sha in to_delete])
+
+    return to_keep
+
+
 def publish_to_symbol_store(context: Context, paths: list[str], cfg: FilesConfiguration) -> None:
     """Indexes PDB/DLL/EXE files into the symbol store via symstore.exe."""
     logger.info("Publish to symbol store")
@@ -397,6 +439,12 @@ def parse_arguments() -> argparse.Namespace:
         default=2,
         help="Number of checkpoints to retain; older ones (and their deltas) get pruned (default: 2)",
     )
+    parser.add_argument(
+        "--keep_hash_caches",
+        type=int,
+        default=100,
+        help="Number of hash cache entries to retain in hash_caches/index.json; older ones get pruned (default: 100)",
+    )
     parser.add_argument(
         "--checkpoint-branch",
         type=str,
@@ -460,8 +508,7 @@ def main() -> None:
         logger.info(f"Finished scan. Found {len(new_state)} files.")
 
         logger.info("Resolve diff baseline from git ancestry")
-        ancestry = get_local_ancestry(context.root_path)
-        baseline_sha, is_exact = resolve_nearest_published_ancestor(index, ancestry)
+        baseline_sha, is_exact = resolve_nearest_published_ancestor(index, context.ancestry)
 
         old_state: dict[str, str]
         if baseline_sha is None:
@@ -573,8 +620,29 @@ def main() -> None:
 
         logger.info("=== Finished processing binary files ===")
 
-        logger.info("Save hash cache")
-        context.hash_cache.save()
+        if not context.args.skip_upload:
+            logger.info("Upload hash cache")
+            context.s3_client.upload_bytes(
+                context.args.s3_bucket_name,
+                f"hash_caches/{context.current_sha}.json",
+                json.dumps(context.hash_cache.to_manifest()).encode(),
+                content_type="application/json",
+            )
+
+            if context.current_sha not in context.hash_cache_index:
+                context.hash_cache_index.append(context.current_sha)
+
+            context.hash_cache_index = prune_old_hash_caches(context, context.hash_cache_index, context.args.keep_hash_caches)
+
+            logger.info("Upload hash_caches/index.json")
+            context.s3_client.upload_bytes(
+                context.args.s3_bucket_name,
+                "hash_caches/index.json",
+                json.dumps(context.hash_cache_index).encode(),
+                content_type="application/json",
+            )
+        else:
+            logger.info("Skipping hash cache upload (--skip-upload)")
     finally:
         context.finalize()
 

{uepyscripts-1.4.1 → uepyscripts-1.4.3}/src/uepyscripts/tools/ugs/ugs_types.py

@@ -7,3 +7,13 @@ class VersionManifest(TypedDict):
     files: Manifest  # full state at this version: {relative_path: sha256}
     changed: list[str]  # files added or modified since the previous version
     removed: list[str]  # files removed since the previous version
+
+
+class HashCacheEntry(TypedDict):
+    fingerprint: str  # "mtime_ns:size"
+    hash: str  # blake3 hex digest
+
+
+class HashCacheManifest(TypedDict):
+    commit_sha: str  # commit this cache was captured at
+    files: dict[str, HashCacheEntry]

uepyscripts-1.4.1/src/uepyscripts/tools/ugs/git_utils.py

@@ -1,49 +0,0 @@
-import os
-import subprocess
-from pathlib import Path
-
-ANCESTRY_LOOKBACK = 2000
-
-
-def get_current_sha(cwd: Path) -> str:
-    sha = os.environ.get("GIT_COMMIT")  # Jenkins sets this automatically
-    if sha:
-        return sha
-    return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd, text=True).strip()
-
-
-def get_current_branch(cwd: Path) -> str:
-    branch = os.environ.get("GIT_BRANCH")  # Jenkins' Git plugin sets this, sometimes as "origin/<branch>"
-    if branch:
-        return branch.split("/", 1)[1] if "/" in branch else branch
-    return subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, text=True).strip()
-
-
-def get_local_ancestry(cwd: Path, limit: int = ANCESTRY_LOOKBACK) -> list[str]:
-    out = subprocess.check_output(["git", "log", f"-{limit}", "--format=%H", "HEAD"], cwd=cwd, text=True)
-    return out.splitlines()
-
-
-def get_remote_reachable_shas(cwd: Path, remote: str = "origin") -> set[str]:
-    """All commit SHAs reachable from any current branch tip on `remote`, after
-    pruning local remote-tracking refs for branches deleted on the remote."""
-    subprocess.run(["git", "fetch", "--prune", remote], cwd=cwd, check=True, capture_output=True, text=True)
-    out = subprocess.check_output(["git", "rev-list", f"--remotes={remote}"], cwd=cwd, text=True)
-    return set(out.splitlines())
-
-
-def resolve_nearest_published_ancestor(index: list[str], ancestry: list[str]) -> tuple[str | None, bool]:
-    """Finds the nearest ancestor SHA (including HEAD itself) present in the
-    index. Returns None if nothing has ever been published reachable from HEAD
-    within the walked ancestry. The bool is True when the match was found by
-    walking the ancestry itself (an exact match), False when falling back to
-    the most recent published version because nothing in the ancestry matched."""
-    index_set = set(index)
-    for sha in ancestry:
-        if sha in index_set:
-            return sha, True
-
-    if index:
-        return index[-1], False
-
-    return None, True