github-backup 0.60.0__py3-none-any.whl → 0.61.1__py3-none-any.whl

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.60.0"
+__version__ = "0.61.1"

github_backup/cli.py

@@ -46,6 +46,16 @@ def main():
             "Use -t/--token or -f/--token-fine to authenticate."
         )
 
+    # Issue #477: Fine-grained PATs cannot download all attachment types from
+    # private repos. Image attachments will be retried via Markdown API workaround.
+    if args.include_attachments and args.token_fine:
+        logger.warning(
+            "Using --attachments with fine-grained token. Due to GitHub platform "
+            "limitations, file attachments (PDFs, etc.) from private repos may fail. "
+            "Image attachments will be retried via workaround. For full attachment "
+            "support, use --token-classic instead."
+        )
+
     if args.quiet:
         logger.setLevel(logging.WARNING)
 

github_backup/github_backup.py

@@ -310,6 +310,12 @@ def parse_args(args=None):
         dest="include_milestones",
         help="include milestones in backup",
     )
+    parser.add_argument(
+        "--security-advisories",
+        action="store_true",
+        dest="include_security_advisories",
+        help="include security advisories in backup",
+    )
     parser.add_argument(
         "--repositories",
         action="store_true",
@@ -1056,6 +1062,65 @@ def download_attachment_file(url, path, auth, as_app=False, fine=False):
     return metadata
 
 
+def get_jwt_signed_url_via_markdown_api(url, token, repo_context):
+    """Convert a user-attachments/assets URL to a JWT-signed URL via Markdown API.
+
+    GitHub's Markdown API renders image URLs and returns HTML containing
+    JWT-signed private-user-images.githubusercontent.com URLs that work
+    without token authentication.
+
+    This is a workaround for issue #477 where fine-grained PATs cannot
+    download user-attachments URLs from private repos directly.
+
+    Limitations:
+    - Only works for /assets/ URLs (images)
+    - Does NOT work for /files/ URLs (PDFs, text files, etc.)
+    - JWT URLs expire after ~5 minutes
+
+    Args:
+        url: The github.com/user-attachments/assets/UUID URL
+        token: Raw fine-grained PAT (github_pat_...)
+        repo_context: Repository context as "owner/repo"
+
+    Returns:
+        str: JWT-signed URL from private-user-images.githubusercontent.com
+        None: If conversion fails
+    """
+
+    try:
+        payload = json.dumps(
+            {"text": f"![img]({url})", "mode": "gfm", "context": repo_context}
+        ).encode("utf-8")
+
+        request = Request("https://api.github.com/markdown", data=payload, method="POST")
+        request.add_header("Authorization", f"token {token}")
+        request.add_header("Content-Type", "application/json")
+        request.add_header("Accept", "application/vnd.github+json")
+
+        html = urlopen(request, timeout=30).read().decode("utf-8")
+
+        # Parse JWT-signed URL from HTML response
+        # Format: <img src="https://private-user-images.githubusercontent.com/...?jwt=..." ...>
+        if match := re.search(
+            r'src="(https://private-user-images\.githubusercontent\.com/[^"]+)"', html
+        ):
+            jwt_url = match.group(1)
+            logger.debug("Converted attachment URL to JWT-signed URL via Markdown API")
+            return jwt_url
+
+        logger.debug("Markdown API response did not contain JWT-signed URL")
+        return None
+
+    except HTTPError as e:
+        logger.debug(
+            "Markdown API request failed with HTTP {0}: {1}".format(e.code, e.reason)
+        )
+        return None
+    except Exception as e:
+        logger.debug("Markdown API request failed: {0}".format(str(e)))
+        return None
+
+
 def extract_attachment_urls(item_data, issue_number=None, repository_full_name=None):
     """Extract GitHub-hosted attachment URLs from issue/PR body and comments.
 
@@ -1409,15 +1474,46 @@ def download_attachments(
         filename = get_attachment_filename(url)
         filepath = os.path.join(attachments_dir, filename)
 
-        # Download and get metadata
-        metadata = download_attachment_file(
-            url,
-            filepath,
-            get_auth(args, encode=not args.as_app),
-            as_app=args.as_app,
-            fine=args.token_fine is not None,
+        # Issue #477: Fine-grained PATs cannot download user-attachments/assets
+        # from private repos directly (404). Use Markdown API workaround to get
+        # a JWT-signed URL. Only works for /assets/ (images), not /files/.
+        needs_jwt = (
+            args.token_fine is not None
+            and repository.get("private", False)
+            and "github.com/user-attachments/assets/" in url
         )
 
+        if not needs_jwt:
+            # NORMAL download path
+            metadata = download_attachment_file(
+                url,
+                filepath,
+                get_auth(args, encode=not args.as_app),
+                as_app=args.as_app,
+                fine=args.token_fine is not None,
+            )
+        elif jwt_url := get_jwt_signed_url_via_markdown_api(
+            url, args.token_fine, repository["full_name"]
+        ):
+            # JWT needed and extracted, download via JWT
+            metadata = download_attachment_file(
+                jwt_url, filepath, auth=None, as_app=False, fine=False
+            )
+            metadata["url"] = url  # Apply back the original URL
+            metadata["jwt_workaround"] = True
+        else:
+            # Markdown API workaround failed - skip download we know will fail
+            metadata = {
+                "url": url,
+                "success": False,
+                "skipped_at": datetime.now(timezone.utc).isoformat(),
+                "error": "Fine-grained token cannot download private repo attachments. "
+                "Markdown API workaround failed. Use --token-classic instead.",
+            }
+            logger.warning(
+                "Skipping attachment {0}: {1}".format(url, metadata["error"])
+            )
+
         # If download succeeded but we got an extension from Content-Disposition,
         # we may need to rename the file to add the extension
         if metadata["success"] and metadata.get("original_filename"):
@@ -1718,6 +1814,9 @@ def backup_repositories(args, output_directory, repositories):
             if args.include_milestones or args.include_everything:
                 backup_milestones(args, repo_cwd, repository, repos_template)
 
+            if args.include_security_advisories or args.include_everything:
+                backup_security_advisories(args, repo_cwd, repository, repos_template)
+
             if args.include_labels or args.include_everything:
                 backup_labels(args, repo_cwd, repository, repos_template)
 
@@ -1934,6 +2033,43 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
         )
 
 
+def backup_security_advisories(args, repo_cwd, repository, repos_template):
+    advisory_cwd = os.path.join(repo_cwd, "security-advisories")
+    if args.skip_existing and os.path.isdir(advisory_cwd):
+        return
+
+    logger.info("Retrieving {0} security advisories".format(repository["full_name"]))
+    mkdir_p(repo_cwd, advisory_cwd)
+
+    template = "{0}/{1}/security-advisories".format(
+        repos_template, repository["full_name"]
+    )
+
+    _advisories = retrieve_data(args, template)
+
+    advisories = {}
+    for advisory in _advisories:
+        advisories[advisory["ghsa_id"]] = advisory
+
+    written_count = 0
+    for ghsa_id, advisory in list(advisories.items()):
+        advisory_file = "{0}/{1}.json".format(advisory_cwd, ghsa_id)
+        if json_dump_if_changed(advisory, advisory_file):
+            written_count += 1
+
+    total = len(advisories)
+    if written_count == total:
+        logger.info("Saved {0} security advisories to disk".format(total))
+    elif written_count == 0:
+        logger.info("{0} security advisories unchanged, skipped write".format(total))
+    else:
+        logger.info(
+            "Saved {0} of {1} security advisories to disk ({2} unchanged)".format(
+                written_count, total, total - written_count
+            )
+        )
+
+
 def backup_labels(args, repo_cwd, repository, repos_template):
     label_cwd = os.path.join(repo_cwd, "labels")
     output_file = "{0}/labels.json".format(label_cwd)

{github_backup-0.60.0.dist-info → github_backup-0.61.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: github-backup
-Version: 0.60.0
+Version: 0.61.1
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez
@@ -73,9 +73,9 @@ CLI Help output::
                   [--watched] [--followers] [--following] [--all]
                   [--issues] [--issue-comments] [--issue-events] [--pulls]
                   [--pull-comments] [--pull-commits] [--pull-details]
-                  [--labels] [--hooks] [--milestones] [--repositories]
-                  [--bare] [--no-prune] [--lfs] [--wikis] [--gists]
-                  [--starred-gists] [--skip-archived] [--skip-existing]
+                  [--labels] [--hooks] [--milestones] [--security-advisories]
+                  [--repositories] [--bare] [--no-prune] [--lfs] [--wikis]
+                  [--gists] [--starred-gists] [--skip-archived] [--skip-existing]
                   [-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
                   [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
@@ -85,7 +85,7 @@ CLI Help output::
                   [--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
                   [--throttle-limit THROTTLE_LIMIT]
                   [--throttle-pause THROTTLE_PAUSE]
-                  [--exclude [EXCLUDE ...]]
+                  [--exclude [EXCLUDE ...]] [--retries MAX_RETRIES]
                   USER
 
     Backup a github account
@@ -131,6 +131,8 @@ CLI Help output::
       --hooks               include hooks in backup (works only when
                             authenticated)
       --milestones          include milestones in backup
+      --security-advisories
+                            include security advisories in backup
       --repositories        include repository clone in backup
       --bare                clone bare repositories
       --no-prune            disable prune option for git fetch
@@ -309,6 +311,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
 
 **Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
 
+**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
+
 
 Run in Docker container
 -----------------------
@@ -431,7 +435,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
     export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
     GH_USER=YOUR-GITHUB-USER
 
-    github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
+    github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
     
 Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
 

github_backup-0.61.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+github_backup/__init__.py,sha256=3UxN07YtETzDhpq7DYM8JY8aOjsIEIETvTVen65rK2k,23
+github_backup/__main__.py,sha256=3yorCiC63maaxbjdtaeAs-G-yAAxKSO3mprhGfa1Fho,271
+github_backup/cli.py,sha256=gPYU0CviuHB19y8RHhcD0p8xHS3UcJbBIK6HVfWazTc,2902
+github_backup/github_backup.py,sha256=ApN6orMb-BlJJVI_PTZZkzteEzuZBhTySydlRWU-Bi0,87911
+github_backup-0.61.1.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
+github_backup-0.61.1.dist-info/METADATA,sha256=-iHVrHGbT0z0R9VqXbltR7pRxDV18aOnfFdofA73K3s,27503
+github_backup-0.61.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+github_backup-0.61.1.dist-info/entry_points.txt,sha256=cLPKK7MqC77QVb6HnT2mnyL_f-ciavJG_Y9vVHlCF68,57
+github_backup-0.61.1.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
+github_backup-0.61.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+github_backup-0.61.1.dist-info/RECORD,,

github_backup-0.60.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-github_backup/__init__.py,sha256=XCDU0PzRSGNchH5cFRqniZdytnPgXmacSgzeC0BxXiI,23
-github_backup/__main__.py,sha256=3yorCiC63maaxbjdtaeAs-G-yAAxKSO3mprhGfa1Fho,271
-github_backup/cli.py,sha256=ifnOne5AaEI7laofgIq3geURJtvbA-bzxSQgLcmZgd0,2345
-github_backup/github_backup.py,sha256=DJ1mFWELLAZBbM8UXmsHOxin3OBGFIFZk_SXQdhv_7w,82562
-github_backup-0.60.0.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
-github_backup-0.60.0.dist-info/METADATA,sha256=Lwu1h4UYm6z8OUq9iieqgS_7ITlsB-oQujHB0h302PM,26529
-github_backup-0.60.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-github_backup-0.60.0.dist-info/entry_points.txt,sha256=cLPKK7MqC77QVb6HnT2mnyL_f-ciavJG_Y9vVHlCF68,57
-github_backup-0.60.0.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
-github_backup-0.60.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-github_backup-0.60.0.dist-info/RECORD,,