github-backup 0.50.3__py3-none-any.whl → 0.51.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
github_backup/__init__.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.50.3"
1
+ __version__ = "0.51.1"
@@ -420,6 +420,12 @@ def parse_args(args=None):
420
420
  dest="include_assets",
421
421
  help="include assets alongside release information; only applies if including releases",
422
422
  )
423
+ parser.add_argument(
424
+ "--attachments",
425
+ action="store_true",
426
+ dest="include_attachments",
427
+ help="download user-attachments from issues and pull requests",
428
+ )
423
429
  parser.add_argument(
424
430
  "--throttle-limit",
425
431
  dest="throttle_limit",
@@ -814,7 +820,9 @@ class S3HTTPRedirectHandler(HTTPRedirectHandler):
814
820
  request = super(S3HTTPRedirectHandler, self).redirect_request(
815
821
  req, fp, code, msg, headers, newurl
816
822
  )
817
- del request.headers["Authorization"]
823
+ # Only delete Authorization header if it exists (attachments may not have it)
824
+ if "Authorization" in request.headers:
825
+ del request.headers["Authorization"]
818
826
  return request
819
827
 
820
828
 
@@ -867,6 +875,585 @@ def download_file(url, path, auth, as_app=False, fine=False):
867
875
  )
868
876
 
869
877
 
878
+ def download_attachment_file(url, path, auth, as_app=False, fine=False):
879
+ """Download attachment file directly (not via GitHub API).
880
+
881
+ Similar to download_file() but for direct file URLs, not API endpoints.
882
+ Attachment URLs (user-images, user-attachments) are direct downloads,
883
+ not API endpoints, so we skip _construct_request() which adds API params.
884
+
885
+ URL Format Support & Authentication Requirements:
886
+
887
+ | URL Format | Auth Required | Notes |
888
+ |----------------------------------------------|---------------|--------------------------|
889
+ | github.com/user-attachments/assets/* | Private only | Modern format (2024+) |
890
+ | github.com/user-attachments/files/* | Private only | Modern format (2024+) |
891
+ | user-images.githubusercontent.com/* | No (public) | Legacy CDN, all eras |
892
+ | private-user-images.githubusercontent.com/* | JWT in URL | Legacy private (5min) |
893
+ | github.com/{owner}/{repo}/files/* | Repo filter | Old repo files |
894
+
895
+ - Modern user-attachments: Requires GitHub token auth for private repos
896
+ - Legacy public CDN: No auth needed/accepted (returns 400 with auth header)
897
+ - Legacy private CDN: Uses JWT token embedded in URL, no GitHub token needed
898
+ - Repo files: Filtered to current repository only during extraction
899
+
900
+ Returns dict with metadata:
901
+ - success: bool
902
+ - http_status: int (200, 404, etc.)
903
+ - content_type: str or None
904
+ - original_filename: str or None (from Content-Disposition)
905
+ - size_bytes: int or None
906
+ - error: str or None
907
+ """
908
+ import re
909
+ from datetime import datetime, timezone
910
+
911
+ metadata = {
912
+ "url": url,
913
+ "success": False,
914
+ "http_status": None,
915
+ "content_type": None,
916
+ "original_filename": None,
917
+ "size_bytes": None,
918
+ "downloaded_at": datetime.now(timezone.utc).isoformat(),
919
+ "error": None,
920
+ }
921
+
922
+ # Create simple request (no API query params)
923
+ request = Request(url)
924
+ request.add_header("Accept", "application/octet-stream")
925
+
926
+ # Add authentication header only for modern github.com/user-attachments URLs
927
+ # Legacy CDN URLs (user-images.githubusercontent.com) are public and don't need/accept auth
928
+ # Private CDN URLs (private-user-images) use JWT tokens embedded in the URL
929
+ if auth is not None and "github.com/user-attachments/" in url:
930
+ if not as_app:
931
+ if fine:
932
+ # Fine-grained token: plain token with "token " prefix
933
+ request.add_header("Authorization", "token " + auth)
934
+ else:
935
+ # Classic token: base64-encoded with "Basic " prefix
936
+ request.add_header("Authorization", "Basic ".encode("ascii") + auth)
937
+ else:
938
+ # App authentication
939
+ auth = auth.encode("ascii")
940
+ request.add_header("Authorization", "token ".encode("ascii") + auth)
941
+
942
+ # Reuse S3HTTPRedirectHandler from download_file()
943
+ opener = build_opener(S3HTTPRedirectHandler)
944
+
945
+ temp_path = path + ".temp"
946
+
947
+ try:
948
+ response = opener.open(request)
949
+ metadata["http_status"] = response.getcode()
950
+
951
+ # Extract Content-Type
952
+ content_type = response.headers.get("Content-Type", "").split(";")[0].strip()
953
+ if content_type:
954
+ metadata["content_type"] = content_type
955
+
956
+ # Extract original filename from Content-Disposition header
957
+ # Format: attachment; filename=example.mov or attachment;filename="example.mov"
958
+ content_disposition = response.headers.get("Content-Disposition", "")
959
+ if content_disposition:
960
+ # Match: filename=something or filename="something" or filename*=UTF-8''something
961
+ match = re.search(r'filename\*?=["\']?([^"\';\r\n]+)', content_disposition)
962
+ if match:
963
+ original_filename = match.group(1).strip()
964
+ # Handle RFC 5987 encoding: filename*=UTF-8''example.mov
965
+ if "UTF-8''" in original_filename:
966
+ original_filename = original_filename.split("UTF-8''")[1]
967
+ metadata["original_filename"] = original_filename
968
+
969
+ # Fallback: Extract filename from final URL after redirects
970
+ # This handles user-attachments/assets URLs which redirect to S3 with filename.ext
971
+ if not metadata["original_filename"]:
972
+ from urllib.parse import urlparse, unquote
973
+
974
+ final_url = response.geturl()
975
+ parsed = urlparse(final_url)
976
+ # Get filename from path (last component before query string)
977
+ path_parts = parsed.path.split("/")
978
+ if path_parts:
979
+ # URL might be encoded, decode it
980
+ filename_from_url = unquote(path_parts[-1])
981
+ # Only use if it has an extension
982
+ if "." in filename_from_url:
983
+ metadata["original_filename"] = filename_from_url
984
+
985
+ # Download file to temporary location
986
+ chunk_size = 16 * 1024
987
+ bytes_downloaded = 0
988
+ with open(temp_path, "wb") as f:
989
+ while True:
990
+ chunk = response.read(chunk_size)
991
+ if not chunk:
992
+ break
993
+ f.write(chunk)
994
+ bytes_downloaded += len(chunk)
995
+
996
+ # Atomic rename to final location
997
+ os.rename(temp_path, path)
998
+
999
+ metadata["size_bytes"] = bytes_downloaded
1000
+ metadata["success"] = True
1001
+
1002
+ except HTTPError as exc:
1003
+ metadata["http_status"] = exc.code
1004
+ metadata["error"] = str(exc.reason)
1005
+ logger.warning(
1006
+ "Skipping download of attachment {0} due to HTTPError: {1}".format(
1007
+ url, exc.reason
1008
+ )
1009
+ )
1010
+ except URLError as e:
1011
+ metadata["error"] = str(e.reason)
1012
+ logger.warning(
1013
+ "Skipping download of attachment {0} due to URLError: {1}".format(
1014
+ url, e.reason
1015
+ )
1016
+ )
1017
+ except socket.error as e:
1018
+ metadata["error"] = str(e.strerror) if hasattr(e, "strerror") else str(e)
1019
+ logger.warning(
1020
+ "Skipping download of attachment {0} due to socket error: {1}".format(
1021
+ url, e.strerror if hasattr(e, "strerror") else str(e)
1022
+ )
1023
+ )
1024
+ except Exception as e:
1025
+ metadata["error"] = str(e)
1026
+ logger.warning(
1027
+ "Skipping download of attachment {0} due to error: {1}".format(url, str(e))
1028
+ )
1029
+ # Clean up temp file if it was partially created
1030
+ if os.path.exists(temp_path):
1031
+ try:
1032
+ os.remove(temp_path)
1033
+ except Exception:
1034
+ pass
1035
+
1036
+ return metadata
1037
+
1038
+
1039
+ def extract_attachment_urls(item_data, issue_number=None, repository_full_name=None):
1040
+ """Extract GitHub-hosted attachment URLs from issue/PR body and comments.
1041
+
1042
+ What qualifies as an attachment?
1043
+ There is no "attachment" concept in the GitHub API - it's a user behavior pattern
1044
+ we've identified through analysis of real-world repositories. We define attachments as:
1045
+
1046
+ - User-uploaded files hosted on GitHub's CDN domains
1047
+ - Found outside of code blocks (not examples/documentation)
1048
+ - Matches known GitHub attachment URL patterns
1049
+
1050
+ This intentionally captures bare URLs pasted by users, not just markdown/HTML syntax.
1051
+ Some false positives (example URLs in documentation) may occur - these fail gracefully
1052
+ with HTTP 404 and are logged in the manifest.
1053
+
1054
+ Supported URL formats:
1055
+ - Modern: github.com/user-attachments/{assets,files}/*
1056
+ - Legacy: user-images.githubusercontent.com/* (including private-user-images)
1057
+ - Repo files: github.com/{owner}/{repo}/files/* (filtered to current repo)
1058
+ - Repo assets: github.com/{owner}/{repo}/assets/* (filtered to current repo)
1059
+
1060
+ Repository filtering (repo files/assets only):
1061
+ - Direct match: URL is for current repository → included
1062
+ - Redirect match: URL redirects to current repository → included (handles renames/transfers)
1063
+ - Different repo: URL is for different repository → excluded
1064
+
1065
+ Code block filtering:
1066
+ - Removes fenced code blocks (```) and inline code (`) before extraction
1067
+ - Prevents extracting URLs from code examples and documentation snippets
1068
+
1069
+ Args:
1070
+ item_data: Issue or PR data dict
1071
+ issue_number: Issue/PR number for logging
1072
+ repository_full_name: Full repository name (owner/repo) for filtering repo-scoped URLs
1073
+ """
1074
+ import re
1075
+
1076
+ urls = []
1077
+
1078
+ # Define all GitHub attachment patterns
1079
+ # Stop at markdown punctuation: whitespace, ), `, ", >, <
1080
+ # Trailing sentence punctuation (. ! ? , ; : ' ") is stripped in post-processing
1081
+ patterns = [
1082
+ r'https://github\.com/user-attachments/(?:assets|files)/[^\s\)`"<>]+', # Modern
1083
+ r'https://(?:private-)?user-images\.githubusercontent\.com/[^\s\)`"<>]+', # Legacy CDN
1084
+ ]
1085
+
1086
+ # Add repo-scoped patterns (will be filtered by repository later)
1087
+ # These patterns match ANY repo, then we filter to current repo with redirect checking
1088
+ repo_files_pattern = r'https://github\.com/[^/]+/[^/]+/files/\d+/[^\s\)`"<>]+'
1089
+ repo_assets_pattern = r'https://github\.com/[^/]+/[^/]+/assets/\d+/[^\s\)`"<>]+'
1090
+ patterns.append(repo_files_pattern)
1091
+ patterns.append(repo_assets_pattern)
1092
+
1093
+ def clean_url(url):
1094
+ """Remove trailing sentence and markdown punctuation that's not part of the URL."""
1095
+ return url.rstrip(".!?,;:'\")")
1096
+
1097
+ def remove_code_blocks(text):
1098
+ """Remove markdown code blocks (fenced and inline) from text.
1099
+
1100
+ This prevents extracting URLs from code examples like:
1101
+ - Fenced code blocks: ```code```
1102
+ - Inline code: `code`
1103
+ """
1104
+ # Remove fenced code blocks first (```...```)
1105
+ # DOTALL flag makes . match newlines
1106
+ text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
1107
+
1108
+ # Remove inline code (`...`)
1109
+ # Non-greedy match between backticks
1110
+ text = re.sub(r"`[^`]*`", "", text)
1111
+
1112
+ return text
1113
+
1114
+ def is_repo_scoped_url(url):
1115
+ """Check if URL is a repo-scoped attachment (files or assets)."""
1116
+ return bool(
1117
+ re.match(r"https://github\.com/[^/]+/[^/]+/(?:files|assets)/\d+/", url)
1118
+ )
1119
+
1120
+ def check_redirect_to_current_repo(url, current_repo):
1121
+ """Check if URL redirects to current repository.
1122
+
1123
+ Returns True if:
1124
+ - URL is already for current repo
1125
+ - URL redirects (301/302) to current repo (handles renames/transfers)
1126
+
1127
+ Returns False otherwise (URL is for a different repo).
1128
+ """
1129
+ # Extract owner/repo from URL
1130
+ match = re.match(r"https://github\.com/([^/]+)/([^/]+)/", url)
1131
+ if not match:
1132
+ return False
1133
+
1134
+ url_owner, url_repo = match.groups()
1135
+ url_repo_full = f"{url_owner}/{url_repo}"
1136
+
1137
+ # Direct match - no need to check redirect
1138
+ if url_repo_full.lower() == current_repo.lower():
1139
+ return True
1140
+
1141
+ # Different repo - check if it redirects to current repo
1142
+ # This handles repository transfers and renames
1143
+ try:
1144
+ import urllib.request
1145
+ import urllib.error
1146
+
1147
+ # Make HEAD request with redirect following disabled
1148
+ # We need to manually handle redirects to see the Location header
1149
+ request = urllib.request.Request(url, method="HEAD")
1150
+ request.add_header("User-Agent", "python-github-backup")
1151
+
1152
+ # Create opener that does NOT follow redirects
1153
+ class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
1154
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
1155
+ return None # Don't follow redirects
1156
+
1157
+ opener = urllib.request.build_opener(NoRedirectHandler)
1158
+
1159
+ try:
1160
+ _ = opener.open(request, timeout=10)
1161
+ # Got 200 - URL works as-is but for different repo
1162
+ return False
1163
+ except urllib.error.HTTPError as e:
1164
+ # Check if it's a redirect (301, 302, 307, 308)
1165
+ if e.code in (301, 302, 307, 308):
1166
+ location = e.headers.get("Location", "")
1167
+ # Check if redirect points to current repo
1168
+ if location:
1169
+ redirect_match = re.match(
1170
+ r"https://github\.com/([^/]+)/([^/]+)/", location
1171
+ )
1172
+ if redirect_match:
1173
+ redirect_owner, redirect_repo = redirect_match.groups()
1174
+ redirect_repo_full = f"{redirect_owner}/{redirect_repo}"
1175
+ return redirect_repo_full.lower() == current_repo.lower()
1176
+ return False
1177
+ except Exception:
1178
+ # On any error (timeout, network issue, etc.), be conservative
1179
+ # and exclude the URL to avoid downloading from wrong repos
1180
+ return False
1181
+
1182
+ # Extract from body
1183
+ body = item_data.get("body") or ""
1184
+ # Remove code blocks before searching for URLs
1185
+ body_cleaned = remove_code_blocks(body)
1186
+ for pattern in patterns:
1187
+ found_urls = re.findall(pattern, body_cleaned)
1188
+ urls.extend([clean_url(url) for url in found_urls])
1189
+
1190
+ # Extract from issue comments
1191
+ if "comment_data" in item_data:
1192
+ for comment in item_data["comment_data"]:
1193
+ comment_body = comment.get("body") or ""
1194
+ # Remove code blocks before searching for URLs
1195
+ comment_cleaned = remove_code_blocks(comment_body)
1196
+ for pattern in patterns:
1197
+ found_urls = re.findall(pattern, comment_cleaned)
1198
+ urls.extend([clean_url(url) for url in found_urls])
1199
+
1200
+ # Extract from PR regular comments
1201
+ if "comment_regular_data" in item_data:
1202
+ for comment in item_data["comment_regular_data"]:
1203
+ comment_body = comment.get("body") or ""
1204
+ # Remove code blocks before searching for URLs
1205
+ comment_cleaned = remove_code_blocks(comment_body)
1206
+ for pattern in patterns:
1207
+ found_urls = re.findall(pattern, comment_cleaned)
1208
+ urls.extend([clean_url(url) for url in found_urls])
1209
+
1210
+ regex_urls = list(set(urls)) # dedupe
1211
+
1212
+ # Filter repo-scoped URLs to current repository only
1213
+ # This handles repository transfers/renames via redirect checking
1214
+ if repository_full_name:
1215
+ filtered_urls = []
1216
+ for url in regex_urls:
1217
+ if is_repo_scoped_url(url):
1218
+ # Check if URL belongs to current repo (or redirects to it)
1219
+ if check_redirect_to_current_repo(url, repository_full_name):
1220
+ filtered_urls.append(url)
1221
+ # else: skip URLs from other repositories
1222
+ else:
1223
+ # Non-repo-scoped URLs (user-attachments, CDN) - always include
1224
+ filtered_urls.append(url)
1225
+ regex_urls = filtered_urls
1226
+
1227
+ return regex_urls
1228
+
1229
+
1230
+ def get_attachment_filename(url):
1231
+ """Get filename from attachment URL, handling all GitHub formats.
1232
+
1233
+ Formats:
1234
+ - github.com/user-attachments/assets/{uuid} → uuid (add extension later)
1235
+ - github.com/user-attachments/files/{id}/{filename} → filename
1236
+ - github.com/{owner}/{repo}/files/{id}/{filename} → filename
1237
+ - user-images.githubusercontent.com/{user}/{hash}.{ext} → hash.ext
1238
+ - private-user-images.githubusercontent.com/...?jwt=... → extract from path
1239
+ """
1240
+ from urllib.parse import urlparse
1241
+
1242
+ parsed = urlparse(url)
1243
+ path_parts = parsed.path.split("/")
1244
+
1245
+ # Modern: /user-attachments/files/{id}/{filename}
1246
+ if "user-attachments/files" in parsed.path:
1247
+ return path_parts[-1]
1248
+
1249
+ # Modern: /user-attachments/assets/{uuid}
1250
+ elif "user-attachments/assets" in parsed.path:
1251
+ return path_parts[-1] # extension added later via detect_and_add_extension
1252
+
1253
+ # Repo files: /{owner}/{repo}/files/{id}/{filename}
1254
+ elif "/files/" in parsed.path and len(path_parts) >= 2:
1255
+ return path_parts[-1]
1256
+
1257
+ # Legacy: user-images.githubusercontent.com/{user}/{hash-with-ext}
1258
+ elif "githubusercontent.com" in parsed.netloc:
1259
+ return path_parts[-1] # Already has extension usually
1260
+
1261
+ # Fallback: use last path component
1262
+ return path_parts[-1] if path_parts[-1] else "unknown_attachment"
1263
+
1264
+
1265
+ def resolve_filename_collision(filepath):
1266
+ """Resolve filename collisions using counter suffix pattern.
1267
+
1268
+ If filepath exists, returns a new filepath with counter suffix.
1269
+ Pattern: report.pdf → report_1.pdf → report_2.pdf
1270
+
1271
+ Also protects against manifest.json collisions by treating it as reserved.
1272
+
1273
+ Args:
1274
+ filepath: Full path to file that might exist
1275
+
1276
+ Returns:
1277
+ filepath that doesn't collide (may be same as input if no collision)
1278
+ """
1279
+ directory = os.path.dirname(filepath)
1280
+ filename = os.path.basename(filepath)
1281
+
1282
+ # Protect manifest.json - it's a reserved filename
1283
+ if filename == "manifest.json":
1284
+ name, ext = os.path.splitext(filename)
1285
+ counter = 1
1286
+ while True:
1287
+ new_filename = f"{name}_{counter}{ext}"
1288
+ new_filepath = os.path.join(directory, new_filename)
1289
+ if not os.path.exists(new_filepath):
1290
+ return new_filepath
1291
+ counter += 1
1292
+
1293
+ if not os.path.exists(filepath):
1294
+ return filepath
1295
+
1296
+ name, ext = os.path.splitext(filename)
1297
+
1298
+ counter = 1
1299
+ while True:
1300
+ new_filename = f"{name}_{counter}{ext}"
1301
+ new_filepath = os.path.join(directory, new_filename)
1302
+ if not os.path.exists(new_filepath):
1303
+ return new_filepath
1304
+ counter += 1
1305
+
1306
+
1307
+ def download_attachments(
1308
+ args, item_cwd, item_data, number, repository, item_type="issue"
1309
+ ):
1310
+ """Download user-attachments from issue/PR body and comments with manifest.
1311
+
1312
+ Args:
1313
+ args: Command line arguments
1314
+ item_cwd: Working directory (issue_cwd or pulls_cwd)
1315
+ item_data: Issue or PR data dict
1316
+ number: Issue or PR number
1317
+ repository: Repository dict
1318
+ item_type: "issue" or "pull" for logging/manifest
1319
+ """
1320
+ import json
1321
+ from datetime import datetime, timezone
1322
+
1323
+ item_type_display = "issue" if item_type == "issue" else "pull request"
1324
+
1325
+ urls = extract_attachment_urls(
1326
+ item_data, issue_number=number, repository_full_name=repository["full_name"]
1327
+ )
1328
+ if not urls:
1329
+ return
1330
+
1331
+ attachments_dir = os.path.join(item_cwd, "attachments", str(number))
1332
+ manifest_path = os.path.join(attachments_dir, "manifest.json")
1333
+
1334
+ # Load existing manifest to prevent duplicate downloads
1335
+ existing_urls = set()
1336
+ existing_metadata = []
1337
+ if os.path.exists(manifest_path):
1338
+ try:
1339
+ with open(manifest_path, "r") as f:
1340
+ existing_manifest = json.load(f)
1341
+ all_metadata = existing_manifest.get("attachments", [])
1342
+ # Only skip URLs that were successfully downloaded OR failed with permanent errors
1343
+ # Retry transient failures (5xx, timeouts, network errors)
1344
+ for item in all_metadata:
1345
+ if item.get("success"):
1346
+ existing_urls.add(item["url"])
1347
+ else:
1348
+ # Check if this is a permanent failure (don't retry) or transient (retry)
1349
+ http_status = item.get("http_status")
1350
+ if http_status in [404, 410, 451]:
1351
+ # Permanent failures - don't retry
1352
+ existing_urls.add(item["url"])
1353
+ # Transient failures (5xx, auth errors, timeouts) will be retried
1354
+ existing_metadata = all_metadata
1355
+ except (json.JSONDecodeError, IOError):
1356
+ # If manifest is corrupted, re-download everything
1357
+ logger.warning(
1358
+ "Corrupted manifest for {0} #{1}, will re-download".format(
1359
+ item_type_display, number
1360
+ )
1361
+ )
1362
+ existing_urls = set()
1363
+ existing_metadata = []
1364
+
1365
+ # Filter to only new URLs
1366
+ new_urls = [url for url in urls if url not in existing_urls]
1367
+
1368
+ if not new_urls and existing_urls:
1369
+ logger.debug(
1370
+ "Skipping attachments for {0} #{1} (all {2} already downloaded)".format(
1371
+ item_type_display, number, len(urls)
1372
+ )
1373
+ )
1374
+ return
1375
+
1376
+ if new_urls:
1377
+ logger.info(
1378
+ "Downloading {0} new attachment(s) for {1} #{2}".format(
1379
+ len(new_urls), item_type_display, number
1380
+ )
1381
+ )
1382
+
1383
+ mkdir_p(item_cwd, attachments_dir)
1384
+
1385
+ # Collect metadata for manifest (start with existing)
1386
+ attachment_metadata_list = existing_metadata[:]
1387
+
1388
+ for url in new_urls:
1389
+ filename = get_attachment_filename(url)
1390
+ filepath = os.path.join(attachments_dir, filename)
1391
+
1392
+ # Download and get metadata
1393
+ metadata = download_attachment_file(
1394
+ url,
1395
+ filepath,
1396
+ get_auth(args, encode=not args.as_app),
1397
+ as_app=args.as_app,
1398
+ fine=args.token_fine is not None,
1399
+ )
1400
+
1401
+ # If download succeeded but we got an extension from Content-Disposition,
1402
+ # we may need to rename the file to add the extension
1403
+ if metadata["success"] and metadata.get("original_filename"):
1404
+ original_ext = os.path.splitext(metadata["original_filename"])[1]
1405
+ current_ext = os.path.splitext(filepath)[1]
1406
+
1407
+ # Add extension if not present
1408
+ if original_ext and current_ext != original_ext:
1409
+ final_filepath = filepath + original_ext
1410
+ # Check for collision again with new extension
1411
+ final_filepath = resolve_filename_collision(final_filepath)
1412
+ logger.debug(
1413
+ "Adding extension {0} to {1}".format(original_ext, filepath)
1414
+ )
1415
+
1416
+ # Rename to add extension (already atomic from download)
1417
+ try:
1418
+ os.rename(filepath, final_filepath)
1419
+ metadata["saved_as"] = os.path.basename(final_filepath)
1420
+ except Exception as e:
1421
+ logger.warning(
1422
+ "Could not add extension to {0}: {1}".format(filepath, str(e))
1423
+ )
1424
+ metadata["saved_as"] = os.path.basename(filepath)
1425
+ else:
1426
+ metadata["saved_as"] = os.path.basename(filepath)
1427
+ elif metadata["success"]:
1428
+ metadata["saved_as"] = os.path.basename(filepath)
1429
+ else:
1430
+ metadata["saved_as"] = None
1431
+
1432
+ attachment_metadata_list.append(metadata)
1433
+
1434
+ # Write manifest
1435
+ if attachment_metadata_list:
1436
+ manifest = {
1437
+ "issue_number": number,
1438
+ "issue_type": item_type,
1439
+ "repository": f"{args.user}/{args.repository}"
1440
+ if hasattr(args, "repository") and args.repository
1441
+ else args.user,
1442
+ "manifest_updated_at": datetime.now(timezone.utc).isoformat(),
1443
+ "attachments": attachment_metadata_list,
1444
+ }
1445
+
1446
+ manifest_path = os.path.join(attachments_dir, "manifest.json")
1447
+ with open(manifest_path + ".temp", "w") as f:
1448
+ json.dump(manifest, f, indent=2)
1449
+ os.rename(manifest_path + ".temp", manifest_path) # Atomic write
1450
+ logger.debug(
1451
+ "Wrote manifest for {0} #{1}: {2} attachments".format(
1452
+ item_type_display, number, len(attachment_metadata_list)
1453
+ )
1454
+ )
1455
+
1456
+
870
1457
  def get_authenticated_user(args):
871
1458
  template = "https://{0}/user".format(get_github_api_host(args))
872
1459
  data = retrieve_data(args, template, single_request=True)
@@ -1157,6 +1744,10 @@ def backup_issues(args, repo_cwd, repository, repos_template):
1157
1744
  if args.include_issue_events or args.include_everything:
1158
1745
  template = events_template.format(number)
1159
1746
  issues[number]["event_data"] = retrieve_data(args, template)
1747
+ if args.include_attachments:
1748
+ download_attachments(
1749
+ args, issue_cwd, issues[number], number, repository, item_type="issue"
1750
+ )
1160
1751
 
1161
1752
  with codecs.open(issue_file + ".temp", "w", encoding="utf-8") as f:
1162
1753
  json_dump(issue, f)
@@ -1228,6 +1819,10 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
1228
1819
  if args.include_pull_commits or args.include_everything:
1229
1820
  template = commits_template.format(number)
1230
1821
  pulls[number]["commit_data"] = retrieve_data(args, template)
1822
+ if args.include_attachments:
1823
+ download_attachments(
1824
+ args, pulls_cwd, pulls[number], number, repository, item_type="pull"
1825
+ )
1231
1826
 
1232
1827
  with codecs.open(pull_file + ".temp", "w", encoding="utf-8") as f:
1233
1828
  json_dump(pull, f)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: github-backup
3
- Version: 0.50.3
3
+ Version: 0.51.1
4
4
  Summary: backup a github user or organization
5
5
  Home-page: http://github.com/josegonzalez/python-github-backup
6
6
  Author: Jose Diaz-Gonzalez
@@ -9,11 +9,12 @@ License: MIT
9
9
  Classifier: Development Status :: 5 - Production/Stable
10
10
  Classifier: Topic :: System :: Archiving :: Backup
11
11
  Classifier: License :: OSI Approved :: MIT License
12
- Classifier: Programming Language :: Python :: 3.8
13
- Classifier: Programming Language :: Python :: 3.9
14
12
  Classifier: Programming Language :: Python :: 3.10
15
13
  Classifier: Programming Language :: Python :: 3.11
16
14
  Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
17
+ Requires-Python: >=3.10
17
18
  Description-Content-Type: text/x-rst
18
19
  License-File: LICENSE.txt
19
20
  Dynamic: author
@@ -24,6 +25,7 @@ Dynamic: description-content-type
24
25
  Dynamic: home-page
25
26
  Dynamic: license
26
27
  Dynamic: license-file
28
+ Dynamic: requires-python
27
29
  Dynamic: summary
28
30
 
29
31
  =============
@@ -37,8 +39,8 @@ The package can be used to backup an *entire* `Github <https://github.com/>`_ or
37
39
  Requirements
38
40
  ============
39
41
 
42
+ - Python 3.10 or higher
40
43
  - GIT 1.9+
41
- - Python
42
44
 
43
45
  Installation
44
46
  ============
@@ -78,7 +80,7 @@ CLI Help output::
78
80
  [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
79
81
  [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
80
82
  [--releases] [--latest-releases NUMBER_OF_LATEST_RELEASES]
81
- [--skip-prerelease] [--assets]
83
+ [--skip-prerelease] [--assets] [--attachments]
82
84
  [--exclude [REPOSITORY [REPOSITORY ...]]
83
85
  [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
84
86
  USER
@@ -161,6 +163,9 @@ CLI Help output::
161
163
  --skip-prerelease skip prerelease and draft versions; only applies if including releases
162
164
  --assets include assets alongside release information; only
163
165
  applies if including releases
166
+ --attachments download user-attachments from issues and pull requests
167
+ to issues/attachments/{issue_number}/ and
168
+ pulls/attachments/{pull_number}/ directories
164
169
  --exclude [REPOSITORY [REPOSITORY ...]]
165
170
  names of repositories to exclude from backup.
166
171
  --throttle-limit THROTTLE_LIMIT
@@ -241,6 +246,29 @@ When you use the ``--lfs`` option, you will need to make sure you have Git LFS i
241
246
  Instructions on how to do this can be found on https://git-lfs.github.com.
242
247
 
243
248
 
249
+ About Attachments
250
+ -----------------
251
+
252
+ When you use the ``--attachments`` option with ``--issues`` or ``--pulls``, the tool will download user-uploaded attachments (images, videos, documents, etc.) from issue and pull request descriptions and comments. In some circumstances attachments contain valuable data related to the topic, and without their backup important information or context might be lost inadvertently.
253
+
254
+ Attachments are saved to ``issues/attachments/{issue_number}/`` and ``pulls/attachments/{pull_number}/`` directories, where ``{issue_number}`` is the GitHub issue number (e.g., issue #123 saves to ``issues/attachments/123/``). Each attachment directory contains:
255
+
256
+ - The downloaded attachment files (named by their GitHub identifier with appropriate file extensions)
257
+ - If multiple attachments have the same filename, conflicts are resolved with numeric suffixes (e.g., ``report.pdf``, ``report_1.pdf``, ``report_2.pdf``)
258
+ - A ``manifest.json`` file documenting all downloads, including URLs, file metadata, and download status
259
+
260
+ The tool automatically extracts file extensions from HTTP headers to ensure files can be more easily opened by your operating system.
261
+
262
+ **Supported URL formats:**
263
+
264
+ - Modern: ``github.com/user-attachments/{assets,files}/*``
265
+ - Legacy: ``user-images.githubusercontent.com/*`` and ``private-user-images.githubusercontent.com/*``
266
+ - Repo files: ``github.com/{owner}/{repo}/files/*`` (filtered to current repository)
267
+ - Repo assets: ``github.com/{owner}/{repo}/assets/*`` (filtered to current repository)
268
+
269
+ **Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
270
+
271
+
244
272
  Run in Docker container
245
273
  -----------------------
246
274
 
@@ -331,7 +359,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
331
359
  export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
332
360
  GH_USER=YOUR-GITHUB-USER
333
361
 
334
- github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
362
+ github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
335
363
 
336
364
  Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
337
365
 
@@ -0,0 +1,9 @@
1
+ github_backup/__init__.py,sha256=1aQCHIs4QPNUtw_wChzJwhINvlLDSxS3S3qxxd-DKeU,23
2
+ github_backup/github_backup.py,sha256=2N0crWKzRfkb5ZOlI5pn2NRxSXOmApmdB68noATWAbk,74783
3
+ github_backup-0.51.1.data/scripts/github-backup,sha256=I79TNA67phytRmI9nm_LlD9h2zauXFOsOn9tYyAy9pU,1546
4
+ github_backup-0.51.1.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
5
+ github_backup-0.51.1.dist-info/METADATA,sha256=rjm9_2D5ivEA1y9NUJhVbKA3C113jd0Jq3ATaGrvdhA,21544
6
+ github_backup-0.51.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
7
+ github_backup-0.51.1.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
8
+ github_backup-0.51.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
9
+ github_backup-0.51.1.dist-info/RECORD,,
@@ -1,9 +0,0 @@
1
- github_backup/__init__.py,sha256=uG9Fg41SFOkUm9XwkxGLSN_5MUansanjnAPwUcghDGU,23
2
- github_backup/github_backup.py,sha256=tc6yz1r_KjIl2BQden_dVD7SCy5oi1bJdYak7K9sqnY,50494
3
- github_backup-0.50.3.data/scripts/github-backup,sha256=I79TNA67phytRmI9nm_LlD9h2zauXFOsOn9tYyAy9pU,1546
4
- github_backup-0.50.3.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
5
- github_backup-0.50.3.dist-info/METADATA,sha256=B_Za11wHbQCHSXXXw6xaubC1CmGMB4wd9QXuEEL8ago,19402
6
- github_backup-0.50.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
7
- github_backup-0.50.3.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
8
- github_backup-0.50.3.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
9
- github_backup-0.50.3.dist-info/RECORD,,