github-backup 0.51.3__tar.gz → 0.53.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.51.3 → github_backup-0.53.0}/CHANGES.rst +117 -1
- {github_backup-0.51.3/github_backup.egg-info → github_backup-0.53.0}/PKG-INFO +1 -1
- github_backup-0.53.0/github_backup/__init__.py +1 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/github_backup/github_backup.py +146 -41
- {github_backup-0.51.3 → github_backup-0.53.0/github_backup.egg-info}/PKG-INFO +1 -1
- {github_backup-0.51.3 → github_backup-0.53.0}/github_backup.egg-info/SOURCES.txt +2 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/release-requirements.txt +4 -4
- github_backup-0.53.0/tests/test_http_451.py +143 -0
- github_backup-0.53.0/tests/test_json_dump_if_changed.py +198 -0
- github_backup-0.51.3/github_backup/__init__.py +0 -1
- {github_backup-0.51.3 → github_backup-0.53.0}/LICENSE.txt +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/MANIFEST.in +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/README +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/README.rst +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/bin/github-backup +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/requirements.txt +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/setup.cfg +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/setup.py +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/tests/test_attachments.py +0 -0
- {github_backup-0.51.3 → github_backup-0.53.0}/tests/test_pagination.py +0 -0
|
@@ -1,9 +1,125 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.
|
|
4
|
+
0.53.0 (2025-11-30)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
|
|
8
|
+
Fix
|
|
9
|
+
~~~
|
|
10
|
+
- Case-sensitive username filtering causing silent backup failures.
|
|
11
|
+
[Rodos]
|
|
12
|
+
|
|
13
|
+
GitHub's API accepts usernames in any case but returns canonical case.
|
|
14
|
+
The case-sensitive comparison in filter_repositories() filtered out all
|
|
15
|
+
repositories when user-provided case didn't match GitHub's canonical case.
|
|
16
|
+
|
|
17
|
+
Changed to case-insensitive comparison.
|
|
18
|
+
|
|
19
|
+
Fixes #198
|
|
20
|
+
|
|
21
|
+
Other
|
|
22
|
+
~~~~~
|
|
23
|
+
- Avoid rewriting unchanged JSON files for labels, milestones, releases,
|
|
24
|
+
hooks, followers, and following. [Rodos]
|
|
25
|
+
|
|
26
|
+
This change reduces unnecessary writes when backing up metadata that changes
|
|
27
|
+
infrequently. The implementation compares existing file content before writing
|
|
28
|
+
and skips the write if the content is identical, preserving file timestamps.
|
|
29
|
+
|
|
30
|
+
Key changes:
|
|
31
|
+
- Added json_dump_if_changed() helper that compares content before writing
|
|
32
|
+
- Uses atomic writes (temp file + rename) for all metadata files
|
|
33
|
+
- NOT applied to issues/pulls (they use incremental_by_files logic)
|
|
34
|
+
- Made log messages consistent and past tense ("Saved" instead of "Saving")
|
|
35
|
+
- Added informative logging showing skip counts
|
|
36
|
+
|
|
37
|
+
Fixes #133
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
0.52.0 (2025-11-28)
|
|
41
|
+
-------------------
|
|
42
|
+
- Skip DMCA'd repos which return a 451 response. [Rodos]
|
|
43
|
+
|
|
44
|
+
Log a warning and the link to the DMCA notice. Continue backing up
|
|
45
|
+
other repositories instead of crashing.
|
|
46
|
+
|
|
47
|
+
Closes #163
|
|
48
|
+
- Chore(deps): bump restructuredtext-lint in the python-packages group.
|
|
49
|
+
[dependabot[bot]]
|
|
50
|
+
|
|
51
|
+
Bumps the python-packages group with 1 update: [restructuredtext-lint](https://github.com/twolfson/restructuredtext-lint).
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
Updates `restructuredtext-lint` from 1.4.0 to 2.0.2
|
|
55
|
+
- [Changelog](https://github.com/twolfson/restructuredtext-lint/blob/master/CHANGELOG.rst)
|
|
56
|
+
- [Commits](https://github.com/twolfson/restructuredtext-lint/compare/1.4.0...2.0.2)
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
updated-dependencies:
|
|
60
|
+
- dependency-name: restructuredtext-lint
|
|
61
|
+
dependency-version: 2.0.2
|
|
62
|
+
dependency-type: direct:production
|
|
63
|
+
update-type: version-update:semver-major
|
|
64
|
+
dependency-group: python-packages
|
|
65
|
+
...
|
|
66
|
+
- Chore(deps): bump actions/checkout from 5 to 6. [dependabot[bot]]
|
|
67
|
+
|
|
68
|
+
Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6.
|
|
69
|
+
- [Release notes](https://github.com/actions/checkout/releases)
|
|
70
|
+
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
|
|
71
|
+
- [Commits](https://github.com/actions/checkout/compare/v5...v6)
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
updated-dependencies:
|
|
75
|
+
- dependency-name: actions/checkout
|
|
76
|
+
dependency-version: '6'
|
|
77
|
+
dependency-type: direct:production
|
|
78
|
+
update-type: version-update:semver-major
|
|
79
|
+
...
|
|
80
|
+
- Chore(deps): bump the python-packages group with 3 updates.
|
|
81
|
+
[dependabot[bot]]
|
|
82
|
+
|
|
83
|
+
Bumps the python-packages group with 3 updates: [click](https://github.com/pallets/click), [pytest](https://github.com/pytest-dev/pytest) and [keyring](https://github.com/jaraco/keyring).
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
Updates `click` from 8.3.0 to 8.3.1
|
|
87
|
+
- [Release notes](https://github.com/pallets/click/releases)
|
|
88
|
+
- [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst)
|
|
89
|
+
- [Commits](https://github.com/pallets/click/compare/8.3.0...8.3.1)
|
|
90
|
+
|
|
91
|
+
Updates `pytest` from 8.3.3 to 9.0.1
|
|
92
|
+
- [Release notes](https://github.com/pytest-dev/pytest/releases)
|
|
93
|
+
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
|
|
94
|
+
- [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...9.0.1)
|
|
95
|
+
|
|
96
|
+
Updates `keyring` from 25.6.0 to 25.7.0
|
|
97
|
+
- [Release notes](https://github.com/jaraco/keyring/releases)
|
|
98
|
+
- [Changelog](https://github.com/jaraco/keyring/blob/main/NEWS.rst)
|
|
99
|
+
- [Commits](https://github.com/jaraco/keyring/compare/v25.6.0...v25.7.0)
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
updated-dependencies:
|
|
103
|
+
- dependency-name: click
|
|
104
|
+
dependency-version: 8.3.1
|
|
105
|
+
dependency-type: direct:production
|
|
106
|
+
update-type: version-update:semver-patch
|
|
107
|
+
dependency-group: python-packages
|
|
108
|
+
- dependency-name: pytest
|
|
109
|
+
dependency-version: 9.0.1
|
|
110
|
+
dependency-type: direct:production
|
|
111
|
+
update-type: version-update:semver-major
|
|
112
|
+
dependency-group: python-packages
|
|
113
|
+
- dependency-name: keyring
|
|
114
|
+
dependency-version: 25.7.0
|
|
115
|
+
dependency-type: direct:production
|
|
116
|
+
update-type: version-update:semver-minor
|
|
117
|
+
dependency-group: python-packages
|
|
118
|
+
...
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
0.51.3 (2025-11-18)
|
|
122
|
+
-------------------
|
|
7
123
|
- Test: Add pagination tests for cursor and page-based Link headers.
|
|
8
124
|
[Rodos]
|
|
9
125
|
- Use cursor based pagination. [Helio Machado]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.53.0"
|
|
@@ -37,6 +37,15 @@ FNULL = open(os.devnull, "w")
|
|
|
37
37
|
FILE_URI_PREFIX = "file://"
|
|
38
38
|
logger = logging.getLogger(__name__)
|
|
39
39
|
|
|
40
|
+
|
|
41
|
+
class RepositoryUnavailableError(Exception):
|
|
42
|
+
"""Raised when a repository is unavailable due to legal reasons (e.g., DMCA takedown)."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, message, dmca_url=None):
|
|
45
|
+
super().__init__(message)
|
|
46
|
+
self.dmca_url = dmca_url
|
|
47
|
+
|
|
48
|
+
|
|
40
49
|
# Setup SSL context with fallback chain
|
|
41
50
|
https_ctx = ssl.create_default_context()
|
|
42
51
|
if https_ctx.get_ca_certs():
|
|
@@ -612,6 +621,19 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
|
|
|
612
621
|
|
|
613
622
|
status_code = int(r.getcode())
|
|
614
623
|
|
|
624
|
+
# Handle DMCA takedown (HTTP 451) - raise exception to skip entire repository
|
|
625
|
+
if status_code == 451:
|
|
626
|
+
dmca_url = None
|
|
627
|
+
try:
|
|
628
|
+
response_data = json.loads(r.read().decode("utf-8"))
|
|
629
|
+
dmca_url = response_data.get("block", {}).get("html_url")
|
|
630
|
+
except Exception:
|
|
631
|
+
pass
|
|
632
|
+
raise RepositoryUnavailableError(
|
|
633
|
+
"Repository unavailable due to legal reasons (HTTP 451)",
|
|
634
|
+
dmca_url=dmca_url
|
|
635
|
+
)
|
|
636
|
+
|
|
615
637
|
# Check if we got correct data
|
|
616
638
|
try:
|
|
617
639
|
response = json.loads(r.read().decode("utf-8"))
|
|
@@ -1565,7 +1587,9 @@ def filter_repositories(args, unfiltered_repositories):
|
|
|
1565
1587
|
repositories = []
|
|
1566
1588
|
for r in unfiltered_repositories:
|
|
1567
1589
|
# gists can be anonymous, so need to safely check owner
|
|
1568
|
-
|
|
1590
|
+
# Use case-insensitive comparison to match GitHub's case-insensitive username behavior
|
|
1591
|
+
owner_login = r.get("owner", {}).get("login", "")
|
|
1592
|
+
if owner_login.lower() == args.user.lower() or r.get("is_starred"):
|
|
1569
1593
|
repositories.append(r)
|
|
1570
1594
|
|
|
1571
1595
|
name_regex = None
|
|
@@ -1668,40 +1692,47 @@ def backup_repositories(args, output_directory, repositories):
|
|
|
1668
1692
|
|
|
1669
1693
|
continue # don't try to back anything else for a gist; it doesn't exist
|
|
1670
1694
|
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1695
|
+
try:
|
|
1696
|
+
download_wiki = args.include_wiki or args.include_everything
|
|
1697
|
+
if repository["has_wiki"] and download_wiki:
|
|
1698
|
+
fetch_repository(
|
|
1699
|
+
repository["name"],
|
|
1700
|
+
repo_url.replace(".git", ".wiki.git"),
|
|
1701
|
+
os.path.join(repo_cwd, "wiki"),
|
|
1702
|
+
skip_existing=args.skip_existing,
|
|
1703
|
+
bare_clone=args.bare_clone,
|
|
1704
|
+
lfs_clone=args.lfs_clone,
|
|
1705
|
+
no_prune=args.no_prune,
|
|
1706
|
+
)
|
|
1707
|
+
if args.include_issues or args.include_everything:
|
|
1708
|
+
backup_issues(args, repo_cwd, repository, repos_template)
|
|
1684
1709
|
|
|
1685
|
-
|
|
1686
|
-
|
|
1710
|
+
if args.include_pulls or args.include_everything:
|
|
1711
|
+
backup_pulls(args, repo_cwd, repository, repos_template)
|
|
1687
1712
|
|
|
1688
|
-
|
|
1689
|
-
|
|
1713
|
+
if args.include_milestones or args.include_everything:
|
|
1714
|
+
backup_milestones(args, repo_cwd, repository, repos_template)
|
|
1690
1715
|
|
|
1691
|
-
|
|
1692
|
-
|
|
1716
|
+
if args.include_labels or args.include_everything:
|
|
1717
|
+
backup_labels(args, repo_cwd, repository, repos_template)
|
|
1693
1718
|
|
|
1694
|
-
|
|
1695
|
-
|
|
1719
|
+
if args.include_hooks or args.include_everything:
|
|
1720
|
+
backup_hooks(args, repo_cwd, repository, repos_template)
|
|
1696
1721
|
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1722
|
+
if args.include_releases or args.include_everything:
|
|
1723
|
+
backup_releases(
|
|
1724
|
+
args,
|
|
1725
|
+
repo_cwd,
|
|
1726
|
+
repository,
|
|
1727
|
+
repos_template,
|
|
1728
|
+
include_assets=args.include_assets or args.include_everything,
|
|
1729
|
+
)
|
|
1730
|
+
except RepositoryUnavailableError as e:
|
|
1731
|
+
logger.warning(f"Repository {repository['full_name']} is unavailable (HTTP 451)")
|
|
1732
|
+
if e.dmca_url:
|
|
1733
|
+
logger.warning(f"DMCA notice: {e.dmca_url}")
|
|
1734
|
+
logger.info(f"Skipping remaining resources for {repository['full_name']}")
|
|
1735
|
+
continue
|
|
1705
1736
|
|
|
1706
1737
|
if args.incremental:
|
|
1707
1738
|
if last_update == "0000-00-00T00:00:00Z":
|
|
@@ -1869,11 +1900,21 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
|
|
|
1869
1900
|
for milestone in _milestones:
|
|
1870
1901
|
milestones[milestone["number"]] = milestone
|
|
1871
1902
|
|
|
1872
|
-
|
|
1903
|
+
written_count = 0
|
|
1873
1904
|
for number, milestone in list(milestones.items()):
|
|
1874
1905
|
milestone_file = "{0}/{1}.json".format(milestone_cwd, number)
|
|
1875
|
-
|
|
1876
|
-
|
|
1906
|
+
if json_dump_if_changed(milestone, milestone_file):
|
|
1907
|
+
written_count += 1
|
|
1908
|
+
|
|
1909
|
+
total = len(milestones)
|
|
1910
|
+
if written_count == total:
|
|
1911
|
+
logger.info("Saved {0} milestones to disk".format(total))
|
|
1912
|
+
elif written_count == 0:
|
|
1913
|
+
logger.info("{0} milestones unchanged, skipped write".format(total))
|
|
1914
|
+
else:
|
|
1915
|
+
logger.info("Saved {0} of {1} milestones to disk ({2} unchanged)".format(
|
|
1916
|
+
written_count, total, total - written_count
|
|
1917
|
+
))
|
|
1877
1918
|
|
|
1878
1919
|
|
|
1879
1920
|
def backup_labels(args, repo_cwd, repository, repos_template):
|
|
@@ -1926,19 +1967,17 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
|
|
|
1926
1967
|
reverse=True,
|
|
1927
1968
|
)
|
|
1928
1969
|
releases = releases[: args.number_of_latest_releases]
|
|
1929
|
-
logger.info("Saving the latest {0} releases to disk".format(len(releases)))
|
|
1930
|
-
else:
|
|
1931
|
-
logger.info("Saving {0} releases to disk".format(len(releases)))
|
|
1932
1970
|
|
|
1933
1971
|
# for each release, store it
|
|
1972
|
+
written_count = 0
|
|
1934
1973
|
for release in releases:
|
|
1935
1974
|
release_name = release["tag_name"]
|
|
1936
1975
|
release_name_safe = release_name.replace("/", "__")
|
|
1937
1976
|
output_filepath = os.path.join(
|
|
1938
1977
|
release_cwd, "{0}.json".format(release_name_safe)
|
|
1939
1978
|
)
|
|
1940
|
-
|
|
1941
|
-
|
|
1979
|
+
if json_dump_if_changed(release, output_filepath):
|
|
1980
|
+
written_count += 1
|
|
1942
1981
|
|
|
1943
1982
|
if include_assets:
|
|
1944
1983
|
assets = retrieve_data(args, release["assets_url"])
|
|
@@ -1955,6 +1994,17 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
|
|
|
1955
1994
|
fine=True if args.token_fine is not None else False,
|
|
1956
1995
|
)
|
|
1957
1996
|
|
|
1997
|
+
# Log the results
|
|
1998
|
+
total = len(releases)
|
|
1999
|
+
if written_count == total:
|
|
2000
|
+
logger.info("Saved {0} releases to disk".format(total))
|
|
2001
|
+
elif written_count == 0:
|
|
2002
|
+
logger.info("{0} releases unchanged, skipped write".format(total))
|
|
2003
|
+
else:
|
|
2004
|
+
logger.info("Saved {0} of {1} releases to disk ({2} unchanged)".format(
|
|
2005
|
+
written_count, total, total - written_count
|
|
2006
|
+
))
|
|
2007
|
+
|
|
1958
2008
|
|
|
1959
2009
|
def fetch_repository(
|
|
1960
2010
|
name,
|
|
@@ -2079,9 +2129,10 @@ def _backup_data(args, name, template, output_file, output_directory):
|
|
|
2079
2129
|
mkdir_p(output_directory)
|
|
2080
2130
|
data = retrieve_data(args, template)
|
|
2081
2131
|
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2132
|
+
if json_dump_if_changed(data, output_file):
|
|
2133
|
+
logger.info("Saved {0} {1} to disk".format(len(data), name))
|
|
2134
|
+
else:
|
|
2135
|
+
logger.info("{0} {1} unchanged, skipped write".format(len(data), name))
|
|
2085
2136
|
|
|
2086
2137
|
|
|
2087
2138
|
def json_dump(data, output_file):
|
|
@@ -2093,3 +2144,57 @@ def json_dump(data, output_file):
|
|
|
2093
2144
|
indent=4,
|
|
2094
2145
|
separators=(",", ": "),
|
|
2095
2146
|
)
|
|
2147
|
+
|
|
2148
|
+
|
|
2149
|
+
def json_dump_if_changed(data, output_file_path):
|
|
2150
|
+
"""
|
|
2151
|
+
Write JSON data to file only if content has changed.
|
|
2152
|
+
|
|
2153
|
+
Compares the serialized JSON data with the existing file content
|
|
2154
|
+
and only writes if different. This prevents unnecessary file
|
|
2155
|
+
modification timestamp updates and disk writes.
|
|
2156
|
+
|
|
2157
|
+
Uses atomic writes (temp file + rename) to prevent corruption
|
|
2158
|
+
if the process is interrupted during the write.
|
|
2159
|
+
|
|
2160
|
+
Args:
|
|
2161
|
+
data: The data to serialize as JSON
|
|
2162
|
+
output_file_path: The path to the output file
|
|
2163
|
+
|
|
2164
|
+
Returns:
|
|
2165
|
+
True if file was written (content changed or new file)
|
|
2166
|
+
False if write was skipped (content unchanged)
|
|
2167
|
+
"""
|
|
2168
|
+
# Serialize new data with consistent formatting matching json_dump()
|
|
2169
|
+
new_content = json.dumps(
|
|
2170
|
+
data,
|
|
2171
|
+
ensure_ascii=False,
|
|
2172
|
+
sort_keys=True,
|
|
2173
|
+
indent=4,
|
|
2174
|
+
separators=(",", ": "),
|
|
2175
|
+
)
|
|
2176
|
+
|
|
2177
|
+
# Check if file exists and compare content
|
|
2178
|
+
if os.path.exists(output_file_path):
|
|
2179
|
+
try:
|
|
2180
|
+
with codecs.open(output_file_path, "r", encoding="utf-8") as f:
|
|
2181
|
+
existing_content = f.read()
|
|
2182
|
+
if existing_content == new_content:
|
|
2183
|
+
logger.debug(
|
|
2184
|
+
"Content unchanged, skipping write: {0}".format(output_file_path)
|
|
2185
|
+
)
|
|
2186
|
+
return False
|
|
2187
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
2188
|
+
# If we can't read the existing file, write the new one
|
|
2189
|
+
logger.debug(
|
|
2190
|
+
"Error reading existing file {0}, will overwrite: {1}".format(
|
|
2191
|
+
output_file_path, e
|
|
2192
|
+
)
|
|
2193
|
+
)
|
|
2194
|
+
|
|
2195
|
+
# Write the file atomically using temp file + rename
|
|
2196
|
+
temp_file = output_file_path + ".temp"
|
|
2197
|
+
with codecs.open(temp_file, "w", encoding="utf-8") as f:
|
|
2198
|
+
f.write(new_content)
|
|
2199
|
+
os.rename(temp_file, output_file_path) # Atomic on POSIX systems
|
|
2200
|
+
return True
|
|
@@ -3,16 +3,16 @@ black==25.11.0
|
|
|
3
3
|
bleach==6.3.0
|
|
4
4
|
certifi==2025.11.12
|
|
5
5
|
charset-normalizer==3.4.4
|
|
6
|
-
click==8.3.
|
|
6
|
+
click==8.3.1
|
|
7
7
|
colorama==0.4.6
|
|
8
8
|
docutils==0.22.3
|
|
9
9
|
flake8==7.3.0
|
|
10
10
|
gitchangelog==3.0.4
|
|
11
|
-
pytest==
|
|
11
|
+
pytest==9.0.1
|
|
12
12
|
idna==3.11
|
|
13
13
|
importlib-metadata==8.7.0
|
|
14
14
|
jaraco.classes==3.4.0
|
|
15
|
-
keyring==25.
|
|
15
|
+
keyring==25.7.0
|
|
16
16
|
markdown-it-py==4.0.0
|
|
17
17
|
mccabe==0.7.0
|
|
18
18
|
mdurl==0.1.2
|
|
@@ -28,7 +28,7 @@ Pygments==2.19.2
|
|
|
28
28
|
readme-renderer==44.0
|
|
29
29
|
requests==2.32.5
|
|
30
30
|
requests-toolbelt==1.0.0
|
|
31
|
-
restructuredtext-lint==
|
|
31
|
+
restructuredtext-lint==2.0.2
|
|
32
32
|
rfc3986==2.0.0
|
|
33
33
|
rich==14.2.0
|
|
34
34
|
setuptools==80.9.0
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Tests for HTTP 451 (DMCA takedown) handling."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from unittest.mock import Mock, patch
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from github_backup import github_backup
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestHTTP451Exception:
|
|
12
|
+
"""Test suite for HTTP 451 DMCA takedown exception handling."""
|
|
13
|
+
|
|
14
|
+
def test_repository_unavailable_error_raised(self):
|
|
15
|
+
"""HTTP 451 should raise RepositoryUnavailableError with DMCA URL."""
|
|
16
|
+
# Create mock args
|
|
17
|
+
args = Mock()
|
|
18
|
+
args.as_app = False
|
|
19
|
+
args.token_fine = None
|
|
20
|
+
args.token_classic = None
|
|
21
|
+
args.username = None
|
|
22
|
+
args.password = None
|
|
23
|
+
args.osx_keychain_item_name = None
|
|
24
|
+
args.osx_keychain_item_account = None
|
|
25
|
+
args.throttle_limit = None
|
|
26
|
+
args.throttle_pause = 0
|
|
27
|
+
|
|
28
|
+
# Mock HTTPError 451 response
|
|
29
|
+
mock_response = Mock()
|
|
30
|
+
mock_response.getcode.return_value = 451
|
|
31
|
+
|
|
32
|
+
dmca_data = {
|
|
33
|
+
"message": "Repository access blocked",
|
|
34
|
+
"block": {
|
|
35
|
+
"reason": "dmca",
|
|
36
|
+
"created_at": "2024-11-12T14:38:04Z",
|
|
37
|
+
"html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
|
|
41
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
42
|
+
mock_response.reason = "Unavailable For Legal Reasons"
|
|
43
|
+
|
|
44
|
+
def mock_get_response(request, auth, template):
|
|
45
|
+
return mock_response, []
|
|
46
|
+
|
|
47
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
48
|
+
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
|
49
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
|
|
50
|
+
|
|
51
|
+
# Check exception has DMCA URL
|
|
52
|
+
assert exc_info.value.dmca_url == "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
|
53
|
+
assert "451" in str(exc_info.value)
|
|
54
|
+
|
|
55
|
+
def test_repository_unavailable_error_without_dmca_url(self):
|
|
56
|
+
"""HTTP 451 without DMCA details should still raise exception."""
|
|
57
|
+
args = Mock()
|
|
58
|
+
args.as_app = False
|
|
59
|
+
args.token_fine = None
|
|
60
|
+
args.token_classic = None
|
|
61
|
+
args.username = None
|
|
62
|
+
args.password = None
|
|
63
|
+
args.osx_keychain_item_name = None
|
|
64
|
+
args.osx_keychain_item_account = None
|
|
65
|
+
args.throttle_limit = None
|
|
66
|
+
args.throttle_pause = 0
|
|
67
|
+
|
|
68
|
+
mock_response = Mock()
|
|
69
|
+
mock_response.getcode.return_value = 451
|
|
70
|
+
mock_response.read.return_value = b'{"message": "Blocked"}'
|
|
71
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
72
|
+
mock_response.reason = "Unavailable For Legal Reasons"
|
|
73
|
+
|
|
74
|
+
def mock_get_response(request, auth, template):
|
|
75
|
+
return mock_response, []
|
|
76
|
+
|
|
77
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
78
|
+
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
|
79
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
|
|
80
|
+
|
|
81
|
+
# Exception raised even without DMCA URL
|
|
82
|
+
assert exc_info.value.dmca_url is None
|
|
83
|
+
assert "451" in str(exc_info.value)
|
|
84
|
+
|
|
85
|
+
def test_repository_unavailable_error_with_malformed_json(self):
|
|
86
|
+
"""HTTP 451 with malformed JSON should still raise exception."""
|
|
87
|
+
args = Mock()
|
|
88
|
+
args.as_app = False
|
|
89
|
+
args.token_fine = None
|
|
90
|
+
args.token_classic = None
|
|
91
|
+
args.username = None
|
|
92
|
+
args.password = None
|
|
93
|
+
args.osx_keychain_item_name = None
|
|
94
|
+
args.osx_keychain_item_account = None
|
|
95
|
+
args.throttle_limit = None
|
|
96
|
+
args.throttle_pause = 0
|
|
97
|
+
|
|
98
|
+
mock_response = Mock()
|
|
99
|
+
mock_response.getcode.return_value = 451
|
|
100
|
+
mock_response.read.return_value = b"invalid json {"
|
|
101
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
102
|
+
mock_response.reason = "Unavailable For Legal Reasons"
|
|
103
|
+
|
|
104
|
+
def mock_get_response(request, auth, template):
|
|
105
|
+
return mock_response, []
|
|
106
|
+
|
|
107
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
108
|
+
with pytest.raises(github_backup.RepositoryUnavailableError):
|
|
109
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
|
|
110
|
+
|
|
111
|
+
def test_other_http_errors_unchanged(self):
|
|
112
|
+
"""Other HTTP errors should still raise generic Exception."""
|
|
113
|
+
args = Mock()
|
|
114
|
+
args.as_app = False
|
|
115
|
+
args.token_fine = None
|
|
116
|
+
args.token_classic = None
|
|
117
|
+
args.username = None
|
|
118
|
+
args.password = None
|
|
119
|
+
args.osx_keychain_item_name = None
|
|
120
|
+
args.osx_keychain_item_account = None
|
|
121
|
+
args.throttle_limit = None
|
|
122
|
+
args.throttle_pause = 0
|
|
123
|
+
|
|
124
|
+
mock_response = Mock()
|
|
125
|
+
mock_response.getcode.return_value = 404
|
|
126
|
+
mock_response.read.return_value = b'{"message": "Not Found"}'
|
|
127
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
128
|
+
mock_response.reason = "Not Found"
|
|
129
|
+
|
|
130
|
+
def mock_get_response(request, auth, template):
|
|
131
|
+
return mock_response, []
|
|
132
|
+
|
|
133
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
134
|
+
# Should raise generic Exception, not RepositoryUnavailableError
|
|
135
|
+
with pytest.raises(Exception) as exc_info:
|
|
136
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/notfound/issues"))
|
|
137
|
+
|
|
138
|
+
assert not isinstance(exc_info.value, github_backup.RepositoryUnavailableError)
|
|
139
|
+
assert "404" in str(exc_info.value)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
pytest.main([__file__, "-v"])
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Tests for json_dump_if_changed functionality."""
|
|
2
|
+
|
|
3
|
+
import codecs
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import tempfile
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from github_backup import github_backup
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestJsonDumpIfChanged:
|
|
14
|
+
"""Test suite for json_dump_if_changed function."""
|
|
15
|
+
|
|
16
|
+
def test_writes_new_file(self):
|
|
17
|
+
"""Should write file when it doesn't exist."""
|
|
18
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
19
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
20
|
+
test_data = {"key": "value", "number": 42}
|
|
21
|
+
|
|
22
|
+
result = github_backup.json_dump_if_changed(test_data, output_file)
|
|
23
|
+
|
|
24
|
+
assert result is True
|
|
25
|
+
assert os.path.exists(output_file)
|
|
26
|
+
|
|
27
|
+
# Verify content matches expected format
|
|
28
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
29
|
+
content = f.read()
|
|
30
|
+
loaded = json.loads(content)
|
|
31
|
+
assert loaded == test_data
|
|
32
|
+
|
|
33
|
+
def test_skips_unchanged_file(self):
|
|
34
|
+
"""Should skip write when content is identical."""
|
|
35
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
36
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
37
|
+
test_data = {"key": "value", "number": 42}
|
|
38
|
+
|
|
39
|
+
# First write
|
|
40
|
+
result1 = github_backup.json_dump_if_changed(test_data, output_file)
|
|
41
|
+
assert result1 is True
|
|
42
|
+
|
|
43
|
+
# Get the initial mtime
|
|
44
|
+
mtime1 = os.path.getmtime(output_file)
|
|
45
|
+
|
|
46
|
+
# Second write with same data
|
|
47
|
+
result2 = github_backup.json_dump_if_changed(test_data, output_file)
|
|
48
|
+
assert result2 is False
|
|
49
|
+
|
|
50
|
+
# File should not have been modified
|
|
51
|
+
mtime2 = os.path.getmtime(output_file)
|
|
52
|
+
assert mtime1 == mtime2
|
|
53
|
+
|
|
54
|
+
def test_writes_when_content_changed(self):
|
|
55
|
+
"""Should write file when content has changed."""
|
|
56
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
57
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
58
|
+
test_data1 = {"key": "value1"}
|
|
59
|
+
test_data2 = {"key": "value2"}
|
|
60
|
+
|
|
61
|
+
# First write
|
|
62
|
+
result1 = github_backup.json_dump_if_changed(test_data1, output_file)
|
|
63
|
+
assert result1 is True
|
|
64
|
+
|
|
65
|
+
# Second write with different data
|
|
66
|
+
result2 = github_backup.json_dump_if_changed(test_data2, output_file)
|
|
67
|
+
assert result2 is True
|
|
68
|
+
|
|
69
|
+
# Verify new content
|
|
70
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
71
|
+
loaded = json.load(f)
|
|
72
|
+
assert loaded == test_data2
|
|
73
|
+
|
|
74
|
+
def test_uses_consistent_formatting(self):
|
|
75
|
+
"""Should use same JSON formatting as json_dump."""
|
|
76
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
77
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
78
|
+
test_data = {"z": "last", "a": "first", "m": "middle"}
|
|
79
|
+
|
|
80
|
+
github_backup.json_dump_if_changed(test_data, output_file)
|
|
81
|
+
|
|
82
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
83
|
+
content = f.read()
|
|
84
|
+
|
|
85
|
+
# Check for consistent formatting:
|
|
86
|
+
# - sorted keys
|
|
87
|
+
# - 4-space indent
|
|
88
|
+
# - comma-colon-space separator
|
|
89
|
+
expected = json.dumps(
|
|
90
|
+
test_data,
|
|
91
|
+
ensure_ascii=False,
|
|
92
|
+
sort_keys=True,
|
|
93
|
+
indent=4,
|
|
94
|
+
separators=(",", ": "),
|
|
95
|
+
)
|
|
96
|
+
assert content == expected
|
|
97
|
+
|
|
98
|
+
def test_atomic_write_always_used(self):
|
|
99
|
+
"""Should always use temp file and rename for atomic writes."""
|
|
100
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
101
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
102
|
+
test_data = {"key": "value"}
|
|
103
|
+
|
|
104
|
+
result = github_backup.json_dump_if_changed(test_data, output_file)
|
|
105
|
+
|
|
106
|
+
assert result is True
|
|
107
|
+
assert os.path.exists(output_file)
|
|
108
|
+
|
|
109
|
+
# Temp file should not exist after atomic write
|
|
110
|
+
temp_file = output_file + ".temp"
|
|
111
|
+
assert not os.path.exists(temp_file)
|
|
112
|
+
|
|
113
|
+
# Verify content
|
|
114
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
115
|
+
loaded = json.load(f)
|
|
116
|
+
assert loaded == test_data
|
|
117
|
+
|
|
118
|
+
def test_handles_unicode_content(self):
|
|
119
|
+
"""Should correctly handle Unicode content."""
|
|
120
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
121
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
122
|
+
test_data = {
|
|
123
|
+
"emoji": "🚀",
|
|
124
|
+
"chinese": "你好",
|
|
125
|
+
"arabic": "مرحبا",
|
|
126
|
+
"cyrillic": "Привет",
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
result = github_backup.json_dump_if_changed(test_data, output_file)
|
|
130
|
+
assert result is True
|
|
131
|
+
|
|
132
|
+
# Verify Unicode is preserved
|
|
133
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
134
|
+
loaded = json.load(f)
|
|
135
|
+
assert loaded == test_data
|
|
136
|
+
|
|
137
|
+
# Second write should skip
|
|
138
|
+
result2 = github_backup.json_dump_if_changed(test_data, output_file)
|
|
139
|
+
assert result2 is False
|
|
140
|
+
|
|
141
|
+
def test_handles_complex_nested_data(self):
|
|
142
|
+
"""Should handle complex nested data structures."""
|
|
143
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
144
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
145
|
+
test_data = {
|
|
146
|
+
"users": [
|
|
147
|
+
{"id": 1, "name": "Alice", "tags": ["admin", "user"]},
|
|
148
|
+
{"id": 2, "name": "Bob", "tags": ["user"]},
|
|
149
|
+
],
|
|
150
|
+
"metadata": {"version": "1.0", "nested": {"deep": {"value": 42}}},
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
result = github_backup.json_dump_if_changed(test_data, output_file)
|
|
154
|
+
assert result is True
|
|
155
|
+
|
|
156
|
+
# Verify structure is preserved
|
|
157
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
158
|
+
loaded = json.load(f)
|
|
159
|
+
assert loaded == test_data
|
|
160
|
+
|
|
161
|
+
def test_overwrites_on_unicode_decode_error(self):
|
|
162
|
+
"""Should overwrite if existing file has invalid UTF-8."""
|
|
163
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
164
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
165
|
+
test_data = {"key": "value"}
|
|
166
|
+
|
|
167
|
+
# Write invalid UTF-8 bytes
|
|
168
|
+
with open(output_file, "wb") as f:
|
|
169
|
+
f.write(b"\xff\xfe invalid utf-8")
|
|
170
|
+
|
|
171
|
+
# Should catch UnicodeDecodeError and overwrite
|
|
172
|
+
result = github_backup.json_dump_if_changed(test_data, output_file)
|
|
173
|
+
assert result is True
|
|
174
|
+
|
|
175
|
+
# Verify new content was written
|
|
176
|
+
with codecs.open(output_file, "r", encoding="utf-8") as f:
|
|
177
|
+
loaded = json.load(f)
|
|
178
|
+
assert loaded == test_data
|
|
179
|
+
|
|
180
|
+
def test_key_order_independence(self):
|
|
181
|
+
"""Should treat differently-ordered dicts as same if keys/values match."""
|
|
182
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
183
|
+
output_file = os.path.join(tmpdir, "test.json")
|
|
184
|
+
|
|
185
|
+
# Write first dict
|
|
186
|
+
data1 = {"z": 1, "a": 2, "m": 3}
|
|
187
|
+
github_backup.json_dump_if_changed(data1, output_file)
|
|
188
|
+
|
|
189
|
+
# Try to write same data but different order
|
|
190
|
+
data2 = {"a": 2, "m": 3, "z": 1}
|
|
191
|
+
result = github_backup.json_dump_if_changed(data2, output_file)
|
|
192
|
+
|
|
193
|
+
# Should skip because content is the same (keys are sorted)
|
|
194
|
+
assert result is False
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
if __name__ == "__main__":
|
|
198
|
+
pytest.main([__file__, "-v"])
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.51.3"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|