github-backup 0.51.2__py3-none-any.whl → 0.52.0__py3-none-any.whl

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.51.2"
+__version__ = "0.52.0"

github_backup/github_backup.py

@@ -37,6 +37,15 @@ FNULL = open(os.devnull, "w")
 FILE_URI_PREFIX = "file://"
 logger = logging.getLogger(__name__)
 
+
+class RepositoryUnavailableError(Exception):
+    """Raised when a repository is unavailable due to legal reasons (e.g., DMCA takedown)."""
+
+    def __init__(self, message, dmca_url=None):
+        super().__init__(message)
+        self.dmca_url = dmca_url
+
+
 # Setup SSL context with fallback chain
 https_ctx = ssl.create_default_context()
 if https_ctx.get_ca_certs():
@@ -592,27 +601,39 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
     auth = get_auth(args, encode=not args.as_app)
     query_args = get_query_args(query_args)
     per_page = 100
-    page = 0
+    next_url = None
 
     while True:
         if single_request:
-            request_page, request_per_page = None, None
+            request_per_page = None
         else:
-            page = page + 1
-            request_page, request_per_page = page, per_page
+            request_per_page = per_page
 
         request = _construct_request(
             request_per_page,
-            request_page,
             query_args,
-            template,
+            next_url or template,
             auth,
             as_app=args.as_app,
             fine=True if args.token_fine is not None else False,
         )  # noqa
-        r, errors = _get_response(request, auth, template)
+        r, errors = _get_response(request, auth, next_url or template)
 
         status_code = int(r.getcode())
+
+        # Handle DMCA takedown (HTTP 451) - raise exception to skip entire repository
+        if status_code == 451:
+            dmca_url = None
+            try:
+                response_data = json.loads(r.read().decode("utf-8"))
+                dmca_url = response_data.get("block", {}).get("html_url")
+            except Exception:
+                pass
+            raise RepositoryUnavailableError(
+                "Repository unavailable due to legal reasons (HTTP 451)",
+                dmca_url=dmca_url
+            )
+
         # Check if we got correct data
         try:
             response = json.loads(r.read().decode("utf-8"))
@@ -644,15 +665,14 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             retries += 1
             time.sleep(5)
             request = _construct_request(
-                per_page,
-                page,
+                request_per_page,
                 query_args,
-                template,
+                next_url or template,
                 auth,
                 as_app=args.as_app,
                 fine=True if args.token_fine is not None else False,
             )  # noqa
-            r, errors = _get_response(request, auth, template)
+            r, errors = _get_response(request, auth, next_url or template)
 
             status_code = int(r.getcode())
             try:
@@ -682,7 +702,16 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             if type(response) is list:
                 for resp in response:
                     yield resp
-                if len(response) < per_page:
+                # Parse Link header for next page URL (cursor-based pagination)
+                link_header = r.headers.get("Link", "")
+                next_url = None
+                if link_header:
+                    # Parse Link header: <https://api.github.com/...?per_page=100&after=cursor>; rel="next"
+                    for link in link_header.split(","):
+                        if 'rel="next"' in link:
+                            next_url = link[link.find("<") + 1:link.find(">")]
+                            break
+                if not next_url:
                     break
             elif type(response) is dict and single_request:
                 yield response
@@ -735,22 +764,27 @@ def _get_response(request, auth, template):
 
 
 def _construct_request(
-    per_page, page, query_args, template, auth, as_app=None, fine=False
+    per_page, query_args, template, auth, as_app=None, fine=False
 ):
-    all_query_args = {}
-    if per_page:
-        all_query_args["per_page"] = per_page
-    if page:
-        all_query_args["page"] = page
-    if query_args:
-        all_query_args.update(query_args)
-
-    request_url = template
-    if all_query_args:
-        querystring = urlencode(all_query_args)
-        request_url = template + "?" + querystring
+    # If template is already a full URL with query params (from Link header), use it directly
+    if "?" in template and template.startswith("http"):
+        request_url = template
+        # Extract query string for logging
+        querystring = template.split("?", 1)[1]
     else:
-        querystring = ""
+        # Build URL with query parameters
+        all_query_args = {}
+        if per_page:
+            all_query_args["per_page"] = per_page
+        if query_args:
+            all_query_args.update(query_args)
+
+        request_url = template
+        if all_query_args:
+            querystring = urlencode(all_query_args)
+            request_url = template + "?" + querystring
+        else:
+            querystring = ""
 
     request = Request(request_url)
     if auth is not None:
@@ -766,7 +800,7 @@ def _construct_request(
                 "Accept", "application/vnd.github.machine-man-preview+json"
             )
 
-    log_url = template
+    log_url = template if "?" not in template else template.split("?")[0]
     if querystring:
         log_url += "?" + querystring
     logger.info("Requesting {}".format(log_url))
@@ -843,8 +877,7 @@ def download_file(url, path, auth, as_app=False, fine=False):
         return
 
     request = _construct_request(
-        per_page=100,
-        page=1,
+        per_page=None,
         query_args={},
         template=url,
         auth=auth,
@@ -1657,40 +1690,47 @@ def backup_repositories(args, output_directory, repositories):
 
                 continue  # don't try to back anything else for a gist; it doesn't exist
 
-        download_wiki = args.include_wiki or args.include_everything
-        if repository["has_wiki"] and download_wiki:
-            fetch_repository(
-                repository["name"],
-                repo_url.replace(".git", ".wiki.git"),
-                os.path.join(repo_cwd, "wiki"),
-                skip_existing=args.skip_existing,
-                bare_clone=args.bare_clone,
-                lfs_clone=args.lfs_clone,
-                no_prune=args.no_prune,
-            )
-        if args.include_issues or args.include_everything:
-            backup_issues(args, repo_cwd, repository, repos_template)
+        try:
+            download_wiki = args.include_wiki or args.include_everything
+            if repository["has_wiki"] and download_wiki:
+                fetch_repository(
+                    repository["name"],
+                    repo_url.replace(".git", ".wiki.git"),
+                    os.path.join(repo_cwd, "wiki"),
+                    skip_existing=args.skip_existing,
+                    bare_clone=args.bare_clone,
+                    lfs_clone=args.lfs_clone,
+                    no_prune=args.no_prune,
+                )
+            if args.include_issues or args.include_everything:
+                backup_issues(args, repo_cwd, repository, repos_template)
 
-        if args.include_pulls or args.include_everything:
-            backup_pulls(args, repo_cwd, repository, repos_template)
+            if args.include_pulls or args.include_everything:
+                backup_pulls(args, repo_cwd, repository, repos_template)
 
-        if args.include_milestones or args.include_everything:
-            backup_milestones(args, repo_cwd, repository, repos_template)
+            if args.include_milestones or args.include_everything:
+                backup_milestones(args, repo_cwd, repository, repos_template)
 
-        if args.include_labels or args.include_everything:
-            backup_labels(args, repo_cwd, repository, repos_template)
+            if args.include_labels or args.include_everything:
+                backup_labels(args, repo_cwd, repository, repos_template)
 
-        if args.include_hooks or args.include_everything:
-            backup_hooks(args, repo_cwd, repository, repos_template)
+            if args.include_hooks or args.include_everything:
+                backup_hooks(args, repo_cwd, repository, repos_template)
 
-        if args.include_releases or args.include_everything:
-            backup_releases(
-                args,
-                repo_cwd,
-                repository,
-                repos_template,
-                include_assets=args.include_assets or args.include_everything,
-            )
+            if args.include_releases or args.include_everything:
+                backup_releases(
+                    args,
+                    repo_cwd,
+                    repository,
+                    repos_template,
+                    include_assets=args.include_assets or args.include_everything,
+                )
+        except RepositoryUnavailableError as e:
+            logger.warning(f"Repository {repository['full_name']} is unavailable (HTTP 451)")
+            if e.dmca_url:
+                logger.warning(f"DMCA notice: {e.dmca_url}")
+            logger.info(f"Skipping remaining resources for {repository['full_name']}")
+            continue
 
     if args.incremental:
         if last_update == "0000-00-00T00:00:00Z":

{github_backup-0.51.2.dist-info → github_backup-0.52.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: github-backup
-Version: 0.51.2
+Version: 0.52.0
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez

github_backup-0.52.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+github_backup/__init__.py,sha256=uXy9jtvBCb6cwZyNng8d7tehPtkx2Xcdr7Lm3SEzEF8,23
+github_backup/github_backup.py,sha256=eo8P69vIgeP_ntzY4DdGrzFnEKPIoqBrgBX3Dw_2lIo,77487
+github_backup-0.52.0.data/scripts/github-backup,sha256=I79TNA67phytRmI9nm_LlD9h2zauXFOsOn9tYyAy9pU,1546
+github_backup-0.52.0.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
+github_backup-0.52.0.dist-info/METADATA,sha256=92rNj52N87cGNzBGRJYCyaotJIEgZ5ZgZ-DUugpuvlI,21544
+github_backup-0.52.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+github_backup-0.52.0.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
+github_backup-0.52.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+github_backup-0.52.0.dist-info/RECORD,,

github_backup-0.51.2.dist-info/RECORD

@@ -1,9 +0,0 @@
-github_backup/__init__.py,sha256=2AuLzuHMBBPVOM-m0Woty11pTO9ExwPKWRWtj1LvS-g,23
-github_backup/github_backup.py,sha256=ousskr9hBtfKgAh1x78vnSefQhPGqa9K2fLxWLUKdyY,75428
-github_backup-0.51.2.data/scripts/github-backup,sha256=I79TNA67phytRmI9nm_LlD9h2zauXFOsOn9tYyAy9pU,1546
-github_backup-0.51.2.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
-github_backup-0.51.2.dist-info/METADATA,sha256=zoLPP4sNYwAsnFUXTZR1K1NB5J-i2xQidc9DZ3hbkfk,21544
-github_backup-0.51.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-github_backup-0.51.2.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
-github_backup-0.51.2.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-github_backup-0.51.2.dist-info/RECORD,,