dump-github 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

dump_github/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0"
+__version__ = "0.2.1"

dump_github/lib.py

@@ -1,9 +1,22 @@
+import re
+import warnings
 from urllib.parse import quote
 
 import requests
 
 GITHUB_API = "https://api.github.com"
+GITHUB_WEB = "https://github.com"
 PER_PAGE = 100
+WEB_USER_AGENT = "dump-github (+https://github.com/Core2002/dump_github)"
+
+
+class GitHubAPIError(RuntimeError):
+    """GitHub could not be read through the REST API (or the web fallback)."""
+
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        super().__init__(message)
 
 
 def _github_auth_headers(token):
@@ -28,26 +41,42 @@ def _api_headers(token=""):
 
 
 def _api_get(path, token=""):
-    """GET one GitHub REST API path and return the decoded JSON, or None on failure."""
+    """GET one GitHub REST API path and return the decoded JSON.
+
+    Raises GitHubAPIError instead of silently returning an empty result, so
+    callers can distinguish "no repositories" from "request failed".
+    """
     try:
         response = requests.get(
             GITHUB_API + path, headers=_api_headers(token), timeout=30
         )
-    except requests.RequestException:
-        return None
+    except requests.RequestException as exc:
+        raise GitHubAPIError(None, "request failed: {}".format(exc)) from exc
     if response.status_code != 200:
-        return None
+        message = "HTTP {}".format(response.status_code)
+        try:
+            payload = response.json()
+        except ValueError:
+            payload = None
+        if isinstance(payload, dict) and payload.get("message"):
+            message = "{} (HTTP {})".format(payload["message"], response.status_code)
+        raise GitHubAPIError(response.status_code, message)
     try:
         return response.json()
-    except ValueError:
-        return None
+    except ValueError as exc:
+        raise GitHubAPIError(
+            response.status_code, "invalid JSON response from GitHub"
+        ) from exc
 
 
 def get_authenticated_login(token=""):
     """Return the login of the token owner, or None if there is no usable token."""
     if not _github_auth_headers(token):
         return None
-    account = _api_get("/user", token)
+    try:
+        account = _api_get("/user", token)
+    except GitHubAPIError:
+        return None
     if not isinstance(account, dict):
         return None
     return account.get("login") or None
@@ -56,12 +85,18 @@ def get_authenticated_login(token=""):
 def get_account_type(username, token=""):
     """Return ``"Organization"``/``"User"`` for ``username``, or None if unknown."""
     quoted = quote(username, safe="")
-    account = _api_get("/users/{}".format(quoted), token)
+    try:
+        account = _api_get("/users/{}".format(quoted), token)
+    except GitHubAPIError:
+        account = None
     if isinstance(account, dict) and account.get("type"):
         return account["type"]
     # ``/users/{org}`` is not guaranteed to resolve for organisations, so fall
     # back to the org endpoint, which unambiguously identifies one.
-    org = _api_get("/orgs/{}".format(quoted), token)
+    try:
+        org = _api_get("/orgs/{}".format(quoted), token)
+    except GitHubAPIError:
+        org = None
     if isinstance(org, dict) and org.get("login"):
         return "Organization"
     return None
@@ -131,13 +166,133 @@ def get_page_of_repos(username, page, token="", authenticated_login=None,
     return [repo for repo in repos if _is_owned_by(repo, username)]
 
 
-def search_user_repos(username, token=""):
-    username = username.strip()
-    authenticated_login = get_authenticated_login(token)
-    account_type = get_account_type(username, token)
+# --- public web fallback -------------------------------------------------
+#
+# api.github.com allows only 60 unauthenticated requests per hour *per IP*.
+# Shared/NATed networks and CI runners exhaust that quickly, which used to make
+# ``dump-github user`` silently return nothing.  github.com's repository pages
+# are not subject to that API quota, so they are used as a read-only fallback
+# for public repositories when no token is configured.
+
+_USER_REPO_TAG_RE = re.compile(r'<a\b[^>]*itemprop="name codeRepository"[^>]*>')
+_WEB_REPO_HREF_RE = re.compile(r'href="/([^"/]+)/([^"/?#]+)"')
+_WEB_NEXT_RE = re.compile(r'rel="next"')
+_WEB_NON_REPO_PATHS = {
+    "followers",
+    "following",
+    "people",
+    "packages",
+    "projects",
+    "repositories",
+    "settings",
+    "sponsoring",
+}
+
+
+def _web_get(url):
+    try:
+        response = requests.get(
+            url, headers={"User-Agent": WEB_USER_AGENT}, timeout=30
+        )
+    except requests.RequestException as exc:
+        raise GitHubAPIError(None, "web request failed: {}".format(exc)) from exc
+    if response.status_code != 200:
+        raise GitHubAPIError(
+            response.status_code,
+            "github.com returned HTTP {}".format(response.status_code),
+        )
+    return response.text
+
+
+def _web_repo(owner, name):
+    return {
+        "name": name,
+        "full_name": "{}/{}".format(owner, name),
+        "owner": {"login": owner},
+        "clone_url": "{}/{}/{}.git".format(GITHUB_WEB, owner, name),
+        # The default branch is not exposed by the web listing; zip downloads
+        # then use the HEAD archive instead.
+    }
+
+
+def _parse_user_repos_page(html, username):
+    repos = []
+    seen = set()
+    for tag in _USER_REPO_TAG_RE.findall(html):
+        match = _WEB_REPO_HREF_RE.search(tag)
+        if not match:
+            continue
+        owner, name = match.group(1), match.group(2)
+        if owner.lower() != username.lower() or name in seen:
+            continue
+        seen.add(name)
+        repos.append(_web_repo(owner, name))
+    return repos
+
+
+def _parse_org_repos_page(html, org):
+    repos = []
+    seen = set()
+    for owner, name in _WEB_REPO_HREF_RE.findall(html):
+        if owner.lower() != org.lower():
+            continue
+        if name.lower() in _WEB_NON_REPO_PATHS or name in seen:
+            continue
+        seen.add(name)
+        repos.append(_web_repo(owner, name))
+    return repos
+
+
+def _web_page_url(username, page, is_org):
+    quoted = quote(username, safe="")
+    if is_org:
+        return "{}/orgs/{}/repositories?page={}".format(GITHUB_WEB, quoted, page)
+    return "{}/{}?tab=repositories&page={}".format(GITHUB_WEB, quoted, page)
+
+
+def _parse_web_page(html, username, is_org):
+    if is_org:
+        return _parse_org_repos_page(html, username)
+    return _parse_user_repos_page(html, username)
+
+
+def _search_via_web(username, account_type):
+    """List public repositories from github.com when the REST API is unavailable."""
+    is_org = (account_type or "").lower() == "organization"
+    first = _web_get(_web_page_url(username, 1, is_org))
+    repos = _parse_web_page(first, username, is_org)
+    if not repos and account_type is None:
+        # The account may be an organisation: its profile page redirects to
+        # github.com/{org} and does not contain the plain repository list.
+        try:
+            org_first = _web_get(_web_page_url(username, 1, True))
+        except GitHubAPIError as exc:
+            if exc.status_code == 404:
+                return []
+            raise
+        org_repos = _parse_org_repos_page(org_first, username)
+        if not org_repos:
+            return []
+        is_org = True
+        repos = org_repos
+        first = org_first
+    if not repos:
+        return []
+    page = 1
+    while _WEB_NEXT_RE.search(first):
+        page += 1
+        html = _web_get(_web_page_url(username, page, is_org))
+        page_repos = _parse_web_page(html, username, is_org)
+        if not page_repos:
+            break
+        repos.extend(page_repos)
+        first = html
+    return repos
+
+
+def _search_via_api(username, token, authenticated_login, account_type):
     page = 0
     res_repos = []
-
     while True:
         page += 1
         repos = _fetch_repos_page(
@@ -148,10 +303,42 @@ def search_user_repos(username, token=""):
         res_repos += repos
         if len(repos) < PER_PAGE:
             break
+    return res_repos
+
+
+def search_user_repos(username, token=""):
+    username = username.strip()
+    authenticated_login = get_authenticated_login(token)
+    account_type = get_account_type(username, token)
+    try:
+        repos = _search_via_api(
+            username, token, authenticated_login, account_type
+        )
+    except GitHubAPIError as exc:
+        if token or exc.status_code == 404:
+            raise
+        warnings.warn(
+            "GitHub API request failed ({}); falling back to the public GitHub "
+            "web listing. Only public repositories can be listed without a "
+            "token.".format(exc),
+            RuntimeWarning,
+            stacklevel=2,
+        )
+        repos = _search_via_web(username, account_type)
 
     # Never return repositories that belong to somebody else (e.g. a user repo
     # returned for an organisation, or an organisation the token owner is in).
-    return [repo for repo in res_repos if _is_owned_by(repo, username)]
+    result = []
+    seen = set()
+    for repo in repos:
+        if not _is_owned_by(repo, username):
+            continue
+        key = (repo.get("full_name") or repo.get("name") or "").lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(repo)
+    return result
 
 
 # Backward-compatible alias for the historical typo

dump_github/main.py

@@ -1,9 +1,10 @@
 import argparse
 import os
 import subprocess
+import sys
 
 from ._version import __version__
-from .lib import search_user_repos
+from .lib import GitHubAPIError, search_user_repos
 
 # One-shot credential helper: git asks for credentials, the helper answers with
 # the token from the environment.  The token is therefore never part of the
@@ -67,7 +68,7 @@ def clone_repo(repo, token=""):
 
 
 def download_zip(user_name, reop, token="", limit_size=100):
-    ref = reop.get("default_branch") or "main"
+    ref = reop.get("default_branch")
     # Always archive the repository the API actually listed.  Using the
     # positional username here would point at the wrong owner whenever the
     # repo does not belong to that user.
@@ -88,7 +89,9 @@ def download_zip(user_name, reop, token="", limit_size=100):
     if max_size > 0:
         curl_cmd.extend(["--max-filesize", str(max_size)])
     if token:
-        url = "https://api.github.com/repos/{}/zipball/{}".format(full_name, ref)
+        url = "https://api.github.com/repos/{}/zipball".format(full_name)
+        if ref:
+            url += "/{}".format(ref)
         curl_cmd.extend(
             [
                 "-H",
@@ -101,7 +104,12 @@ def download_zip(user_name, reop, token="", limit_size=100):
             ]
         )
     else:
-        url = "https://github.com/{}/archive/refs/heads/{}.zip".format(full_name, ref)
+        # The web listing does not know the default branch; GitHub resolves
+        # HEAD to it.
+        if ref:
+            url = "https://github.com/{}/archive/refs/heads/{}.zip".format(full_name, ref)
+        else:
+            url = "https://github.com/{}/archive/HEAD.zip".format(full_name)
         curl_cmd.extend(["-o", file_name, url])
     print("Download {} : {}".format(reop["name"], url))
     subprocess.run(curl_cmd, check=False)
@@ -117,7 +125,20 @@ def download_zip(user_name, reop, token="", limit_size=100):
 def main(argv=None):
     args = parser.parse_args(argv)
     token = (args.token or "").strip() or (os.environ.get("GITHUB_TOKEN") or "").strip()
-    repos = search_user_repos(args.username, token)
+    try:
+        repos = search_user_repos(args.username, token)
+    except GitHubAPIError as exc:
+        print(
+            "error: cannot list repositories for {}: {}".format(args.username, exc),
+            file=sys.stderr,
+        )
+        raise SystemExit(1)
+    if not repos:
+        print(
+            "No public repositories found for {}.".format(args.username),
+            file=sys.stderr,
+        )
+        return
     for repo in repos:
         if args.print:
             print(repo["clone_url"])

{dump_github-0.2.0.dist-info → dump_github-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dump-github
-Version: 0.2.0
+Version: 0.2.1
 Summary: Back up users' GitHub repositories in case they need them later.
 Author-email: core2002 <core2002@aliyun.com>
 License-Expression: MIT
@@ -81,6 +81,15 @@ GitHub’s API requires a proper `Authorization: Bearer <token>` header. This to
 - **Private repositories**: Without a token, only public repos are listed and can be cloned or archived. With a token that has the right scopes, you can include private repos you are allowed to access (for example when backing up your own account).
 - **Rate limits**: Authenticated API requests get a much higher limit than anonymous requests.
 
+**Without a token**
+
+Only public repositories can be listed and downloaded. The anonymous GitHub API allows
+60 requests per hour per IP, which shared/NATed networks and CI runners exhaust quickly;
+if `api.github.com` is rate-limited or otherwise unavailable, the tool automatically
+falls back to the public GitHub web listing
+(`github.com/{user}?tab=repositories` / `github.com/orgs/{org}/repositories`), prints a
+warning and keeps going. Private repositories still require a token.
+
 **How the token is used**
 
 - **Listing repos**: sent as `Bearer` on the REST API and scoped strictly to the requested account:
@@ -97,9 +106,12 @@ Create a token under GitHub → **Settings** → **Developer settings** → **Pe
 **Examples**
 
 ```bash
-# Public repos only (no token)
+# Public repos only (no token; falls back to the web listing if the API is rate-limited)
 dump-github Core2002 -p
 
+# Public backup without any credentials
+dump-github Core2002 -d
+
 # Token on the command line (private + higher API limits)
 dump-github username --token ghp_xxxxxxxxxxxx
 

dump_github-0.2.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+dump_github/__init__.py,sha256=7bV8yfP0ii_gvNXKfDzFhBj2V5nC82mcJrlZqs9fRZ0,92
+dump_github/_version.py,sha256=HfjVOrpTnmZ-xVFCYSVmX50EXaBQeJteUHG-PD6iQs8,22
+dump_github/lib.py,sha256=elhVYzMSK2abVKPWqQUukQCCNdxdkCjGQPiW7kauGkA,11662
+dump_github/main.py,sha256=unifRRUhyvu_2e7UMmoIxtRbVOANUGUUIQJbhaDQ_I8,4768
+dump_github-0.2.1.dist-info/licenses/LICENSE,sha256=5tQa-lQLqvLhjTuELEk4bm70I5hcdjUeOn4qf3MK84E,1064
+dump_github-0.2.1.dist-info/METADATA,sha256=-Ol1P5UNwJe6OjSiLD6LkFHQKbUO-jHYI5yEV-D7Ru8,7373
+dump_github-0.2.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+dump_github-0.2.1.dist-info/entry_points.txt,sha256=2OAP2WnA1p54LWyCw3tHe44Pf4aR7Thsxf9uWKWfWl8,49
+dump_github-0.2.1.dist-info/top_level.txt,sha256=6oWfdkP2U1AN9zL6AUIUTxe6hYIeo_Nf8M8SWBxElUw,12
+dump_github-0.2.1.dist-info/RECORD,,

dump_github-0.2.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-dump_github/__init__.py,sha256=7bV8yfP0ii_gvNXKfDzFhBj2V5nC82mcJrlZqs9fRZ0,92
-dump_github/_version.py,sha256=Zn1KFblwuFHiDRdRAiRnDBRkbPttWh44jKa5zG2ov0E,22
-dump_github/lib.py,sha256=l35zDGPRRKdMv6KzzcqJXnCmOMLrPY07wnsijcwbfCA,5485
-dump_github/main.py,sha256=lYSma1C2m-7Pl5r4zwbEfeZyKqgVf2H2jFwD-98hnYg,4122
-dump_github-0.2.0.dist-info/licenses/LICENSE,sha256=5tQa-lQLqvLhjTuELEk4bm70I5hcdjUeOn4qf3MK84E,1064
-dump_github-0.2.0.dist-info/METADATA,sha256=vjimbmFSa9hnjYn8L58iX79KvNbL3tq104VVsIXg09I,6766
-dump_github-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-dump_github-0.2.0.dist-info/entry_points.txt,sha256=2OAP2WnA1p54LWyCw3tHe44Pf4aR7Thsxf9uWKWfWl8,49
-dump_github-0.2.0.dist-info/top_level.txt,sha256=6oWfdkP2U1AN9zL6AUIUTxe6hYIeo_Nf8M8SWBxElUw,12
-dump_github-0.2.0.dist-info/RECORD,,