stardown 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stardown-0.3.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 wydyippee
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,143 @@
1
+ Metadata-Version: 2.4
2
+ Name: stardown
3
+ Version: 0.3.0
4
+ Summary: StarDown (GitHub Starred Downloader) — async fetch with rich live progress
5
+ Author: wydyippee
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/wydyippee/StarDown
8
+ Project-URL: Repository, https://github.com/wydyippee/StarDown
9
+ Project-URL: Issues, https://github.com/wydyippee/StarDown/issues
10
+ Keywords: github,stars,backup,downloader,archive
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Utilities
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: httpx>=0.27
24
+ Requires-Dist: rich>=13
25
+ Provides-Extra: linux
26
+ Requires-Dist: uvloop>=0.19; extra == "linux"
27
+ Provides-Extra: test
28
+ Requires-Dist: pytest>=8; extra == "test"
29
+ Dynamic: license-file
30
+
31
+ <div align="center">
32
+
33
+ # ★ StarDown ★
34
+
35
+ **Stars, on your disk, fast.**
36
+
37
+ [![Python](https://img.shields.io/badge/Python-3.11%2B-blue?style=flat-square&logo=python&logoColor=white)](https://www.python.org/)
38
+ [![License](https://img.shields.io/badge/License-MIT-green?style=flat-square)](LICENSE)
39
+
40
+ _List and bulk-download everything you've ever starred — with live progress bars that don't suck._
41
+
42
+ </div>
43
+
44
+ ---
45
+
46
+ ## Why StarDown?
47
+
48
+ Stars are bookmarks. Bookmarks rot. Repos get deleted, renamed, DMCA'd, or force-pushed into oblivion. **StarDown turns your star list into a local archive** — snapshots or full git clones, pulled down in parallel with a gorgeous live dashboard.
49
+
50
+ - ⚡ **Async engine** — one shared HTTP session, bounded concurrency, 128 KiB streaming chunks, atomic writes, auto-resume
51
+ - 📊 **Rich live progress** — per-repo bars with speed + ETA, plus a global counter (plain fallback in pipes/CI)
52
+ - 🎯 **Two download modes** — `tar` snapshots (fast, tiny) or `git` clones (full history)
53
+ - 🔎 **Filter before you pull** — by language, topic, search text, forks, archived
54
+ - 🔑 **Auth-aware** — token support for private stars + 30x the rate limit
55
+ - 🪶 **Zero friction** — one command install, double-click launchers included
56
+
57
+ ## Install
58
+
59
+ ```sh
60
+ pip install -e .
61
+ ```
62
+
63
+ That's it. You now own the `stardown` command everywhere.
64
+
65
+ > No install? No problem — use the zero-setup launchers (they fetch deps on first run):
66
+ >
67
+ > ```bat
68
+ > .\run.bat download --user octocat
69
+ > ```
70
+ >
71
+ > ```sh
72
+ > ./run.sh download --user octocat
73
+ > ```
74
+
75
+ ## Quickstart
76
+
77
+ ```sh
78
+ # Peek at your stars in a pretty table
79
+ stardown list --user octocat
80
+
81
+ # Archive them all — fast snapshots, 16 at a time
82
+ stardown download --user octocat --out stars
83
+
84
+ # Just your Python CLI obsessions, as JSON
85
+ stardown list --user octocat --language python --topic cli --format json --out stars.json
86
+
87
+ # Your own stars via token (also unlocks private repos)
88
+ export GH_TOKEN=ghp_...
89
+ stardown download --me --out stars
90
+ ```
91
+
92
+ ## Usage
93
+
94
+ ```n/a
95
+ stardown list --user NAME [--me] [--language LANG] [--topic TOPIC]
96
+ [--search TEXT] [--no-forks] [--no-archived] [--limit N]
97
+ [--format table|json|csv] [--out FILE]
98
+
99
+ stardown download --user NAME [--out DIR] [--method tar|git]
100
+ [--workers 16] [--keep-tar] [--no-extract] [--force]
101
+ [--ssh] [--full] [--no-progress] [filters...]
102
+ ```
103
+
104
+ | Flag | What it does |
105
+ | ----------------------------- | ---------------------------------------------------------------------------- |
106
+ | `--method tar` _(default)_ | Snapshot archives via the API — 5–10x faster, no `.git` weight |
107
+ | `--method git` | Real clones (`--depth 1` unless `--full`), reruns `pull --ff-only` to update |
108
+ | `--workers N` | Parallel downloads (default 16, max 64) |
109
+ | `--keep-tar` / `--no-extract` | Keep the `.tar.gz` files / skip extracting |
110
+ | `--force` | Re-download even if already cached (`.stardown-done` marker) |
111
+ | `--no-progress` | Plain output for scripts and CI |
112
+
113
+ Auth: you probably don't need to think about it. StarDown takes `--token` if you hand it one, else `$GH_TOKEN`, else it quietly checks your `gh` login and git's own credential store — and tells you when it finds something. No login anywhere? It goes anonymous (60 API calls/hr instead of 5,000). `--no-auto-auth` skips the snooping.
114
+
115
+ ## Under the hood
116
+
117
+ - Single `httpx.AsyncClient` with keepalive reuse (fewer handshakes, fewer FDs)
118
+ - `Semaphore`-bounded concurrency — fast without tripping `EMFILE`
119
+ - Crash-safe writes: stream to `.part`, resume with `Range`, atomic `os.replace` on completion
120
+ - Blocking tar extraction offloaded to a thread so the event loop never stalls
121
+ - `uvloop` auto-engaged on Linux when installed (`pip install -e .[linux]`)
122
+ - Sync `git` path kept in `clone.py` as the simple fallback
123
+
124
+ ## Dev
125
+
126
+ ```sh
127
+ pip install -e .[test]
128
+ python -m pytest tests/ -q
129
+ ```
130
+
131
+ ```n/a
132
+ src/stardown/ api.py · async_download.py · clone.py · cli.py · ui.py
133
+ tests/ offline unit tests (no network)
134
+ run.bat · run.sh zero-install launchers
135
+ ```
136
+
137
+ ## Credits
138
+
139
+ Built by [**wydyippee**](https://github.com/wydyippee) ★ — for folk who star first and read later.
140
+
141
+ ## License
142
+
143
+ MIT — see [LICENSE](LICENSE). StarDown save your sources? Smash that star. 🌟
@@ -0,0 +1,113 @@
1
+ <div align="center">
2
+
3
+ # ★ StarDown ★
4
+
5
+ **Stars, on your disk, fast.**
6
+
7
+ [![Python](https://img.shields.io/badge/Python-3.11%2B-blue?style=flat-square&logo=python&logoColor=white)](https://www.python.org/)
8
+ [![License](https://img.shields.io/badge/License-MIT-green?style=flat-square)](LICENSE)
9
+
10
+ _List and bulk-download everything you've ever starred — with live progress bars that don't suck._
11
+
12
+ </div>
13
+
14
+ ---
15
+
16
+ ## Why StarDown?
17
+
18
+ Stars are bookmarks. Bookmarks rot. Repos get deleted, renamed, DMCA'd, or force-pushed into oblivion. **StarDown turns your star list into a local archive** — snapshots or full git clones, pulled down in parallel with a gorgeous live dashboard.
19
+
20
+ - ⚡ **Async engine** — one shared HTTP session, bounded concurrency, 128 KiB streaming chunks, atomic writes, auto-resume
21
+ - 📊 **Rich live progress** — per-repo bars with speed + ETA, plus a global counter (plain fallback in pipes/CI)
22
+ - 🎯 **Two download modes** — `tar` snapshots (fast, tiny) or `git` clones (full history)
23
+ - 🔎 **Filter before you pull** — by language, topic, search text, forks, archived
24
+ - 🔑 **Auth-aware** — token support for private stars + 30x the rate limit
25
+ - 🪶 **Zero friction** — one command install, double-click launchers included
26
+
27
+ ## Install
28
+
29
+ ```sh
30
+ pip install -e .
31
+ ```
32
+
33
+ That's it. You now own the `stardown` command everywhere.
34
+
35
+ > No install? No problem — use the zero-setup launchers (they fetch deps on first run):
36
+ >
37
+ > ```bat
38
+ > .\run.bat download --user octocat
39
+ > ```
40
+ >
41
+ > ```sh
42
+ > ./run.sh download --user octocat
43
+ > ```
44
+
45
+ ## Quickstart
46
+
47
+ ```sh
48
+ # Peek at your stars in a pretty table
49
+ stardown list --user octocat
50
+
51
+ # Archive them all — fast snapshots, 16 at a time
52
+ stardown download --user octocat --out stars
53
+
54
+ # Just your Python CLI obsessions, as JSON
55
+ stardown list --user octocat --language python --topic cli --format json --out stars.json
56
+
57
+ # Your own stars via token (also unlocks private repos)
58
+ export GH_TOKEN=ghp_...
59
+ stardown download --me --out stars
60
+ ```
61
+
62
+ ## Usage
63
+
64
+ ```n/a
65
+ stardown list --user NAME [--me] [--language LANG] [--topic TOPIC]
66
+ [--search TEXT] [--no-forks] [--no-archived] [--limit N]
67
+ [--format table|json|csv] [--out FILE]
68
+
69
+ stardown download --user NAME [--out DIR] [--method tar|git]
70
+ [--workers 16] [--keep-tar] [--no-extract] [--force]
71
+ [--ssh] [--full] [--no-progress] [filters...]
72
+ ```
73
+
74
+ | Flag | What it does |
75
+ | ----------------------------- | ---------------------------------------------------------------------------- |
76
+ | `--method tar` _(default)_ | Snapshot archives via the API — 5–10x faster, no `.git` weight |
77
+ | `--method git` | Real clones (`--depth 1` unless `--full`), reruns `pull --ff-only` to update |
78
+ | `--workers N` | Parallel downloads (default 16, max 64) |
79
+ | `--keep-tar` / `--no-extract` | Keep the `.tar.gz` files / skip extracting |
80
+ | `--force` | Re-download even if already cached (`.stardown-done` marker) |
81
+ | `--no-progress` | Plain output for scripts and CI |
82
+
83
+ Auth: you probably don't need to think about it. StarDown takes `--token` if you hand it one, else `$GH_TOKEN`, else it quietly checks your `gh` login and git's own credential store — and tells you when it finds something. No login anywhere? It goes anonymous (60 API calls/hr instead of 5,000). `--no-auto-auth` skips the snooping.
84
+
85
+ ## Under the hood
86
+
87
+ - Single `httpx.AsyncClient` with keepalive reuse (fewer handshakes, fewer FDs)
88
+ - `Semaphore`-bounded concurrency — fast without tripping `EMFILE`
89
+ - Crash-safe writes: stream to `.part`, resume with `Range`, atomic `os.replace` on completion
90
+ - Blocking tar extraction offloaded to a thread so the event loop never stalls
91
+ - `uvloop` auto-engaged on Linux when installed (`pip install -e .[linux]`)
92
+ - Sync `git` path kept in `clone.py` as the simple fallback
93
+
94
+ ## Dev
95
+
96
+ ```sh
97
+ pip install -e .[test]
98
+ python -m pytest tests/ -q
99
+ ```
100
+
101
+ ```n/a
102
+ src/stardown/ api.py · async_download.py · clone.py · cli.py · ui.py
103
+ tests/ offline unit tests (no network)
104
+ run.bat · run.sh zero-install launchers
105
+ ```
106
+
107
+ ## Credits
108
+
109
+ Built by [**wydyippee**](https://github.com/wydyippee) ★ — for folk who star first and read later.
110
+
111
+ ## License
112
+
113
+ MIT — see [LICENSE](LICENSE). StarDown save your sources? Smash that star. 🌟
@@ -0,0 +1,43 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "stardown"
7
+ version = "0.3.0"
8
+ description = "StarDown (GitHub Starred Downloader) — async fetch with rich live progress"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ authors = [{name = "wydyippee"}]
12
+ license = {text = "MIT"}
13
+ keywords = ["github", "stars", "backup", "downloader", "archive"]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Environment :: Console",
17
+ "Intended Audience :: Developers",
18
+ "License :: OSI Approved :: MIT License",
19
+ "Operating System :: OS Independent",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Topic :: Utilities",
24
+ ]
25
+ dependencies = [
26
+ "httpx>=0.27",
27
+ "rich>=13",
28
+ ]
29
+
30
+ [project.urls]
31
+ Homepage = "https://github.com/wydyippee/StarDown"
32
+ Repository = "https://github.com/wydyippee/StarDown"
33
+ Issues = "https://github.com/wydyippee/StarDown/issues"
34
+
35
+ [project.optional-dependencies]
36
+ linux = ["uvloop>=0.19"]
37
+ test = ["pytest>=8"]
38
+
39
+ [project.scripts]
40
+ stardown = "stardown.cli:main"
41
+
42
+ [tool.setuptools.packages.find]
43
+ where = ["."]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,4 @@
1
+ """StarDown — back up your GitHub stars before they rot."""
2
+
3
+ __version__ = "0.3.0"
4
+ __all__ = ["__version__"]
@@ -0,0 +1,4 @@
1
+ from .cli import main
2
+
3
+ if __name__ == "__main__":
4
+ raise SystemExit(main())
@@ -0,0 +1,121 @@
1
+ """GitHub API calls. Sync stdlib, on purpose.
2
+
3
+ Listing stars is a handful of paginated GETs per run — nothing that justifies
4
+ an async client here. The hot path (streaming tarballs) lives in
5
+ async_download.py.
6
+ """
7
+
8
+ import json
9
+ import urllib.error
10
+ import urllib.request
11
+
12
+ API = "https://api.github.com"
13
+
14
+
15
+ def _headers(token=None):
16
+ headers = {
17
+ "Accept": "application/vnd.github+json",
18
+ "User-Agent": "StarDown/0.3.0",
19
+ "X-GitHub-Api-Version": "2022-11-28",
20
+ }
21
+ if token:
22
+ headers["Authorization"] = f"Bearer {token}"
23
+ return headers
24
+
25
+
26
+ def _get_json(url, token, timeout):
27
+ req = urllib.request.Request(url, headers=_headers(token))
28
+ try:
29
+ with urllib.request.urlopen(req, timeout=timeout) as res:
30
+ return json.load(res), dict(res.headers)
31
+ except urllib.error.HTTPError as e:
32
+ try:
33
+ body = e.read().decode("utf-8", "replace")[:500]
34
+ except Exception:
35
+ body = ""
36
+ if e.code == 404:
37
+ raise RuntimeError(f"GitHub 404 for {url}: user/repo not found. {body}")
38
+ if e.code in (401, 403):
39
+ # 403 almost always means the anonymous rate limit, not bad auth.
40
+ # Worth spelling out because it bites everyone exactly once.
41
+ reset = e.headers.get("x-ratelimit-reset", "")
42
+ msg = f"GitHub {e.code} for {url}."
43
+ if "rate limit" in body.lower() or e.code == 403:
44
+ msg += " Rate limited — pass --token or set GH_TOKEN to raise limit from 60/hr."
45
+ if reset:
46
+ msg += f" Resets at epoch {reset}."
47
+ else:
48
+ msg += f" Check token. {body}"
49
+ raise RuntimeError(msg)
50
+ raise RuntimeError(f"GitHub {e.code} for {url}: {body}")
51
+
52
+
53
+ def normalize(r):
54
+ # Trim the API payload down to what the rest of the tool actually touches.
55
+ full = r.get("full_name", "")
56
+ branch = r.get("default_branch") or ""
57
+ return {
58
+ "full_name": full,
59
+ "html_url": r.get("html_url", ""),
60
+ "clone_url": r.get("clone_url", ""),
61
+ "ssh_url": r.get("ssh_url", ""),
62
+ "tarball_url": f"{API}/repos/{full}/tarball/{branch}" if full else "",
63
+ "default_branch": branch,
64
+ "description": r.get("description") or "",
65
+ "language": r.get("language") or "",
66
+ "topics": r.get("topics") or [],
67
+ "stars": r.get("stargazers_count", 0),
68
+ "fork": bool(r.get("fork", False)),
69
+ "archived": bool(r.get("archived", False)),
70
+ "pushed_at": r.get("pushed_at") or "",
71
+ }
72
+
73
+
74
+ def get_authenticated_user(token, timeout=20):
75
+ data, _ = _get_json(f"{API}/user", token, timeout)
76
+ login = data.get("login", "")
77
+ if not login:
78
+ raise RuntimeError("Could not resolve authenticated user from token.")
79
+ return login
80
+
81
+
82
+ def fetch_starred(user, token=None, per_page=100, limit=0, timeout=20):
83
+ # Walk pages until a short page arrives. limit=0 means "all of them".
84
+ out = []
85
+ page = 1
86
+ per_page = max(1, min(100, per_page))
87
+ while True:
88
+ url = f"{API}/users/{user}/starred?per_page={per_page}&page={page}"
89
+ data, _ = _get_json(url, token, timeout)
90
+ if not isinstance(data, list):
91
+ raise RuntimeError(f"Unexpected response for {url}: {str(data)[:200]}")
92
+ out.extend(normalize(r) for r in data)
93
+ if limit and len(out) >= limit:
94
+ return out[:limit]
95
+ if len(data) < per_page or page >= 100:
96
+ break
97
+ page += 1
98
+ return out
99
+
100
+
101
+ def filter_repos(repos, language=None, topic=None, query=None,
102
+ include_forks=True, include_archived=True):
103
+ lang = language.lower() if language else None
104
+ top = topic.lower() if topic else None
105
+ q = query.lower() if query else None
106
+
107
+ def keep(r):
108
+ if not include_forks and r.get("fork"):
109
+ return False
110
+ if not include_archived and r.get("archived"):
111
+ return False
112
+ if lang and (r.get("language") or "").lower() != lang:
113
+ return False
114
+ if top and top not in [t.lower() for t in r.get("topics", [])]:
115
+ return False
116
+ if q and q not in r.get("full_name", "").lower() \
117
+ and q not in (r.get("description") or "").lower():
118
+ return False
119
+ return True
120
+
121
+ return [r for r in repos if keep(r)]
@@ -0,0 +1,216 @@
1
+ """The fast path: stream repo tarballs concurrently, fall back to git.
2
+
3
+ Why it's shaped like this:
4
+ - one shared AsyncClient so connections get reused instead of re-handshaked
5
+ - a semaphore around everything so we can't run the FD table dry
6
+ - big read chunks, because syscalls aren't free
7
+ - .part files + atomic rename, so a killed run never leaves a half file
8
+ wearing a finished file's name
9
+ - tar extraction in a thread — tarfile would stall the loop otherwise
10
+ - uvloop when it's around (Linux), plain asyncio everywhere else
11
+ """
12
+
13
+ import asyncio
14
+ import os
15
+ import tarfile
16
+ import time
17
+ from pathlib import Path
18
+
19
+ import httpx
20
+
21
+ CHUNK = 128 * 1024
22
+ RETRIES = 3
23
+
24
+
25
+ def _maybe_uvloop():
26
+ try:
27
+ import uvloop
28
+ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
29
+ except ImportError:
30
+ pass
31
+
32
+
33
+ def _paths(repo, out_dir):
34
+ dest = Path(out_dir) / repo["full_name"]
35
+ archive = dest.parent / (dest.name + ".tar.gz")
36
+ part = archive.with_suffix(".tar.gz.part")
37
+ done = dest / ".stardown-done"
38
+ return dest, archive, part, done
39
+
40
+
41
+ def _extract_strip_top(archive, dest):
42
+ # GitHub wraps every tarball in one top-level dir (owner-repo-sha).
43
+ # Nobody wants that dir, so peel it off on the way in. Blocking — the
44
+ # caller runs this in a thread.
45
+ dest.mkdir(parents=True, exist_ok=True)
46
+ with tarfile.open(archive, "r:gz") as tf:
47
+ members = tf.getmembers()
48
+ prefix = os.path.commonprefix([m.name for m in members if m.name])
49
+ if "/" in prefix:
50
+ # commonprefix works on characters, not path segments — back up
51
+ # to the last slash or we'd mangle sibling names.
52
+ prefix = prefix.rsplit("/", 1)[0] + "/"
53
+ else:
54
+ prefix = ""
55
+ for m in members:
56
+ name = m.name
57
+ if prefix and name.startswith(prefix):
58
+ name = name[len(prefix):]
59
+ if not name or name in ("/", "."):
60
+ continue
61
+ m.name = name
62
+ tf.extract(m, dest, filter="data")
63
+
64
+
65
+ async def _git(args, timeout):
66
+ p = await asyncio.create_subprocess_exec(
67
+ "git", *args,
68
+ stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
69
+ )
70
+ try:
71
+ out, err = await asyncio.wait_for(p.communicate(), timeout)
72
+ except asyncio.TimeoutError:
73
+ p.kill()
74
+ raise RuntimeError("git timed out")
75
+ if p.returncode != 0:
76
+ raise RuntimeError((err or out).decode("utf-8", "replace").strip()[-300:])
77
+
78
+
79
+ async def _git_one(repo, out_dir, sem, use_ssh, shallow, timeout):
80
+ dest = Path(out_dir) / repo["full_name"]
81
+ async with sem:
82
+ if (dest / ".git").is_dir():
83
+ await _git(["-C", str(dest), "pull", "--ff-only", "--quiet"], timeout)
84
+ return "updated"
85
+ url = repo["ssh_url"] if use_ssh else repo["clone_url"]
86
+ if not url:
87
+ raise RuntimeError("no clone URL")
88
+ dest.parent.mkdir(parents=True, exist_ok=True)
89
+ args = ["clone", "--quiet"]
90
+ if shallow:
91
+ args += ["--depth", "1"]
92
+ await _git([*args, url, str(dest)], timeout)
93
+ return "cloned"
94
+
95
+
96
+ async def _tar_one(client, repo, out_dir, sem, progress, overall,
97
+ keep_tar, extract, force, retries):
98
+ name = repo["full_name"]
99
+ dest, archive, part, done = _paths(repo, out_dir)
100
+ if done.exists() and not force and dest.is_dir():
101
+ if progress is not None and overall is not None:
102
+ progress.advance(overall)
103
+ return "cached"
104
+ url = repo.get("tarball_url") or (
105
+ f"https://api.github.com/repos/{name}/tarball")
106
+ last_err = "unknown"
107
+ async with sem:
108
+ for attempt in range(1, retries + 1):
109
+ task = None
110
+ try:
111
+ dest.parent.mkdir(parents=True, exist_ok=True)
112
+ resume_from = part.stat().st_size if part.exists() else 0
113
+ headers = {}
114
+ if resume_from:
115
+ headers["Range"] = f"bytes={resume_from}-"
116
+ async with client.stream("GET", url, headers=headers) as resp:
117
+ if resp.status_code == 404:
118
+ raise RuntimeError("repo/tarball not found (private? need --token)")
119
+ if resp.status_code in (403, 429):
120
+ retry_after = resp.headers.get("retry-after")
121
+ wait = int(retry_after) if str(retry_after or "").isdigit() else 2 ** attempt
122
+ raise RuntimeError(f"rate limited, retry in {wait}s")
123
+ if resp.status_code not in (200, 206):
124
+ raise RuntimeError(f"HTTP {resp.status_code}")
125
+ if resp.status_code == 200 and resume_from:
126
+ resume_from = 0 # server shrugged at Range; start over
127
+ total = int(resp.headers.get("content-length") or 0)
128
+ if resp.status_code == 206:
129
+ total += resume_from
130
+ if progress is not None:
131
+ task = progress.add_task("dl", name=name,
132
+ total=total or None,
133
+ completed=resume_from)
134
+ mode = "ab" if resume_from else "wb"
135
+ loop = asyncio.get_running_loop()
136
+ with open(part, mode) as f:
137
+ async for chunk in resp.aiter_bytes(CHUNK):
138
+ if not chunk:
139
+ continue
140
+ # Synchronous write inside an async loop looks
141
+ # wrong until you remember the network is ~1000x
142
+ # slower than the disk. It never shows up.
143
+ f.write(chunk)
144
+ if progress is not None and task is not None:
145
+ progress.update(task, advance=len(chunk))
146
+ os.replace(part, archive)
147
+ if extract:
148
+ await loop.run_in_executor(None, _extract_strip_top, archive, dest)
149
+ if not keep_tar:
150
+ archive.unlink(missing_ok=True)
151
+ done.write_text(f"{name} {time.time():.0f}\n")
152
+ if progress is not None:
153
+ if task is not None:
154
+ progress.update(task, visible=False)
155
+ if overall is not None:
156
+ progress.advance(overall)
157
+ return "resumed" if resume_from else "downloaded"
158
+ except Exception as e:
159
+ last_err = str(e)[:200]
160
+ if progress is not None and task is not None:
161
+ progress.update(task, visible=False)
162
+ if attempt < retries:
163
+ await asyncio.sleep(2 ** attempt)
164
+ if progress is not None and overall is not None:
165
+ progress.advance(overall)
166
+ raise RuntimeError(last_err)
167
+
168
+
169
+ async def _amain(repos, out_dir, workers, method, token, progress,
170
+ use_ssh, shallow, git_timeout, keep_tar, extract, force):
171
+ _maybe_uvloop()
172
+ sem = asyncio.Semaphore(max(1, min(64, workers)))
173
+ headers = {"User-Agent": "StarDown/0.3.0",
174
+ "Accept": "application/vnd.github+json"}
175
+ if token:
176
+ headers["Authorization"] = f"Bearer {token}"
177
+ limits = httpx.Limits(max_connections=max(1, workers),
178
+ max_keepalive_connections=max(1, workers))
179
+ timeout = httpx.Timeout(connect=10.0, read=60.0, write=10.0, pool=10.0)
180
+ overall = progress.add_task("all", name="total",
181
+ total=len(repos)) if progress is not None else None
182
+ results = []
183
+
184
+ if method == "git":
185
+ async def one_git(r):
186
+ try:
187
+ s = await _git_one(r, out_dir, sem, use_ssh, shallow, git_timeout)
188
+ results.append((r["full_name"], s, ""))
189
+ except Exception as e:
190
+ results.append((r["full_name"], "failed", str(e)[:300]))
191
+ finally:
192
+ if progress is not None and overall is not None:
193
+ progress.advance(overall)
194
+ await asyncio.gather(*(one_git(r) for r in repos))
195
+ return results
196
+
197
+ async with httpx.AsyncClient(headers=headers, limits=limits,
198
+ timeout=timeout, follow_redirects=True) as client:
199
+ async def one_tar(r):
200
+ try:
201
+ s = await _tar_one(client, r, out_dir, sem, progress, overall,
202
+ keep_tar, extract, force, RETRIES)
203
+ results.append((r["full_name"], s, ""))
204
+ except Exception as e:
205
+ results.append((r["full_name"], "failed", str(e)[:300]))
206
+ await asyncio.gather(*(one_tar(r) for r in repos))
207
+ return results
208
+
209
+
210
+ def download_repos(repos, out_dir, workers=16, method="tar", token=None,
211
+ progress=None, use_ssh=False, shallow=True, git_timeout=300,
212
+ keep_tar=False, extract=True, force=False):
213
+ # The CLI is sync; this is the one place asyncio gets entered.
214
+ return asyncio.run(_amain(repos, out_dir, workers, method, token, progress,
215
+ use_ssh, shallow, git_timeout,
216
+ keep_tar, extract, force))