stardown 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stardown-0.3.0/LICENSE +21 -0
- stardown-0.3.0/PKG-INFO +143 -0
- stardown-0.3.0/README.md +113 -0
- stardown-0.3.0/pyproject.toml +43 -0
- stardown-0.3.0/setup.cfg +4 -0
- stardown-0.3.0/stardown/__init__.py +4 -0
- stardown-0.3.0/stardown/__main__.py +4 -0
- stardown-0.3.0/stardown/api.py +121 -0
- stardown-0.3.0/stardown/async_download.py +216 -0
- stardown-0.3.0/stardown/auth.py +80 -0
- stardown-0.3.0/stardown/cli.py +151 -0
- stardown-0.3.0/stardown/clone.py +55 -0
- stardown-0.3.0/stardown/ui.py +78 -0
- stardown-0.3.0/stardown.egg-info/PKG-INFO +143 -0
- stardown-0.3.0/stardown.egg-info/SOURCES.txt +21 -0
- stardown-0.3.0/stardown.egg-info/dependency_links.txt +1 -0
- stardown-0.3.0/stardown.egg-info/entry_points.txt +2 -0
- stardown-0.3.0/stardown.egg-info/requires.txt +8 -0
- stardown-0.3.0/stardown.egg-info/top_level.txt +3 -0
- stardown-0.3.0/tests/test_api_filter.py +33 -0
- stardown-0.3.0/tests/test_async_engine.py +42 -0
- stardown-0.3.0/tests/test_auth.py +85 -0
- stardown-0.3.0/tests/test_download_e2e.py +57 -0
stardown-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 wydyippee
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
stardown-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stardown
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: StarDown (GitHub Starred Downloader) — async fetch with rich live progress
|
|
5
|
+
Author: wydyippee
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/wydyippee/StarDown
|
|
8
|
+
Project-URL: Repository, https://github.com/wydyippee/StarDown
|
|
9
|
+
Project-URL: Issues, https://github.com/wydyippee/StarDown/issues
|
|
10
|
+
Keywords: github,stars,backup,downloader,archive
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Utilities
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Requires-Dist: rich>=13
|
|
25
|
+
Provides-Extra: linux
|
|
26
|
+
Requires-Dist: uvloop>=0.19; extra == "linux"
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
<div align="center">
|
|
32
|
+
|
|
33
|
+
# ★ StarDown ★
|
|
34
|
+
|
|
35
|
+
**Stars, on your disk, fast.**
|
|
36
|
+
|
|
37
|
+
[](https://www.python.org/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
_List and bulk-download everything you've ever starred — with live progress bars that don't suck._
|
|
41
|
+
|
|
42
|
+
</div>
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Why StarDown?
|
|
47
|
+
|
|
48
|
+
Stars are bookmarks. Bookmarks rot. Repos get deleted, renamed, DMCA'd, or force-pushed into oblivion. **StarDown turns your star list into a local archive** — snapshots or full git clones, pulled down in parallel with a gorgeous live dashboard.
|
|
49
|
+
|
|
50
|
+
- ⚡ **Async engine** — one shared HTTP session, bounded concurrency, 128 KiB streaming chunks, atomic writes, auto-resume
|
|
51
|
+
- 📊 **Rich live progress** — per-repo bars with speed + ETA, plus a global counter (plain fallback in pipes/CI)
|
|
52
|
+
- 🎯 **Two download modes** — `tar` snapshots (fast, tiny) or `git` clones (full history)
|
|
53
|
+
- 🔎 **Filter before you pull** — by language, topic, search text, forks, archived
|
|
54
|
+
- 🔑 **Auth-aware** — token support for private stars + 30x the rate limit
|
|
55
|
+
- 🪶 **Zero friction** — one command install, double-click launchers included
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
pip install -e .
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
That's it. You now own the `stardown` command everywhere.
|
|
64
|
+
|
|
65
|
+
> No install? No problem — use the zero-setup launchers (they fetch deps on first run):
|
|
66
|
+
>
|
|
67
|
+
> ```bat
|
|
68
|
+
> .\run.bat download --user octocat
|
|
69
|
+
> ```
|
|
70
|
+
>
|
|
71
|
+
> ```sh
|
|
72
|
+
> ./run.sh download --user octocat
|
|
73
|
+
> ```
|
|
74
|
+
|
|
75
|
+
## Quickstart
|
|
76
|
+
|
|
77
|
+
```sh
|
|
78
|
+
# Peek at your stars in a pretty table
|
|
79
|
+
stardown list --user octocat
|
|
80
|
+
|
|
81
|
+
# Archive them all — fast snapshots, 16 at a time
|
|
82
|
+
stardown download --user octocat --out stars
|
|
83
|
+
|
|
84
|
+
# Just your Python CLI obsessions, as JSON
|
|
85
|
+
stardown list --user octocat --language python --topic cli --format json --out stars.json
|
|
86
|
+
|
|
87
|
+
# Your own stars via token (also unlocks private repos)
|
|
88
|
+
export GH_TOKEN=ghp_...
|
|
89
|
+
stardown download --me --out stars
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Usage
|
|
93
|
+
|
|
94
|
+
```n/a
|
|
95
|
+
stardown list --user NAME [--me] [--language LANG] [--topic TOPIC]
|
|
96
|
+
[--search TEXT] [--no-forks] [--no-archived] [--limit N]
|
|
97
|
+
[--format table|json|csv] [--out FILE]
|
|
98
|
+
|
|
99
|
+
stardown download --user NAME [--out DIR] [--method tar|git]
|
|
100
|
+
[--workers 16] [--keep-tar] [--no-extract] [--force]
|
|
101
|
+
[--ssh] [--full] [--no-progress] [filters...]
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
| Flag | What it does |
|
|
105
|
+
| ----------------------------- | ---------------------------------------------------------------------------- |
|
|
106
|
+
| `--method tar` _(default)_ | Snapshot archives via the API — 5–10x faster, no `.git` weight |
|
|
107
|
+
| `--method git` | Real clones (`--depth 1` unless `--full`), reruns `pull --ff-only` to update |
|
|
108
|
+
| `--workers N` | Parallel downloads (default 16, max 64) |
|
|
109
|
+
| `--keep-tar` / `--no-extract` | Keep the `.tar.gz` files / skip extracting |
|
|
110
|
+
| `--force` | Re-download even if already cached (`.stardown-done` marker) |
|
|
111
|
+
| `--no-progress` | Plain output for scripts and CI |
|
|
112
|
+
|
|
113
|
+
Auth: you probably don't need to think about it. StarDown takes `--token` if you hand it one, else `$GH_TOKEN`, else it quietly checks your `gh` login and git's own credential store — and tells you when it finds something. No login anywhere? It goes anonymous (60 API calls/hr instead of 5,000). `--no-auto-auth` skips the snooping.
|
|
114
|
+
|
|
115
|
+
## Under the hood
|
|
116
|
+
|
|
117
|
+
- Single `httpx.AsyncClient` with keepalive reuse (fewer handshakes, fewer FDs)
|
|
118
|
+
- `Semaphore`-bounded concurrency — fast without tripping `EMFILE`
|
|
119
|
+
- Crash-safe writes: stream to `.part`, resume with `Range`, atomic `os.replace` on completion
|
|
120
|
+
- Blocking tar extraction offloaded to a thread so the event loop never stalls
|
|
121
|
+
- `uvloop` auto-engaged on Linux when installed (`pip install -e .[linux]`)
|
|
122
|
+
- Sync `git` path kept in `clone.py` as the simple fallback
|
|
123
|
+
|
|
124
|
+
## Dev
|
|
125
|
+
|
|
126
|
+
```sh
|
|
127
|
+
pip install -e .[test]
|
|
128
|
+
python -m pytest tests/ -q
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
```n/a
|
|
132
|
+
src/stardown/ api.py · async_download.py · clone.py · cli.py · ui.py
|
|
133
|
+
tests/ offline unit tests (no network)
|
|
134
|
+
run.bat · run.sh zero-install launchers
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Credits
|
|
138
|
+
|
|
139
|
+
Built by [**wydyippee**](https://github.com/wydyippee) ★ — for folk who star first and read later.
|
|
140
|
+
|
|
141
|
+
## License
|
|
142
|
+
|
|
143
|
+
MIT — see [LICENSE](LICENSE). StarDown save your sources? Smash that star. 🌟
|
stardown-0.3.0/README.md
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# ★ StarDown ★
|
|
4
|
+
|
|
5
|
+
**Stars, on your disk, fast.**
|
|
6
|
+
|
|
7
|
+
[](https://www.python.org/)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
10
|
+
_List and bulk-download everything you've ever starred — with live progress bars that don't suck._
|
|
11
|
+
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## Why StarDown?
|
|
17
|
+
|
|
18
|
+
Stars are bookmarks. Bookmarks rot. Repos get deleted, renamed, DMCA'd, or force-pushed into oblivion. **StarDown turns your star list into a local archive** — snapshots or full git clones, pulled down in parallel with a gorgeous live dashboard.
|
|
19
|
+
|
|
20
|
+
- ⚡ **Async engine** — one shared HTTP session, bounded concurrency, 128 KiB streaming chunks, atomic writes, auto-resume
|
|
21
|
+
- 📊 **Rich live progress** — per-repo bars with speed + ETA, plus a global counter (plain fallback in pipes/CI)
|
|
22
|
+
- 🎯 **Two download modes** — `tar` snapshots (fast, tiny) or `git` clones (full history)
|
|
23
|
+
- 🔎 **Filter before you pull** — by language, topic, search text, forks, archived
|
|
24
|
+
- 🔑 **Auth-aware** — token support for private stars + 30x the rate limit
|
|
25
|
+
- 🪶 **Zero friction** — one command install, double-click launchers included
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
pip install -e .
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
That's it. You now own the `stardown` command everywhere.
|
|
34
|
+
|
|
35
|
+
> No install? No problem — use the zero-setup launchers (they fetch deps on first run):
|
|
36
|
+
>
|
|
37
|
+
> ```bat
|
|
38
|
+
> .\run.bat download --user octocat
|
|
39
|
+
> ```
|
|
40
|
+
>
|
|
41
|
+
> ```sh
|
|
42
|
+
> ./run.sh download --user octocat
|
|
43
|
+
> ```
|
|
44
|
+
|
|
45
|
+
## Quickstart
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
# Peek at your stars in a pretty table
|
|
49
|
+
stardown list --user octocat
|
|
50
|
+
|
|
51
|
+
# Archive them all — fast snapshots, 16 at a time
|
|
52
|
+
stardown download --user octocat --out stars
|
|
53
|
+
|
|
54
|
+
# Just your Python CLI obsessions, as JSON
|
|
55
|
+
stardown list --user octocat --language python --topic cli --format json --out stars.json
|
|
56
|
+
|
|
57
|
+
# Your own stars via token (also unlocks private repos)
|
|
58
|
+
export GH_TOKEN=ghp_...
|
|
59
|
+
stardown download --me --out stars
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Usage
|
|
63
|
+
|
|
64
|
+
```n/a
|
|
65
|
+
stardown list --user NAME [--me] [--language LANG] [--topic TOPIC]
|
|
66
|
+
[--search TEXT] [--no-forks] [--no-archived] [--limit N]
|
|
67
|
+
[--format table|json|csv] [--out FILE]
|
|
68
|
+
|
|
69
|
+
stardown download --user NAME [--out DIR] [--method tar|git]
|
|
70
|
+
[--workers 16] [--keep-tar] [--no-extract] [--force]
|
|
71
|
+
[--ssh] [--full] [--no-progress] [filters...]
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
| Flag | What it does |
|
|
75
|
+
| ----------------------------- | ---------------------------------------------------------------------------- |
|
|
76
|
+
| `--method tar` _(default)_ | Snapshot archives via the API — 5–10x faster, no `.git` weight |
|
|
77
|
+
| `--method git` | Real clones (`--depth 1` unless `--full`), reruns `pull --ff-only` to update |
|
|
78
|
+
| `--workers N` | Parallel downloads (default 16, max 64) |
|
|
79
|
+
| `--keep-tar` / `--no-extract` | Keep the `.tar.gz` files / skip extracting |
|
|
80
|
+
| `--force` | Re-download even if already cached (`.stardown-done` marker) |
|
|
81
|
+
| `--no-progress` | Plain output for scripts and CI |
|
|
82
|
+
|
|
83
|
+
Auth: you probably don't need to think about it. StarDown takes `--token` if you hand it one, else `$GH_TOKEN`, else it quietly checks your `gh` login and git's own credential store — and tells you when it finds something. No login anywhere? It goes anonymous (60 API calls/hr instead of 5,000). `--no-auto-auth` skips the snooping.
|
|
84
|
+
|
|
85
|
+
## Under the hood
|
|
86
|
+
|
|
87
|
+
- Single `httpx.AsyncClient` with keepalive reuse (fewer handshakes, fewer FDs)
|
|
88
|
+
- `Semaphore`-bounded concurrency — fast without tripping `EMFILE`
|
|
89
|
+
- Crash-safe writes: stream to `.part`, resume with `Range`, atomic `os.replace` on completion
|
|
90
|
+
- Blocking tar extraction offloaded to a thread so the event loop never stalls
|
|
91
|
+
- `uvloop` auto-engaged on Linux when installed (`pip install -e .[linux]`)
|
|
92
|
+
- Sync `git` path kept in `clone.py` as the simple fallback
|
|
93
|
+
|
|
94
|
+
## Dev
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
pip install -e .[test]
|
|
98
|
+
python -m pytest tests/ -q
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```n/a
|
|
102
|
+
src/stardown/ api.py · async_download.py · clone.py · cli.py · ui.py
|
|
103
|
+
tests/ offline unit tests (no network)
|
|
104
|
+
run.bat · run.sh zero-install launchers
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Credits
|
|
108
|
+
|
|
109
|
+
Built by [**wydyippee**](https://github.com/wydyippee) ★ — for folk who star first and read later.
|
|
110
|
+
|
|
111
|
+
## License
|
|
112
|
+
|
|
113
|
+
MIT — see [LICENSE](LICENSE). StarDown save your sources? Smash that star. 🌟
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stardown"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "StarDown (GitHub Starred Downloader) — async fetch with rich live progress"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [{name = "wydyippee"}]
|
|
12
|
+
license = {text = "MIT"}
|
|
13
|
+
keywords = ["github", "stars", "backup", "downloader", "archive"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Topic :: Utilities",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"httpx>=0.27",
|
|
27
|
+
"rich>=13",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/wydyippee/StarDown"
|
|
32
|
+
Repository = "https://github.com/wydyippee/StarDown"
|
|
33
|
+
Issues = "https://github.com/wydyippee/StarDown/issues"
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
linux = ["uvloop>=0.19"]
|
|
37
|
+
test = ["pytest>=8"]
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
stardown = "stardown.cli:main"
|
|
41
|
+
|
|
42
|
+
[tool.setuptools.packages.find]
|
|
43
|
+
where = ["."]
|
stardown-0.3.0/setup.cfg
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""GitHub API calls. Sync stdlib, on purpose.
|
|
2
|
+
|
|
3
|
+
Listing stars is a handful of paginated GETs per run — nothing that justifies
|
|
4
|
+
an async client here. The hot path (streaming tarballs) lives in
|
|
5
|
+
async_download.py.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
|
|
12
|
+
API = "https://api.github.com"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _headers(token=None):
|
|
16
|
+
headers = {
|
|
17
|
+
"Accept": "application/vnd.github+json",
|
|
18
|
+
"User-Agent": "StarDown/0.3.0",
|
|
19
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
20
|
+
}
|
|
21
|
+
if token:
|
|
22
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
23
|
+
return headers
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _get_json(url, token, timeout):
|
|
27
|
+
req = urllib.request.Request(url, headers=_headers(token))
|
|
28
|
+
try:
|
|
29
|
+
with urllib.request.urlopen(req, timeout=timeout) as res:
|
|
30
|
+
return json.load(res), dict(res.headers)
|
|
31
|
+
except urllib.error.HTTPError as e:
|
|
32
|
+
try:
|
|
33
|
+
body = e.read().decode("utf-8", "replace")[:500]
|
|
34
|
+
except Exception:
|
|
35
|
+
body = ""
|
|
36
|
+
if e.code == 404:
|
|
37
|
+
raise RuntimeError(f"GitHub 404 for {url}: user/repo not found. {body}")
|
|
38
|
+
if e.code in (401, 403):
|
|
39
|
+
# 403 almost always means the anonymous rate limit, not bad auth.
|
|
40
|
+
# Worth spelling out because it bites everyone exactly once.
|
|
41
|
+
reset = e.headers.get("x-ratelimit-reset", "")
|
|
42
|
+
msg = f"GitHub {e.code} for {url}."
|
|
43
|
+
if "rate limit" in body.lower() or e.code == 403:
|
|
44
|
+
msg += " Rate limited — pass --token or set GH_TOKEN to raise limit from 60/hr."
|
|
45
|
+
if reset:
|
|
46
|
+
msg += f" Resets at epoch {reset}."
|
|
47
|
+
else:
|
|
48
|
+
msg += f" Check token. {body}"
|
|
49
|
+
raise RuntimeError(msg)
|
|
50
|
+
raise RuntimeError(f"GitHub {e.code} for {url}: {body}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def normalize(r):
|
|
54
|
+
# Trim the API payload down to what the rest of the tool actually touches.
|
|
55
|
+
full = r.get("full_name", "")
|
|
56
|
+
branch = r.get("default_branch") or ""
|
|
57
|
+
return {
|
|
58
|
+
"full_name": full,
|
|
59
|
+
"html_url": r.get("html_url", ""),
|
|
60
|
+
"clone_url": r.get("clone_url", ""),
|
|
61
|
+
"ssh_url": r.get("ssh_url", ""),
|
|
62
|
+
"tarball_url": f"{API}/repos/{full}/tarball/{branch}" if full else "",
|
|
63
|
+
"default_branch": branch,
|
|
64
|
+
"description": r.get("description") or "",
|
|
65
|
+
"language": r.get("language") or "",
|
|
66
|
+
"topics": r.get("topics") or [],
|
|
67
|
+
"stars": r.get("stargazers_count", 0),
|
|
68
|
+
"fork": bool(r.get("fork", False)),
|
|
69
|
+
"archived": bool(r.get("archived", False)),
|
|
70
|
+
"pushed_at": r.get("pushed_at") or "",
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def get_authenticated_user(token, timeout=20):
|
|
75
|
+
data, _ = _get_json(f"{API}/user", token, timeout)
|
|
76
|
+
login = data.get("login", "")
|
|
77
|
+
if not login:
|
|
78
|
+
raise RuntimeError("Could not resolve authenticated user from token.")
|
|
79
|
+
return login
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def fetch_starred(user, token=None, per_page=100, limit=0, timeout=20):
|
|
83
|
+
# Walk pages until a short page arrives. limit=0 means "all of them".
|
|
84
|
+
out = []
|
|
85
|
+
page = 1
|
|
86
|
+
per_page = max(1, min(100, per_page))
|
|
87
|
+
while True:
|
|
88
|
+
url = f"{API}/users/{user}/starred?per_page={per_page}&page={page}"
|
|
89
|
+
data, _ = _get_json(url, token, timeout)
|
|
90
|
+
if not isinstance(data, list):
|
|
91
|
+
raise RuntimeError(f"Unexpected response for {url}: {str(data)[:200]}")
|
|
92
|
+
out.extend(normalize(r) for r in data)
|
|
93
|
+
if limit and len(out) >= limit:
|
|
94
|
+
return out[:limit]
|
|
95
|
+
if len(data) < per_page or page >= 100:
|
|
96
|
+
break
|
|
97
|
+
page += 1
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def filter_repos(repos, language=None, topic=None, query=None,
|
|
102
|
+
include_forks=True, include_archived=True):
|
|
103
|
+
lang = language.lower() if language else None
|
|
104
|
+
top = topic.lower() if topic else None
|
|
105
|
+
q = query.lower() if query else None
|
|
106
|
+
|
|
107
|
+
def keep(r):
|
|
108
|
+
if not include_forks and r.get("fork"):
|
|
109
|
+
return False
|
|
110
|
+
if not include_archived and r.get("archived"):
|
|
111
|
+
return False
|
|
112
|
+
if lang and (r.get("language") or "").lower() != lang:
|
|
113
|
+
return False
|
|
114
|
+
if top and top not in [t.lower() for t in r.get("topics", [])]:
|
|
115
|
+
return False
|
|
116
|
+
if q and q not in r.get("full_name", "").lower() \
|
|
117
|
+
and q not in (r.get("description") or "").lower():
|
|
118
|
+
return False
|
|
119
|
+
return True
|
|
120
|
+
|
|
121
|
+
return [r for r in repos if keep(r)]
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""The fast path: stream repo tarballs concurrently, fall back to git.
|
|
2
|
+
|
|
3
|
+
Why it's shaped like this:
|
|
4
|
+
- one shared AsyncClient so connections get reused instead of re-handshaked
|
|
5
|
+
- a semaphore around everything so we can't run the FD table dry
|
|
6
|
+
- big read chunks, because syscalls aren't free
|
|
7
|
+
- .part files + atomic rename, so a killed run never leaves a half file
|
|
8
|
+
wearing a finished file's name
|
|
9
|
+
- tar extraction in a thread — tarfile would stall the loop otherwise
|
|
10
|
+
- uvloop when it's around (Linux), plain asyncio everywhere else
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import os
|
|
15
|
+
import tarfile
|
|
16
|
+
import time
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import httpx
|
|
20
|
+
|
|
21
|
+
CHUNK = 128 * 1024
|
|
22
|
+
RETRIES = 3
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _maybe_uvloop():
|
|
26
|
+
try:
|
|
27
|
+
import uvloop
|
|
28
|
+
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
|
|
29
|
+
except ImportError:
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _paths(repo, out_dir):
|
|
34
|
+
dest = Path(out_dir) / repo["full_name"]
|
|
35
|
+
archive = dest.parent / (dest.name + ".tar.gz")
|
|
36
|
+
part = archive.with_suffix(".tar.gz.part")
|
|
37
|
+
done = dest / ".stardown-done"
|
|
38
|
+
return dest, archive, part, done
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _extract_strip_top(archive, dest):
|
|
42
|
+
# GitHub wraps every tarball in one top-level dir (owner-repo-sha).
|
|
43
|
+
# Nobody wants that dir, so peel it off on the way in. Blocking — the
|
|
44
|
+
# caller runs this in a thread.
|
|
45
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
with tarfile.open(archive, "r:gz") as tf:
|
|
47
|
+
members = tf.getmembers()
|
|
48
|
+
prefix = os.path.commonprefix([m.name for m in members if m.name])
|
|
49
|
+
if "/" in prefix:
|
|
50
|
+
# commonprefix works on characters, not path segments — back up
|
|
51
|
+
# to the last slash or we'd mangle sibling names.
|
|
52
|
+
prefix = prefix.rsplit("/", 1)[0] + "/"
|
|
53
|
+
else:
|
|
54
|
+
prefix = ""
|
|
55
|
+
for m in members:
|
|
56
|
+
name = m.name
|
|
57
|
+
if prefix and name.startswith(prefix):
|
|
58
|
+
name = name[len(prefix):]
|
|
59
|
+
if not name or name in ("/", "."):
|
|
60
|
+
continue
|
|
61
|
+
m.name = name
|
|
62
|
+
tf.extract(m, dest, filter="data")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def _git(args, timeout):
|
|
66
|
+
p = await asyncio.create_subprocess_exec(
|
|
67
|
+
"git", *args,
|
|
68
|
+
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
|
|
69
|
+
)
|
|
70
|
+
try:
|
|
71
|
+
out, err = await asyncio.wait_for(p.communicate(), timeout)
|
|
72
|
+
except asyncio.TimeoutError:
|
|
73
|
+
p.kill()
|
|
74
|
+
raise RuntimeError("git timed out")
|
|
75
|
+
if p.returncode != 0:
|
|
76
|
+
raise RuntimeError((err or out).decode("utf-8", "replace").strip()[-300:])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def _git_one(repo, out_dir, sem, use_ssh, shallow, timeout):
|
|
80
|
+
dest = Path(out_dir) / repo["full_name"]
|
|
81
|
+
async with sem:
|
|
82
|
+
if (dest / ".git").is_dir():
|
|
83
|
+
await _git(["-C", str(dest), "pull", "--ff-only", "--quiet"], timeout)
|
|
84
|
+
return "updated"
|
|
85
|
+
url = repo["ssh_url"] if use_ssh else repo["clone_url"]
|
|
86
|
+
if not url:
|
|
87
|
+
raise RuntimeError("no clone URL")
|
|
88
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
89
|
+
args = ["clone", "--quiet"]
|
|
90
|
+
if shallow:
|
|
91
|
+
args += ["--depth", "1"]
|
|
92
|
+
await _git([*args, url, str(dest)], timeout)
|
|
93
|
+
return "cloned"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def _tar_one(client, repo, out_dir, sem, progress, overall,
|
|
97
|
+
keep_tar, extract, force, retries):
|
|
98
|
+
name = repo["full_name"]
|
|
99
|
+
dest, archive, part, done = _paths(repo, out_dir)
|
|
100
|
+
if done.exists() and not force and dest.is_dir():
|
|
101
|
+
if progress is not None and overall is not None:
|
|
102
|
+
progress.advance(overall)
|
|
103
|
+
return "cached"
|
|
104
|
+
url = repo.get("tarball_url") or (
|
|
105
|
+
f"https://api.github.com/repos/{name}/tarball")
|
|
106
|
+
last_err = "unknown"
|
|
107
|
+
async with sem:
|
|
108
|
+
for attempt in range(1, retries + 1):
|
|
109
|
+
task = None
|
|
110
|
+
try:
|
|
111
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
resume_from = part.stat().st_size if part.exists() else 0
|
|
113
|
+
headers = {}
|
|
114
|
+
if resume_from:
|
|
115
|
+
headers["Range"] = f"bytes={resume_from}-"
|
|
116
|
+
async with client.stream("GET", url, headers=headers) as resp:
|
|
117
|
+
if resp.status_code == 404:
|
|
118
|
+
raise RuntimeError("repo/tarball not found (private? need --token)")
|
|
119
|
+
if resp.status_code in (403, 429):
|
|
120
|
+
retry_after = resp.headers.get("retry-after")
|
|
121
|
+
wait = int(retry_after) if str(retry_after or "").isdigit() else 2 ** attempt
|
|
122
|
+
raise RuntimeError(f"rate limited, retry in {wait}s")
|
|
123
|
+
if resp.status_code not in (200, 206):
|
|
124
|
+
raise RuntimeError(f"HTTP {resp.status_code}")
|
|
125
|
+
if resp.status_code == 200 and resume_from:
|
|
126
|
+
resume_from = 0 # server shrugged at Range; start over
|
|
127
|
+
total = int(resp.headers.get("content-length") or 0)
|
|
128
|
+
if resp.status_code == 206:
|
|
129
|
+
total += resume_from
|
|
130
|
+
if progress is not None:
|
|
131
|
+
task = progress.add_task("dl", name=name,
|
|
132
|
+
total=total or None,
|
|
133
|
+
completed=resume_from)
|
|
134
|
+
mode = "ab" if resume_from else "wb"
|
|
135
|
+
loop = asyncio.get_running_loop()
|
|
136
|
+
with open(part, mode) as f:
|
|
137
|
+
async for chunk in resp.aiter_bytes(CHUNK):
|
|
138
|
+
if not chunk:
|
|
139
|
+
continue
|
|
140
|
+
# Synchronous write inside an async loop looks
|
|
141
|
+
# wrong until you remember the network is ~1000x
|
|
142
|
+
# slower than the disk. It never shows up.
|
|
143
|
+
f.write(chunk)
|
|
144
|
+
if progress is not None and task is not None:
|
|
145
|
+
progress.update(task, advance=len(chunk))
|
|
146
|
+
os.replace(part, archive)
|
|
147
|
+
if extract:
|
|
148
|
+
await loop.run_in_executor(None, _extract_strip_top, archive, dest)
|
|
149
|
+
if not keep_tar:
|
|
150
|
+
archive.unlink(missing_ok=True)
|
|
151
|
+
done.write_text(f"{name} {time.time():.0f}\n")
|
|
152
|
+
if progress is not None:
|
|
153
|
+
if task is not None:
|
|
154
|
+
progress.update(task, visible=False)
|
|
155
|
+
if overall is not None:
|
|
156
|
+
progress.advance(overall)
|
|
157
|
+
return "resumed" if resume_from else "downloaded"
|
|
158
|
+
except Exception as e:
|
|
159
|
+
last_err = str(e)[:200]
|
|
160
|
+
if progress is not None and task is not None:
|
|
161
|
+
progress.update(task, visible=False)
|
|
162
|
+
if attempt < retries:
|
|
163
|
+
await asyncio.sleep(2 ** attempt)
|
|
164
|
+
if progress is not None and overall is not None:
|
|
165
|
+
progress.advance(overall)
|
|
166
|
+
raise RuntimeError(last_err)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
async def _amain(repos, out_dir, workers, method, token, progress,
|
|
170
|
+
use_ssh, shallow, git_timeout, keep_tar, extract, force):
|
|
171
|
+
_maybe_uvloop()
|
|
172
|
+
sem = asyncio.Semaphore(max(1, min(64, workers)))
|
|
173
|
+
headers = {"User-Agent": "StarDown/0.3.0",
|
|
174
|
+
"Accept": "application/vnd.github+json"}
|
|
175
|
+
if token:
|
|
176
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
177
|
+
limits = httpx.Limits(max_connections=max(1, workers),
|
|
178
|
+
max_keepalive_connections=max(1, workers))
|
|
179
|
+
timeout = httpx.Timeout(connect=10.0, read=60.0, write=10.0, pool=10.0)
|
|
180
|
+
overall = progress.add_task("all", name="total",
|
|
181
|
+
total=len(repos)) if progress is not None else None
|
|
182
|
+
results = []
|
|
183
|
+
|
|
184
|
+
if method == "git":
|
|
185
|
+
async def one_git(r):
|
|
186
|
+
try:
|
|
187
|
+
s = await _git_one(r, out_dir, sem, use_ssh, shallow, git_timeout)
|
|
188
|
+
results.append((r["full_name"], s, ""))
|
|
189
|
+
except Exception as e:
|
|
190
|
+
results.append((r["full_name"], "failed", str(e)[:300]))
|
|
191
|
+
finally:
|
|
192
|
+
if progress is not None and overall is not None:
|
|
193
|
+
progress.advance(overall)
|
|
194
|
+
await asyncio.gather(*(one_git(r) for r in repos))
|
|
195
|
+
return results
|
|
196
|
+
|
|
197
|
+
async with httpx.AsyncClient(headers=headers, limits=limits,
|
|
198
|
+
timeout=timeout, follow_redirects=True) as client:
|
|
199
|
+
async def one_tar(r):
|
|
200
|
+
try:
|
|
201
|
+
s = await _tar_one(client, r, out_dir, sem, progress, overall,
|
|
202
|
+
keep_tar, extract, force, RETRIES)
|
|
203
|
+
results.append((r["full_name"], s, ""))
|
|
204
|
+
except Exception as e:
|
|
205
|
+
results.append((r["full_name"], "failed", str(e)[:300]))
|
|
206
|
+
await asyncio.gather(*(one_tar(r) for r in repos))
|
|
207
|
+
return results
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def download_repos(repos, out_dir, workers=16, method="tar", token=None,
|
|
211
|
+
progress=None, use_ssh=False, shallow=True, git_timeout=300,
|
|
212
|
+
keep_tar=False, extract=True, force=False):
|
|
213
|
+
# The CLI is sync; this is the one place asyncio gets entered.
|
|
214
|
+
return asyncio.run(_amain(repos, out_dir, workers, method, token, progress,
|
|
215
|
+
use_ssh, shallow, git_timeout,
|
|
216
|
+
keep_tar, extract, force))
|