pkgreach 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pkgreach/__init__.py +53 -0
- pkgreach/__main__.py +36 -0
- pkgreach/analyze.py +242 -0
- pkgreach/collect.py +301 -0
- pkgreach/config.py +120 -0
- pkgreach/http.py +129 -0
- pkgreach/platforms/__init__.py +103 -0
- pkgreach/platforms/crates.py +67 -0
- pkgreach/platforms/ecosystems.py +36 -0
- pkgreach/platforms/github.py +84 -0
- pkgreach/platforms/npm.py +90 -0
- pkgreach/platforms/pypi.py +74 -0
- pkgreach/report.py +290 -0
- pkgreach/scaffold.py +137 -0
- pkgreach/tables.py +120 -0
- pkgreach/tools.py +89 -0
- pkgreach-0.0.2.dist-info/METADATA +127 -0
- pkgreach-0.0.2.dist-info/RECORD +21 -0
- pkgreach-0.0.2.dist-info/WHEEL +4 -0
- pkgreach-0.0.2.dist-info/entry_points.txt +2 -0
- pkgreach-0.0.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""PyPI: daily downloads from pypistats.org, metadata from PyPI's JSON API.
|
|
2
|
+
|
|
3
|
+
- **Listing** an account's packages: PyPI's XML-RPC ``user_packages`` (the only
|
|
4
|
+
official listing; deprecated, but answered). Owners and maintainers alike.
|
|
5
|
+
- **Downloads**: ``pypistats.org/api/packages/{name}/overall``, daily, about
|
|
6
|
+
the last six months, as two series: ``all`` (with mirrors) and
|
|
7
|
+
``without_mirrors``. The analysis prefers the latter, since mirror traffic is
|
|
8
|
+
pure noise. Older days are gone: hence a history kept weekly.
|
|
9
|
+
- **Metadata**: ``pypi.org/pypi/{name}/json`` (latest version, summary, source
|
|
10
|
+
repository, first upload, number of releases).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
import xmlrpc.client
|
|
17
|
+
from collections.abc import Iterator
|
|
18
|
+
from datetime import date
|
|
19
|
+
|
|
20
|
+
XMLRPC_URL = "https://pypi.org/pypi"
|
|
21
|
+
STATS_URL = "https://pypistats.org/api/packages/{name}/overall"
|
|
22
|
+
JSON_URL = "https://pypi.org/pypi/{name}/json"
|
|
23
|
+
#: pypistats categories to the series names stored.
|
|
24
|
+
SERIES = {"with_mirrors": "all", "without_mirrors": "without_mirrors"}
|
|
25
|
+
_REPO_KEYS = ("source", "source code", "repository", "github", "code", "homepage")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def normalize(name: str) -> str:
|
|
29
|
+
"""PEP 503: ``Flask_Login`` -> ``flask-login`` (pypistats only knows this form)."""
|
|
30
|
+
return re.sub(r"[-_.]+", "-", name).lower()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def list_packages(user: str, *, http) -> Iterator[dict]:
|
|
34
|
+
"""The packages ``user`` owns or maintains on PyPI."""
|
|
35
|
+
body = xmlrpc.client.dumps((user,), "user_packages").encode("utf-8")
|
|
36
|
+
text = http(XMLRPC_URL, headers={"Content-Type": "text/xml"}, data=body)
|
|
37
|
+
(pairs,), _ = xmlrpc.client.loads(text)
|
|
38
|
+
for role, name in pairs:
|
|
39
|
+
yield {"package": name, "role": role.lower()}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _repository(info: dict) -> str:
|
|
43
|
+
urls = {k.lower(): v for k, v in (info.get("project_urls") or {}).items()}
|
|
44
|
+
for k in _REPO_KEYS:
|
|
45
|
+
if urls.get(k, "").startswith("https://"):
|
|
46
|
+
return urls[k]
|
|
47
|
+
return ""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def fetch(record: dict, *, http, today: date, since: date | None = None) -> dict:
|
|
51
|
+
"""Daily downloads (both series), versions, and metadata of one package."""
|
|
52
|
+
name = record["package"]
|
|
53
|
+
stats = http(STATS_URL.format(name=name)).get("data", [])
|
|
54
|
+
lo = since.isoformat() if since else ""
|
|
55
|
+
downloads = [
|
|
56
|
+
{"date": r["date"], "series": SERIES[r["category"]], "count": r["downloads"]}
|
|
57
|
+
for r in stats
|
|
58
|
+
if r.get("category") in SERIES and r["date"] >= lo
|
|
59
|
+
]
|
|
60
|
+
meta = http(JSON_URL.format(name=name))
|
|
61
|
+
info, releases = meta.get("info", {}), meta.get("releases", {})
|
|
62
|
+
uploads = [f["upload_time"] for files in releases.values() for f in files]
|
|
63
|
+
return {
|
|
64
|
+
"downloads": downloads,
|
|
65
|
+
"history_from": min((r["date"] for r in stats), default=""),
|
|
66
|
+
"metrics": {"versions": len(releases)},
|
|
67
|
+
"info": {
|
|
68
|
+
"url": f"https://pypi.org/project/{name}/",
|
|
69
|
+
"repository": _repository(info),
|
|
70
|
+
"description": info.get("summary") or "",
|
|
71
|
+
"latest_version": info.get("version") or "",
|
|
72
|
+
"created": min(uploads, default="")[:10],
|
|
73
|
+
},
|
|
74
|
+
}
|
pkgreach/report.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""The report: a static site (one ``index.html`` plus ``summary.json``).
|
|
2
|
+
|
|
3
|
+
:func:`build_site` reads the history, summarizes it (the ``analyze=`` seam)
|
|
4
|
+
and writes the page into ``out``, any
|
|
5
|
+
``MutableMapping[str, str]``: a folder by default, which any static host
|
|
6
|
+
serves (GitHub Pages, your own server, an S3 bucket through a ``dol`` store).
|
|
7
|
+
|
|
8
|
+
Nothing here knows a platform by name: titles, the kind of table (daily
|
|
9
|
+
downloads or snapshot metrics) and the headline metrics come from the
|
|
10
|
+
summary, which takes them from each :class:`~pkgreach.platforms.Platform`.
|
|
11
|
+
|
|
12
|
+
The page is generated, not an app: the data is baked in, nothing is fetched at
|
|
13
|
+
view time, and the only script sorts tables. Every string from a registry is
|
|
14
|
+
escaped, and only ``http(s)`` links are emitted.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import html
|
|
20
|
+
import json
|
|
21
|
+
from collections.abc import Callable, Mapping, MutableMapping
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from pkgreach.analyze import summarize
|
|
25
|
+
from pkgreach.tables import SUMMARY_KEY
|
|
26
|
+
|
|
27
|
+
Summary = dict[str, Any]
|
|
28
|
+
DFLT_TITLE = "Package reach"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _e(s: Any) -> str:
|
|
32
|
+
return html.escape(str(s), quote=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _link(url: str, text: str) -> str:
|
|
36
|
+
if url.startswith(("https://", "http://")):
|
|
37
|
+
return f'<a href="{_e(url)}">{_e(text)}</a>'
|
|
38
|
+
return _e(text)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _num(n: Any) -> str:
|
|
42
|
+
if n is None or n == "":
|
|
43
|
+
return "–"
|
|
44
|
+
n = int(n)
|
|
45
|
+
if abs(n) >= 10_000:
|
|
46
|
+
return f"{n / 1000:,.0f}k"
|
|
47
|
+
return f"{n:,}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _pct(x: float | None) -> str:
|
|
51
|
+
if x is None:
|
|
52
|
+
return '<span class="muted">–</span>'
|
|
53
|
+
cls = "up" if x > 0 else "down" if x < 0 else "muted"
|
|
54
|
+
return f'<span class="{cls}">{x:+.0%}</span>'
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _times(x: float | None) -> str:
|
|
58
|
+
if x is None:
|
|
59
|
+
return "–"
|
|
60
|
+
return f"{x:,.0f}×" if x >= 10 else f"{x:g}×"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _signed(n: int | None) -> str:
|
|
64
|
+
if not n:
|
|
65
|
+
return ""
|
|
66
|
+
cls = "up" if n > 0 else "down"
|
|
67
|
+
return f' <span class="{cls} small">{n:+,}</span>'
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _spark(values: list[int], *, w: int = 120, h: int = 28) -> str:
|
|
71
|
+
if not values or not any(values):
|
|
72
|
+
return '<span class="muted small">no downloads</span>'
|
|
73
|
+
top = max(values)
|
|
74
|
+
step = w / max(1, len(values) - 1)
|
|
75
|
+
pts = " ".join(
|
|
76
|
+
f"{i * step:.1f},{h - 2 - (v / top) * (h - 4):.1f}"
|
|
77
|
+
for i, v in enumerate(values)
|
|
78
|
+
)
|
|
79
|
+
return (
|
|
80
|
+
f'<svg class="spark" viewBox="0 0 {w} {h}" width="{w}" height="{h}" '
|
|
81
|
+
f'role="img" aria-label="weekly downloads, last {len(values)} weeks">'
|
|
82
|
+
f'<polyline points="{pts}" fill="none" stroke="currentColor" '
|
|
83
|
+
'stroke-width="1.5" stroke-linejoin="round"/></svg>'
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _metric(row: Mapping, name: str) -> dict:
|
|
88
|
+
return row["metrics"].get(name, {})
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _label(metric: str) -> str:
|
|
92
|
+
return metric.replace("_", " ")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _errors_note(summary: Mapping) -> str:
|
|
96
|
+
n = summary.get("last_run_errors")
|
|
97
|
+
if not n:
|
|
98
|
+
return ""
|
|
99
|
+
return f' · <span class="down">last run: {n} errors (see last_run.json)</span>'
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _card(p: Mapping) -> str:
|
|
103
|
+
if p["kind"] == "snapshot":
|
|
104
|
+
t, head = p["metric_totals"], p["headline"]
|
|
105
|
+
rest = " · ".join(f"{_num(t.get(m))} {_label(m)}" for m in head[1:])
|
|
106
|
+
body = (
|
|
107
|
+
f'<div class="big">{_num(t.get(head[0]) if head else None)}</div>'
|
|
108
|
+
f'<div class="small">{_label(head[0]) if head else ""} across '
|
|
109
|
+
f"{p['packages']} {'· ' + rest if rest else ''}</div>"
|
|
110
|
+
)
|
|
111
|
+
else:
|
|
112
|
+
prev = p["downloads_prev_30d"]
|
|
113
|
+
growth = p["downloads_30d"] / prev - 1 if prev else None
|
|
114
|
+
deps = p["metric_totals"].get("dependent_packages")
|
|
115
|
+
body = (
|
|
116
|
+
f'<div class="big">{_num(p["downloads_30d"])}</div>'
|
|
117
|
+
f'<div class="small">downloads, last 30 days {_pct(growth)}</div>'
|
|
118
|
+
f'<div class="small">{p["packages"]} packages · '
|
|
119
|
+
f"{_num(deps)} dependent packages · noise floor "
|
|
120
|
+
f"{_num(p['noise_floor_30d'])}/30d</div>"
|
|
121
|
+
)
|
|
122
|
+
return f'<section class="card"><h3>{_e(p["title"])}</h3>{body}</section>'
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _registry_table(rows: list[Mapping]) -> str:
|
|
126
|
+
rows = sorted(rows, key=lambda r: -r["downloads"]["30d"])
|
|
127
|
+
head = (
|
|
128
|
+
"<tr><th data-sort>package</th><th data-sort=num>30d</th>"
|
|
129
|
+
"<th data-sort=num>vs prior</th><th data-sort=num title='30-day downloads "
|
|
130
|
+
"over the noise floor'>× floor</th><th data-sort=num title='dependent "
|
|
131
|
+
"packages / repositories'>dependents</th><th>26 weeks</th></tr>"
|
|
132
|
+
)
|
|
133
|
+
body = []
|
|
134
|
+
for r in rows:
|
|
135
|
+
dp = _metric(r, "dependent_packages").get("value")
|
|
136
|
+
dr = _metric(r, "dependent_repos").get("value")
|
|
137
|
+
deps = f"{_num(dp)} / {_num(dr)}" if dp is not None or dr is not None else "–"
|
|
138
|
+
xf = r.get("x_floor")
|
|
139
|
+
body.append(
|
|
140
|
+
f"<tr><td>{_link(r['url'], r['package'])}</td>"
|
|
141
|
+
f'<td data-v="{r["downloads"]["30d"]}">{_num(r["downloads"]["30d"])}</td>'
|
|
142
|
+
f'<td data-v="{r["growth_30d"] if r["growth_30d"] is not None else ""}">'
|
|
143
|
+
f"{_pct(r['growth_30d'])}</td>"
|
|
144
|
+
f'<td data-v="{xf if xf is not None else ""}">'
|
|
145
|
+
f"{_times(xf)}</td>"
|
|
146
|
+
f'<td data-v="{(dp or 0) * 1_000_000 + (dr or 0)}">{deps}</td>'
|
|
147
|
+
f"<td>{_spark(r['weekly'])}</td></tr>"
|
|
148
|
+
)
|
|
149
|
+
return f'<table class="sortable"><thead>{head}</thead><tbody>{"".join(body)}</tbody></table>'
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _snapshot_table(rows: list[Mapping], headline: list[str]) -> str:
|
|
153
|
+
first = headline[0] if headline else ""
|
|
154
|
+
rows = sorted(rows, key=lambda r: -(_metric(r, first).get("value") or 0))
|
|
155
|
+
head = "<tr><th data-sort>name</th>" + "".join(
|
|
156
|
+
f"<th data-sort=num>{_e(_label(m))}</th>" for m in headline
|
|
157
|
+
)
|
|
158
|
+
body = []
|
|
159
|
+
for r in rows:
|
|
160
|
+
cells = []
|
|
161
|
+
for m in headline:
|
|
162
|
+
d = _metric(r, m)
|
|
163
|
+
cells.append(
|
|
164
|
+
f'<td data-v="{d.get("value", "")}">{_num(d.get("value"))}'
|
|
165
|
+
f"{_signed(d.get('change_30d'))}</td>"
|
|
166
|
+
)
|
|
167
|
+
body.append(
|
|
168
|
+
f"<tr><td>{_link(r['url'], r['package'])}</td>{''.join(cells)}</tr>"
|
|
169
|
+
)
|
|
170
|
+
return (
|
|
171
|
+
f'<table class="sortable"><thead>{head}</tr></thead>'
|
|
172
|
+
f"<tbody>{''.join(body)}</tbody></table>"
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
_CSS = """
|
|
177
|
+
:root{--bg:#fbfbfa;--fg:#1d1d1f;--muted:#6e6e73;--line:#e3e3e0;--card:#fff;
|
|
178
|
+
--accent:#2f6fde;--up:#1f8a4c;--down:#c0392b}
|
|
179
|
+
@media (prefers-color-scheme:dark){:root:not([data-theme=light]){--bg:#141416;
|
|
180
|
+
--fg:#ececee;--muted:#9a9aa1;--line:#2c2c30;--card:#1c1c1f;--accent:#7aa7ff;
|
|
181
|
+
--up:#5cc98a;--down:#ff7b6b}}
|
|
182
|
+
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);
|
|
183
|
+
font:15px/1.45 system-ui,-apple-system,Segoe UI,sans-serif}
|
|
184
|
+
main{max-width:980px;margin:0 auto;padding:16px}
|
|
185
|
+
h1{font-size:1.5rem;margin:.2em 0}h2{margin:1.6em 0 .4em;font-size:1.15rem}
|
|
186
|
+
h3{margin:0 0 .3em;font-size:1rem;color:var(--muted);font-weight:600}
|
|
187
|
+
a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}
|
|
188
|
+
.muted{color:var(--muted)}.small{font-size:.85rem}.up{color:var(--up)}.down{color:var(--down)}
|
|
189
|
+
.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(210px,100%),1fr));gap:12px}
|
|
190
|
+
.card{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
|
|
191
|
+
.big{font-size:1.7rem;font-weight:650;font-variant-numeric:tabular-nums}
|
|
192
|
+
.scroll{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--card)}
|
|
193
|
+
table{border-collapse:collapse;width:100%;font-variant-numeric:tabular-nums}
|
|
194
|
+
th,td{padding:6px 10px;border-bottom:1px solid var(--line);text-align:right;white-space:nowrap}
|
|
195
|
+
th:first-child,td:first-child{text-align:left;position:sticky;left:0;background:var(--card)}
|
|
196
|
+
th{font-size:.8rem;color:var(--muted);font-weight:600}th[data-sort]{cursor:pointer}
|
|
197
|
+
th[aria-sort=descending]::after{content:" ▾"}th[aria-sort=ascending]::after{content:" ▴"}
|
|
198
|
+
.spark{color:var(--accent);vertical-align:middle}
|
|
199
|
+
details{margin:1.5em 0;background:var(--card);border:1px solid var(--line);border-radius:10px;padding:10px 14px}
|
|
200
|
+
summary{cursor:pointer;font-weight:600}
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
_JS = """
|
|
204
|
+
document.querySelectorAll('table.sortable th[data-sort]').forEach((th,i)=>{
|
|
205
|
+
th.addEventListener('click',()=>{const t=th.closest('table'),b=t.tBodies[0];
|
|
206
|
+
const col=[...th.parentNode.children].indexOf(th),num=th.dataset.sort==='num';
|
|
207
|
+
const dir=th.getAttribute('aria-sort')==='descending'?1:-1;
|
|
208
|
+
t.querySelectorAll('th').forEach(h=>h.removeAttribute('aria-sort'));
|
|
209
|
+
th.setAttribute('aria-sort',dir<0?'descending':'ascending');
|
|
210
|
+
const val=r=>{const c=r.children[col];return num?parseFloat(c.dataset.v||'-1e18'):c.textContent.toLowerCase()};
|
|
211
|
+
[...b.rows].sort((x,y)=>{const a=val(x),z=val(y);return (a>z?1:a<z?-1:0)*dir}).forEach(r=>b.appendChild(r))})});
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
_HOW_TO_READ = """
|
|
215
|
+
<details><summary>How to read these numbers</summary>
|
|
216
|
+
<p><b>Downloads are mostly not people.</b> Mirrors, CI pipelines, security
|
|
217
|
+
scanners and bots fetch every release of every package, so a package nobody
|
|
218
|
+
uses still gets a steady trickle. The <b>noise floor</b> is the 25th percentile
|
|
219
|
+
of 30-day downloads among the packages followed here that were downloaded at all; <b>× floor</b> says how
|
|
220
|
+
far above it a package is. A package near 1× is probably only seen by robots.</p>
|
|
221
|
+
<p><b>The strongest signs of real use</b> are the ones a robot does not create:
|
|
222
|
+
<b>dependents</b> (other packages and repositories that declare it, from
|
|
223
|
+
ecosyste.ms), stars and forks, and issues opened by people other than you.
|
|
224
|
+
Growth over the prior 30 days matters more than the level.</p>
|
|
225
|
+
<p>PyPI counts exclude known mirrors (pypistats.org's <i>without_mirrors</i>).
|
|
226
|
+
Registries keep only a few months of daily counts, so this history starts at
|
|
227
|
+
each package's first collection.</p></details>
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def render_html(summary: Summary, *, title: str = DFLT_TITLE) -> str:
|
|
232
|
+
"""The report page for a :func:`~pkgreach.analyze.summarize` result."""
|
|
233
|
+
platforms = summary["platforms"]
|
|
234
|
+
# daily-download platforms first, by downloads; then snapshot ones
|
|
235
|
+
order = sorted(
|
|
236
|
+
platforms,
|
|
237
|
+
key=lambda n: (platforms[n]["kind"] != "daily", -platforms[n]["downloads_30d"]),
|
|
238
|
+
)
|
|
239
|
+
cards = "".join(_card(platforms[p]) for p in order)
|
|
240
|
+
sections = []
|
|
241
|
+
for p in order:
|
|
242
|
+
meta = platforms[p]
|
|
243
|
+
rows = [r for r in summary["packages"] if r["platform"] == p]
|
|
244
|
+
if meta["kind"] == "snapshot":
|
|
245
|
+
table = _snapshot_table(rows, meta["headline"])
|
|
246
|
+
else:
|
|
247
|
+
table = _registry_table(rows)
|
|
248
|
+
sections.append(
|
|
249
|
+
f"<h2>{_e(meta['title'])} "
|
|
250
|
+
f'<span class="muted small">{len(rows)}</span></h2>'
|
|
251
|
+
f'<div class="scroll">{table}</div>'
|
|
252
|
+
)
|
|
253
|
+
return f"""<!doctype html>
|
|
254
|
+
<html lang="en"><head><meta charset="utf-8">
|
|
255
|
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
256
|
+
<title>{_e(title)}</title><style>{_CSS}</style></head>
|
|
257
|
+
<body><main>
|
|
258
|
+
<h1>{_e(title)}</h1>
|
|
259
|
+
<p class="muted small">As of {_e(summary["as_of"])} · made with
|
|
260
|
+
<a href="https://github.com/thorwhalen/pkgreach">pkgreach</a> ·
|
|
261
|
+
<a href="summary.json">summary.json</a>{_errors_note(summary)}</p>
|
|
262
|
+
<div class="cards">{cards}</div>
|
|
263
|
+
{_HOW_TO_READ}
|
|
264
|
+
{"".join(sections)}
|
|
265
|
+
<p class="muted small">Downloads: pypistats.org, api.npmjs.org, crates.io.
|
|
266
|
+
Dependents: <a href="https://ecosyste.ms">ecosyste.ms</a>, licensed
|
|
267
|
+
<a href="https://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>,
|
|
268
|
+
aggregated here. Stars and forks: GitHub, public repositories only.</p>
|
|
269
|
+
</main><script>{_JS}</script></body></html>
|
|
270
|
+
"""
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def build_site(
|
|
274
|
+
store: MutableMapping,
|
|
275
|
+
out: MutableMapping,
|
|
276
|
+
*,
|
|
277
|
+
analyze: Callable[[MutableMapping], Summary] = summarize,
|
|
278
|
+
title: str = DFLT_TITLE,
|
|
279
|
+
) -> Summary:
|
|
280
|
+
"""Summarize the history in ``store`` and write the site into ``out``.
|
|
281
|
+
|
|
282
|
+
``analyze`` is the seam for another summary (it must return the shape of
|
|
283
|
+
:func:`~pkgreach.analyze.summarize`, which ``summary.json`` documents).
|
|
284
|
+
"""
|
|
285
|
+
# seam candidate: render= (another page from the same summary), once a
|
|
286
|
+
# second renderer exists
|
|
287
|
+
summary = analyze(store)
|
|
288
|
+
out[SUMMARY_KEY] = json.dumps(summary, indent=1) + "\n"
|
|
289
|
+
out["index.html"] = render_html(summary, title=title)
|
|
290
|
+
return summary
|
pkgreach/scaffold.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Setting up a repository: the config and the weekly workflow.
|
|
2
|
+
|
|
3
|
+
``pkgreach init <dir> --pypi alice --npm alice`` writes ``pkgreach.toml`` and
|
|
4
|
+
``.github/workflows/pkgreach.yml``: every Sunday, the workflow collects into
|
|
5
|
+
the history (kept on the ``pkgreach-data`` branch of the same repository),
|
|
6
|
+
builds the report and publishes it on the repository's GitHub Pages.
|
|
7
|
+
|
|
8
|
+
Everything pkgreach collects is public already (download counts, dependents,
|
|
9
|
+
stars of public repositories), so the repository can be public, and needs no
|
|
10
|
+
secret: the workflow's own token reads GitHub and pushes the history.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Iterable, Mapping
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from pkgreach.platforms import PLATFORMS
|
|
19
|
+
|
|
20
|
+
ACTION_REPO = "thorwhalen/pkgreach"
|
|
21
|
+
FALLBACK_ACTION_REF = f"{ACTION_REPO}@main"
|
|
22
|
+
#: 05:23 UTC on Sundays: off the top of the hour (GitHub delays on-the-hour
|
|
23
|
+
#: crons). Registries keep 3 to 18 months of daily counts, so weekly runs lose
|
|
24
|
+
#: nothing even when a few are missed.
|
|
25
|
+
DFLT_CRON = "23 5 * * 0"
|
|
26
|
+
WORKFLOW_PATH = ".github/workflows/pkgreach.yml"
|
|
27
|
+
CONFIG_PATH = "pkgreach.toml"
|
|
28
|
+
|
|
29
|
+
WORKFLOW_TEMPLATE = """\
|
|
30
|
+
# Collects package stats every Sunday and publishes the report on GitHub Pages.
|
|
31
|
+
# Generated by `pkgreach init`. Docs: https://github.com/thorwhalen/pkgreach
|
|
32
|
+
#
|
|
33
|
+
# Needs no secret. The history is kept on the `pkgreach-data` branch.
|
|
34
|
+
# One-time setup: Settings -> Pages -> Source: "GitHub Actions".
|
|
35
|
+
name: pkgreach
|
|
36
|
+
on:
|
|
37
|
+
schedule:
|
|
38
|
+
- cron: "{cron}"
|
|
39
|
+
workflow_dispatch:
|
|
40
|
+
permissions:
|
|
41
|
+
contents: write
|
|
42
|
+
pages: write
|
|
43
|
+
id-token: write
|
|
44
|
+
concurrency:
|
|
45
|
+
group: pkgreach
|
|
46
|
+
cancel-in-progress: false
|
|
47
|
+
jobs:
|
|
48
|
+
collect:
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
timeout-minutes: 30
|
|
51
|
+
environment:
|
|
52
|
+
name: github-pages
|
|
53
|
+
url: ${{{{ steps.deploy.outputs.page_url }}}}
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/checkout@v4
|
|
56
|
+
- uses: {action_ref}
|
|
57
|
+
id: reach
|
|
58
|
+
- uses: actions/upload-pages-artifact@v3
|
|
59
|
+
with:
|
|
60
|
+
path: ${{{{ steps.reach.outputs.site-dir }}}}
|
|
61
|
+
- uses: actions/deploy-pages@v4
|
|
62
|
+
id: deploy
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
CONFIG_HEADER = """\
|
|
66
|
+
# pkgreach settings: one table per platform. Every key is optional; see
|
|
67
|
+
# https://github.com/thorwhalen/pkgreach#configuration
|
|
68
|
+
#
|
|
69
|
+
# users = accounts whose packages are all followed
|
|
70
|
+
# packages = packages followed by name, on top of the users' ones
|
|
71
|
+
# exclude = glob patterns of package names to drop
|
|
72
|
+
# where = metadata a package must match, e.g. {{ fork = false }}
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def default_action_ref() -> str:
|
|
77
|
+
"""``thorwhalen/pkgreach@<installed version>``, or ``@main`` from a checkout."""
|
|
78
|
+
try:
|
|
79
|
+
from importlib.metadata import version
|
|
80
|
+
|
|
81
|
+
return f"{ACTION_REPO}@{version('pkgreach')}"
|
|
82
|
+
except Exception: # noqa: BLE001 - not installed: a source checkout
|
|
83
|
+
return FALLBACK_ACTION_REF
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _toml_list(items: Iterable[str]) -> str:
|
|
87
|
+
return "[" + ", ".join(f'"{i}"' for i in items) + "]"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _toml_inline(d: Mapping) -> str:
|
|
91
|
+
def v(x):
|
|
92
|
+
return str(x).lower() if isinstance(x, bool) else f'"{x}"'
|
|
93
|
+
|
|
94
|
+
return "{ " + ", ".join(f"{k} = {v(x)}" for k, x in d.items()) + " }"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def config_text(users: Mapping[str, Iterable[str]], *, platforms=None) -> str:
|
|
98
|
+
"""The ``pkgreach.toml`` following ``users`` per platform (each platform's
|
|
99
|
+
``default_where`` included)."""
|
|
100
|
+
platforms = PLATFORMS if platforms is None else platforms
|
|
101
|
+
tables = []
|
|
102
|
+
for platform, names in users.items():
|
|
103
|
+
lines = [f"[{platform}]", f"users = {_toml_list(names)}"]
|
|
104
|
+
where = getattr(platforms.get(platform), "default_where", None)
|
|
105
|
+
if where:
|
|
106
|
+
lines.append(f"where = {_toml_inline(where)}")
|
|
107
|
+
tables.append("\n".join(lines))
|
|
108
|
+
return CONFIG_HEADER.format() + "\n" + "\n\n".join(tables) + "\n"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def init_repo(
|
|
112
|
+
directory: str | Path,
|
|
113
|
+
*,
|
|
114
|
+
users: Mapping[str, Iterable[str]],
|
|
115
|
+
cron: str = DFLT_CRON,
|
|
116
|
+
action_ref: str | None = None,
|
|
117
|
+
overwrite: bool = False,
|
|
118
|
+
) -> list[str]:
|
|
119
|
+
"""Write the config and the workflow; return the paths written."""
|
|
120
|
+
if not users:
|
|
121
|
+
raise ValueError("Name at least one account, e.g. --pypi you --npm you.")
|
|
122
|
+
root = Path(directory)
|
|
123
|
+
files = {
|
|
124
|
+
CONFIG_PATH: config_text(users),
|
|
125
|
+
WORKFLOW_PATH: WORKFLOW_TEMPLATE.format(
|
|
126
|
+
cron=cron, action_ref=action_ref or default_action_ref()
|
|
127
|
+
),
|
|
128
|
+
}
|
|
129
|
+
written = []
|
|
130
|
+
for rel, text in files.items():
|
|
131
|
+
path = root / rel
|
|
132
|
+
if path.exists() and not overwrite:
|
|
133
|
+
continue
|
|
134
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
135
|
+
path.write_text(text, encoding="utf-8")
|
|
136
|
+
written.append(rel)
|
|
137
|
+
return written
|
pkgreach/tables.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""The schema of a pkgreach data directory: long-form CSV tables, one per kind.
|
|
2
|
+
|
|
3
|
+
Every platform writes the same tables, with a ``platform`` column, so adding a
|
|
4
|
+
platform never changes the schema. The CSV machinery (year partitions,
|
|
5
|
+
idempotent merges, the ``MutableMapping`` store) is ghtraffic's
|
|
6
|
+
(:mod:`ghtraffic.tables`): the history of a package's downloads has the same
|
|
7
|
+
shape, and the same merge problem, as the history of a repo's views.
|
|
8
|
+
|
|
9
|
+
- ``downloads``: daily counts, merged by **max** per ``(date, platform,
|
|
10
|
+
package, series)``. Registries recompute recent days, and windows overlap
|
|
11
|
+
between runs; max makes re-runs and overlaps harmless. ``series`` separates
|
|
12
|
+
variants of one count (``all``; ``without_mirrors`` on PyPI).
|
|
13
|
+
- ``metrics``: slowly changing numbers (dependents, stars, versions, total
|
|
14
|
+
downloads), one row per **change** per ``(platform, package, metric)``.
|
|
15
|
+
- ``packages``: current metadata of every package seen (kept after it is
|
|
16
|
+
dropped from the selection, with its ``last_seen``). ``history_from`` is the
|
|
17
|
+
first day its downloads were collected for: before it, a day is unknown;
|
|
18
|
+
from it on, a missing ``downloads`` row means zero.
|
|
19
|
+
- ``runs``: one row per collection run.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
# The one import of ghtraffic: every other module goes through here, so moving the
|
|
25
|
+
# store to its own package (thorwhalen/ghtraffic#7) changes this line only.
|
|
26
|
+
from ghtraffic.tables import (
|
|
27
|
+
Files,
|
|
28
|
+
TableSpec,
|
|
29
|
+
dump_csv,
|
|
30
|
+
parse_csv,
|
|
31
|
+
merge_rows,
|
|
32
|
+
read_table,
|
|
33
|
+
table_files,
|
|
34
|
+
write_rows,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"Files",
|
|
39
|
+
"TableSpec",
|
|
40
|
+
"dump_csv",
|
|
41
|
+
"parse_csv",
|
|
42
|
+
"TABLES",
|
|
43
|
+
"merge_rows",
|
|
44
|
+
"read_table",
|
|
45
|
+
"table_files",
|
|
46
|
+
"write_rows",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
SCHEMA_VERSION = 1
|
|
50
|
+
MANIFEST_KEY = "manifest.json"
|
|
51
|
+
LAST_RUN_KEY = "last_run.json"
|
|
52
|
+
SUMMARY_KEY = "summary.json"
|
|
53
|
+
DATA_LICENSE_KEY = "DATA-LICENSE.md"
|
|
54
|
+
DATA_LICENSE = """\
|
|
55
|
+
# Where this data comes from, and on what terms
|
|
56
|
+
|
|
57
|
+
- `metrics.csv`, metrics `dependent_packages` and `dependent_repos`: from
|
|
58
|
+
[ecosyste.ms](https://ecosyste.ms), licensed
|
|
59
|
+
[CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). Collected weekly
|
|
60
|
+
and kept as a change log; reuse them under the same licence, crediting ecosyste.ms.
|
|
61
|
+
- `downloads/`: daily download counts from pypistats.org (PyPI), api.npmjs.org (npm)
|
|
62
|
+
and crates.io.
|
|
63
|
+
- Other metrics and `packages.csv`: the registries' public metadata, and the
|
|
64
|
+
GitHub REST API for public repositories.
|
|
65
|
+
|
|
66
|
+
Collected by pkgreach (https://github.com/thorwhalen/pkgreach).
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
TABLES: dict[str, TableSpec] = {
|
|
70
|
+
t.name: t
|
|
71
|
+
for t in [
|
|
72
|
+
TableSpec(
|
|
73
|
+
"downloads",
|
|
74
|
+
("date", "platform", "package", "series", "count"),
|
|
75
|
+
key=("date", "platform", "package", "series"),
|
|
76
|
+
merge="max",
|
|
77
|
+
int_columns=("count",),
|
|
78
|
+
time_column="date",
|
|
79
|
+
description="Daily downloads per package; days with zero downloads "
|
|
80
|
+
"are absent (from the package's history_from on, absence means zero).",
|
|
81
|
+
),
|
|
82
|
+
TableSpec(
|
|
83
|
+
"metrics",
|
|
84
|
+
("date", "platform", "package", "metric", "value"),
|
|
85
|
+
key=("date", "platform", "package", "metric"),
|
|
86
|
+
merge="change_only",
|
|
87
|
+
entity=("platform", "package", "metric"),
|
|
88
|
+
int_columns=("value",),
|
|
89
|
+
description="Slowly changing numbers (dependents, stars, forks, "
|
|
90
|
+
"versions, total downloads), one row per change.",
|
|
91
|
+
sort=("platform", "package", "metric", "date"),
|
|
92
|
+
),
|
|
93
|
+
TableSpec(
|
|
94
|
+
"packages",
|
|
95
|
+
(
|
|
96
|
+
"platform",
|
|
97
|
+
"package",
|
|
98
|
+
"url",
|
|
99
|
+
"repository",
|
|
100
|
+
"description",
|
|
101
|
+
"latest_version",
|
|
102
|
+
"created",
|
|
103
|
+
"history_from",
|
|
104
|
+
"first_seen",
|
|
105
|
+
"last_seen",
|
|
106
|
+
),
|
|
107
|
+
key=("platform", "package"),
|
|
108
|
+
merge="upsert",
|
|
109
|
+
description="Current metadata of every package seen.",
|
|
110
|
+
),
|
|
111
|
+
TableSpec(
|
|
112
|
+
"runs",
|
|
113
|
+
("started_at", "duration_s", "platform", "packages", "errors"),
|
|
114
|
+
key=("started_at", "platform"),
|
|
115
|
+
merge="upsert",
|
|
116
|
+
int_columns=("packages", "errors"),
|
|
117
|
+
description="One row per collection run and platform.",
|
|
118
|
+
),
|
|
119
|
+
]
|
|
120
|
+
}
|