pkgreach 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,74 @@
1
+ """PyPI: daily downloads from pypistats.org, metadata from PyPI's JSON API.
2
+
3
+ - **Listing** an account's packages: PyPI's XML-RPC ``user_packages`` (the only
4
+ official listing; deprecated, but answered). Owners and maintainers alike.
5
+ - **Downloads**: ``pypistats.org/api/packages/{name}/overall``, daily, about
6
+ the last six months, as two series: ``all`` (with mirrors) and
7
+ ``without_mirrors``. The analysis prefers the latter, since mirror traffic is
8
+ pure noise. Older days are gone: hence a history kept weekly.
9
+ - **Metadata**: ``pypi.org/pypi/{name}/json`` (latest version, summary, source
10
+ repository, first upload, number of releases).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import re
16
+ import xmlrpc.client
17
+ from collections.abc import Iterator
18
+ from datetime import date
19
+
20
+ XMLRPC_URL = "https://pypi.org/pypi"
21
+ STATS_URL = "https://pypistats.org/api/packages/{name}/overall"
22
+ JSON_URL = "https://pypi.org/pypi/{name}/json"
23
+ #: pypistats categories to the series names stored.
24
+ SERIES = {"with_mirrors": "all", "without_mirrors": "without_mirrors"}
25
+ _REPO_KEYS = ("source", "source code", "repository", "github", "code", "homepage")
26
+
27
+
28
+ def normalize(name: str) -> str:
29
+ """PEP 503: ``Flask_Login`` -> ``flask-login`` (pypistats only knows this form)."""
30
+ return re.sub(r"[-_.]+", "-", name).lower()
31
+
32
+
33
+ def list_packages(user: str, *, http) -> Iterator[dict]:
34
+ """The packages ``user`` owns or maintains on PyPI."""
35
+ body = xmlrpc.client.dumps((user,), "user_packages").encode("utf-8")
36
+ text = http(XMLRPC_URL, headers={"Content-Type": "text/xml"}, data=body)
37
+ (pairs,), _ = xmlrpc.client.loads(text)
38
+ for role, name in pairs:
39
+ yield {"package": name, "role": role.lower()}
40
+
41
+
42
+ def _repository(info: dict) -> str:
43
+ urls = {k.lower(): v for k, v in (info.get("project_urls") or {}).items()}
44
+ for k in _REPO_KEYS:
45
+ if urls.get(k, "").startswith("https://"):
46
+ return urls[k]
47
+ return ""
48
+
49
+
50
+ def fetch(record: dict, *, http, today: date, since: date | None = None) -> dict:
51
+ """Daily downloads (both series), versions, and metadata of one package."""
52
+ name = record["package"]
53
+ stats = http(STATS_URL.format(name=name)).get("data", [])
54
+ lo = since.isoformat() if since else ""
55
+ downloads = [
56
+ {"date": r["date"], "series": SERIES[r["category"]], "count": r["downloads"]}
57
+ for r in stats
58
+ if r.get("category") in SERIES and r["date"] >= lo
59
+ ]
60
+ meta = http(JSON_URL.format(name=name))
61
+ info, releases = meta.get("info", {}), meta.get("releases", {})
62
+ uploads = [f["upload_time"] for files in releases.values() for f in files]
63
+ return {
64
+ "downloads": downloads,
65
+ "history_from": min((r["date"] for r in stats), default=""),
66
+ "metrics": {"versions": len(releases)},
67
+ "info": {
68
+ "url": f"https://pypi.org/project/{name}/",
69
+ "repository": _repository(info),
70
+ "description": info.get("summary") or "",
71
+ "latest_version": info.get("version") or "",
72
+ "created": min(uploads, default="")[:10],
73
+ },
74
+ }
pkgreach/report.py ADDED
@@ -0,0 +1,290 @@
1
+ """The report: a static site (one ``index.html`` plus ``summary.json``).
2
+
3
+ :func:`build_site` reads the history, summarizes it (the ``analyze=`` seam)
4
+ and writes the page into ``out``, any
5
+ ``MutableMapping[str, str]``: a folder by default, which any static host
6
+ serves (GitHub Pages, your own server, an S3 bucket through a ``dol`` store).
7
+
8
+ Nothing here knows a platform by name: titles, the kind of table (daily
9
+ downloads or snapshot metrics) and the headline metrics come from the
10
+ summary, which takes them from each :class:`~pkgreach.platforms.Platform`.
11
+
12
+ The page is generated, not an app: the data is baked in, nothing is fetched at
13
+ view time, and the only script sorts tables. Every string from a registry is
14
+ escaped, and only ``http(s)`` links are emitted.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import html
20
+ import json
21
+ from collections.abc import Callable, Mapping, MutableMapping
22
+ from typing import Any
23
+
24
+ from pkgreach.analyze import summarize
25
+ from pkgreach.tables import SUMMARY_KEY
26
+
27
+ Summary = dict[str, Any]
28
+ DFLT_TITLE = "Package reach"
29
+
30
+
31
+ def _e(s: Any) -> str:
32
+ return html.escape(str(s), quote=True)
33
+
34
+
35
+ def _link(url: str, text: str) -> str:
36
+ if url.startswith(("https://", "http://")):
37
+ return f'<a href="{_e(url)}">{_e(text)}</a>'
38
+ return _e(text)
39
+
40
+
41
+ def _num(n: Any) -> str:
42
+ if n is None or n == "":
43
+ return "–"
44
+ n = int(n)
45
+ if abs(n) >= 10_000:
46
+ return f"{n / 1000:,.0f}k"
47
+ return f"{n:,}"
48
+
49
+
50
+ def _pct(x: float | None) -> str:
51
+ if x is None:
52
+ return '<span class="muted">–</span>'
53
+ cls = "up" if x > 0 else "down" if x < 0 else "muted"
54
+ return f'<span class="{cls}">{x:+.0%}</span>'
55
+
56
+
57
+ def _times(x: float | None) -> str:
58
+ if x is None:
59
+ return "–"
60
+ return f"{x:,.0f}×" if x >= 10 else f"{x:g}×"
61
+
62
+
63
+ def _signed(n: int | None) -> str:
64
+ if not n:
65
+ return ""
66
+ cls = "up" if n > 0 else "down"
67
+ return f' <span class="{cls} small">{n:+,}</span>'
68
+
69
+
70
+ def _spark(values: list[int], *, w: int = 120, h: int = 28) -> str:
71
+ if not values or not any(values):
72
+ return '<span class="muted small">no downloads</span>'
73
+ top = max(values)
74
+ step = w / max(1, len(values) - 1)
75
+ pts = " ".join(
76
+ f"{i * step:.1f},{h - 2 - (v / top) * (h - 4):.1f}"
77
+ for i, v in enumerate(values)
78
+ )
79
+ return (
80
+ f'<svg class="spark" viewBox="0 0 {w} {h}" width="{w}" height="{h}" '
81
+ f'role="img" aria-label="weekly downloads, last {len(values)} weeks">'
82
+ f'<polyline points="{pts}" fill="none" stroke="currentColor" '
83
+ 'stroke-width="1.5" stroke-linejoin="round"/></svg>'
84
+ )
85
+
86
+
87
+ def _metric(row: Mapping, name: str) -> dict:
88
+ return row["metrics"].get(name, {})
89
+
90
+
91
+ def _label(metric: str) -> str:
92
+ return metric.replace("_", " ")
93
+
94
+
95
+ def _errors_note(summary: Mapping) -> str:
96
+ n = summary.get("last_run_errors")
97
+ if not n:
98
+ return ""
99
+ return f' · <span class="down">last run: {n} errors (see last_run.json)</span>'
100
+
101
+
102
+ def _card(p: Mapping) -> str:
103
+ if p["kind"] == "snapshot":
104
+ t, head = p["metric_totals"], p["headline"]
105
+ rest = " · ".join(f"{_num(t.get(m))} {_label(m)}" for m in head[1:])
106
+ body = (
107
+ f'<div class="big">{_num(t.get(head[0]) if head else None)}</div>'
108
+ f'<div class="small">{_label(head[0]) if head else ""} across '
109
+ f"{p['packages']} {'· ' + rest if rest else ''}</div>"
110
+ )
111
+ else:
112
+ prev = p["downloads_prev_30d"]
113
+ growth = p["downloads_30d"] / prev - 1 if prev else None
114
+ deps = p["metric_totals"].get("dependent_packages")
115
+ body = (
116
+ f'<div class="big">{_num(p["downloads_30d"])}</div>'
117
+ f'<div class="small">downloads, last 30 days {_pct(growth)}</div>'
118
+ f'<div class="small">{p["packages"]} packages · '
119
+ f"{_num(deps)} dependent packages · noise floor "
120
+ f"{_num(p['noise_floor_30d'])}/30d</div>"
121
+ )
122
+ return f'<section class="card"><h3>{_e(p["title"])}</h3>{body}</section>'
123
+
124
+
125
+ def _registry_table(rows: list[Mapping]) -> str:
126
+ rows = sorted(rows, key=lambda r: -r["downloads"]["30d"])
127
+ head = (
128
+ "<tr><th data-sort>package</th><th data-sort=num>30d</th>"
129
+ "<th data-sort=num>vs prior</th><th data-sort=num title='30-day downloads "
130
+ "over the noise floor'>× floor</th><th data-sort=num title='dependent "
131
+ "packages / repositories'>dependents</th><th>26 weeks</th></tr>"
132
+ )
133
+ body = []
134
+ for r in rows:
135
+ dp = _metric(r, "dependent_packages").get("value")
136
+ dr = _metric(r, "dependent_repos").get("value")
137
+ deps = f"{_num(dp)} / {_num(dr)}" if dp is not None or dr is not None else "–"
138
+ xf = r.get("x_floor")
139
+ body.append(
140
+ f"<tr><td>{_link(r['url'], r['package'])}</td>"
141
+ f'<td data-v="{r["downloads"]["30d"]}">{_num(r["downloads"]["30d"])}</td>'
142
+ f'<td data-v="{r["growth_30d"] if r["growth_30d"] is not None else ""}">'
143
+ f"{_pct(r['growth_30d'])}</td>"
144
+ f'<td data-v="{xf if xf is not None else ""}">'
145
+ f"{_times(xf)}</td>"
146
+ f'<td data-v="{(dp or 0) * 1_000_000 + (dr or 0)}">{deps}</td>'
147
+ f"<td>{_spark(r['weekly'])}</td></tr>"
148
+ )
149
+ return f'<table class="sortable"><thead>{head}</thead><tbody>{"".join(body)}</tbody></table>'
150
+
151
+
152
+ def _snapshot_table(rows: list[Mapping], headline: list[str]) -> str:
153
+ first = headline[0] if headline else ""
154
+ rows = sorted(rows, key=lambda r: -(_metric(r, first).get("value") or 0))
155
+ head = "<tr><th data-sort>name</th>" + "".join(
156
+ f"<th data-sort=num>{_e(_label(m))}</th>" for m in headline
157
+ )
158
+ body = []
159
+ for r in rows:
160
+ cells = []
161
+ for m in headline:
162
+ d = _metric(r, m)
163
+ cells.append(
164
+ f'<td data-v="{d.get("value", "")}">{_num(d.get("value"))}'
165
+ f"{_signed(d.get('change_30d'))}</td>"
166
+ )
167
+ body.append(
168
+ f"<tr><td>{_link(r['url'], r['package'])}</td>{''.join(cells)}</tr>"
169
+ )
170
+ return (
171
+ f'<table class="sortable"><thead>{head}</tr></thead>'
172
+ f"<tbody>{''.join(body)}</tbody></table>"
173
+ )
174
+
175
+
176
+ _CSS = """
177
+ :root{--bg:#fbfbfa;--fg:#1d1d1f;--muted:#6e6e73;--line:#e3e3e0;--card:#fff;
178
+ --accent:#2f6fde;--up:#1f8a4c;--down:#c0392b}
179
+ @media (prefers-color-scheme:dark){:root:not([data-theme=light]){--bg:#141416;
180
+ --fg:#ececee;--muted:#9a9aa1;--line:#2c2c30;--card:#1c1c1f;--accent:#7aa7ff;
181
+ --up:#5cc98a;--down:#ff7b6b}}
182
+ *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);
183
+ font:15px/1.45 system-ui,-apple-system,Segoe UI,sans-serif}
184
+ main{max-width:980px;margin:0 auto;padding:16px}
185
+ h1{font-size:1.5rem;margin:.2em 0}h2{margin:1.6em 0 .4em;font-size:1.15rem}
186
+ h3{margin:0 0 .3em;font-size:1rem;color:var(--muted);font-weight:600}
187
+ a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}
188
+ .muted{color:var(--muted)}.small{font-size:.85rem}.up{color:var(--up)}.down{color:var(--down)}
189
+ .cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(210px,100%),1fr));gap:12px}
190
+ .card{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
191
+ .big{font-size:1.7rem;font-weight:650;font-variant-numeric:tabular-nums}
192
+ .scroll{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--card)}
193
+ table{border-collapse:collapse;width:100%;font-variant-numeric:tabular-nums}
194
+ th,td{padding:6px 10px;border-bottom:1px solid var(--line);text-align:right;white-space:nowrap}
195
+ th:first-child,td:first-child{text-align:left;position:sticky;left:0;background:var(--card)}
196
+ th{font-size:.8rem;color:var(--muted);font-weight:600}th[data-sort]{cursor:pointer}
197
+ th[aria-sort=descending]::after{content:" ▾"}th[aria-sort=ascending]::after{content:" ▴"}
198
+ .spark{color:var(--accent);vertical-align:middle}
199
+ details{margin:1.5em 0;background:var(--card);border:1px solid var(--line);border-radius:10px;padding:10px 14px}
200
+ summary{cursor:pointer;font-weight:600}
201
+ """
202
+
203
+ _JS = """
204
+ document.querySelectorAll('table.sortable th[data-sort]').forEach((th,i)=>{
205
+ th.addEventListener('click',()=>{const t=th.closest('table'),b=t.tBodies[0];
206
+ const col=[...th.parentNode.children].indexOf(th),num=th.dataset.sort==='num';
207
+ const dir=th.getAttribute('aria-sort')==='descending'?1:-1;
208
+ t.querySelectorAll('th').forEach(h=>h.removeAttribute('aria-sort'));
209
+ th.setAttribute('aria-sort',dir<0?'descending':'ascending');
210
+ const val=r=>{const c=r.children[col];return num?parseFloat(c.dataset.v||'-1e18'):c.textContent.toLowerCase()};
211
+ [...b.rows].sort((x,y)=>{const a=val(x),z=val(y);return (a>z?1:a<z?-1:0)*dir}).forEach(r=>b.appendChild(r))})});
212
+ """
213
+
214
+ _HOW_TO_READ = """
215
+ <details><summary>How to read these numbers</summary>
216
+ <p><b>Downloads are mostly not people.</b> Mirrors, CI pipelines, security
217
+ scanners and bots fetch every release of every package, so a package nobody
218
+ uses still gets a steady trickle. The <b>noise floor</b> is the 25th percentile
219
+ of 30-day downloads among the packages followed here that were downloaded at all; <b>× floor</b> says how
220
+ far above it a package is. A package near 1× is probably only seen by robots.</p>
221
+ <p><b>The strongest signs of real use</b> are the ones a robot does not create:
222
+ <b>dependents</b> (other packages and repositories that declare it, from
223
+ ecosyste.ms), stars and forks, and issues opened by people other than you.
224
+ Growth over the prior 30 days matters more than the level.</p>
225
+ <p>PyPI counts exclude known mirrors (pypistats.org's <i>without_mirrors</i>).
226
+ Registries keep only a few months of daily counts, so this history starts at
227
+ each package's first collection.</p></details>
228
+ """
229
+
230
+
231
+ def render_html(summary: Summary, *, title: str = DFLT_TITLE) -> str:
232
+ """The report page for a :func:`~pkgreach.analyze.summarize` result."""
233
+ platforms = summary["platforms"]
234
+ # daily-download platforms first, by downloads; then snapshot ones
235
+ order = sorted(
236
+ platforms,
237
+ key=lambda n: (platforms[n]["kind"] != "daily", -platforms[n]["downloads_30d"]),
238
+ )
239
+ cards = "".join(_card(platforms[p]) for p in order)
240
+ sections = []
241
+ for p in order:
242
+ meta = platforms[p]
243
+ rows = [r for r in summary["packages"] if r["platform"] == p]
244
+ if meta["kind"] == "snapshot":
245
+ table = _snapshot_table(rows, meta["headline"])
246
+ else:
247
+ table = _registry_table(rows)
248
+ sections.append(
249
+ f"<h2>{_e(meta['title'])} "
250
+ f'<span class="muted small">{len(rows)}</span></h2>'
251
+ f'<div class="scroll">{table}</div>'
252
+ )
253
+ return f"""<!doctype html>
254
+ <html lang="en"><head><meta charset="utf-8">
255
+ <meta name="viewport" content="width=device-width,initial-scale=1">
256
+ <title>{_e(title)}</title><style>{_CSS}</style></head>
257
+ <body><main>
258
+ <h1>{_e(title)}</h1>
259
+ <p class="muted small">As of {_e(summary["as_of"])} · made with
260
+ <a href="https://github.com/thorwhalen/pkgreach">pkgreach</a> ·
261
+ <a href="summary.json">summary.json</a>{_errors_note(summary)}</p>
262
+ <div class="cards">{cards}</div>
263
+ {_HOW_TO_READ}
264
+ {"".join(sections)}
265
+ <p class="muted small">Downloads: pypistats.org, api.npmjs.org, crates.io.
266
+ Dependents: <a href="https://ecosyste.ms">ecosyste.ms</a>, licensed
267
+ <a href="https://creativecommons.org/licenses/by-sa/4.0/">CC BY-SA 4.0</a>,
268
+ aggregated here. Stars and forks: GitHub, public repositories only.</p>
269
+ </main><script>{_JS}</script></body></html>
270
+ """
271
+
272
+
273
+ def build_site(
274
+ store: MutableMapping,
275
+ out: MutableMapping,
276
+ *,
277
+ analyze: Callable[[MutableMapping], Summary] = summarize,
278
+ title: str = DFLT_TITLE,
279
+ ) -> Summary:
280
+ """Summarize the history in ``store`` and write the site into ``out``.
281
+
282
+ ``analyze`` is the seam for another summary (it must return the shape of
283
+ :func:`~pkgreach.analyze.summarize`, which ``summary.json`` documents).
284
+ """
285
+ # seam candidate: render= (another page from the same summary), once a
286
+ # second renderer exists
287
+ summary = analyze(store)
288
+ out[SUMMARY_KEY] = json.dumps(summary, indent=1) + "\n"
289
+ out["index.html"] = render_html(summary, title=title)
290
+ return summary
pkgreach/scaffold.py ADDED
@@ -0,0 +1,137 @@
1
+ """Setting up a repository: the config and the weekly workflow.
2
+
3
+ ``pkgreach init <dir> --pypi alice --npm alice`` writes ``pkgreach.toml`` and
4
+ ``.github/workflows/pkgreach.yml``: every Sunday, the workflow collects into
5
+ the history (kept on the ``pkgreach-data`` branch of the same repository),
6
+ builds the report and publishes it on the repository's GitHub Pages.
7
+
8
+ Everything pkgreach collects is public already (download counts, dependents,
9
+ stars of public repositories), so the repository can be public, and needs no
10
+ secret: the workflow's own token reads GitHub and pushes the history.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from collections.abc import Iterable, Mapping
16
+ from pathlib import Path
17
+
18
+ from pkgreach.platforms import PLATFORMS
19
+
20
+ ACTION_REPO = "thorwhalen/pkgreach"
21
+ FALLBACK_ACTION_REF = f"{ACTION_REPO}@main"
22
+ #: 05:23 UTC on Sundays: off the top of the hour (GitHub delays on-the-hour
23
+ #: crons). Registries keep 3 to 18 months of daily counts, so weekly runs lose
24
+ #: nothing even when a few are missed.
25
+ DFLT_CRON = "23 5 * * 0"
26
+ WORKFLOW_PATH = ".github/workflows/pkgreach.yml"
27
+ CONFIG_PATH = "pkgreach.toml"
28
+
29
+ WORKFLOW_TEMPLATE = """\
30
+ # Collects package stats every Sunday and publishes the report on GitHub Pages.
31
+ # Generated by `pkgreach init`. Docs: https://github.com/thorwhalen/pkgreach
32
+ #
33
+ # Needs no secret. The history is kept on the `pkgreach-data` branch.
34
+ # One-time setup: Settings -> Pages -> Source: "GitHub Actions".
35
+ name: pkgreach
36
+ on:
37
+ schedule:
38
+ - cron: "{cron}"
39
+ workflow_dispatch:
40
+ permissions:
41
+ contents: write
42
+ pages: write
43
+ id-token: write
44
+ concurrency:
45
+ group: pkgreach
46
+ cancel-in-progress: false
47
+ jobs:
48
+ collect:
49
+ runs-on: ubuntu-latest
50
+ timeout-minutes: 30
51
+ environment:
52
+ name: github-pages
53
+ url: ${{{{ steps.deploy.outputs.page_url }}}}
54
+ steps:
55
+ - uses: actions/checkout@v4
56
+ - uses: {action_ref}
57
+ id: reach
58
+ - uses: actions/upload-pages-artifact@v3
59
+ with:
60
+ path: ${{{{ steps.reach.outputs.site-dir }}}}
61
+ - uses: actions/deploy-pages@v4
62
+ id: deploy
63
+ """
64
+
65
+ CONFIG_HEADER = """\
66
+ # pkgreach settings: one table per platform. Every key is optional; see
67
+ # https://github.com/thorwhalen/pkgreach#configuration
68
+ #
69
+ # users = accounts whose packages are all followed
70
+ # packages = packages followed by name, on top of the users' ones
71
+ # exclude = glob patterns of package names to drop
72
+ # where = metadata a package must match, e.g. {{ fork = false }}
73
+ """
74
+
75
+
76
+ def default_action_ref() -> str:
77
+ """``thorwhalen/pkgreach@<installed version>``, or ``@main`` from a checkout."""
78
+ try:
79
+ from importlib.metadata import version
80
+
81
+ return f"{ACTION_REPO}@{version('pkgreach')}"
82
+ except Exception: # noqa: BLE001 - not installed: a source checkout
83
+ return FALLBACK_ACTION_REF
84
+
85
+
86
+ def _toml_list(items: Iterable[str]) -> str:
87
+ return "[" + ", ".join(f'"{i}"' for i in items) + "]"
88
+
89
+
90
+ def _toml_inline(d: Mapping) -> str:
91
+ def v(x):
92
+ return str(x).lower() if isinstance(x, bool) else f'"{x}"'
93
+
94
+ return "{ " + ", ".join(f"{k} = {v(x)}" for k, x in d.items()) + " }"
95
+
96
+
97
+ def config_text(users: Mapping[str, Iterable[str]], *, platforms=None) -> str:
98
+ """The ``pkgreach.toml`` following ``users`` per platform (each platform's
99
+ ``default_where`` included)."""
100
+ platforms = PLATFORMS if platforms is None else platforms
101
+ tables = []
102
+ for platform, names in users.items():
103
+ lines = [f"[{platform}]", f"users = {_toml_list(names)}"]
104
+ where = getattr(platforms.get(platform), "default_where", None)
105
+ if where:
106
+ lines.append(f"where = {_toml_inline(where)}")
107
+ tables.append("\n".join(lines))
108
+ return CONFIG_HEADER.format() + "\n" + "\n\n".join(tables) + "\n"
109
+
110
+
111
+ def init_repo(
112
+ directory: str | Path,
113
+ *,
114
+ users: Mapping[str, Iterable[str]],
115
+ cron: str = DFLT_CRON,
116
+ action_ref: str | None = None,
117
+ overwrite: bool = False,
118
+ ) -> list[str]:
119
+ """Write the config and the workflow; return the paths written."""
120
+ if not users:
121
+ raise ValueError("Name at least one account, e.g. --pypi you --npm you.")
122
+ root = Path(directory)
123
+ files = {
124
+ CONFIG_PATH: config_text(users),
125
+ WORKFLOW_PATH: WORKFLOW_TEMPLATE.format(
126
+ cron=cron, action_ref=action_ref or default_action_ref()
127
+ ),
128
+ }
129
+ written = []
130
+ for rel, text in files.items():
131
+ path = root / rel
132
+ if path.exists() and not overwrite:
133
+ continue
134
+ path.parent.mkdir(parents=True, exist_ok=True)
135
+ path.write_text(text, encoding="utf-8")
136
+ written.append(rel)
137
+ return written
pkgreach/tables.py ADDED
@@ -0,0 +1,120 @@
1
+ """The schema of a pkgreach data directory: long-form CSV tables, one per kind.
2
+
3
+ Every platform writes the same tables, with a ``platform`` column, so adding a
4
+ platform never changes the schema. The CSV machinery (year partitions,
5
+ idempotent merges, the ``MutableMapping`` store) is ghtraffic's
6
+ (:mod:`ghtraffic.tables`): the history of a package's downloads has the same
7
+ shape, and the same merge problem, as the history of a repo's views.
8
+
9
+ - ``downloads``: daily counts, merged by **max** per ``(date, platform,
10
+ package, series)``. Registries recompute recent days, and windows overlap
11
+ between runs; max makes re-runs and overlaps harmless. ``series`` separates
12
+ variants of one count (``all``; ``without_mirrors`` on PyPI).
13
+ - ``metrics``: slowly changing numbers (dependents, stars, versions, total
14
+ downloads), one row per **change** per ``(platform, package, metric)``.
15
+ - ``packages``: current metadata of every package seen (kept after it is
16
+ dropped from the selection, with its ``last_seen``). ``history_from`` is the
17
+ first day its downloads were collected for: before it, a day is unknown;
18
+ from it on, a missing ``downloads`` row means zero.
19
+ - ``runs``: one row per collection run.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ # The one import of ghtraffic: every other module goes through here, so moving the
25
+ # store to its own package (thorwhalen/ghtraffic#7) changes this line only.
26
+ from ghtraffic.tables import (
27
+ Files,
28
+ TableSpec,
29
+ dump_csv,
30
+ parse_csv,
31
+ merge_rows,
32
+ read_table,
33
+ table_files,
34
+ write_rows,
35
+ )
36
+
37
+ __all__ = [
38
+ "Files",
39
+ "TableSpec",
40
+ "dump_csv",
41
+ "parse_csv",
42
+ "TABLES",
43
+ "merge_rows",
44
+ "read_table",
45
+ "table_files",
46
+ "write_rows",
47
+ ]
48
+
49
+ SCHEMA_VERSION = 1
50
+ MANIFEST_KEY = "manifest.json"
51
+ LAST_RUN_KEY = "last_run.json"
52
+ SUMMARY_KEY = "summary.json"
53
+ DATA_LICENSE_KEY = "DATA-LICENSE.md"
54
+ DATA_LICENSE = """\
55
+ # Where this data comes from, and on what terms
56
+
57
+ - `metrics.csv`, metrics `dependent_packages` and `dependent_repos`: from
58
+ [ecosyste.ms](https://ecosyste.ms), licensed
59
+ [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). Collected weekly
60
+ and kept as a change log; reuse them under the same licence, crediting ecosyste.ms.
61
+ - `downloads/`: daily download counts from pypistats.org (PyPI), api.npmjs.org (npm)
62
+ and crates.io.
63
+ - Other metrics and `packages.csv`: the registries' public metadata, and the
64
+ GitHub REST API for public repositories.
65
+
66
+ Collected by pkgreach (https://github.com/thorwhalen/pkgreach).
67
+ """
68
+
69
+ TABLES: dict[str, TableSpec] = {
70
+ t.name: t
71
+ for t in [
72
+ TableSpec(
73
+ "downloads",
74
+ ("date", "platform", "package", "series", "count"),
75
+ key=("date", "platform", "package", "series"),
76
+ merge="max",
77
+ int_columns=("count",),
78
+ time_column="date",
79
+ description="Daily downloads per package; days with zero downloads "
80
+ "are absent (from the package's history_from on, absence means zero).",
81
+ ),
82
+ TableSpec(
83
+ "metrics",
84
+ ("date", "platform", "package", "metric", "value"),
85
+ key=("date", "platform", "package", "metric"),
86
+ merge="change_only",
87
+ entity=("platform", "package", "metric"),
88
+ int_columns=("value",),
89
+ description="Slowly changing numbers (dependents, stars, forks, "
90
+ "versions, total downloads), one row per change.",
91
+ sort=("platform", "package", "metric", "date"),
92
+ ),
93
+ TableSpec(
94
+ "packages",
95
+ (
96
+ "platform",
97
+ "package",
98
+ "url",
99
+ "repository",
100
+ "description",
101
+ "latest_version",
102
+ "created",
103
+ "history_from",
104
+ "first_seen",
105
+ "last_seen",
106
+ ),
107
+ key=("platform", "package"),
108
+ merge="upsert",
109
+ description="Current metadata of every package seen.",
110
+ ),
111
+ TableSpec(
112
+ "runs",
113
+ ("started_at", "duration_s", "platform", "packages", "errors"),
114
+ key=("started_at", "platform"),
115
+ merge="upsert",
116
+ int_columns=("packages", "errors"),
117
+ description="One row per collection run and platform.",
118
+ ),
119
+ ]
120
+ }