pkgreach 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pkgreach/__init__.py +53 -0
- pkgreach/__main__.py +36 -0
- pkgreach/analyze.py +242 -0
- pkgreach/collect.py +301 -0
- pkgreach/config.py +120 -0
- pkgreach/http.py +129 -0
- pkgreach/platforms/__init__.py +103 -0
- pkgreach/platforms/crates.py +67 -0
- pkgreach/platforms/ecosystems.py +36 -0
- pkgreach/platforms/github.py +84 -0
- pkgreach/platforms/npm.py +90 -0
- pkgreach/platforms/pypi.py +74 -0
- pkgreach/report.py +290 -0
- pkgreach/scaffold.py +137 -0
- pkgreach/tables.py +120 -0
- pkgreach/tools.py +89 -0
- pkgreach-0.0.2.dist-info/METADATA +127 -0
- pkgreach-0.0.2.dist-info/RECORD +21 -0
- pkgreach-0.0.2.dist-info/WHEEL +4 -0
- pkgreach-0.0.2.dist-info/entry_points.txt +2 -0
- pkgreach-0.0.2.dist-info/licenses/LICENSE +21 -0
pkgreach/tools.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""The operations, as plain functions: JSON-able arguments in, JSON-able dicts out.
|
|
2
|
+
|
|
3
|
+
This is the single source of truth for every surface: the CLI dispatches
|
|
4
|
+
:data:`COMMANDS` with ``cw`` (:mod:`pkgreach.__main__`), and an MCP or HTTP
|
|
5
|
+
adapter would reference ``pkgreach.tools:collect`` and so on. Each function
|
|
6
|
+
does the wiring a caller should not have to: reading the settings file,
|
|
7
|
+
opening the folders, giving the transport the GitHub token from the
|
|
8
|
+
environment.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
from collections.abc import Mapping, Sequence
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from pkgreach.collect import collect as _collect
|
|
18
|
+
from pkgreach.config import DFLT_CONFIG_FILENAME, DFLT_DATA_DIR, load_settings
|
|
19
|
+
from pkgreach.http import make_http
|
|
20
|
+
from pkgreach.platforms import PLATFORMS
|
|
21
|
+
from pkgreach.platforms.github import auth_headers, token_from_env
|
|
22
|
+
from pkgreach.report import DFLT_TITLE, build_site
|
|
23
|
+
from pkgreach.scaffold import DFLT_CRON, default_action_ref, init_repo
|
|
24
|
+
from pkgreach.tables import Files
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CollectionFailed(RuntimeError):
|
|
28
|
+
"""Raised by :func:`collect` with ``fail_on_errors`` when something failed."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def collect(
|
|
32
|
+
*,
|
|
33
|
+
config: str = DFLT_CONFIG_FILENAME,
|
|
34
|
+
data_dir: str = str(DFLT_DATA_DIR),
|
|
35
|
+
fail_on_errors: bool = False,
|
|
36
|
+
environ: Mapping[str, str] = os.environ,
|
|
37
|
+
) -> dict:
|
|
38
|
+
"""Fetch this week's numbers for the packages in ``config`` into ``data_dir``.
|
|
39
|
+
|
|
40
|
+
The GitHub token, if any, comes from ``GITHUB_TOKEN`` or ``GH_TOKEN``.
|
|
41
|
+
"""
|
|
42
|
+
settings = load_settings(config, known_platforms=PLATFORMS)
|
|
43
|
+
http = make_http(headers_by_host=auth_headers(token_from_env(environ)))
|
|
44
|
+
report = _collect(settings, store=Files(Path(data_dir)), http=http, log=print)
|
|
45
|
+
if fail_on_errors and report.errors:
|
|
46
|
+
raise CollectionFailed(f"{len(report.errors)} errors; see last_run.json")
|
|
47
|
+
return {
|
|
48
|
+
"packages": report.packages,
|
|
49
|
+
"errors": len(report.errors),
|
|
50
|
+
"files_changed": report.files_changed,
|
|
51
|
+
"duration_s": report.duration_s,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def report(
|
|
56
|
+
*, data_dir: str = str(DFLT_DATA_DIR), out: str = "site", title: str = DFLT_TITLE
|
|
57
|
+
) -> dict:
|
|
58
|
+
"""Write the report site (``index.html``, ``summary.json``) into ``out``."""
|
|
59
|
+
s = build_site(Files(Path(data_dir)), Files(Path(out)), title=title)
|
|
60
|
+
return {"out": out, "packages": len(s["packages"]), "as_of": s["as_of"]}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def init(
|
|
64
|
+
directory: str,
|
|
65
|
+
*,
|
|
66
|
+
pypi: Sequence[str] = (),
|
|
67
|
+
npm: Sequence[str] = (),
|
|
68
|
+
crates: Sequence[str] = (),
|
|
69
|
+
github: Sequence[str] = (),
|
|
70
|
+
cron: str = DFLT_CRON,
|
|
71
|
+
action_ref: str = "",
|
|
72
|
+
overwrite: bool = False,
|
|
73
|
+
) -> dict:
|
|
74
|
+
"""Write ``pkgreach.toml`` and a weekly workflow into the repo at ``directory``.
|
|
75
|
+
|
|
76
|
+
Name your accounts per platform (``--pypi you --github you your-org``).
|
|
77
|
+
"""
|
|
78
|
+
users = {"pypi": pypi, "npm": npm, "crates": crates, "github": github}
|
|
79
|
+
written = init_repo(
|
|
80
|
+
directory,
|
|
81
|
+
users={k: list(v) for k, v in users.items() if v},
|
|
82
|
+
cron=cron,
|
|
83
|
+
action_ref=action_ref or default_action_ref(),
|
|
84
|
+
overwrite=overwrite,
|
|
85
|
+
)
|
|
86
|
+
return {"written": written}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
COMMANDS = [collect, report, init]
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pkgreach
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Track how your packages are really used across PyPI, npm, crates.io and GitHub: downloads against a noise floor, dependents, stars; a weekly GitHub Action keeps the history and publishes a report.
|
|
5
|
+
Project-URL: Homepage, https://github.com/thorwhalen/pkgreach
|
|
6
|
+
Project-URL: Repository, https://github.com/thorwhalen/pkgreach
|
|
7
|
+
Project-URL: Dashboard, https://thorwhalen.github.io/pkgreach/
|
|
8
|
+
Author: Thor Whalen
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: crates,dashboard,dependents,downloads,github,github-actions,npm,pypi,statistics
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: cw<0.2,>=0.1.14
|
|
14
|
+
Requires-Dist: ghtraffic<0.1,>=0.0.4
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
18
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
19
|
+
Provides-Extra: docs
|
|
20
|
+
Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
|
|
21
|
+
Requires-Dist: sphinx>=6.0; extra == 'docs'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# pkgreach
|
|
25
|
+
|
|
26
|
+
How are your packages really used? `pkgreach` tracks your packages on PyPI, npm, crates.io and GitHub every week, keeps the history the registries throw away, and publishes a report that separates robots from people.
|
|
27
|
+
|
|
28
|
+
**Live example (the author's packages):** https://thorwhalen.github.io/pkgreach/
|
|
29
|
+
|
|
30
|
+
## Why
|
|
31
|
+
|
|
32
|
+
Registries keep a few months of daily download counts (crates.io 90 days, PyPI about 6 months, npm 18 months), and none of them separates people from robots: mirrors, CI pipelines and scanners download every release of every package. So `pkgreach` reports downloads against a **noise floor** (what your least-used packages get), next to the signals a robot does not create: **dependents** (packages and repositories that declare yours, from [ecosyste.ms](https://ecosyste.ms)), and **stars and forks** of your public repositories.
|
|
33
|
+
|
|
34
|
+
## Quick start: a weekly GitHub Action
|
|
35
|
+
|
|
36
|
+
In any repository you own (it can be public: everything collected is public already):
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install pkgreach
|
|
40
|
+
pkgreach init . --pypi YOUR_PYPI_USER --npm YOUR_NPM_USER --github YOUR_GITHUB_USER YOUR_ORG
|
|
41
|
+
git add -A && git commit -m "pkgreach" && git push
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`pkgreach init` writes `pkgreach.toml` (what to follow) and `.github/workflows/pkgreach.yml`. Then turn on Pages once (Settings → Pages → Source: **GitHub Actions**) and run the workflow (`gh workflow run pkgreach.yml`). Every Sunday after that it collects, commits the history to the `pkgreach-data` branch, and publishes the report on your GitHub Pages. No secret is needed.
|
|
45
|
+
|
|
46
|
+
The workflow is three steps:
|
|
47
|
+
|
|
48
|
+
```yaml
|
|
49
|
+
- uses: actions/checkout@v4
|
|
50
|
+
- uses: thorwhalen/pkgreach@<version> # pin the release you installed
|
|
51
|
+
id: reach
|
|
52
|
+
- uses: actions/upload-pages-artifact@v3
|
|
53
|
+
with:
|
|
54
|
+
path: ${{ steps.reach.outputs.site-dir }}
|
|
55
|
+
- uses: actions/deploy-pages@v4
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Configuration
|
|
59
|
+
|
|
60
|
+
`pkgreach.toml` has one table per platform: `pypi`, `npm`, `crates`, `github`. Every key is optional; unknown keys are refused, so a typo never silently does nothing.
|
|
61
|
+
|
|
62
|
+
```toml
|
|
63
|
+
[pypi]
|
|
64
|
+
users = ["alice"] # every package these accounts own or maintain
|
|
65
|
+
packages = ["a-package-of-a-friend"] # plus these
|
|
66
|
+
exclude = ["alice-test-*"] # minus these (glob patterns)
|
|
67
|
+
|
|
68
|
+
[npm]
|
|
69
|
+
users = ["alice"]
|
|
70
|
+
|
|
71
|
+
[crates]
|
|
72
|
+
users = ["alice"] # crates.io login
|
|
73
|
+
|
|
74
|
+
[github]
|
|
75
|
+
users = ["alice", "alice-org"] # public repos of these users and orgs
|
|
76
|
+
where = { fork = false, archived = false } # characteristics a package must have
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`where` matches the metadata of listed packages field by field (GitHub repos have `fork`, `archived`, `language`; PyPI packages have `role`, `owner` or `maintainer`); packages named in `packages` are always kept. PyPI names are normalized (`Flask_Login` is `flask-login`). Top-level keys: `max_workers` (default 4) and `dependents` (default `true`: ask ecosyste.ms).
|
|
80
|
+
|
|
81
|
+
The Action's inputs: `config`, `data-branch` (default `pkgreach-data`), `site-dir`, `title`, `token` (reads GitHub; defaults to the workflow's token), `commit`, `fail-on-errors` (default `false`: a failed package is listed in `last_run.json` and flagged on the page, and the run stays green).
|
|
82
|
+
|
|
83
|
+
## Run it locally
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pkgreach collect --config pkgreach.toml # into ~/.local/share/pkgreach/data
|
|
87
|
+
pkgreach report --out site # site/index.html and site/summary.json
|
|
88
|
+
pkgreach init . --pypi me --github me my-org # a config and a weekly workflow
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
From Python, every part is a keyword argument with a working default:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from pkgreach import collect, load_settings, build_site, Files, PLATFORMS
|
|
95
|
+
|
|
96
|
+
collect(load_settings("pkgreach.toml"), store=Files("data"))
|
|
97
|
+
build_site(Files("data"), Files("site"))
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
- `collect(platforms=...)` and `summarize(platforms=...)`: add a platform with `Platform(name, list_packages, fetch, kind=..., headline=...)`; see `pkgreach/platforms/__init__.py` for the contract. A `"snapshot"` platform (current values only, like Hugging Face likes) is shown by its `headline` metrics.
|
|
101
|
+
- `collect(select=lambda platform, record: ...)`: any selection a TOML file cannot express.
|
|
102
|
+
- `collect(dependents=...)`: another source of dependents.
|
|
103
|
+
- `build_site(analyze=...)`: your own summary; `summary.json` is the default one, for any other tool.
|
|
104
|
+
- `make_http(headers_by_host=...)`: credentials per host (the CLI gives GitHub `GITHUB_TOKEN` or `GH_TOKEN`).
|
|
105
|
+
- `store=` / `out=`: any `MutableMapping[str, str]`, such as a [dol](https://github.com/i2mint/dol) store (S3, a database).
|
|
106
|
+
|
|
107
|
+
## What is collected
|
|
108
|
+
|
|
109
|
+
| platform | daily downloads | other numbers | source |
|
|
110
|
+
|---|---|---|---|
|
|
111
|
+
| PyPI | all, and without mirrors | versions, dependents | pypistats.org, PyPI JSON, ecosyste.ms |
|
|
112
|
+
| npm | all | dependents | api.npmjs.org, ecosyste.ms |
|
|
113
|
+
| crates.io | all | total downloads, versions, dependents | crates.io, ecosyste.ms |
|
|
114
|
+
| GitHub | none | stars, forks, open issues and PRs | GitHub REST (public repos only) |
|
|
115
|
+
|
|
116
|
+
The history is plain CSV, one table per kind, split by year (`downloads/YYYY.csv`, `metrics.csv`, `packages.csv`, `runs.csv`), merged idempotently so re-running a collection never changes the result. `manifest.json` lists every file and its columns. Design and alternatives: [docs/adr/0001-seams-and-scope.md](docs/adr/0001-seams-and-scope.md). What each platform offers, verified: [docs/research/platform-apis.md](docs/research/platform-apis.md).
|
|
117
|
+
|
|
118
|
+
**GitHub traffic** (views, clones, referrers) is private and needs a token with push access. That is [ghtraffic](https://github.com/thorwhalen/ghtraffic)'s job, in a private repository; pkgreach reuses its storage layer.
|
|
119
|
+
|
|
120
|
+
Dependents data is from [ecosyste.ms](https://ecosyste.ms), licensed [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/); the report credits it and the history carries a `DATA-LICENSE.md`. A repository that stops being listed (made private, deleted, or dropped from the selection) leaves the current tables at the next run; what was committed before stays in the branch's git history.
|
|
121
|
+
|
|
122
|
+
## Limits
|
|
123
|
+
|
|
124
|
+
- History starts at your first run, minus what the registry still keeps.
|
|
125
|
+
- One run of about 400 packages takes about 7 minutes, mostly waiting on pypistats.org's limit of 60 requests a minute.
|
|
126
|
+
- Listing a PyPI account uses PyPI's XML-RPC `user_packages`, deprecated but still answered.
|
|
127
|
+
- Dependents come from ecosyste.ms, which indexes new packages with some delay: a fresh package shows none at first.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
pkgreach/__init__.py,sha256=WuAu6U7d8SBA6-PJsQFcER_xhTR-4UMbpyqxF1Cgmxk,1978
|
|
2
|
+
pkgreach/__main__.py,sha256=vzH65JOSvKg09CkwGQb5ewbtcYM1q8RXMVyo6e7RFJY,1136
|
|
3
|
+
pkgreach/analyze.py,sha256=_b-Y4xWNhfeTkeNBt0doSU76REcAWpC4VchPq2SAw-0,9739
|
|
4
|
+
pkgreach/collect.py,sha256=JU14kFyRMFJPFx5edFYBMjNLaez-qbewcv9c0-qRgB0,11686
|
|
5
|
+
pkgreach/config.py,sha256=JuK11-9HnqDDugr5JpmJGkYan0FxSEOkS_BFtfPC8OU,4332
|
|
6
|
+
pkgreach/http.py,sha256=wcXv2JPhvl9IY-eDn5LVZizs5My75rPge4LseqrQGbU,5126
|
|
7
|
+
pkgreach/report.py,sha256=RSY4rQSbk8OtcIa3R7Mzocjb4o-nGfyIq22vp7BN6sU,12137
|
|
8
|
+
pkgreach/scaffold.py,sha256=ISDMqcrvA-i1ufZlZUrVDnAWIsNoRPsvWZiTMYOk0pY,4631
|
|
9
|
+
pkgreach/tables.py,sha256=pizrzEmjIRx3HsLGwXIq1JkfqMoGN-bJmRcmQeqw64M,4350
|
|
10
|
+
pkgreach/tools.py,sha256=rZNNszRI6zT64AYbndgIS0r-WskL-_JF-I9xhRlVQCY,3147
|
|
11
|
+
pkgreach/platforms/__init__.py,sha256=LVoSTzDLwYnszyGFcKy2Qls49BAqhmw1XLHVof5TVSg,4180
|
|
12
|
+
pkgreach/platforms/crates.py,sha256=Bymf0V4nTXTCmA-40vSMS3-zGdz8IOdG-_1Qj-65i7Q,2442
|
|
13
|
+
pkgreach/platforms/ecosystems.py,sha256=2ElK5CbPnv6Z_PiWCuXDh7DiYeavUDVITw7DrUYw6no,1342
|
|
14
|
+
pkgreach/platforms/github.py,sha256=rFU3cy9rFdctyeKpzn3l9fasrxjN7ytrquzjRcE-Lyg,3179
|
|
15
|
+
pkgreach/platforms/npm.py,sha256=3Fb2QSazutVWBu_gBuv32uiwrH6W0qKjzZ9lM_OLsGU,3641
|
|
16
|
+
pkgreach/platforms/pypi.py,sha256=OCj2EfliB9CZHQIVjF0Zo3f-Eov4XcvwjGYihMxDRjo,3083
|
|
17
|
+
pkgreach-0.0.2.dist-info/METADATA,sha256=YQY0-lV14JCVA_AYg24ABNZ0RL-YtDXvoA9BPWmllJM,7393
|
|
18
|
+
pkgreach-0.0.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
19
|
+
pkgreach-0.0.2.dist-info/entry_points.txt,sha256=TR2x9HMTe-eZWAPb5SPmb_i8xsKEm-WV22yH1Q0LV74,52
|
|
20
|
+
pkgreach-0.0.2.dist-info/licenses/LICENSE,sha256=ekE13yCE9fWWEJfP1NPnUJabZ_Wwrc6coKJV2ZZx27w,1068
|
|
21
|
+
pkgreach-0.0.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Thor Whalen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|