pkgreach 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pkgreach-0.0.2/.gitattributes +1 -0
- pkgreach-0.0.2/.gitignore +120 -0
- pkgreach-0.0.2/LICENSE +21 -0
- pkgreach-0.0.2/PKG-INFO +127 -0
- pkgreach-0.0.2/README.md +104 -0
- pkgreach-0.0.2/action.yml +88 -0
- pkgreach-0.0.2/pkgreach/__init__.py +53 -0
- pkgreach-0.0.2/pkgreach/__main__.py +36 -0
- pkgreach-0.0.2/pkgreach/analyze.py +242 -0
- pkgreach-0.0.2/pkgreach/collect.py +301 -0
- pkgreach-0.0.2/pkgreach/config.py +120 -0
- pkgreach-0.0.2/pkgreach/http.py +129 -0
- pkgreach-0.0.2/pkgreach/platforms/__init__.py +103 -0
- pkgreach-0.0.2/pkgreach/platforms/crates.py +67 -0
- pkgreach-0.0.2/pkgreach/platforms/ecosystems.py +36 -0
- pkgreach-0.0.2/pkgreach/platforms/github.py +84 -0
- pkgreach-0.0.2/pkgreach/platforms/npm.py +90 -0
- pkgreach-0.0.2/pkgreach/platforms/pypi.py +74 -0
- pkgreach-0.0.2/pkgreach/report.py +290 -0
- pkgreach-0.0.2/pkgreach/scaffold.py +137 -0
- pkgreach-0.0.2/pkgreach/tables.py +120 -0
- pkgreach-0.0.2/pkgreach/tools.py +89 -0
- pkgreach-0.0.2/pkgreach.toml +16 -0
- pkgreach-0.0.2/pyproject.toml +178 -0
- pkgreach-0.0.2/tests/test_pkgreach.py +319 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
*.ipynb linguist-documentation
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
.claude/handoffs/
|
|
2
|
+
.claude/scratch/
|
|
3
|
+
|
|
4
|
+
# Byte-compiled / optimized / DLL files
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*$py.class
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
.DS_Store
|
|
11
|
+
# C extensions
|
|
12
|
+
*.so
|
|
13
|
+
|
|
14
|
+
# TLS certificates
|
|
15
|
+
## Ignore all PEM files anywhere
|
|
16
|
+
*.pem
|
|
17
|
+
## Also ignore any certs directory
|
|
18
|
+
certs/
|
|
19
|
+
|
|
20
|
+
# Distribution / packaging
|
|
21
|
+
.Python
|
|
22
|
+
build/
|
|
23
|
+
develop-eggs/
|
|
24
|
+
dist/
|
|
25
|
+
downloads/
|
|
26
|
+
eggs/
|
|
27
|
+
.eggs/
|
|
28
|
+
lib/
|
|
29
|
+
lib64/
|
|
30
|
+
parts/
|
|
31
|
+
sdist/
|
|
32
|
+
var/
|
|
33
|
+
wheels/
|
|
34
|
+
node_modules/
|
|
35
|
+
*.egg-info/
|
|
36
|
+
.installed.cfg
|
|
37
|
+
*.egg
|
|
38
|
+
MANIFEST
|
|
39
|
+
_build
|
|
40
|
+
|
|
41
|
+
# PyInstaller
|
|
42
|
+
# Usually these files are written by a python script from a template
|
|
43
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
44
|
+
*.manifest
|
|
45
|
+
*.spec
|
|
46
|
+
|
|
47
|
+
# Installer logs
|
|
48
|
+
pip-log.txt
|
|
49
|
+
pip-delete-this-directory.txt
|
|
50
|
+
|
|
51
|
+
# Unit test / coverage reports
|
|
52
|
+
htmlcov/
|
|
53
|
+
.tox/
|
|
54
|
+
.coverage
|
|
55
|
+
.coverage.*
|
|
56
|
+
.cache
|
|
57
|
+
nosetests.xml
|
|
58
|
+
coverage.xml
|
|
59
|
+
*.cover
|
|
60
|
+
.hypothesis/
|
|
61
|
+
.pytest_cache/
|
|
62
|
+
|
|
63
|
+
# Translations
|
|
64
|
+
*.mo
|
|
65
|
+
*.pot
|
|
66
|
+
|
|
67
|
+
# Django stuff:
|
|
68
|
+
*.log
|
|
69
|
+
local_settings.py
|
|
70
|
+
db.sqlite3
|
|
71
|
+
|
|
72
|
+
# Flask stuff:
|
|
73
|
+
instance/
|
|
74
|
+
.webassets-cache
|
|
75
|
+
|
|
76
|
+
# Scrapy stuff:
|
|
77
|
+
.scrapy
|
|
78
|
+
|
|
79
|
+
# Sphinx documentation
|
|
80
|
+
docs/_build/
|
|
81
|
+
|
|
82
|
+
# PyBuilder
|
|
83
|
+
target/
|
|
84
|
+
|
|
85
|
+
# Jupyter Notebook
|
|
86
|
+
.ipynb_checkpoints
|
|
87
|
+
|
|
88
|
+
# pyenv
|
|
89
|
+
.python-version
|
|
90
|
+
|
|
91
|
+
# celery beat schedule file
|
|
92
|
+
celerybeat-schedule
|
|
93
|
+
|
|
94
|
+
# SageMath parsed files
|
|
95
|
+
*.sage.py
|
|
96
|
+
|
|
97
|
+
# Environments
|
|
98
|
+
.env
|
|
99
|
+
.venv
|
|
100
|
+
env/
|
|
101
|
+
venv/
|
|
102
|
+
ENV/
|
|
103
|
+
env.bak/
|
|
104
|
+
venv.bak/
|
|
105
|
+
|
|
106
|
+
# Spyder project settings
|
|
107
|
+
.spyderproject
|
|
108
|
+
.spyproject
|
|
109
|
+
|
|
110
|
+
# Rope project settings
|
|
111
|
+
.ropeproject
|
|
112
|
+
|
|
113
|
+
# mkdocs documentation
|
|
114
|
+
/site
|
|
115
|
+
|
|
116
|
+
# mypy
|
|
117
|
+
.mypy_cache/
|
|
118
|
+
|
|
119
|
+
# PyCharm
|
|
120
|
+
.idea
|
pkgreach-0.0.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Thor Whalen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pkgreach-0.0.2/PKG-INFO
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pkgreach
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Track how your packages are really used across PyPI, npm, crates.io and GitHub: downloads against a noise floor, dependents, stars; a weekly GitHub Action keeps the history and publishes a report.
|
|
5
|
+
Project-URL: Homepage, https://github.com/thorwhalen/pkgreach
|
|
6
|
+
Project-URL: Repository, https://github.com/thorwhalen/pkgreach
|
|
7
|
+
Project-URL: Dashboard, https://thorwhalen.github.io/pkgreach/
|
|
8
|
+
Author: Thor Whalen
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: crates,dashboard,dependents,downloads,github,github-actions,npm,pypi,statistics
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: cw<0.2,>=0.1.14
|
|
14
|
+
Requires-Dist: ghtraffic<0.1,>=0.0.4
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
18
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
19
|
+
Provides-Extra: docs
|
|
20
|
+
Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
|
|
21
|
+
Requires-Dist: sphinx>=6.0; extra == 'docs'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# pkgreach
|
|
25
|
+
|
|
26
|
+
How are your packages really used? `pkgreach` tracks your packages on PyPI, npm, crates.io and GitHub every week, keeps the history the registries throw away, and publishes a report that separates robots from people.
|
|
27
|
+
|
|
28
|
+
**Live example (the author's packages):** https://thorwhalen.github.io/pkgreach/
|
|
29
|
+
|
|
30
|
+
## Why
|
|
31
|
+
|
|
32
|
+
Registries keep a few months of daily download counts (crates.io 90 days, PyPI about 6 months, npm 18 months), and none of them separates people from robots: mirrors, CI pipelines and scanners download every release of every package. So `pkgreach` reports downloads against a **noise floor** (what your least-used packages get), next to the signals a robot does not create: **dependents** (packages and repositories that declare yours, from [ecosyste.ms](https://ecosyste.ms)), and **stars and forks** of your public repositories.
|
|
33
|
+
|
|
34
|
+
## Quick start: a weekly GitHub Action
|
|
35
|
+
|
|
36
|
+
In any repository you own (it can be public: everything collected is public already):
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install pkgreach
|
|
40
|
+
pkgreach init . --pypi YOUR_PYPI_USER --npm YOUR_NPM_USER --github YOUR_GITHUB_USER YOUR_ORG
|
|
41
|
+
git add -A && git commit -m "pkgreach" && git push
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`pkgreach init` writes `pkgreach.toml` (what to follow) and `.github/workflows/pkgreach.yml`. Then turn on Pages once (Settings → Pages → Source: **GitHub Actions**) and run the workflow (`gh workflow run pkgreach.yml`). Every Sunday after that it collects, commits the history to the `pkgreach-data` branch, and publishes the report on your GitHub Pages. No secret is needed.
|
|
45
|
+
|
|
46
|
+
The workflow is three steps:
|
|
47
|
+
|
|
48
|
+
```yaml
|
|
49
|
+
- uses: actions/checkout@v4
|
|
50
|
+
- uses: thorwhalen/pkgreach@<version> # pin the release you installed
|
|
51
|
+
id: reach
|
|
52
|
+
- uses: actions/upload-pages-artifact@v3
|
|
53
|
+
with:
|
|
54
|
+
path: ${{ steps.reach.outputs.site-dir }}
|
|
55
|
+
- uses: actions/deploy-pages@v4
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Configuration
|
|
59
|
+
|
|
60
|
+
`pkgreach.toml` has one table per platform: `pypi`, `npm`, `crates`, `github`. Every key is optional; unknown keys are refused, so a typo never silently does nothing.
|
|
61
|
+
|
|
62
|
+
```toml
|
|
63
|
+
[pypi]
|
|
64
|
+
users = ["alice"] # every package these accounts own or maintain
|
|
65
|
+
packages = ["a-package-of-a-friend"] # plus these
|
|
66
|
+
exclude = ["alice-test-*"] # minus these (glob patterns)
|
|
67
|
+
|
|
68
|
+
[npm]
|
|
69
|
+
users = ["alice"]
|
|
70
|
+
|
|
71
|
+
[crates]
|
|
72
|
+
users = ["alice"] # crates.io login
|
|
73
|
+
|
|
74
|
+
[github]
|
|
75
|
+
users = ["alice", "alice-org"] # public repos of these users and orgs
|
|
76
|
+
where = { fork = false, archived = false } # characteristics a package must have
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`where` matches the metadata of listed packages field by field (GitHub repos have `fork`, `archived`, `language`; PyPI packages have `role`, `owner` or `maintainer`); packages named in `packages` are always kept. PyPI names are normalized (`Flask_Login` is `flask-login`). Top-level keys: `max_workers` (default 4) and `dependents` (default `true`: ask ecosyste.ms).
|
|
80
|
+
|
|
81
|
+
The Action's inputs: `config`, `data-branch` (default `pkgreach-data`), `site-dir`, `title`, `token` (reads GitHub; defaults to the workflow's token), `commit`, `fail-on-errors` (default `false`: a failed package is listed in `last_run.json` and flagged on the page, and the run stays green).
|
|
82
|
+
|
|
83
|
+
## Run it locally
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pkgreach collect --config pkgreach.toml # into ~/.local/share/pkgreach/data
|
|
87
|
+
pkgreach report --out site # site/index.html and site/summary.json
|
|
88
|
+
pkgreach init . --pypi me --github me my-org # a config and a weekly workflow
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
From Python, every part is a keyword argument with a working default:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from pkgreach import collect, load_settings, build_site, Files, PLATFORMS
|
|
95
|
+
|
|
96
|
+
collect(load_settings("pkgreach.toml"), store=Files("data"))
|
|
97
|
+
build_site(Files("data"), Files("site"))
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
- `collect(platforms=...)` and `summarize(platforms=...)`: add a platform with `Platform(name, list_packages, fetch, kind=..., headline=...)`; see `pkgreach/platforms/__init__.py` for the contract. A `"snapshot"` platform (current values only, like Hugging Face likes) is shown by its `headline` metrics.
|
|
101
|
+
- `collect(select=lambda platform, record: ...)`: any selection a TOML file cannot express.
|
|
102
|
+
- `collect(dependents=...)`: another source of dependents.
|
|
103
|
+
- `build_site(analyze=...)`: your own summary; `summary.json` is the default one, for any other tool.
|
|
104
|
+
- `make_http(headers_by_host=...)`: credentials per host (the CLI gives GitHub `GITHUB_TOKEN` or `GH_TOKEN`).
|
|
105
|
+
- `store=` / `out=`: any `MutableMapping[str, str]`, such as a [dol](https://github.com/i2mint/dol) store (S3, a database).
|
|
106
|
+
|
|
107
|
+
## What is collected
|
|
108
|
+
|
|
109
|
+
| platform | daily downloads | other numbers | source |
|
|
110
|
+
|---|---|---|---|
|
|
111
|
+
| PyPI | all, and without mirrors | versions, dependents | pypistats.org, PyPI JSON, ecosyste.ms |
|
|
112
|
+
| npm | all | dependents | api.npmjs.org, ecosyste.ms |
|
|
113
|
+
| crates.io | all | total downloads, versions, dependents | crates.io, ecosyste.ms |
|
|
114
|
+
| GitHub | none | stars, forks, open issues and PRs | GitHub REST (public repos only) |
|
|
115
|
+
|
|
116
|
+
The history is plain CSV, one table per kind, split by year (`downloads/YYYY.csv`, `metrics.csv`, `packages.csv`, `runs.csv`), merged idempotently so re-running a collection never changes the result. `manifest.json` lists every file and its columns. Design and alternatives: [docs/adr/0001-seams-and-scope.md](docs/adr/0001-seams-and-scope.md). What each platform offers, verified: [docs/research/platform-apis.md](docs/research/platform-apis.md).
|
|
117
|
+
|
|
118
|
+
**GitHub traffic** (views, clones, referrers) is private and needs a token with push access. That is [ghtraffic](https://github.com/thorwhalen/ghtraffic)'s job, in a private repository; pkgreach reuses its storage layer.
|
|
119
|
+
|
|
120
|
+
Dependents data is from [ecosyste.ms](https://ecosyste.ms), licensed [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/); the report credits it and the history carries a `DATA-LICENSE.md`. A repository that stops being listed (made private, deleted, or dropped from the selection) leaves the current tables at the next run; what was committed before stays in the branch's git history.
|
|
121
|
+
|
|
122
|
+
## Limits
|
|
123
|
+
|
|
124
|
+
- History starts at your first run, minus what the registry still keeps.
|
|
125
|
+
- One run of about 400 packages takes about 7 minutes, mostly waiting on pypistats.org's limit of 60 requests a minute.
|
|
126
|
+
- Listing a PyPI account uses PyPI's XML-RPC `user_packages`, deprecated but still answered.
|
|
127
|
+
- Dependents come from ecosyste.ms, which indexes new packages with some delay: a fresh package shows none at first.
|
pkgreach-0.0.2/README.md
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# pkgreach
|
|
2
|
+
|
|
3
|
+
How are your packages really used? `pkgreach` tracks your packages on PyPI, npm, crates.io and GitHub every week, keeps the history the registries throw away, and publishes a report that separates robots from people.
|
|
4
|
+
|
|
5
|
+
**Live example (the author's packages):** https://thorwhalen.github.io/pkgreach/
|
|
6
|
+
|
|
7
|
+
## Why
|
|
8
|
+
|
|
9
|
+
Registries keep a few months of daily download counts (crates.io 90 days, PyPI about 6 months, npm 18 months), and none of them separates people from robots: mirrors, CI pipelines and scanners download every release of every package. So `pkgreach` reports downloads against a **noise floor** (what your least-used packages get), next to the signals a robot does not create: **dependents** (packages and repositories that declare yours, from [ecosyste.ms](https://ecosyste.ms)), and **stars and forks** of your public repositories.
|
|
10
|
+
|
|
11
|
+
## Quick start: a weekly GitHub Action
|
|
12
|
+
|
|
13
|
+
In any repository you own (it can be public: everything collected is public already):
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install pkgreach
|
|
17
|
+
pkgreach init . --pypi YOUR_PYPI_USER --npm YOUR_NPM_USER --github YOUR_GITHUB_USER YOUR_ORG
|
|
18
|
+
git add -A && git commit -m "pkgreach" && git push
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
`pkgreach init` writes `pkgreach.toml` (what to follow) and `.github/workflows/pkgreach.yml`. Then turn on Pages once (Settings → Pages → Source: **GitHub Actions**) and run the workflow (`gh workflow run pkgreach.yml`). Every Sunday after that it collects, commits the history to the `pkgreach-data` branch, and publishes the report on your GitHub Pages. No secret is needed.
|
|
22
|
+
|
|
23
|
+
The workflow is three steps:
|
|
24
|
+
|
|
25
|
+
```yaml
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
- uses: thorwhalen/pkgreach@<version> # pin the release you installed
|
|
28
|
+
id: reach
|
|
29
|
+
- uses: actions/upload-pages-artifact@v3
|
|
30
|
+
with:
|
|
31
|
+
path: ${{ steps.reach.outputs.site-dir }}
|
|
32
|
+
- uses: actions/deploy-pages@v4
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Configuration
|
|
36
|
+
|
|
37
|
+
`pkgreach.toml` has one table per platform: `pypi`, `npm`, `crates`, `github`. Every key is optional; unknown keys are refused, so a typo never silently does nothing.
|
|
38
|
+
|
|
39
|
+
```toml
|
|
40
|
+
[pypi]
|
|
41
|
+
users = ["alice"] # every package these accounts own or maintain
|
|
42
|
+
packages = ["a-package-of-a-friend"] # plus these
|
|
43
|
+
exclude = ["alice-test-*"] # minus these (glob patterns)
|
|
44
|
+
|
|
45
|
+
[npm]
|
|
46
|
+
users = ["alice"]
|
|
47
|
+
|
|
48
|
+
[crates]
|
|
49
|
+
users = ["alice"] # crates.io login
|
|
50
|
+
|
|
51
|
+
[github]
|
|
52
|
+
users = ["alice", "alice-org"] # public repos of these users and orgs
|
|
53
|
+
where = { fork = false, archived = false } # characteristics a package must have
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`where` matches the metadata of listed packages field by field (GitHub repos have `fork`, `archived`, `language`; PyPI packages have `role`, `owner` or `maintainer`); packages named in `packages` are always kept. PyPI names are normalized (`Flask_Login` is `flask-login`). Top-level keys: `max_workers` (default 4) and `dependents` (default `true`: ask ecosyste.ms).
|
|
57
|
+
|
|
58
|
+
The Action's inputs: `config`, `data-branch` (default `pkgreach-data`), `site-dir`, `title`, `token` (reads GitHub; defaults to the workflow's token), `commit`, `fail-on-errors` (default `false`: a failed package is listed in `last_run.json` and flagged on the page, and the run stays green).
|
|
59
|
+
|
|
60
|
+
## Run it locally
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pkgreach collect --config pkgreach.toml # into ~/.local/share/pkgreach/data
|
|
64
|
+
pkgreach report --out site # site/index.html and site/summary.json
|
|
65
|
+
pkgreach init . --pypi me --github me my-org # a config and a weekly workflow
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
From Python, every part is a keyword argument with a working default:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from pkgreach import collect, load_settings, build_site, Files, PLATFORMS
|
|
72
|
+
|
|
73
|
+
collect(load_settings("pkgreach.toml"), store=Files("data"))
|
|
74
|
+
build_site(Files("data"), Files("site"))
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
- `collect(platforms=...)` and `summarize(platforms=...)`: add a platform with `Platform(name, list_packages, fetch, kind=..., headline=...)`; see `pkgreach/platforms/__init__.py` for the contract. A `"snapshot"` platform (current values only, like Hugging Face likes) is shown by its `headline` metrics.
|
|
78
|
+
- `collect(select=lambda platform, record: ...)`: any selection a TOML file cannot express.
|
|
79
|
+
- `collect(dependents=...)`: another source of dependents.
|
|
80
|
+
- `build_site(analyze=...)`: your own summary; `summary.json` is the default one, for any other tool.
|
|
81
|
+
- `make_http(headers_by_host=...)`: credentials per host (the CLI gives GitHub `GITHUB_TOKEN` or `GH_TOKEN`).
|
|
82
|
+
- `store=` / `out=`: any `MutableMapping[str, str]`, such as a [dol](https://github.com/i2mint/dol) store (S3, a database).
|
|
83
|
+
|
|
84
|
+
## What is collected
|
|
85
|
+
|
|
86
|
+
| platform | daily downloads | other numbers | source |
|
|
87
|
+
|---|---|---|---|
|
|
88
|
+
| PyPI | all, and without mirrors | versions, dependents | pypistats.org, PyPI JSON, ecosyste.ms |
|
|
89
|
+
| npm | all | dependents | api.npmjs.org, ecosyste.ms |
|
|
90
|
+
| crates.io | all | total downloads, versions, dependents | crates.io, ecosyste.ms |
|
|
91
|
+
| GitHub | none | stars, forks, open issues and PRs | GitHub REST (public repos only) |
|
|
92
|
+
|
|
93
|
+
The history is plain CSV, one table per kind, split by year (`downloads/YYYY.csv`, `metrics.csv`, `packages.csv`, `runs.csv`), merged idempotently so re-running a collection never changes the result. `manifest.json` lists every file and its columns. Design and alternatives: [docs/adr/0001-seams-and-scope.md](docs/adr/0001-seams-and-scope.md). What each platform offers, verified: [docs/research/platform-apis.md](docs/research/platform-apis.md).
|
|
94
|
+
|
|
95
|
+
**GitHub traffic** (views, clones, referrers) is private and needs a token with push access. That is [ghtraffic](https://github.com/thorwhalen/ghtraffic)'s job, in a private repository; pkgreach reuses its storage layer.
|
|
96
|
+
|
|
97
|
+
Dependents data is from [ecosyste.ms](https://ecosyste.ms), licensed [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/); the report credits it and the history carries a `DATA-LICENSE.md`. A repository that stops being listed (made private, deleted, or dropped from the selection) leaves the current tables at the next run; what was committed before stays in the branch's git history.
|
|
98
|
+
|
|
99
|
+
## Limits
|
|
100
|
+
|
|
101
|
+
- History starts at your first run, minus what the registry still keeps.
|
|
102
|
+
- One run of about 400 packages takes about 7 minutes, mostly waiting on pypistats.org's limit of 60 requests a minute.
|
|
103
|
+
- Listing a PyPI account uses PyPI's XML-RPC `user_packages`, deprecated but still answered.
|
|
104
|
+
- Dependents come from ecosyste.ms, which indexes new packages with some delay: a fresh package shows none at first.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
name: pkgreach
|
|
2
|
+
description: Track downloads, dependents and stars of your packages on PyPI, npm, crates.io and GitHub, keep the history, and build a report site.
|
|
3
|
+
branding:
|
|
4
|
+
icon: trending-up
|
|
5
|
+
color: green
|
|
6
|
+
inputs:
|
|
7
|
+
config:
|
|
8
|
+
description: Path to pkgreach.toml in the checked-out repository.
|
|
9
|
+
default: pkgreach.toml
|
|
10
|
+
data-branch:
|
|
11
|
+
description: Branch of this repository that keeps the history (created on first run).
|
|
12
|
+
default: pkgreach-data
|
|
13
|
+
site-dir:
|
|
14
|
+
description: Where to write the report site (upload it with actions/upload-pages-artifact).
|
|
15
|
+
default: _pkgreach_site
|
|
16
|
+
title:
|
|
17
|
+
description: Title of the report page.
|
|
18
|
+
default: Package reach
|
|
19
|
+
token:
|
|
20
|
+
description: Token for reading GitHub (public data only; it only raises the rate limit).
|
|
21
|
+
default: ${{ github.token }}
|
|
22
|
+
commit:
|
|
23
|
+
description: Commit and push the updated history (pushed with the checkout's credentials).
|
|
24
|
+
default: "true"
|
|
25
|
+
fail-on-errors:
|
|
26
|
+
description: Fail the job if any package failed (the report is still built and pushed).
|
|
27
|
+
default: "false"
|
|
28
|
+
outputs:
|
|
29
|
+
site-dir:
|
|
30
|
+
description: The folder holding the report site.
|
|
31
|
+
value: ${{ inputs.site-dir }}
|
|
32
|
+
runs:
|
|
33
|
+
using: composite
|
|
34
|
+
steps:
|
|
35
|
+
- name: Install pkgreach
|
|
36
|
+
shell: bash
|
|
37
|
+
run: |
|
|
38
|
+
set -euo pipefail
|
|
39
|
+
python3 -c 'import sys; sys.exit(sys.version_info < (3, 11))' || {
|
|
40
|
+
echo "::error::pkgreach needs Python 3.11+; add actions/setup-python before this step."; exit 1; }
|
|
41
|
+
python3 -m venv "$RUNNER_TEMP/pkgreach-venv"
|
|
42
|
+
"$RUNNER_TEMP/pkgreach-venv/bin/pip" install --quiet "$GITHUB_ACTION_PATH"
|
|
43
|
+
|
|
44
|
+
- name: Collect, commit, report
|
|
45
|
+
shell: bash
|
|
46
|
+
env:
|
|
47
|
+
GH_TOKEN: ${{ inputs.token }}
|
|
48
|
+
CONFIG: ${{ inputs.config }}
|
|
49
|
+
BRANCH: ${{ inputs.data-branch }}
|
|
50
|
+
SITE_DIR: ${{ inputs.site-dir }}
|
|
51
|
+
TITLE: ${{ inputs.title }}
|
|
52
|
+
COMMIT: ${{ inputs.commit }}
|
|
53
|
+
FAIL_ON_ERRORS: ${{ inputs.fail-on-errors }}
|
|
54
|
+
run: |
|
|
55
|
+
set -euo pipefail
|
|
56
|
+
PKGREACH="$RUNNER_TEMP/pkgreach-venv/bin/pkgreach"
|
|
57
|
+
DATA_DIR="$RUNNER_TEMP/pkgreach-data"
|
|
58
|
+
CONFIG="$(realpath "$CONFIG")"
|
|
59
|
+
# The history lives on its own branch, checked out as a worktree.
|
|
60
|
+
# A network or auth failure must not look like "no branch yet".
|
|
61
|
+
if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null; then
|
|
62
|
+
git fetch -q --depth=1 origin "$BRANCH"
|
|
63
|
+
git worktree add -q -B "$BRANCH" "$DATA_DIR" FETCH_HEAD
|
|
64
|
+
else
|
|
65
|
+
rc=$?
|
|
66
|
+
[ "$rc" = 2 ] || { echo "::error::cannot reach origin (git ls-remote exit $rc)"; exit 1; }
|
|
67
|
+
git worktree add -q --orphan -b "$BRANCH" "$DATA_DIR"
|
|
68
|
+
fi
|
|
69
|
+
flags=()
|
|
70
|
+
[ "$FAIL_ON_ERRORS" = "true" ] && flags+=(--fail-on-errors)
|
|
71
|
+
status=0
|
|
72
|
+
"$PKGREACH" collect --config "$CONFIG" --data-dir "$DATA_DIR" "${flags[@]}" || status=$?
|
|
73
|
+
if [ "$COMMIT" = "true" ]; then
|
|
74
|
+
cd "$DATA_DIR"
|
|
75
|
+
git config user.name "github-actions[bot]"
|
|
76
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
77
|
+
git add -A
|
|
78
|
+
if ! git diff --cached --quiet; then
|
|
79
|
+
git commit -q -m "pkgreach: collect $(date -u +%F) [skip ci]"
|
|
80
|
+
fi
|
|
81
|
+
# nothing ever committed (a first run that collected nothing): no push
|
|
82
|
+
if git rev-parse -q --verify HEAD >/dev/null; then
|
|
83
|
+
git push -q origin "HEAD:refs/heads/$BRANCH"
|
|
84
|
+
fi
|
|
85
|
+
cd - >/dev/null
|
|
86
|
+
fi
|
|
87
|
+
"$PKGREACH" report --data-dir "$DATA_DIR" --out "$SITE_DIR" --title "$TITLE"
|
|
88
|
+
exit $status
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Track how your packages are really used, across PyPI, npm, crates.io and GitHub.
|
|
2
|
+
|
|
3
|
+
Registries keep a few months of daily download counts at most, and none of
|
|
4
|
+
them separates people from robots. ``pkgreach`` collects the counts every week
|
|
5
|
+
into a git-friendly history, adds the signals a robot does not create
|
|
6
|
+
(dependent packages and repositories, stars, forks), and builds a static
|
|
7
|
+
report that puts each package against a noise floor.
|
|
8
|
+
|
|
9
|
+
>>> from pkgreach import collect, load_settings, build_site, Files # doctest: +SKIP
|
|
10
|
+
>>> collect(load_settings("pkgreach.toml"), store=Files("data")) # doctest: +SKIP
|
|
11
|
+
>>> build_site(Files("data"), Files("site")) # doctest: +SKIP
|
|
12
|
+
|
|
13
|
+
Seams (each one keyword argument, with a working default):
|
|
14
|
+
|
|
15
|
+
- ``collect(platforms=)``: which publishing platforms (:data:`PLATFORMS`).
|
|
16
|
+
- :class:`Selection` per platform, plus ``collect(select=)``: which packages.
|
|
17
|
+
- ``collect(dependents=)``: where dependents come from (ecosyste.ms).
|
|
18
|
+
- ``build_site(analyze=, render=)``: what is computed and how it is shown.
|
|
19
|
+
- ``store`` / ``out``: where the history and the site live (any
|
|
20
|
+
``MutableMapping[str, str]``: a folder by default).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from ghtraffic.tables import Files
|
|
24
|
+
|
|
25
|
+
from pkgreach.analyze import summarize
|
|
26
|
+
from pkgreach.collect import RunReport, collect, select_packages, write_manifest
|
|
27
|
+
from pkgreach.config import Selection, Settings, load_settings, settings_from_mapping
|
|
28
|
+
from pkgreach.http import HttpError, make_http
|
|
29
|
+
from pkgreach.platforms import PLATFORMS, Platform
|
|
30
|
+
from pkgreach.report import build_site, render_html
|
|
31
|
+
from pkgreach.scaffold import init_repo
|
|
32
|
+
from pkgreach.tables import TABLES
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"Files",
|
|
36
|
+
"HttpError",
|
|
37
|
+
"PLATFORMS",
|
|
38
|
+
"Platform",
|
|
39
|
+
"RunReport",
|
|
40
|
+
"Selection",
|
|
41
|
+
"Settings",
|
|
42
|
+
"TABLES",
|
|
43
|
+
"build_site",
|
|
44
|
+
"collect",
|
|
45
|
+
"init_repo",
|
|
46
|
+
"load_settings",
|
|
47
|
+
"make_http",
|
|
48
|
+
"render_html",
|
|
49
|
+
"select_packages",
|
|
50
|
+
"settings_from_mapping",
|
|
51
|
+
"summarize",
|
|
52
|
+
"write_manifest",
|
|
53
|
+
]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# PYTHON_ARGCOMPLETE_OK
|
|
2
|
+
"""Command line: ``pkgreach collect | report | init``, one command per function
|
|
3
|
+
of :data:`pkgreach.tools.COMMANDS` (dispatched by ``cw``).
|
|
4
|
+
|
|
5
|
+
``pkgreach collect --config pkgreach.toml --data-dir data`` merges this week's
|
|
6
|
+
numbers into the history; ``pkgreach report --data-dir data --out site``
|
|
7
|
+
writes the static report; ``pkgreach init . --pypi you --github you`` sets up
|
|
8
|
+
a repository. Without ``--data-dir``, data goes to ``~/.local/share/pkgreach/data``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
import cw
|
|
16
|
+
|
|
17
|
+
from pkgreach.config import SettingsError
|
|
18
|
+
from pkgreach.tools import COMMANDS, CollectionFailed
|
|
19
|
+
|
|
20
|
+
#: Keep injection-only parameters off the command line.
|
|
21
|
+
_CLI_CONFIG = {"collect": {"environ": cw.HIDE}}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(argv=None) -> int:
|
|
25
|
+
"""Entry point of the ``pkgreach`` command; returns the exit code."""
|
|
26
|
+
try:
|
|
27
|
+
return cw.dispatch(
|
|
28
|
+
COMMANDS, argv=argv, config=_CLI_CONFIG, convention=cw.MODERN
|
|
29
|
+
)
|
|
30
|
+
except (CollectionFailed, SettingsError) as e:
|
|
31
|
+
print(f"pkgreach: {e}", file=sys.stderr)
|
|
32
|
+
return 1
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
sys.exit(main())
|