pkgreach 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pkgreach/tools.py ADDED
@@ -0,0 +1,89 @@
1
+ """The operations, as plain functions: JSON-able arguments in, JSON-able dicts out.
2
+
3
+ This is the single source of truth for every surface: the CLI dispatches
4
+ :data:`COMMANDS` with ``cw`` (:mod:`pkgreach.__main__`), and an MCP or HTTP
5
+ adapter would reference ``pkgreach.tools:collect`` and so on. Each function
6
+ does the wiring a caller should not have to: reading the settings file,
7
+ opening the folders, giving the transport the GitHub token from the
8
+ environment.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import os
14
+ from collections.abc import Mapping, Sequence
15
+ from pathlib import Path
16
+
17
+ from pkgreach.collect import collect as _collect
18
+ from pkgreach.config import DFLT_CONFIG_FILENAME, DFLT_DATA_DIR, load_settings
19
+ from pkgreach.http import make_http
20
+ from pkgreach.platforms import PLATFORMS
21
+ from pkgreach.platforms.github import auth_headers, token_from_env
22
+ from pkgreach.report import DFLT_TITLE, build_site
23
+ from pkgreach.scaffold import DFLT_CRON, default_action_ref, init_repo
24
+ from pkgreach.tables import Files
25
+
26
+
27
+ class CollectionFailed(RuntimeError):
28
+ """Raised by :func:`collect` with ``fail_on_errors`` when something failed."""
29
+
30
+
31
+ def collect(
32
+ *,
33
+ config: str = DFLT_CONFIG_FILENAME,
34
+ data_dir: str = str(DFLT_DATA_DIR),
35
+ fail_on_errors: bool = False,
36
+ environ: Mapping[str, str] = os.environ,
37
+ ) -> dict:
38
+ """Fetch this week's numbers for the packages in ``config`` into ``data_dir``.
39
+
40
+ The GitHub token, if any, comes from ``GITHUB_TOKEN`` or ``GH_TOKEN``.
41
+ """
42
+ settings = load_settings(config, known_platforms=PLATFORMS)
43
+ http = make_http(headers_by_host=auth_headers(token_from_env(environ)))
44
+ report = _collect(settings, store=Files(Path(data_dir)), http=http, log=print)
45
+ if fail_on_errors and report.errors:
46
+ raise CollectionFailed(f"{len(report.errors)} errors; see last_run.json")
47
+ return {
48
+ "packages": report.packages,
49
+ "errors": len(report.errors),
50
+ "files_changed": report.files_changed,
51
+ "duration_s": report.duration_s,
52
+ }
53
+
54
+
55
+ def report(
56
+ *, data_dir: str = str(DFLT_DATA_DIR), out: str = "site", title: str = DFLT_TITLE
57
+ ) -> dict:
58
+ """Write the report site (``index.html``, ``summary.json``) into ``out``."""
59
+ s = build_site(Files(Path(data_dir)), Files(Path(out)), title=title)
60
+ return {"out": out, "packages": len(s["packages"]), "as_of": s["as_of"]}
61
+
62
+
63
+ def init(
64
+ directory: str,
65
+ *,
66
+ pypi: Sequence[str] = (),
67
+ npm: Sequence[str] = (),
68
+ crates: Sequence[str] = (),
69
+ github: Sequence[str] = (),
70
+ cron: str = DFLT_CRON,
71
+ action_ref: str = "",
72
+ overwrite: bool = False,
73
+ ) -> dict:
74
+ """Write ``pkgreach.toml`` and a weekly workflow into the repo at ``directory``.
75
+
76
+ Name your accounts per platform (``--pypi you --github you your-org``).
77
+ """
78
+ users = {"pypi": pypi, "npm": npm, "crates": crates, "github": github}
79
+ written = init_repo(
80
+ directory,
81
+ users={k: list(v) for k, v in users.items() if v},
82
+ cron=cron,
83
+ action_ref=action_ref or default_action_ref(),
84
+ overwrite=overwrite,
85
+ )
86
+ return {"written": written}
87
+
88
+
89
+ COMMANDS = [collect, report, init]
@@ -0,0 +1,127 @@
1
+ Metadata-Version: 2.5
2
+ Name: pkgreach
3
+ Version: 0.0.2
4
+ Summary: Track how your packages are really used across PyPI, npm, crates.io and GitHub: downloads against a noise floor, dependents, stars; a weekly GitHub Action keeps the history and publishes a report.
5
+ Project-URL: Homepage, https://github.com/thorwhalen/pkgreach
6
+ Project-URL: Repository, https://github.com/thorwhalen/pkgreach
7
+ Project-URL: Dashboard, https://thorwhalen.github.io/pkgreach/
8
+ Author: Thor Whalen
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: crates,dashboard,dependents,downloads,github,github-actions,npm,pypi,statistics
12
+ Requires-Python: >=3.11
13
+ Requires-Dist: cw<0.2,>=0.1.14
14
+ Requires-Dist: ghtraffic<0.1,>=0.0.4
15
+ Provides-Extra: dev
16
+ Requires-Dist: pytest-cov>=4.0; extra == 'dev'
17
+ Requires-Dist: pytest>=7.0; extra == 'dev'
18
+ Requires-Dist: ruff>=0.1.0; extra == 'dev'
19
+ Provides-Extra: docs
20
+ Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
21
+ Requires-Dist: sphinx>=6.0; extra == 'docs'
22
+ Description-Content-Type: text/markdown
23
+
24
+ # pkgreach
25
+
26
+ How are your packages really used? `pkgreach` tracks your packages on PyPI, npm, crates.io and GitHub every week, keeps the history the registries throw away, and publishes a report that separates robots from people.
27
+
28
+ **Live example (the author's packages):** https://thorwhalen.github.io/pkgreach/
29
+
30
+ ## Why
31
+
32
+ Registries keep a few months of daily download counts (crates.io 90 days, PyPI about 6 months, npm 18 months), and none of them separates people from robots: mirrors, CI pipelines and scanners download every release of every package. So `pkgreach` reports downloads against a **noise floor** (what your least-used packages get), next to the signals a robot does not create: **dependents** (packages and repositories that declare yours, from [ecosyste.ms](https://ecosyste.ms)), and **stars and forks** of your public repositories.
33
+
34
+ ## Quick start: a weekly GitHub Action
35
+
36
+ In any repository you own (it can be public: everything collected is public already):
37
+
38
+ ```bash
39
+ pip install pkgreach
40
+ pkgreach init . --pypi YOUR_PYPI_USER --npm YOUR_NPM_USER --github YOUR_GITHUB_USER YOUR_ORG
41
+ git add -A && git commit -m "pkgreach" && git push
42
+ ```
43
+
44
+ `pkgreach init` writes `pkgreach.toml` (what to follow) and `.github/workflows/pkgreach.yml`. Then turn on Pages once (Settings → Pages → Source: **GitHub Actions**) and run the workflow (`gh workflow run pkgreach.yml`). Every Sunday after that it collects, commits the history to the `pkgreach-data` branch, and publishes the report on your GitHub Pages. No secret is needed.
45
+
46
+ The workflow is three steps:
47
+
48
+ ```yaml
49
+ - uses: actions/checkout@v4
50
+ - uses: thorwhalen/pkgreach@<version> # pin the release you installed
51
+ id: reach
52
+ - uses: actions/upload-pages-artifact@v3
53
+ with:
54
+ path: ${{ steps.reach.outputs.site-dir }}
55
+ - uses: actions/deploy-pages@v4
56
+ ```
57
+
58
+ ## Configuration
59
+
60
+ `pkgreach.toml` has one table per platform: `pypi`, `npm`, `crates`, `github`. Every key is optional; unknown keys are refused, so a typo never silently does nothing.
61
+
62
+ ```toml
63
+ [pypi]
64
+ users = ["alice"] # every package these accounts own or maintain
65
+ packages = ["a-package-of-a-friend"] # plus these
66
+ exclude = ["alice-test-*"] # minus these (glob patterns)
67
+
68
+ [npm]
69
+ users = ["alice"]
70
+
71
+ [crates]
72
+ users = ["alice"] # crates.io login
73
+
74
+ [github]
75
+ users = ["alice", "alice-org"] # public repos of these users and orgs
76
+ where = { fork = false, archived = false } # characteristics a package must have
77
+ ```
78
+
79
+ `where` matches the metadata of listed packages field by field (GitHub repos have `fork`, `archived`, `language`; PyPI packages have `role`, `owner` or `maintainer`); packages named in `packages` are always kept. PyPI names are normalized (`Flask_Login` is `flask-login`). Top-level keys: `max_workers` (default 4) and `dependents` (default `true`: ask ecosyste.ms).
80
+
81
+ The Action's inputs: `config`, `data-branch` (default `pkgreach-data`), `site-dir`, `title`, `token` (reads GitHub; defaults to the workflow's token), `commit`, `fail-on-errors` (default `false`: a failed package is listed in `last_run.json` and flagged on the page, and the run stays green).
82
+
83
+ ## Run it locally
84
+
85
+ ```bash
86
+ pkgreach collect --config pkgreach.toml # into ~/.local/share/pkgreach/data
87
+ pkgreach report --out site # site/index.html and site/summary.json
88
+ pkgreach init . --pypi me --github me my-org # a config and a weekly workflow
89
+ ```
90
+
91
+ From Python, every part is a keyword argument with a working default:
92
+
93
+ ```python
94
+ from pkgreach import collect, load_settings, build_site, Files, PLATFORMS
95
+
96
+ collect(load_settings("pkgreach.toml"), store=Files("data"))
97
+ build_site(Files("data"), Files("site"))
98
+ ```
99
+
100
+ - `collect(platforms=...)` and `summarize(platforms=...)`: add a platform with `Platform(name, list_packages, fetch, kind=..., headline=...)`; see `pkgreach/platforms/__init__.py` for the contract. A `"snapshot"` platform (current values only, like Hugging Face likes) is shown by its `headline` metrics.
101
+ - `collect(select=lambda platform, record: ...)`: any selection a TOML file cannot express.
102
+ - `collect(dependents=...)`: another source of dependents.
103
+ - `build_site(analyze=...)`: your own summary; `summary.json` is the default one, for any other tool.
104
+ - `make_http(headers_by_host=...)`: credentials per host (the CLI gives GitHub `GITHUB_TOKEN` or `GH_TOKEN`).
105
+ - `store=` / `out=`: any `MutableMapping[str, str]`, such as a [dol](https://github.com/i2mint/dol) store (S3, a database).
106
+
107
+ ## What is collected
108
+
109
+ | platform | daily downloads | other numbers | source |
110
+ |---|---|---|---|
111
+ | PyPI | all, and without mirrors | versions, dependents | pypistats.org, PyPI JSON, ecosyste.ms |
112
+ | npm | all | dependents | api.npmjs.org, ecosyste.ms |
113
+ | crates.io | all | total downloads, versions, dependents | crates.io, ecosyste.ms |
114
+ | GitHub | none | stars, forks, open issues and PRs | GitHub REST (public repos only) |
115
+
116
+ The history is plain CSV, one table per kind, split by year (`downloads/YYYY.csv`, `metrics.csv`, `packages.csv`, `runs.csv`), merged idempotently so re-running a collection never changes the result. `manifest.json` lists every file and its columns. Design and alternatives: [docs/adr/0001-seams-and-scope.md](docs/adr/0001-seams-and-scope.md). What each platform offers, verified: [docs/research/platform-apis.md](docs/research/platform-apis.md).
117
+
118
+ **GitHub traffic** (views, clones, referrers) is private and needs a token with push access. That is [ghtraffic](https://github.com/thorwhalen/ghtraffic)'s job, in a private repository; pkgreach reuses its storage layer.
119
+
120
+ Dependents data is from [ecosyste.ms](https://ecosyste.ms), licensed [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/); the report credits it and the history carries a `DATA-LICENSE.md`. A repository that stops being listed (made private, deleted, or dropped from the selection) leaves the current tables at the next run; what was committed before stays in the branch's git history.
121
+
122
+ ## Limits
123
+
124
+ - History starts at your first run, minus what the registry still keeps.
125
+ - One run of about 400 packages takes about 7 minutes, mostly waiting on pypistats.org's limit of 60 requests a minute.
126
+ - Listing a PyPI account uses PyPI's XML-RPC `user_packages`, deprecated but still answered.
127
+ - Dependents come from ecosyste.ms, which indexes new packages with some delay: a fresh package shows none at first.
@@ -0,0 +1,21 @@
1
+ pkgreach/__init__.py,sha256=WuAu6U7d8SBA6-PJsQFcER_xhTR-4UMbpyqxF1Cgmxk,1978
2
+ pkgreach/__main__.py,sha256=vzH65JOSvKg09CkwGQb5ewbtcYM1q8RXMVyo6e7RFJY,1136
3
+ pkgreach/analyze.py,sha256=_b-Y4xWNhfeTkeNBt0doSU76REcAWpC4VchPq2SAw-0,9739
4
+ pkgreach/collect.py,sha256=JU14kFyRMFJPFx5edFYBMjNLaez-qbewcv9c0-qRgB0,11686
5
+ pkgreach/config.py,sha256=JuK11-9HnqDDugr5JpmJGkYan0FxSEOkS_BFtfPC8OU,4332
6
+ pkgreach/http.py,sha256=wcXv2JPhvl9IY-eDn5LVZizs5My75rPge4LseqrQGbU,5126
7
+ pkgreach/report.py,sha256=RSY4rQSbk8OtcIa3R7Mzocjb4o-nGfyIq22vp7BN6sU,12137
8
+ pkgreach/scaffold.py,sha256=ISDMqcrvA-i1ufZlZUrVDnAWIsNoRPsvWZiTMYOk0pY,4631
9
+ pkgreach/tables.py,sha256=pizrzEmjIRx3HsLGwXIq1JkfqMoGN-bJmRcmQeqw64M,4350
10
+ pkgreach/tools.py,sha256=rZNNszRI6zT64AYbndgIS0r-WskL-_JF-I9xhRlVQCY,3147
11
+ pkgreach/platforms/__init__.py,sha256=LVoSTzDLwYnszyGFcKy2Qls49BAqhmw1XLHVof5TVSg,4180
12
+ pkgreach/platforms/crates.py,sha256=Bymf0V4nTXTCmA-40vSMS3-zGdz8IOdG-_1Qj-65i7Q,2442
13
+ pkgreach/platforms/ecosystems.py,sha256=2ElK5CbPnv6Z_PiWCuXDh7DiYeavUDVITw7DrUYw6no,1342
14
+ pkgreach/platforms/github.py,sha256=rFU3cy9rFdctyeKpzn3l9fasrxjN7ytrquzjRcE-Lyg,3179
15
+ pkgreach/platforms/npm.py,sha256=3Fb2QSazutVWBu_gBuv32uiwrH6W0qKjzZ9lM_OLsGU,3641
16
+ pkgreach/platforms/pypi.py,sha256=OCj2EfliB9CZHQIVjF0Zo3f-Eov4XcvwjGYihMxDRjo,3083
17
+ pkgreach-0.0.2.dist-info/METADATA,sha256=YQY0-lV14JCVA_AYg24ABNZ0RL-YtDXvoA9BPWmllJM,7393
18
+ pkgreach-0.0.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
19
+ pkgreach-0.0.2.dist-info/entry_points.txt,sha256=TR2x9HMTe-eZWAPb5SPmb_i8xsKEm-WV22yH1Q0LV74,52
20
+ pkgreach-0.0.2.dist-info/licenses/LICENSE,sha256=ekE13yCE9fWWEJfP1NPnUJabZ_Wwrc6coKJV2ZZx27w,1068
21
+ pkgreach-0.0.2.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ pkgreach = pkgreach.__main__:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Thor Whalen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.