gemhub 1.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. gemhub-1.0.4/.gitignore +28 -0
  2. gemhub-1.0.4/Dockerfile +29 -0
  3. gemhub-1.0.4/LICENSE +21 -0
  4. gemhub-1.0.4/PKG-INFO +302 -0
  5. gemhub-1.0.4/README.md +266 -0
  6. gemhub-1.0.4/docker-compose.yml +50 -0
  7. gemhub-1.0.4/pyproject.toml +114 -0
  8. gemhub-1.0.4/src/gemhub/__init__.py +3 -0
  9. gemhub-1.0.4/src/gemhub/app.py +151 -0
  10. gemhub-1.0.4/src/gemhub/auth.py +93 -0
  11. gemhub-1.0.4/src/gemhub/cache.py +124 -0
  12. gemhub-1.0.4/src/gemhub/cli.py +171 -0
  13. gemhub-1.0.4/src/gemhub/config.py +149 -0
  14. gemhub-1.0.4/src/gemhub/extensions.py +5 -0
  15. gemhub-1.0.4/src/gemhub/gemfile.py +222 -0
  16. gemhub-1.0.4/src/gemhub/gems.py +288 -0
  17. gemhub-1.0.4/src/gemhub/hub.py +32 -0
  18. gemhub-1.0.4/src/gemhub/merge.py +104 -0
  19. gemhub-1.0.4/src/gemhub/metadata.py +228 -0
  20. gemhub-1.0.4/src/gemhub/migrate.py +85 -0
  21. gemhub-1.0.4/src/gemhub/migrations/__init__.py +1 -0
  22. gemhub-1.0.4/src/gemhub/migrations/env.py +39 -0
  23. gemhub-1.0.4/src/gemhub/migrations/script.py.mako +22 -0
  24. gemhub-1.0.4/src/gemhub/migrations/versions/0001_initial.py +66 -0
  25. gemhub-1.0.4/src/gemhub/migrations/versions/0002_private_gems.py +79 -0
  26. gemhub-1.0.4/src/gemhub/models.py +106 -0
  27. gemhub-1.0.4/src/gemhub/names.py +38 -0
  28. gemhub-1.0.4/src/gemhub/private.py +455 -0
  29. gemhub-1.0.4/src/gemhub/proxy.py +349 -0
  30. gemhub-1.0.4/src/gemhub/routes/__init__.py +12 -0
  31. gemhub-1.0.4/src/gemhub/routes/api.py +116 -0
  32. gemhub-1.0.4/src/gemhub/routes/index.py +254 -0
  33. gemhub-1.0.4/src/gemhub/routes/ops.py +87 -0
  34. gemhub-1.0.4/src/gemhub/routes/web.py +190 -0
  35. gemhub-1.0.4/src/gemhub/rubymarshal.py +223 -0
  36. gemhub-1.0.4/src/gemhub/static/css/app.css +250 -0
  37. gemhub-1.0.4/src/gemhub/static/img/gem.svg +4 -0
  38. gemhub-1.0.4/src/gemhub/static/js/app.js +55 -0
  39. gemhub-1.0.4/src/gemhub/templates/base.html +54 -0
  40. gemhub-1.0.4/src/gemhub/templates/error.html +10 -0
  41. gemhub-1.0.4/src/gemhub/templates/gem.html +137 -0
  42. gemhub-1.0.4/src/gemhub/templates/home.html +61 -0
  43. gemhub-1.0.4/src/gemhub/templates/partials/_pagination.html +21 -0
  44. gemhub-1.0.4/src/gemhub/templates/search.html +36 -0
  45. gemhub-1.0.4/src/gemhub/templates/versions.html +30 -0
  46. gemhub-1.0.4/src/gemhub/upstream.py +87 -0
  47. gemhub-1.0.4/src/gemhub/views.py +163 -0
  48. gemhub-1.0.4/tests/conftest.py +70 -0
  49. gemhub-1.0.4/tests/e2e/build_fixture.sh +27 -0
  50. gemhub-1.0.4/tests/e2e/run_private_e2e.py +169 -0
  51. gemhub-1.0.4/tests/e2e/run_ruby_e2e.py +106 -0
  52. gemhub-1.0.4/tests/fake_upstream.py +216 -0
  53. gemhub-1.0.4/tests/gemtools.py +105 -0
  54. gemhub-1.0.4/tests/test_migrations.py +121 -0
  55. gemhub-1.0.4/tests/test_private.py +254 -0
  56. gemhub-1.0.4/tests/test_protocol.py +253 -0
  57. gemhub-1.0.4/tests/test_units.py +110 -0
  58. gemhub-1.0.4/tests/test_web.py +185 -0
@@ -0,0 +1,28 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .venv/
5
+ venv/
6
+ dist/
7
+ build/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .env
11
+ data/
12
+ *.db
13
+
14
+ # Ignore Sublime Text project files
15
+ *.sublime-project
16
+ *.sublime-workspace
17
+
18
+ # Added entries (4 items)
19
+ ANSWER*.md
20
+ QUESTION*.md
21
+ PLAN*.md
22
+ ANSWER*.html
23
+
24
+ # Added entries (1 items)
25
+ ANSWER*
26
+
27
+ # Added entries (1 items)
28
+ BUGS*
@@ -0,0 +1,29 @@
1
+ # syntax=docker/dockerfile:1
2
+ FROM python:3.12-slim AS build
3
+ ARG EXTRAS=postgres
4
+ WORKDIR /src
5
+ COPY pyproject.toml README.md LICENSE ./
6
+ COPY src ./src
7
+ RUN pip install --no-cache-dir build \
8
+ && python -m build --wheel --outdir /dist . \
9
+ && pip wheel --no-cache-dir --wheel-dir /wheels "/dist/$(ls /dist | head -n1)[${EXTRAS}]"
10
+
11
+ FROM python:3.12-slim
12
+ ARG EXTRAS=postgres
13
+ ENV PYTHONUNBUFFERED=1 \
14
+ PIP_NO_CACHE_DIR=1 \
15
+ GEMHUB_DATA_DIR=/data \
16
+ GEMHUB_HOST=0.0.0.0 \
17
+ GEMHUB_PORT=8080
18
+ RUN useradd --system --uid 10001 --create-home --home-dir /home/gemhub gemhub \
19
+ && mkdir -p /data && chown gemhub:gemhub /data
20
+ COPY --from=build /wheels /wheels
21
+ RUN pip install --no-index --find-links=/wheels /wheels/gemhub-*.whl \
22
+ && rm -rf /wheels
23
+ USER gemhub
24
+ VOLUME ["/data"]
25
+ EXPOSE 8080
26
+ HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
27
+ CMD python -c "import urllib.request,os;urllib.request.urlopen('http://127.0.0.1:'+os.environ.get('GEMHUB_PORT','8080')+'/healthz',timeout=4)"
28
+ ENTRYPOINT ["gemhub"]
29
+ CMD ["serve"]
gemhub-1.0.4/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hadi Cahyadi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
gemhub-1.0.4/PKG-INFO ADDED
@@ -0,0 +1,302 @@
1
+ Metadata-Version: 2.5
2
+ Name: gemhub
3
+ Version: 1.0.4
4
+ Summary: Caching RubyGems mirror with a web hub: compact index, legacy index, verified gem files, and a package browser
5
+ Project-URL: Homepage, https://github.com/cumulus13/gemhub
6
+ Project-URL: Issues, https://github.com/cumulus13/gemhub/issues
7
+ Author-email: Hadi Cahyadi <cumulus13@gmail.com>
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: bundler,cache,gem,mirror,proxy,registry,rubygems
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Web Environment
13
+ Classifier: Framework :: Flask
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Topic :: Software Development :: Build Tools
18
+ Classifier: Topic :: System :: Software Distribution
19
+ Requires-Python: >=3.9
20
+ Requires-Dist: alembic<2,>=1.13
21
+ Requires-Dist: flask-sqlalchemy<4,>=3.1
22
+ Requires-Dist: flask<4,>=3.0
23
+ Requires-Dist: pyyaml<7,>=6
24
+ Requires-Dist: redis<9,>=5
25
+ Requires-Dist: requests<3,>=2.31
26
+ Requires-Dist: sqlalchemy<3,>=2.0
27
+ Requires-Dist: waitress<4,>=3
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=8; extra == 'dev'
30
+ Requires-Dist: ruff>=0.5; extra == 'dev'
31
+ Provides-Extra: mysql
32
+ Requires-Dist: pymysql>=1.1; extra == 'mysql'
33
+ Provides-Extra: postgres
34
+ Requires-Dist: psycopg[binary]>=3.1; extra == 'postgres'
35
+ Description-Content-Type: text/markdown
36
+
37
+ # gemhub
38
+
39
+ A caching RubyGems mirror with a web hub, in the spirit of `pypihub`. Point `gem` and Bundler at it and every gem is fetched from rubygems.org once, verified, stored, and served locally from then on. Open it in a browser to search rubygems.org and browse a rubygems.org-style page for each gem.
40
+
41
+ - **Works with the real clients.** `gem install` and Bundler (compact index) are covered by an end-to-end test that runs the actual Ruby tools against the hub.
42
+ - **Verified downloads.** A `.gem` is compared with the SHA-256 in the compact index before it is stored. A mismatch is rejected and never cached.
43
+ - **Survives outages.** If rubygems.org is down, cached indexes and gems keep being served (`stale-if-error`).
44
+ - **Web hub.** Dark and light themes, paginated list of cached gems, search, gem pages with install snippets, versions, dependencies, download statistics (rubygems.org totals plus what this hub served, with a 30-day chart).
45
+ - **Private gems.** `gem push` your own gems to the hub, with API keys, ownership, yank and a guard against dependency-confusion attacks. They install with `gem` and Bundler next to public gems.
46
+ - **Any database, with migrations.** SQLAlchemy plus Alembic: SQLite by default, PostgreSQL tested, MySQL/MariaDB supported by the code. Optional Redis.
47
+ - **Single process, no build step.** No Node, no CSS toolchain, no external fonts or CDNs, so it renders on an isolated network.
48
+
49
+ ## Quick start
50
+
51
+ ```bash
52
+ pip install gemhub # or: pip install "gemhub[postgres]"
53
+ gemhub serve # http://127.0.0.1:8080
54
+ ```
55
+
56
+ Docker, with PostgreSQL and Redis:
57
+
58
+ ```bash
59
+ cp .env.example .env # set POSTGRES_PASSWORD
60
+ docker compose up -d
61
+ ```
62
+
63
+ ### Use it
64
+
65
+ ```bash
66
+ # Bundler, for every project (recommended)
67
+ bundle config set --global mirror.https://rubygems.org http://localhost:8080
68
+
69
+ # Or per project, in the Gemfile
70
+ # source "http://localhost:8080"
71
+
72
+ # RubyGems command line
73
+ gem install rails --clear-sources --source http://localhost:8080
74
+ ```
75
+
76
+ When serving over the network, put TLS in front (Caddy, nginx, Traefik) and set `GEMHUB_PUBLIC_URL` and `GEMHUB_TRUST_PROXY_HOPS=1`.
77
+
78
+ ## Private gems
79
+
80
+ Create a key (the secret is printed once; only its hash is stored):
81
+
82
+ ```bash
83
+ gemhub key create ci --scopes push,yank # scopes: push, yank, admin
84
+ gemhub key list
85
+ gemhub key revoke ci
86
+ ```
87
+
88
+ Publish with the normal RubyGems tools:
89
+
90
+ ```bash
91
+ export GEM_HOST_API_KEY=gh_xxx
92
+ gem build acme_billing.gemspec
93
+ gem push --host http://localhost:8080 acme_billing-1.0.0.gem
94
+ gem yank acme_billing -v 1.0.0 --host http://localhost:8080 # hide a version
95
+ ```
96
+
97
+ Then use it like any other gem: `source "http://localhost:8080"` in the Gemfile, or `gem install acme_billing --clear-sources --source http://localhost:8080`. The web hub marks private gems and includes them in search.
98
+
99
+ Rules the hub enforces:
100
+
101
+ - **Ownership.** The key that first pushes a name owns it; other keys get 403 unless they have the `admin` scope.
102
+ - **No re-pushing.** A version can be pushed once, even after it was yanked, as on rubygems.org.
103
+ - **Private names win.** A name that exists on the hub is always answered by the hub, never by rubygems.org, so a client cannot be tricked into fetching a public gem with the same name.
104
+ - **Public names are protected.** By default pushing a name that already exists on rubygems.org is refused (otherwise you would hijack that name for every client of the hub). Set `GEMHUB_PRIVATE_NAME_POLICY=allow` to permit deliberate shadowing, for example a private fork.
105
+ - **Validated uploads.** The archive is parsed without extracting it, metadata is size-limited, the YAML never instantiates objects, `checksums.yaml.gz` is verified, and `GEMHUB_MAX_GEM_BYTES` (default 100 MB) caps uploads.
106
+ - **Offline friendly.** Private gems keep working when rubygems.org is unreachable.
107
+
108
+ ## Database migrations
109
+
110
+ gemhub uses [Alembic](https://alembic.sqlalchemy.org/), the migration tool of the SQLAlchemy project. Each schema change is a small numbered script; the database records which ones it has applied, so upgrading means "apply the missing scripts" rather than recreating tables, and existing data is kept.
111
+
112
+ ```bash
113
+ gemhub db-status # current revision, latest revision, history
114
+ gemhub migrate # apply pending migrations (they also run on start)
115
+ ```
116
+
117
+ - Migrations run automatically at start unless `GEMHUB_AUTO_MIGRATE=false` (do that if you prefer to run `gemhub migrate` as a separate deploy step).
118
+ - A database created by gemhub 0.1 has no version table; it is detected and adopted at the baseline revision, then upgraded, with its data intact.
119
+ - On PostgreSQL and MySQL a database lock stops several workers from migrating at once.
120
+ - Adding a change: edit `src/gemhub/models.py`, then add a script under `src/gemhub/migrations/versions/`. A test fails if the models and the migrations ever disagree.
121
+
122
+ ## How it works
123
+
124
+ ```mermaid
125
+ flowchart LR
126
+ subgraph Clients
127
+ G[gem]
128
+ B[Bundler]
129
+ U[Browser]
130
+ end
131
+ subgraph gemhub
132
+ P[Protocol routes<br/>versions, names, info, specs, quick, gems]
133
+ W[Web routes<br/>list, search, gem pages]
134
+ PC[(Proxy cache<br/>files on disk)]
135
+ GS[(Gem store<br/>.gem files + checksums)]
136
+ MS[Metadata service]
137
+ R[(Redis<br/>optional)]
138
+ D[(Database)]
139
+ end
140
+ RG[(rubygems.org)]
141
+ G --> P
142
+ B --> P
143
+ U --> W
144
+ P --> PC
145
+ P --> GS
146
+ W --> MS
147
+ MS --> R
148
+ MS --> D
149
+ PC -. revalidate .-> RG
150
+ GS -. first download .-> RG
151
+ MS -. refresh .-> RG
152
+ ```
153
+
154
+ [![How it Works](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture1.jpg)](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture1.jpg)
155
+
156
+ ### Gem download
157
+
158
+ ```mermaid
159
+ sequenceDiagram
160
+ participant C as gem / Bundler
161
+ participant H as gemhub
162
+ participant U as rubygems.org
163
+ C->>H: GET /gems/foo-1.0.0.gem
164
+ alt already stored
165
+ H-->>C: 200 from disk (counted)
166
+ else first request
167
+ H->>U: GET /gems/foo-1.0.0.gem
168
+ U-->>H: file
169
+ H->>U: GET /info/foo (checksum)
170
+ alt SHA-256 matches
171
+ H->>H: store atomically, record in database
172
+ H-->>C: 200
173
+ else mismatch
174
+ H-->>C: 502, nothing stored
175
+ end
176
+ end
177
+ ```
178
+
179
+ [![Gem download](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture2.jpg)](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture2.jpg)
180
+
181
+ ### Metadata freshness for gem pages
182
+
183
+ ```mermaid
184
+ flowchart TD
185
+ A[Open /gems/foo] --> B{In Redis?<br/>TTL 1 hour}
186
+ B -- yes --> Z[Render]
187
+ B -- no --> C{In database and<br/>younger than 6 hours?}
188
+ C -- yes --> D[Refill Redis] --> Z
189
+ C -- no --> E[Fetch from rubygems.org]
190
+ E -- ok --> F[Save to database and Redis] --> Z
191
+ E -- unreachable --> G{Old copy in database?}
192
+ G -- yes --> H[Render with a notice] --> Z
193
+ G -- no --> I[502 page]
194
+ ```
195
+
196
+ [![Metadata freshness for gem pages](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture3.jpg)](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture3.jpg)
197
+
198
+ Without Redis the same short-lived layer is kept in memory inside the process.
199
+
200
+ ## Configuration
201
+
202
+ All settings are environment variables.
203
+
204
+ | Variable | Default | Purpose |
205
+ |---|---|---|
206
+ | `GEMHUB_DATA_DIR` | `./data` | Gem files, protocol cache, default SQLite file |
207
+ | `GEMHUB_DATABASE_URL` | SQLite in data dir | Any SQLAlchemy URL, e.g. `postgresql+psycopg://user:pw@host/db` |
208
+ | `GEMHUB_REDIS_URL` | none | Enables Redis for the short-lived metadata cache |
209
+ | `GEMHUB_UPSTREAM_URL` | `https://rubygems.org` | Upstream server |
210
+ | `GEMHUB_PUBLIC_URL` | derived from the request | Address shown in install snippets |
211
+ | `GEMHUB_HOST` / `GEMHUB_PORT` / `GEMHUB_THREADS` | `127.0.0.1` / `8080` / `16` | Server binding |
212
+ | `GEMHUB_TRUST_PROXY_HOPS` | `0` | Reverse proxies in front (reads `X-Forwarded-*`) |
213
+ | `GEMHUB_REDIS_TTL` | `3600` | Gem page metadata in Redis, seconds |
214
+ | `GEMHUB_DB_TTL` | `21600` | Gem page metadata in the database, seconds |
215
+ | `GEMHUB_VERSIONS_TTL` | `60` | Compact index `/versions` and `/names` |
216
+ | `GEMHUB_INFO_TTL` | `15` | Compact index `/info/<gem>` |
217
+ | `GEMHUB_SPECS_TTL` / `GEMHUB_QUICK_TTL` | `900` / `86400` | Legacy index and per-version specs |
218
+ | `GEMHUB_NEGATIVE_TTL` | `60` | How long a 404 from upstream is remembered |
219
+ | `GEMHUB_STALE_IF_ERROR` | `604800` | Serve a cached copy up to this old when upstream fails |
220
+ | `GEMHUB_CACHE_MAX_BYTES` | `0` | Evict least recently used gems above this size (0 = unlimited) |
221
+ | `GEMHUB_VERIFY_CHECKSUMS` | `true` | Compare downloads with the compact index checksum |
222
+ | `GEMHUB_AUTO_MIGRATE` | `true` | Apply database migrations at start |
223
+ | `GEMHUB_MAX_GEM_BYTES` | `104857600` | Largest accepted `gem push` |
224
+ | `GEMHUB_PRIVATE_NAME_POLICY` | `reject-public` | `reject-public` or `allow` (see Private gems) |
225
+ | `GEMHUB_PAGE_SIZE` | `24` | Gems per page on the home page |
226
+ | `GEMHUB_UPSTREAM_RETRIES`, `GEMHUB_CONNECT_TIMEOUT`, `GEMHUB_READ_TIMEOUT` | `3`, `5`, `30` | Upstream HTTP client |
227
+ | `GEMHUB_LOG_LEVEL` | `INFO` | Logging |
228
+
229
+ `GEMHUB_INFO_TTL` is deliberately short. Bundler compares each `info` file with the checksum listed in `/versions`, so serving an `info` file much older than `/versions` can cause transient checksum mismatches. Raise it only if you accept that trade-off for fewer upstream requests.
230
+
231
+ ## Commands
232
+
233
+ ```bash
234
+ gemhub serve [--host H] [--port P] [--threads N]
235
+ gemhub migrate # apply database migrations (also run on start)
236
+ gemhub db-status
237
+ gemhub key create NAME [--scopes push,yank,admin] | list | revoke NAME
238
+ gemhub refresh rails rack # re-fetch metadata now (names are case-sensitive)
239
+ gemhub purge --older-than-days 90 [--dry-run]
240
+ gemhub stats
241
+ ```
242
+
243
+ ## Endpoints
244
+
245
+ | Path | Notes |
246
+ |---|---|
247
+ | `/versions`, `/names`, `/info/<gem>` | Compact index. `/versions` supports `Range`, updated by verified ranged requests |
248
+ | `/specs.4.8.gz`, `/latest_specs.4.8.gz`, `/prerelease_specs.4.8.gz`, `/quick/Marshal.4.8/*` | Legacy index |
249
+ | `/api/v1/dependencies?gems=a,b` | Passed through; relays whatever upstream answers |
250
+ | `POST /api/v1/gems`, `DELETE /api/v1/gems/yank`, `PUT /api/v1/gems/unyank` | `gem push`, `gem yank`, `gem yank --undo` (API key required) |
251
+ | `/gems/<name>-<version>.gem` | Verified, cached gem files (`/downloads/…` also accepted) |
252
+ | `/api/v1/gems/<gem>.json`, `/api/v1/versions/<gem>.json`, `/api/v1/search.json` | Read-only JSON passthrough |
253
+ | `/`, `/search`, `/gems/<gem>`, `/gems/<gem>/versions` | Web hub |
254
+ | `/healthz`, `/readyz`, `/_hub/stats`, `/robots.txt` | Operations |
255
+
256
+ ## Security notes
257
+
258
+ - Names from URLs are validated before they touch the filesystem or upstream.
259
+ - Metadata from upstream is untrusted: HTML is escaped and only `http(s)` links are rendered.
260
+ - HTML responses carry a nonce-based Content-Security-Policy; there are no inline styles or third-party assets.
261
+ - API keys are 256-bit random values; only their SHA-256 is stored, and a key can be revoked at any time.
262
+ - Gems already cached stay available even if upstream later yanks them. That is usually what a mirror is for, but delete the file (or `gemhub purge`) if you need yanks to propagate.
263
+
264
+ ## Limitations
265
+
266
+ - **Private gems are for the `ruby` platform only.** Platform-specific gems (for example `x86_64-linux`) are rejected on push.
267
+ - **Private gems are not in the legacy index.** They work with `gem install`, `gem push` and Bundler (compact index and quick specs). `gem search --remote`, which reads `specs.4.8.gz`, and the old dependency API do not list them, because merging into those large Marshal files is not implemented.
268
+ - Migrations were exercised on SQLite and PostgreSQL 16. MySQL/MariaDB support is written but has not been run.
269
+ - `waitress` runs one process with threads. Scale out with several containers sharing PostgreSQL, Redis and a shared data volume.
270
+ - `/info` and `/versions` consistency and the ranged `/versions` update were tested against a mock upstream and the real `gem` and Bundler clients, not against production rubygems.org traffic. Try it on your own network before relying on it.
271
+
272
+ ## Development
273
+
274
+ ```bash
275
+ pip install -e ".[dev]"
276
+ pytest
277
+ ruff check src tests && ruff format --check src tests
278
+
279
+ # real gem + Bundler end-to-end (needs Ruby)
280
+ tests/e2e/build_fixture.sh /tmp/fixture
281
+ python tests/e2e/run_ruby_e2e.py /tmp/fixture
282
+ python tests/e2e/run_private_e2e.py /tmp/fixture
283
+
284
+ # run the whole suite on PostgreSQL instead of SQLite
285
+ GEMHUB_TEST_DATABASE_URL=postgresql+psycopg://user:pw@localhost/gemhub_test pytest
286
+ ```
287
+
288
+ ## License
289
+
290
+ [MIT](LICENSE)
291
+
292
+
293
+ ## 👤 Author
294
+
295
+ [Hadi Cahyadi](mailto:cumulus13@gmail.com)
296
+
297
+
298
+ [![Buy Me a Coffee](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/cumulus13)
299
+
300
+ [![Donate via Ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/cumulus13)
301
+
302
+ [Support me on Patreon](https://www.patreon.com/cumulus13)
gemhub-1.0.4/README.md ADDED
@@ -0,0 +1,266 @@
1
+ # gemhub
2
+
3
+ A caching RubyGems mirror with a web hub, in the spirit of `pypihub`. Point `gem` and Bundler at it and every gem is fetched from rubygems.org once, verified, stored, and served locally from then on. Open it in a browser to search rubygems.org and browse a rubygems.org-style page for each gem.
4
+
5
+ - **Works with the real clients.** `gem install` and Bundler (compact index) are covered by an end-to-end test that runs the actual Ruby tools against the hub.
6
+ - **Verified downloads.** A `.gem` is compared with the SHA-256 in the compact index before it is stored. A mismatch is rejected and never cached.
7
+ - **Survives outages.** If rubygems.org is down, cached indexes and gems keep being served (`stale-if-error`).
8
+ - **Web hub.** Dark and light themes, paginated list of cached gems, search, gem pages with install snippets, versions, dependencies, download statistics (rubygems.org totals plus what this hub served, with a 30-day chart).
9
+ - **Private gems.** `gem push` your own gems to the hub, with API keys, ownership, yank and a guard against dependency-confusion attacks. They install with `gem` and Bundler next to public gems.
10
+ - **Any database, with migrations.** SQLAlchemy plus Alembic: SQLite by default, PostgreSQL tested, MySQL/MariaDB supported by the code. Optional Redis.
11
+ - **Single process, no build step.** No Node, no CSS toolchain, no external fonts or CDNs, so it renders on an isolated network.
12
+
13
+ ## Quick start
14
+
15
+ ```bash
16
+ pip install gemhub # or: pip install "gemhub[postgres]"
17
+ gemhub serve # http://127.0.0.1:8080
18
+ ```
19
+
20
+ Docker, with PostgreSQL and Redis:
21
+
22
+ ```bash
23
+ cp .env.example .env # set POSTGRES_PASSWORD
24
+ docker compose up -d
25
+ ```
26
+
27
+ ### Use it
28
+
29
+ ```bash
30
+ # Bundler, for every project (recommended)
31
+ bundle config set --global mirror.https://rubygems.org http://localhost:8080
32
+
33
+ # Or per project, in the Gemfile
34
+ # source "http://localhost:8080"
35
+
36
+ # RubyGems command line
37
+ gem install rails --clear-sources --source http://localhost:8080
38
+ ```
39
+
40
+ When serving over the network, put TLS in front (Caddy, nginx, Traefik) and set `GEMHUB_PUBLIC_URL` and `GEMHUB_TRUST_PROXY_HOPS=1`.
41
+
42
+ ## Private gems
43
+
44
+ Create a key (the secret is printed once; only its hash is stored):
45
+
46
+ ```bash
47
+ gemhub key create ci --scopes push,yank # scopes: push, yank, admin
48
+ gemhub key list
49
+ gemhub key revoke ci
50
+ ```
51
+
52
+ Publish with the normal RubyGems tools:
53
+
54
+ ```bash
55
+ export GEM_HOST_API_KEY=gh_xxx
56
+ gem build acme_billing.gemspec
57
+ gem push --host http://localhost:8080 acme_billing-1.0.0.gem
58
+ gem yank acme_billing -v 1.0.0 --host http://localhost:8080 # hide a version
59
+ ```
60
+
61
+ Then use it like any other gem: `source "http://localhost:8080"` in the Gemfile, or `gem install acme_billing --clear-sources --source http://localhost:8080`. The web hub marks private gems and includes them in search.
62
+
63
+ Rules the hub enforces:
64
+
65
+ - **Ownership.** The key that first pushes a name owns it; other keys get 403 unless they have the `admin` scope.
66
+ - **No re-pushing.** A version can be pushed once, even after it was yanked, as on rubygems.org.
67
+ - **Private names win.** A name that exists on the hub is always answered by the hub, never by rubygems.org, so a client cannot be tricked into fetching a public gem with the same name.
68
+ - **Public names are protected.** By default pushing a name that already exists on rubygems.org is refused (otherwise you would hijack that name for every client of the hub). Set `GEMHUB_PRIVATE_NAME_POLICY=allow` to permit deliberate shadowing, for example a private fork.
69
+ - **Validated uploads.** The archive is parsed without extracting it, metadata is size-limited, the YAML never instantiates objects, `checksums.yaml.gz` is verified, and `GEMHUB_MAX_GEM_BYTES` (default 100 MB) caps uploads.
70
+ - **Offline friendly.** Private gems keep working when rubygems.org is unreachable.
71
+
72
+ ## Database migrations
73
+
74
+ gemhub uses [Alembic](https://alembic.sqlalchemy.org/), the migration tool of the SQLAlchemy project. Each schema change is a small numbered script; the database records which ones it has applied, so upgrading means "apply the missing scripts" rather than recreating tables, and existing data is kept.
75
+
76
+ ```bash
77
+ gemhub db-status # current revision, latest revision, history
78
+ gemhub migrate # apply pending migrations (they also run on start)
79
+ ```
80
+
81
+ - Migrations run automatically at start unless `GEMHUB_AUTO_MIGRATE=false` (do that if you prefer to run `gemhub migrate` as a separate deploy step).
82
+ - A database created by gemhub 0.1 has no version table; it is detected and adopted at the baseline revision, then upgraded, with its data intact.
83
+ - On PostgreSQL and MySQL a database lock stops several workers from migrating at once.
84
+ - Adding a change: edit `src/gemhub/models.py`, then add a script under `src/gemhub/migrations/versions/`. A test fails if the models and the migrations ever disagree.
85
+
86
+ ## How it works
87
+
88
+ ```mermaid
89
+ flowchart LR
90
+ subgraph Clients
91
+ G[gem]
92
+ B[Bundler]
93
+ U[Browser]
94
+ end
95
+ subgraph gemhub
96
+ P[Protocol routes<br/>versions, names, info, specs, quick, gems]
97
+ W[Web routes<br/>list, search, gem pages]
98
+ PC[(Proxy cache<br/>files on disk)]
99
+ GS[(Gem store<br/>.gem files + checksums)]
100
+ MS[Metadata service]
101
+ R[(Redis<br/>optional)]
102
+ D[(Database)]
103
+ end
104
+ RG[(rubygems.org)]
105
+ G --> P
106
+ B --> P
107
+ U --> W
108
+ P --> PC
109
+ P --> GS
110
+ W --> MS
111
+ MS --> R
112
+ MS --> D
113
+ PC -. revalidate .-> RG
114
+ GS -. first download .-> RG
115
+ MS -. refresh .-> RG
116
+ ```
117
+
118
+ [![How it Works](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture1.jpg)](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture1.jpg)
119
+
120
+ ### Gem download
121
+
122
+ ```mermaid
123
+ sequenceDiagram
124
+ participant C as gem / Bundler
125
+ participant H as gemhub
126
+ participant U as rubygems.org
127
+ C->>H: GET /gems/foo-1.0.0.gem
128
+ alt already stored
129
+ H-->>C: 200 from disk (counted)
130
+ else first request
131
+ H->>U: GET /gems/foo-1.0.0.gem
132
+ U-->>H: file
133
+ H->>U: GET /info/foo (checksum)
134
+ alt SHA-256 matches
135
+ H->>H: store atomically, record in database
136
+ H-->>C: 200
137
+ else mismatch
138
+ H-->>C: 502, nothing stored
139
+ end
140
+ end
141
+ ```
142
+
143
+ [![Gem download](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture2.jpg)](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture2.jpg)
144
+
145
+ ### Metadata freshness for gem pages
146
+
147
+ ```mermaid
148
+ flowchart TD
149
+ A[Open /gems/foo] --> B{In Redis?<br/>TTL 1 hour}
150
+ B -- yes --> Z[Render]
151
+ B -- no --> C{In database and<br/>younger than 6 hours?}
152
+ C -- yes --> D[Refill Redis] --> Z
153
+ C -- no --> E[Fetch from rubygems.org]
154
+ E -- ok --> F[Save to database and Redis] --> Z
155
+ E -- unreachable --> G{Old copy in database?}
156
+ G -- yes --> H[Render with a notice] --> Z
157
+ G -- no --> I[502 page]
158
+ ```
159
+
160
+ [![Metadata freshness for gem pages](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture3.jpg)](https://raw.githubusercontent.com/licface15/images/refs/heads/main/gemhub/architecture3.jpg)
161
+
162
+ Without Redis the same short-lived layer is kept in memory inside the process.
163
+
164
+ ## Configuration
165
+
166
+ All settings are environment variables.
167
+
168
+ | Variable | Default | Purpose |
169
+ |---|---|---|
170
+ | `GEMHUB_DATA_DIR` | `./data` | Gem files, protocol cache, default SQLite file |
171
+ | `GEMHUB_DATABASE_URL` | SQLite in data dir | Any SQLAlchemy URL, e.g. `postgresql+psycopg://user:pw@host/db` |
172
+ | `GEMHUB_REDIS_URL` | none | Enables Redis for the short-lived metadata cache |
173
+ | `GEMHUB_UPSTREAM_URL` | `https://rubygems.org` | Upstream server |
174
+ | `GEMHUB_PUBLIC_URL` | derived from the request | Address shown in install snippets |
175
+ | `GEMHUB_HOST` / `GEMHUB_PORT` / `GEMHUB_THREADS` | `127.0.0.1` / `8080` / `16` | Server binding |
176
+ | `GEMHUB_TRUST_PROXY_HOPS` | `0` | Reverse proxies in front (reads `X-Forwarded-*`) |
177
+ | `GEMHUB_REDIS_TTL` | `3600` | Gem page metadata in Redis, seconds |
178
+ | `GEMHUB_DB_TTL` | `21600` | Gem page metadata in the database, seconds |
179
+ | `GEMHUB_VERSIONS_TTL` | `60` | Compact index `/versions` and `/names` |
180
+ | `GEMHUB_INFO_TTL` | `15` | Compact index `/info/<gem>` |
181
+ | `GEMHUB_SPECS_TTL` / `GEMHUB_QUICK_TTL` | `900` / `86400` | Legacy index and per-version specs |
182
+ | `GEMHUB_NEGATIVE_TTL` | `60` | How long a 404 from upstream is remembered |
183
+ | `GEMHUB_STALE_IF_ERROR` | `604800` | Serve a cached copy up to this old when upstream fails |
184
+ | `GEMHUB_CACHE_MAX_BYTES` | `0` | Evict least recently used gems above this size (0 = unlimited) |
185
+ | `GEMHUB_VERIFY_CHECKSUMS` | `true` | Compare downloads with the compact index checksum |
186
+ | `GEMHUB_AUTO_MIGRATE` | `true` | Apply database migrations at start |
187
+ | `GEMHUB_MAX_GEM_BYTES` | `104857600` | Largest accepted `gem push` |
188
+ | `GEMHUB_PRIVATE_NAME_POLICY` | `reject-public` | `reject-public` or `allow` (see Private gems) |
189
+ | `GEMHUB_PAGE_SIZE` | `24` | Gems per page on the home page |
190
+ | `GEMHUB_UPSTREAM_RETRIES`, `GEMHUB_CONNECT_TIMEOUT`, `GEMHUB_READ_TIMEOUT` | `3`, `5`, `30` | Upstream HTTP client |
191
+ | `GEMHUB_LOG_LEVEL` | `INFO` | Logging |
192
+
193
+ `GEMHUB_INFO_TTL` is deliberately short. Bundler compares each `info` file with the checksum listed in `/versions`, so serving an `info` file much older than `/versions` can cause transient checksum mismatches. Raise it only if you accept that trade-off for fewer upstream requests.
194
+
195
+ ## Commands
196
+
197
+ ```bash
198
+ gemhub serve [--host H] [--port P] [--threads N]
199
+ gemhub migrate # apply database migrations (also run on start)
200
+ gemhub db-status
201
+ gemhub key create NAME [--scopes push,yank,admin] | list | revoke NAME
202
+ gemhub refresh rails rack # re-fetch metadata now (names are case-sensitive)
203
+ gemhub purge --older-than-days 90 [--dry-run]
204
+ gemhub stats
205
+ ```
206
+
207
+ ## Endpoints
208
+
209
+ | Path | Notes |
210
+ |---|---|
211
+ | `/versions`, `/names`, `/info/<gem>` | Compact index. `/versions` supports `Range`, updated by verified ranged requests |
212
+ | `/specs.4.8.gz`, `/latest_specs.4.8.gz`, `/prerelease_specs.4.8.gz`, `/quick/Marshal.4.8/*` | Legacy index |
213
+ | `/api/v1/dependencies?gems=a,b` | Passed through; relays whatever upstream answers |
214
+ | `POST /api/v1/gems`, `DELETE /api/v1/gems/yank`, `PUT /api/v1/gems/unyank` | `gem push`, `gem yank`, `gem yank --undo` (API key required) |
215
+ | `/gems/<name>-<version>.gem` | Verified, cached gem files (`/downloads/…` also accepted) |
216
+ | `/api/v1/gems/<gem>.json`, `/api/v1/versions/<gem>.json`, `/api/v1/search.json` | Read-only JSON passthrough |
217
+ | `/`, `/search`, `/gems/<gem>`, `/gems/<gem>/versions` | Web hub |
218
+ | `/healthz`, `/readyz`, `/_hub/stats`, `/robots.txt` | Operations |
219
+
220
+ ## Security notes
221
+
222
+ - Names from URLs are validated before they touch the filesystem or upstream.
223
+ - Metadata from upstream is untrusted: HTML is escaped and only `http(s)` links are rendered.
224
+ - HTML responses carry a nonce-based Content-Security-Policy; there are no inline styles or third-party assets.
225
+ - API keys are 256-bit random values; only their SHA-256 is stored, and a key can be revoked at any time.
226
+ - Gems already cached stay available even if upstream later yanks them. That is usually what a mirror is for, but delete the file (or `gemhub purge`) if you need yanks to propagate.
227
+
228
+ ## Limitations
229
+
230
+ - **Private gems are for the `ruby` platform only.** Platform-specific gems (for example `x86_64-linux`) are rejected on push.
231
+ - **Private gems are not in the legacy index.** They work with `gem install`, `gem push` and Bundler (compact index and quick specs). `gem search --remote`, which reads `specs.4.8.gz`, and the old dependency API do not list them, because merging into those large Marshal files is not implemented.
232
+ - Migrations were exercised on SQLite and PostgreSQL 16. MySQL/MariaDB support is written but has not been run.
233
+ - `waitress` runs one process with threads. Scale out with several containers sharing PostgreSQL, Redis and a shared data volume.
234
+ - `/info` and `/versions` consistency and the ranged `/versions` update were tested against a mock upstream and the real `gem` and Bundler clients, not against production rubygems.org traffic. Try it on your own network before relying on it.
235
+
236
+ ## Development
237
+
238
+ ```bash
239
+ pip install -e ".[dev]"
240
+ pytest
241
+ ruff check src tests && ruff format --check src tests
242
+
243
+ # real gem + Bundler end-to-end (needs Ruby)
244
+ tests/e2e/build_fixture.sh /tmp/fixture
245
+ python tests/e2e/run_ruby_e2e.py /tmp/fixture
246
+ python tests/e2e/run_private_e2e.py /tmp/fixture
247
+
248
+ # run the whole suite on PostgreSQL instead of SQLite
249
+ GEMHUB_TEST_DATABASE_URL=postgresql+psycopg://user:pw@localhost/gemhub_test pytest
250
+ ```
251
+
252
+ ## License
253
+
254
+ [MIT](LICENSE)
255
+
256
+
257
+ ## 👤 Author
258
+
259
+ [Hadi Cahyadi](mailto:cumulus13@gmail.com)
260
+
261
+
262
+ [![Buy Me a Coffee](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/cumulus13)
263
+
264
+ [![Donate via Ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/cumulus13)
265
+
266
+ [Support me on Patreon](https://www.patreon.com/cumulus13)