gemhub 1.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gemhub/__init__.py +3 -0
- gemhub/app.py +151 -0
- gemhub/auth.py +93 -0
- gemhub/cache.py +124 -0
- gemhub/cli.py +171 -0
- gemhub/config.py +149 -0
- gemhub/extensions.py +5 -0
- gemhub/gemfile.py +222 -0
- gemhub/gems.py +288 -0
- gemhub/hub.py +32 -0
- gemhub/merge.py +104 -0
- gemhub/metadata.py +228 -0
- gemhub/migrate.py +85 -0
- gemhub/migrations/__init__.py +1 -0
- gemhub/migrations/env.py +39 -0
- gemhub/migrations/script.py.mako +22 -0
- gemhub/migrations/versions/0001_initial.py +66 -0
- gemhub/migrations/versions/0002_private_gems.py +79 -0
- gemhub/models.py +106 -0
- gemhub/names.py +38 -0
- gemhub/private.py +455 -0
- gemhub/proxy.py +349 -0
- gemhub/routes/__init__.py +12 -0
- gemhub/routes/api.py +116 -0
- gemhub/routes/index.py +254 -0
- gemhub/routes/ops.py +87 -0
- gemhub/routes/web.py +190 -0
- gemhub/rubymarshal.py +223 -0
- gemhub/static/css/app.css +250 -0
- gemhub/static/img/gem.svg +4 -0
- gemhub/static/js/app.js +55 -0
- gemhub/templates/base.html +54 -0
- gemhub/templates/error.html +10 -0
- gemhub/templates/gem.html +137 -0
- gemhub/templates/home.html +61 -0
- gemhub/templates/partials/_pagination.html +21 -0
- gemhub/templates/search.html +36 -0
- gemhub/templates/versions.html +30 -0
- gemhub/upstream.py +87 -0
- gemhub/views.py +163 -0
- gemhub-1.0.4.dist-info/METADATA +302 -0
- gemhub-1.0.4.dist-info/RECORD +45 -0
- gemhub-1.0.4.dist-info/WHEEL +4 -0
- gemhub-1.0.4.dist-info/entry_points.txt +2 -0
- gemhub-1.0.4.dist-info/licenses/LICENSE +21 -0
gemhub/__init__.py
ADDED
gemhub/app.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Application factory."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import secrets
|
|
7
|
+
import sqlite3
|
|
8
|
+
from typing import Optional
|
|
9
|
+
|
|
10
|
+
from flask import Flask, Response, g, render_template, request
|
|
11
|
+
from sqlalchemy import event
|
|
12
|
+
from sqlalchemy.engine import Engine
|
|
13
|
+
from werkzeug.exceptions import HTTPException
|
|
14
|
+
from werkzeug.middleware.proxy_fix import ProxyFix
|
|
15
|
+
|
|
16
|
+
from . import __version__, views
|
|
17
|
+
from .cache import Cache
|
|
18
|
+
from .config import Settings
|
|
19
|
+
from .extensions import db
|
|
20
|
+
from .gems import GemStore
|
|
21
|
+
from .hub import Hub
|
|
22
|
+
from .merge import MergedIndex
|
|
23
|
+
from .metadata import MetadataService
|
|
24
|
+
from .migrate import upgrade_database
|
|
25
|
+
from .private import PrivateService
|
|
26
|
+
from .proxy import ProxyCache
|
|
27
|
+
from .routes import register_blueprints
|
|
28
|
+
from .upstream import Upstream
|
|
29
|
+
|
|
30
|
+
log = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@event.listens_for(Engine, "connect")
|
|
34
|
+
def _sqlite_pragmas(dbapi_connection, _record) -> None: # pragma: no cover - trivial
|
|
35
|
+
"""WAL + busy timeout so several threads and workers can share one SQLite file."""
|
|
36
|
+
if isinstance(dbapi_connection, sqlite3.Connection):
|
|
37
|
+
cursor = dbapi_connection.cursor()
|
|
38
|
+
cursor.execute("PRAGMA journal_mode=WAL")
|
|
39
|
+
cursor.execute("PRAGMA synchronous=NORMAL")
|
|
40
|
+
cursor.execute("PRAGMA busy_timeout=30000")
|
|
41
|
+
cursor.close()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def configure_logging(level: str) -> None:
|
|
45
|
+
logging.basicConfig(
|
|
46
|
+
level=getattr(logging, level, logging.INFO),
|
|
47
|
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def create_app(settings: Optional[Settings] = None, upstream: Optional[Upstream] = None) -> Flask:
|
|
52
|
+
settings = settings or Settings.from_env()
|
|
53
|
+
configure_logging(settings.log_level)
|
|
54
|
+
settings.data_dir.mkdir(parents=True, exist_ok=True)
|
|
55
|
+
|
|
56
|
+
app = Flask(__name__)
|
|
57
|
+
app.config.update(
|
|
58
|
+
SQLALCHEMY_DATABASE_URI=settings.database_url,
|
|
59
|
+
SQLALCHEMY_ENGINE_OPTIONS={"pool_pre_ping": True},
|
|
60
|
+
SQLALCHEMY_TRACK_MODIFICATIONS=False,
|
|
61
|
+
JSON_AS_ASCII=False,
|
|
62
|
+
TEMPLATES_AUTO_RELOAD=False,
|
|
63
|
+
MAX_CONTENT_LENGTH=settings.max_gem_bytes + 64 * 1024,
|
|
64
|
+
)
|
|
65
|
+
if settings.trust_proxy_hops:
|
|
66
|
+
hops = settings.trust_proxy_hops
|
|
67
|
+
app.wsgi_app = ProxyFix( # type: ignore[method-assign]
|
|
68
|
+
app.wsgi_app, x_for=hops, x_proto=hops, x_host=hops, x_prefix=hops
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
db.init_app(app)
|
|
72
|
+
upstream = upstream or Upstream(settings)
|
|
73
|
+
cache = Cache(settings.redis_url)
|
|
74
|
+
proxy = ProxyCache(settings, upstream)
|
|
75
|
+
gems = GemStore(settings, upstream, proxy, cache)
|
|
76
|
+
metadata = MetadataService(settings, upstream, cache)
|
|
77
|
+
private = PrivateService(settings, upstream, cache)
|
|
78
|
+
merged = MergedIndex(settings)
|
|
79
|
+
app.extensions["gemhub"] = Hub(
|
|
80
|
+
settings, upstream, cache, proxy, gems, metadata, private, merged
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
views.register(app)
|
|
84
|
+
register_blueprints(app)
|
|
85
|
+
_register_hooks(app)
|
|
86
|
+
|
|
87
|
+
if settings.auto_migrate:
|
|
88
|
+
with app.app_context():
|
|
89
|
+
upgrade_database(db.engine)
|
|
90
|
+
log.info(
|
|
91
|
+
"gemhub %s ready: upstream=%s cache=%s data_dir=%s",
|
|
92
|
+
__version__,
|
|
93
|
+
settings.upstream_url,
|
|
94
|
+
cache.backend,
|
|
95
|
+
settings.data_dir,
|
|
96
|
+
)
|
|
97
|
+
return app
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _register_hooks(app: Flask) -> None:
|
|
101
|
+
@app.before_request
|
|
102
|
+
def _nonce() -> None:
|
|
103
|
+
g.csp_nonce = secrets.token_urlsafe(16)
|
|
104
|
+
|
|
105
|
+
@app.context_processor
|
|
106
|
+
def _globals():
|
|
107
|
+
return {"csp_nonce": g.get("csp_nonce", ""), "hub_version": __version__}
|
|
108
|
+
|
|
109
|
+
@app.after_request
|
|
110
|
+
def _security_headers(response: Response) -> Response:
|
|
111
|
+
response.headers.setdefault("X-Content-Type-Options", "nosniff")
|
|
112
|
+
response.headers.setdefault("Referrer-Policy", "same-origin")
|
|
113
|
+
if response.mimetype == "text/html":
|
|
114
|
+
nonce = g.get("csp_nonce", "")
|
|
115
|
+
response.headers["Content-Security-Policy"] = (
|
|
116
|
+
"default-src 'none'; "
|
|
117
|
+
f"script-src 'self' 'nonce-{nonce}'; "
|
|
118
|
+
"style-src 'self'; img-src 'self' data:; connect-src 'self'; "
|
|
119
|
+
"base-uri 'none'; form-action 'self'; frame-ancestors 'none'"
|
|
120
|
+
)
|
|
121
|
+
return response
|
|
122
|
+
|
|
123
|
+
def _wants_html() -> bool:
|
|
124
|
+
return request.accept_mimetypes.best == "text/html"
|
|
125
|
+
|
|
126
|
+
@app.errorhandler(HTTPException)
|
|
127
|
+
def _http_error(exc: HTTPException):
|
|
128
|
+
description = exc.description or ""
|
|
129
|
+
if _wants_html():
|
|
130
|
+
return (
|
|
131
|
+
render_template(
|
|
132
|
+
"error.html", code=exc.code, title=exc.name, description=description
|
|
133
|
+
),
|
|
134
|
+
exc.code,
|
|
135
|
+
)
|
|
136
|
+
return Response(f"{exc.code} {exc.name}: {description}\n", exc.code, mimetype="text/plain")
|
|
137
|
+
|
|
138
|
+
@app.errorhandler(Exception)
|
|
139
|
+
def _unhandled(exc: Exception):
|
|
140
|
+
log.exception("Unhandled error on %s %s", request.method, request.path)
|
|
141
|
+
if _wants_html():
|
|
142
|
+
return (
|
|
143
|
+
render_template(
|
|
144
|
+
"error.html",
|
|
145
|
+
code=500,
|
|
146
|
+
title="Something went wrong",
|
|
147
|
+
description="The hub hit an unexpected error. It has been logged.",
|
|
148
|
+
),
|
|
149
|
+
500,
|
|
150
|
+
)
|
|
151
|
+
return Response("500 Internal Server Error\n", 500, mimetype="text/plain")
|
gemhub/auth.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""API keys for pushing and yanking private gems."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import logging
|
|
7
|
+
import secrets
|
|
8
|
+
from typing import Iterable, Optional, Tuple
|
|
9
|
+
|
|
10
|
+
from flask import abort, request
|
|
11
|
+
from sqlalchemy import select
|
|
12
|
+
|
|
13
|
+
from .extensions import db
|
|
14
|
+
from .models import ApiKey, utcnow
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
VALID_SCOPES = ("push", "yank", "admin")
|
|
19
|
+
KEY_PREFIX = "gh_"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def hash_key(plain: str) -> str:
|
|
23
|
+
# API keys are 256-bit random values, so a plain SHA-256 is sufficient (no password KDF).
|
|
24
|
+
return hashlib.sha256(plain.encode("utf-8")).hexdigest()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def parse_scopes(raw: Iterable[str]) -> str:
|
|
28
|
+
scopes = []
|
|
29
|
+
for scope in raw:
|
|
30
|
+
scope = scope.strip().lower()
|
|
31
|
+
if scope not in VALID_SCOPES:
|
|
32
|
+
raise ValueError(f"unknown scope {scope!r}; choose from {', '.join(VALID_SCOPES)}")
|
|
33
|
+
if scope not in scopes:
|
|
34
|
+
scopes.append(scope)
|
|
35
|
+
if not scopes:
|
|
36
|
+
raise ValueError("at least one scope is required")
|
|
37
|
+
return ",".join(scopes)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def create_key(name: str, scopes: str = "push,yank") -> Tuple[ApiKey, str]:
|
|
41
|
+
"""Create a key and return it with the plaintext, which is shown once and never stored."""
|
|
42
|
+
name = name.strip()
|
|
43
|
+
if not name or len(name) > 100:
|
|
44
|
+
raise ValueError("key name must be 1 to 100 characters")
|
|
45
|
+
if db.session.execute(select(ApiKey).where(ApiKey.name == name)).scalar_one_or_none():
|
|
46
|
+
raise ValueError(f"a key named {name!r} already exists")
|
|
47
|
+
plain = KEY_PREFIX + secrets.token_urlsafe(32)
|
|
48
|
+
key = ApiKey(name=name, key_hash=hash_key(plain), scopes=parse_scopes(scopes.split(",")))
|
|
49
|
+
db.session.add(key)
|
|
50
|
+
db.session.commit()
|
|
51
|
+
return key, plain
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def revoke_key(name: str) -> bool:
|
|
55
|
+
key = db.session.execute(select(ApiKey).where(ApiKey.name == name)).scalar_one_or_none()
|
|
56
|
+
if key is None or key.revoked_at is not None:
|
|
57
|
+
return False
|
|
58
|
+
key.revoked_at = utcnow()
|
|
59
|
+
db.session.commit()
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def has_scope(key: ApiKey, scope: str) -> bool:
|
|
64
|
+
scopes = key.scopes.split(",")
|
|
65
|
+
return scope in scopes or "admin" in scopes
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def authenticate(scope: str) -> Optional[ApiKey]:
|
|
69
|
+
raw = request.headers.get("Authorization", "").strip()
|
|
70
|
+
if raw.lower().startswith("bearer "):
|
|
71
|
+
raw = raw[7:].strip()
|
|
72
|
+
if not raw:
|
|
73
|
+
return None
|
|
74
|
+
key = db.session.execute(
|
|
75
|
+
select(ApiKey).where(ApiKey.key_hash == hash_key(raw))
|
|
76
|
+
).scalar_one_or_none()
|
|
77
|
+
if key is None or key.revoked_at is not None:
|
|
78
|
+
return None
|
|
79
|
+
key.last_used_at = utcnow()
|
|
80
|
+
try:
|
|
81
|
+
db.session.commit()
|
|
82
|
+
except Exception: # last-used bookkeeping must not block a push
|
|
83
|
+
db.session.rollback()
|
|
84
|
+
return key
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def require_key(scope: str) -> ApiKey:
|
|
88
|
+
key = authenticate(scope)
|
|
89
|
+
if key is None:
|
|
90
|
+
abort(401, "Missing or invalid API key. Send it in the Authorization header.")
|
|
91
|
+
if not has_scope(key, scope):
|
|
92
|
+
abort(403, f"This API key does not have the '{scope}' scope.")
|
|
93
|
+
return key
|
gemhub/cache.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Small JSON cache: Redis when configured, an in-process TTL cache otherwise."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from collections import OrderedDict
|
|
10
|
+
from typing import Any, Optional, Tuple
|
|
11
|
+
|
|
12
|
+
import redis
|
|
13
|
+
|
|
14
|
+
log = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class _MemoryTTL:
|
|
18
|
+
def __init__(self, max_entries: int = 10000) -> None:
|
|
19
|
+
self._max = max_entries
|
|
20
|
+
self._data: OrderedDict[str, Tuple[float, str]] = OrderedDict()
|
|
21
|
+
self._lock = threading.Lock()
|
|
22
|
+
|
|
23
|
+
def get(self, key: str) -> Optional[str]:
|
|
24
|
+
now = time.monotonic()
|
|
25
|
+
with self._lock:
|
|
26
|
+
item = self._data.get(key)
|
|
27
|
+
if item is None:
|
|
28
|
+
return None
|
|
29
|
+
if item[0] <= now:
|
|
30
|
+
del self._data[key]
|
|
31
|
+
return None
|
|
32
|
+
return item[1]
|
|
33
|
+
|
|
34
|
+
def set(self, key: str, value: str, ttl: int) -> None:
|
|
35
|
+
with self._lock:
|
|
36
|
+
self._data[key] = (time.monotonic() + ttl, value)
|
|
37
|
+
self._data.move_to_end(key)
|
|
38
|
+
while len(self._data) > self._max:
|
|
39
|
+
self._data.popitem(last=False)
|
|
40
|
+
|
|
41
|
+
def delete(self, key: str) -> None:
|
|
42
|
+
with self._lock:
|
|
43
|
+
self._data.pop(key, None)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Cache:
|
|
47
|
+
"""JSON values with a TTL. Redis failures degrade to the in-process cache, never to errors."""
|
|
48
|
+
|
|
49
|
+
def __init__(self, redis_url: Optional[str], prefix: str = "gemhub:") -> None:
|
|
50
|
+
self.prefix = prefix
|
|
51
|
+
self._memory = _MemoryTTL()
|
|
52
|
+
self._redis: Optional[redis.Redis] = None
|
|
53
|
+
self._redis_down_logged = False
|
|
54
|
+
if redis_url:
|
|
55
|
+
self._redis = redis.Redis.from_url(
|
|
56
|
+
redis_url,
|
|
57
|
+
socket_timeout=2,
|
|
58
|
+
socket_connect_timeout=2,
|
|
59
|
+
health_check_interval=30,
|
|
60
|
+
decode_responses=True,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def backend(self) -> str:
|
|
65
|
+
return "redis" if self._redis is not None else "memory"
|
|
66
|
+
|
|
67
|
+
def _redis_failed(self, exc: Exception) -> None:
|
|
68
|
+
if not self._redis_down_logged:
|
|
69
|
+
log.warning("Redis unavailable, using in-process cache until it recovers: %s", exc)
|
|
70
|
+
self._redis_down_logged = True
|
|
71
|
+
|
|
72
|
+
def _redis_ok(self) -> None:
|
|
73
|
+
if self._redis_down_logged:
|
|
74
|
+
log.info("Redis is reachable again")
|
|
75
|
+
self._redis_down_logged = False
|
|
76
|
+
|
|
77
|
+
def get_json(self, key: str) -> Any:
|
|
78
|
+
full = self.prefix + key
|
|
79
|
+
raw: Optional[str] = None
|
|
80
|
+
if self._redis is not None:
|
|
81
|
+
try:
|
|
82
|
+
raw = self._redis.get(full)
|
|
83
|
+
self._redis_ok()
|
|
84
|
+
except redis.RedisError as exc:
|
|
85
|
+
self._redis_failed(exc)
|
|
86
|
+
raw = self._memory.get(full)
|
|
87
|
+
else:
|
|
88
|
+
raw = self._memory.get(full)
|
|
89
|
+
if raw is None:
|
|
90
|
+
return None
|
|
91
|
+
try:
|
|
92
|
+
return json.loads(raw)
|
|
93
|
+
except ValueError:
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
def set_json(self, key: str, value: Any, ttl: int) -> None:
|
|
97
|
+
full = self.prefix + key
|
|
98
|
+
raw = json.dumps(value, separators=(",", ":"), default=str)
|
|
99
|
+
if self._redis is not None:
|
|
100
|
+
try:
|
|
101
|
+
self._redis.set(full, raw, ex=ttl)
|
|
102
|
+
self._redis_ok()
|
|
103
|
+
return
|
|
104
|
+
except redis.RedisError as exc:
|
|
105
|
+
self._redis_failed(exc)
|
|
106
|
+
self._memory.set(full, raw, ttl)
|
|
107
|
+
|
|
108
|
+
def delete(self, key: str) -> None:
|
|
109
|
+
full = self.prefix + key
|
|
110
|
+
self._memory.delete(full)
|
|
111
|
+
if self._redis is not None:
|
|
112
|
+
try:
|
|
113
|
+
self._redis.delete(full)
|
|
114
|
+
except redis.RedisError as exc:
|
|
115
|
+
self._redis_failed(exc)
|
|
116
|
+
|
|
117
|
+
def ping(self) -> Optional[bool]:
|
|
118
|
+
"""``None`` when Redis is not configured, otherwise whether it answers."""
|
|
119
|
+
if self._redis is None:
|
|
120
|
+
return None
|
|
121
|
+
try:
|
|
122
|
+
return bool(self._redis.ping())
|
|
123
|
+
except redis.RedisError:
|
|
124
|
+
return False
|
gemhub/cli.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Command line entry point: ``gemhub serve | init-db | refresh | purge | stats``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
from sqlalchemy import func, select
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .app import create_app
|
|
13
|
+
from .auth import create_key, revoke_key
|
|
14
|
+
from .config import ConfigError, Settings
|
|
15
|
+
from .extensions import db
|
|
16
|
+
from .metadata import GemNotFound, UpstreamUnavailable
|
|
17
|
+
from .migrate import current_revision, head_revision, history, upgrade_database
|
|
18
|
+
from .models import ApiKey, CachedFile, Gem, PrivateRelease
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
22
|
+
parser = argparse.ArgumentParser(
|
|
23
|
+
prog="gemhub", description="Caching RubyGems mirror with a web hub."
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument("--version", action="version", version=f"gemhub {__version__}")
|
|
26
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
27
|
+
|
|
28
|
+
serve = sub.add_parser("serve", help="run the production server (waitress)")
|
|
29
|
+
serve.add_argument("--host", help="bind address (default: GEMHUB_HOST or 127.0.0.1)")
|
|
30
|
+
serve.add_argument("--port", type=int, help="port (default: GEMHUB_PORT or 8080)")
|
|
31
|
+
serve.add_argument("--threads", type=int, help="worker threads (default: GEMHUB_THREADS or 16)")
|
|
32
|
+
|
|
33
|
+
sub.add_parser("migrate", help="apply database migrations (also run automatically on start)")
|
|
34
|
+
sub.add_parser("init-db", help="alias of migrate")
|
|
35
|
+
sub.add_parser("db-status", help="show the database schema revision and migration history")
|
|
36
|
+
|
|
37
|
+
key = sub.add_parser("key", help="manage API keys for pushing private gems")
|
|
38
|
+
key_sub = key.add_subparsers(dest="key_command", required=True)
|
|
39
|
+
create = key_sub.add_parser("create", help="create a key (the secret is shown once)")
|
|
40
|
+
create.add_argument("name", help="a label, e.g. ci or alice")
|
|
41
|
+
create.add_argument("--scopes", default="push,yank", help="comma list of push, yank, admin")
|
|
42
|
+
key_sub.add_parser("list", help="list keys (secrets are never stored)")
|
|
43
|
+
revoke = key_sub.add_parser("revoke", help="revoke a key")
|
|
44
|
+
revoke.add_argument("name")
|
|
45
|
+
|
|
46
|
+
refresh = sub.add_parser("refresh", help="re-fetch gem metadata from rubygems.org now")
|
|
47
|
+
refresh.add_argument("names", nargs="+", help="gem names (case-sensitive)")
|
|
48
|
+
|
|
49
|
+
purge = sub.add_parser("purge", help="delete cached .gem files not served recently")
|
|
50
|
+
purge.add_argument(
|
|
51
|
+
"--older-than-days", type=int, required=True, help="idle days before removal"
|
|
52
|
+
)
|
|
53
|
+
purge.add_argument("--dry-run", action="store_true", help="only report what would be removed")
|
|
54
|
+
|
|
55
|
+
sub.add_parser("stats", help="print cache statistics")
|
|
56
|
+
return parser
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _human(size: int) -> str:
|
|
60
|
+
value = float(size)
|
|
61
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
62
|
+
if value < 1024 or unit == "TB":
|
|
63
|
+
return f"{value:.1f} {unit}" if unit != "B" else f"{int(value)} B"
|
|
64
|
+
value /= 1024
|
|
65
|
+
return f"{value:.1f} TB"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _key_command(args: argparse.Namespace) -> int:
|
|
69
|
+
if args.key_command == "create":
|
|
70
|
+
try:
|
|
71
|
+
key, plain = create_key(args.name, args.scopes)
|
|
72
|
+
except ValueError as exc:
|
|
73
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
74
|
+
return 1
|
|
75
|
+
print(f"Created key {key.name!r} with scopes {key.scopes}.")
|
|
76
|
+
print("Copy the secret now; it cannot be shown again:\n")
|
|
77
|
+
print(f" {plain}\n")
|
|
78
|
+
print("Use it with: GEM_HOST_API_KEY=<secret> gem push --host <hub url> my_gem.gem")
|
|
79
|
+
return 0
|
|
80
|
+
if args.key_command == "revoke":
|
|
81
|
+
if revoke_key(args.name):
|
|
82
|
+
print(f"revoked {args.name!r}")
|
|
83
|
+
return 0
|
|
84
|
+
print(f"no active key named {args.name!r}", file=sys.stderr)
|
|
85
|
+
return 1
|
|
86
|
+
rows = db.session.execute(select(ApiKey).order_by(ApiKey.id)).scalars().all()
|
|
87
|
+
if not rows:
|
|
88
|
+
print("no keys yet; create one with: gemhub key create NAME")
|
|
89
|
+
for row in rows:
|
|
90
|
+
state = "revoked" if row.revoked_at else "active"
|
|
91
|
+
used = row.last_used_at.isoformat(timespec="seconds") if row.last_used_at else "never"
|
|
92
|
+
print(f"{row.name:<24} {state:<8} scopes={row.scopes:<18} last used {used}")
|
|
93
|
+
return 0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
97
|
+
args = _build_parser().parse_args(argv)
|
|
98
|
+
try:
|
|
99
|
+
settings = Settings.from_env()
|
|
100
|
+
except ConfigError as exc:
|
|
101
|
+
print(f"configuration error: {exc}", file=sys.stderr)
|
|
102
|
+
return 2
|
|
103
|
+
app = create_app(settings)
|
|
104
|
+
|
|
105
|
+
if args.command == "serve":
|
|
106
|
+
from waitress import serve
|
|
107
|
+
|
|
108
|
+
host = args.host or settings.host
|
|
109
|
+
port = args.port or settings.port
|
|
110
|
+
threads = args.threads or settings.threads
|
|
111
|
+
print(
|
|
112
|
+
f"gemhub {__version__} listening on http://{host}:{port} "
|
|
113
|
+
f"(upstream {settings.upstream_url})"
|
|
114
|
+
)
|
|
115
|
+
serve(app, host=host, port=port, threads=threads, ident="gemhub")
|
|
116
|
+
return 0
|
|
117
|
+
|
|
118
|
+
with app.app_context():
|
|
119
|
+
if args.command in ("migrate", "init-db"):
|
|
120
|
+
upgrade_database(db.engine)
|
|
121
|
+
print(f"database at revision {current_revision(db.engine)}")
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
if args.command == "db-status":
|
|
125
|
+
print(f"current: {current_revision(db.engine)}")
|
|
126
|
+
print(f"latest : {head_revision()}")
|
|
127
|
+
for line in history():
|
|
128
|
+
print(f" {line}")
|
|
129
|
+
return 0
|
|
130
|
+
|
|
131
|
+
if args.command == "key":
|
|
132
|
+
return _key_command(args)
|
|
133
|
+
|
|
134
|
+
if args.command == "refresh":
|
|
135
|
+
hub = app.extensions["gemhub"]
|
|
136
|
+
status = 0
|
|
137
|
+
for name in args.names:
|
|
138
|
+
try:
|
|
139
|
+
view = hub.metadata.get_gem(name, force_refresh=True)
|
|
140
|
+
print(f"{name}: refreshed ({view.data.get('version', '?')})")
|
|
141
|
+
except GemNotFound:
|
|
142
|
+
print(f"{name}: not found on upstream", file=sys.stderr)
|
|
143
|
+
status = 1
|
|
144
|
+
except UpstreamUnavailable as exc:
|
|
145
|
+
print(f"{name}: upstream unavailable: {exc}", file=sys.stderr)
|
|
146
|
+
status = 1
|
|
147
|
+
return status
|
|
148
|
+
|
|
149
|
+
if args.command == "purge":
|
|
150
|
+
hub = app.extensions["gemhub"]
|
|
151
|
+
count, freed = hub.gems.purge(args.older_than_days, dry_run=args.dry_run)
|
|
152
|
+
verb = "would remove" if args.dry_run else "removed"
|
|
153
|
+
print(f"{verb} {count} file(s), {_human(freed)}")
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
if args.command == "stats":
|
|
157
|
+
gems = db.session.scalar(select(func.count()).select_from(Gem)) or 0
|
|
158
|
+
files = db.session.scalar(select(func.count()).select_from(CachedFile)) or 0
|
|
159
|
+
size = db.session.scalar(select(func.coalesce(func.sum(CachedFile.size), 0))) or 0
|
|
160
|
+
hits = db.session.scalar(select(func.coalesce(func.sum(CachedFile.hits), 0))) or 0
|
|
161
|
+
print(f"gems with metadata : {gems}")
|
|
162
|
+
print(f"cached .gem files : {files} ({_human(int(size))})")
|
|
163
|
+
print(f"downloads served : {hits}")
|
|
164
|
+
private = db.session.scalar(select(func.count()).select_from(PrivateRelease)) or 0
|
|
165
|
+
print(f"private releases : {private}")
|
|
166
|
+
return 0
|
|
167
|
+
return 1
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
if __name__ == "__main__":
|
|
171
|
+
raise SystemExit(main())
|
gemhub/config.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Runtime configuration, read from ``GEMHUB_*`` environment variables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Mapping, Optional
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
|
|
12
|
+
ENV_PREFIX = "GEMHUB_"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ConfigError(ValueError):
|
|
16
|
+
"""Raised when an environment variable holds an invalid value."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _raw(env: Mapping[str, str], name: str) -> Optional[str]:
|
|
20
|
+
value = env.get(ENV_PREFIX + name)
|
|
21
|
+
if value is None or value.strip() == "":
|
|
22
|
+
return None
|
|
23
|
+
return value.strip()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _int(env: Mapping[str, str], name: str, default: int, minimum: int = 0) -> int:
|
|
27
|
+
raw = _raw(env, name)
|
|
28
|
+
if raw is None:
|
|
29
|
+
return default
|
|
30
|
+
try:
|
|
31
|
+
value = int(raw)
|
|
32
|
+
except ValueError as exc:
|
|
33
|
+
raise ConfigError(f"{ENV_PREFIX}{name} must be an integer, got {raw!r}") from exc
|
|
34
|
+
if value < minimum:
|
|
35
|
+
raise ConfigError(f"{ENV_PREFIX}{name} must be >= {minimum}, got {value}")
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _bool(env: Mapping[str, str], name: str, default: bool) -> bool:
|
|
40
|
+
raw = _raw(env, name)
|
|
41
|
+
if raw is None:
|
|
42
|
+
return default
|
|
43
|
+
lowered = raw.lower()
|
|
44
|
+
if lowered in {"1", "true", "yes", "on"}:
|
|
45
|
+
return True
|
|
46
|
+
if lowered in {"0", "false", "no", "off"}:
|
|
47
|
+
return False
|
|
48
|
+
raise ConfigError(f"{ENV_PREFIX}{name} must be a boolean, got {raw!r}")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _policy(env: Mapping[str, str]) -> str:
|
|
52
|
+
raw = (_raw(env, "PRIVATE_NAME_POLICY") or "reject-public").lower()
|
|
53
|
+
if raw not in {"reject-public", "allow"}:
|
|
54
|
+
raise ConfigError(f"{ENV_PREFIX}PRIVATE_NAME_POLICY must be 'reject-public' or 'allow'")
|
|
55
|
+
return raw
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _normalize_database_url(url: str) -> str:
|
|
59
|
+
# Some platforms still hand out the legacy "postgres://" scheme, which SQLAlchemy rejects.
|
|
60
|
+
if url.startswith("postgres://"):
|
|
61
|
+
return "postgresql://" + url[len("postgres://") :]
|
|
62
|
+
return url
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Settings:
|
|
67
|
+
"""Immutable application settings."""
|
|
68
|
+
|
|
69
|
+
data_dir: Path
|
|
70
|
+
database_url: str
|
|
71
|
+
redis_url: Optional[str]
|
|
72
|
+
upstream_url: str
|
|
73
|
+
public_url: Optional[str]
|
|
74
|
+
host: str
|
|
75
|
+
port: int
|
|
76
|
+
threads: int
|
|
77
|
+
trust_proxy_hops: int
|
|
78
|
+
auto_migrate: bool
|
|
79
|
+
# Metadata freshness: Redis is deliberately shorter-lived than the database.
|
|
80
|
+
redis_ttl: int
|
|
81
|
+
db_ttl: int
|
|
82
|
+
# Protocol caches (seconds before the hub revalidates against upstream).
|
|
83
|
+
versions_ttl: int
|
|
84
|
+
info_ttl: int
|
|
85
|
+
specs_ttl: int
|
|
86
|
+
quick_ttl: int
|
|
87
|
+
negative_ttl: int
|
|
88
|
+
stale_if_error: int
|
|
89
|
+
# Gem file cache.
|
|
90
|
+
cache_max_bytes: int
|
|
91
|
+
verify_checksums: bool
|
|
92
|
+
# Upstream HTTP client.
|
|
93
|
+
connect_timeout: int
|
|
94
|
+
read_timeout: int
|
|
95
|
+
retries: int
|
|
96
|
+
user_agent: str
|
|
97
|
+
# Private gems.
|
|
98
|
+
max_gem_bytes: int
|
|
99
|
+
private_name_policy: str
|
|
100
|
+
# Web UI.
|
|
101
|
+
page_size: int
|
|
102
|
+
log_level: str
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def from_env(cls, env: Optional[Mapping[str, str]] = None) -> Settings:
|
|
106
|
+
env = os.environ if env is None else env
|
|
107
|
+
data_dir = Path(_raw(env, "DATA_DIR") or "./data").expanduser().resolve()
|
|
108
|
+
database_url = _raw(env, "DATABASE_URL") or (
|
|
109
|
+
"sqlite:///" + (data_dir / "gemhub.db").as_posix()
|
|
110
|
+
)
|
|
111
|
+
upstream = (_raw(env, "UPSTREAM_URL") or "https://rubygems.org").rstrip("/")
|
|
112
|
+
if not upstream.startswith(("http://", "https://")):
|
|
113
|
+
raise ConfigError(f"{ENV_PREFIX}UPSTREAM_URL must start with http:// or https://")
|
|
114
|
+
public = _raw(env, "PUBLIC_URL")
|
|
115
|
+
if public is not None:
|
|
116
|
+
public = public.rstrip("/")
|
|
117
|
+
if not public.startswith(("http://", "https://")):
|
|
118
|
+
raise ConfigError(f"{ENV_PREFIX}PUBLIC_URL must start with http:// or https://")
|
|
119
|
+
return cls(
|
|
120
|
+
data_dir=data_dir,
|
|
121
|
+
database_url=_normalize_database_url(database_url),
|
|
122
|
+
redis_url=_raw(env, "REDIS_URL"),
|
|
123
|
+
upstream_url=upstream,
|
|
124
|
+
public_url=public,
|
|
125
|
+
host=_raw(env, "HOST") or "127.0.0.1",
|
|
126
|
+
port=_int(env, "PORT", 8080, 1),
|
|
127
|
+
threads=_int(env, "THREADS", 16, 1),
|
|
128
|
+
trust_proxy_hops=_int(env, "TRUST_PROXY_HOPS", 0),
|
|
129
|
+
auto_migrate=_bool(env, "AUTO_MIGRATE", True),
|
|
130
|
+
redis_ttl=_int(env, "REDIS_TTL", 3600, 1),
|
|
131
|
+
db_ttl=_int(env, "DB_TTL", 6 * 3600, 1),
|
|
132
|
+
versions_ttl=_int(env, "VERSIONS_TTL", 60),
|
|
133
|
+
info_ttl=_int(env, "INFO_TTL", 15),
|
|
134
|
+
specs_ttl=_int(env, "SPECS_TTL", 900),
|
|
135
|
+
quick_ttl=_int(env, "QUICK_TTL", 86400),
|
|
136
|
+
negative_ttl=_int(env, "NEGATIVE_TTL", 60),
|
|
137
|
+
stale_if_error=_int(env, "STALE_IF_ERROR", 7 * 86400),
|
|
138
|
+
cache_max_bytes=_int(env, "CACHE_MAX_BYTES", 0),
|
|
139
|
+
verify_checksums=_bool(env, "VERIFY_CHECKSUMS", True),
|
|
140
|
+
connect_timeout=_int(env, "CONNECT_TIMEOUT", 5, 1),
|
|
141
|
+
read_timeout=_int(env, "READ_TIMEOUT", 30, 1),
|
|
142
|
+
retries=_int(env, "UPSTREAM_RETRIES", 3),
|
|
143
|
+
user_agent=_raw(env, "USER_AGENT")
|
|
144
|
+
or f"gemhub/{__version__} (+https://github.com/cumulus13/gemhub)",
|
|
145
|
+
max_gem_bytes=_int(env, "MAX_GEM_BYTES", 100 * 1024 * 1024, 1024),
|
|
146
|
+
private_name_policy=_policy(env),
|
|
147
|
+
page_size=_int(env, "PAGE_SIZE", 24, 1),
|
|
148
|
+
log_level=(_raw(env, "LOG_LEVEL") or "INFO").upper(),
|
|
149
|
+
)
|