gitlytics 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlytics-0.4.0 → gitlytics-0.4.2}/PKG-INFO +1 -1
- {gitlytics-0.4.0 → gitlytics-0.4.2}/pyproject.toml +1 -1
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/__init__.py +1 -1
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/api.py +116 -36
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/automation.py +14 -5
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/cli.py +15 -3
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/core.py +72 -62
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/process.py +38 -10
- gitlytics-0.4.2/src/gitlytics/static/_headers +6 -0
- gitlytics-0.4.2/src/gitlytics/static/_redirects +1 -0
- gitlytics-0.4.0/src/gitlytics/static/assets/CardModal-DmHoAnrK.js → gitlytics-0.4.2/src/gitlytics/static/assets/CardModal-BnM9BB99.js +1 -1
- gitlytics-0.4.0/src/gitlytics/static/assets/ExportModal-BF_sosAP.js → gitlytics-0.4.2/src/gitlytics/static/assets/ExportModal-B7bldVwi.js +1 -1
- gitlytics-0.4.0/src/gitlytics/static/assets/exports-CrU6Ta4Q.js → gitlytics-0.4.2/src/gitlytics/static/assets/exports-BM_YcnUf.js +3 -3
- gitlytics-0.4.2/src/gitlytics/static/assets/index-BRI0yQ1K.css +1 -0
- gitlytics-0.4.2/src/gitlytics/static/assets/index-DO5EQgEU.js +124 -0
- gitlytics-0.4.0/src/gitlytics/static/assets/index.es-ComYkcii.js → gitlytics-0.4.2/src/gitlytics/static/assets/index.es-DjwqRfc7.js +1 -1
- gitlytics-0.4.0/src/gitlytics/static/assets/jspdf.es.min-vktmDsgQ.js → gitlytics-0.4.2/src/gitlytics/static/assets/jspdf.es.min-B8eoAQ72.js +3 -3
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/index.html +26 -25
- gitlytics-0.4.2/src/gitlytics/static/robots.txt +4 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics.egg-info/PKG-INFO +1 -1
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics.egg-info/SOURCES.txt +9 -7
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_api.py +25 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_cli.py +1 -2
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_process.py +30 -3
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_username.py +6 -9
- gitlytics-0.4.0/src/gitlytics/static/assets/index-D5qdeNvu.css +0 -1
- gitlytics-0.4.0/src/gitlytics/static/assets/index-hb6NMx5Y.js +0 -119
- gitlytics-0.4.0/src/gitlytics/static/robots.txt +0 -5
- {gitlytics-0.4.0 → gitlytics-0.4.2}/LICENSE +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/README.md +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/setup.cfg +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/__main__.py +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/android-chrome-192x192.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/android-chrome-512x512.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/apple-touch-icon.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/assets/html2canvas-pro.esm-9xys3ejh.js +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/assets/html2canvas.esm-DXEQVQnt.js +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/assets/purify.es-CC4Brkr_.js +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/favicon-16x16.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/favicon-32x32.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/favicon-48x48.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/favicon.ico +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/gitlytics-logo.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/google-search-icon-48x48.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/google-search-icon.png +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics/static/sitemap.xml +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics.egg-info/dependency_links.txt +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics.egg-info/entry_points.txt +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics.egg-info/requires.txt +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/src/gitlytics.egg-info/top_level.txt +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_automation.py +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_core.py +0 -0
- {gitlytics-0.4.0 → gitlytics-0.4.2}/tests/test_star_history.py +0 -0
|
@@ -5,13 +5,16 @@ Powers the FastAPI backend — serves traffic data and the React dashboard to th
|
|
|
5
5
|
import hashlib
|
|
6
6
|
import logging
|
|
7
7
|
import os
|
|
8
|
+
import secrets
|
|
8
9
|
import shutil
|
|
10
|
+
import threading as _threading
|
|
9
11
|
import time as _time
|
|
10
12
|
import uuid
|
|
13
|
+
from collections import OrderedDict
|
|
11
14
|
from pathlib import Path
|
|
12
15
|
|
|
13
16
|
import pandas as pd
|
|
14
|
-
from fastapi import FastAPI, HTTPException, Body, File, Header, Response, UploadFile
|
|
17
|
+
from fastapi import FastAPI, HTTPException, Body, File, Header, Request, Response, UploadFile, Depends
|
|
15
18
|
from fastapi.responses import FileResponse, JSONResponse
|
|
16
19
|
from fastapi.middleware.cors import CORSMiddleware
|
|
17
20
|
from fastapi.staticfiles import StaticFiles
|
|
@@ -21,13 +24,14 @@ from gitlytics.core import (
|
|
|
21
24
|
get_user_profile,
|
|
22
25
|
get_public_user,
|
|
23
26
|
get_public_repos,
|
|
27
|
+
validate_github_username,
|
|
24
28
|
fetch_traffic_data,
|
|
25
29
|
fetch_deep_stats_for_top,
|
|
26
30
|
fetch_star_history,
|
|
27
31
|
GitHubRateLimitError,
|
|
28
32
|
StarHistoryFetchError,
|
|
29
33
|
)
|
|
30
|
-
from gitlytics.process import process_uploaded_csv, build_react_payload
|
|
34
|
+
from gitlytics.process import process_uploaded_csv, build_react_payload, _to_bool
|
|
31
35
|
|
|
32
36
|
logger = logging.getLogger(__name__)
|
|
33
37
|
|
|
@@ -38,9 +42,11 @@ app = FastAPI(title="GitHub Traffic API")
|
|
|
38
42
|
_ALLOWED_ORIGINS = [
|
|
39
43
|
"http://localhost",
|
|
40
44
|
"http://localhost:3000",
|
|
45
|
+
"http://localhost:4173",
|
|
41
46
|
"http://localhost:8000",
|
|
42
47
|
"http://127.0.0.1",
|
|
43
48
|
"http://127.0.0.1:3000",
|
|
49
|
+
"http://127.0.0.1:4173",
|
|
44
50
|
"http://127.0.0.1:8000",
|
|
45
51
|
]
|
|
46
52
|
|
|
@@ -55,13 +61,62 @@ app.add_middleware(
|
|
|
55
61
|
)
|
|
56
62
|
|
|
57
63
|
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
# In-process short-term cache for token validation. Cleared when the
|
|
65
|
+
# Python process exits — same lifetime as the CLI / FastAPI worker.
|
|
66
|
+
# OrderedDict + insert-order eviction gives us a bounded LRU without
|
|
67
|
+
# pulling in functools.lru_cache (which can't be cleared on TTL).
|
|
68
|
+
_auth_cache: "OrderedDict[str, tuple]" = OrderedDict()
|
|
69
|
+
_auth_cache_lock = _threading.Lock()
|
|
70
|
+
_AUTH_CACHE_TTL = 300 # 5 minutes — applied only to positive (valid=True) results.
|
|
71
|
+
_AUTH_NEG_TTL = 10 # short TTL for negative results so a transient 5xx doesn't lock the user out.
|
|
72
|
+
_AUTH_CACHE_MAX = 256 # cap entries so a long-running worker can't grow unbounded
|
|
60
73
|
|
|
61
74
|
# Hard cap on CSV upload size — prevents DoS via /api/upload-csv.
|
|
62
75
|
_MAX_UPLOAD_BYTES = 25 * 1024 * 1024
|
|
63
76
|
|
|
64
77
|
|
|
78
|
+
class _SecurityHeadersMiddleware:
|
|
79
|
+
def __init__(self, app):
|
|
80
|
+
self.app = app
|
|
81
|
+
|
|
82
|
+
async def __call__(self, scope, receive, send):
|
|
83
|
+
if scope["type"] != "http":
|
|
84
|
+
await self.app(scope, receive, send)
|
|
85
|
+
return
|
|
86
|
+
|
|
87
|
+
async def send_wrapper(message):
|
|
88
|
+
if message["type"] == "http.response.start":
|
|
89
|
+
headers = list(message.get("headers", []))
|
|
90
|
+
headers.extend([
|
|
91
|
+
(b"x-content-type-options", b"nosniff"),
|
|
92
|
+
(b"x-frame-options", b"DENY"),
|
|
93
|
+
(b"referrer-policy", b"strict-origin-when-cross-origin"),
|
|
94
|
+
])
|
|
95
|
+
message = {**message, "headers": headers}
|
|
96
|
+
await send(message)
|
|
97
|
+
|
|
98
|
+
await self.app(scope, receive, send_wrapper)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
app.add_middleware(_SecurityHeadersMiddleware)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _require_upload_access(
|
|
105
|
+
request: Request,
|
|
106
|
+
x_upload_key: str | None = Header(default=None),
|
|
107
|
+
) -> None:
|
|
108
|
+
client_host = (request.client.host if request.client else "").lower()
|
|
109
|
+
if client_host in ("127.0.0.1", "::1", "testclient", "testserver"):
|
|
110
|
+
return
|
|
111
|
+
expected = (os.environ.get("GITLYTICS_UPLOAD_KEY") or "").strip()
|
|
112
|
+
if not expected:
|
|
113
|
+
raise HTTPException(status_code=403, detail="Upload not allowed from this host.")
|
|
114
|
+
got = (x_upload_key or "").strip()
|
|
115
|
+
if got and secrets.compare_digest(got, expected):
|
|
116
|
+
return
|
|
117
|
+
raise HTTPException(status_code=403, detail="Upload not allowed from this host.")
|
|
118
|
+
|
|
119
|
+
|
|
65
120
|
def _get_token(token: str = None) -> str:
|
|
66
121
|
# C-2: explicit empty string must not fall through to the env token
|
|
67
122
|
if token and token.strip():
|
|
@@ -71,15 +126,28 @@ def _get_token(token: str = None) -> str:
|
|
|
71
126
|
|
|
72
127
|
def _validate_token_cached(token: str):
|
|
73
128
|
# M-1: cache validation results to avoid a double HTTP round-trip on every /api/traffic call
|
|
74
|
-
from gitlytics.core import validate_token as _validate_token
|
|
75
129
|
key = hashlib.sha256(token.encode()).hexdigest()[:16]
|
|
76
130
|
now = _time.time()
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
131
|
+
# Lock guards the read-then-write so two threads can't both miss the
|
|
132
|
+
# cache and double-fetch the same token (FastAPI runs sync routes in
|
|
133
|
+
# a worker threadpool, so this race is reachable under load).
|
|
134
|
+
with _auth_cache_lock:
|
|
135
|
+
cached = _auth_cache.get(key)
|
|
136
|
+
if cached is not None:
|
|
137
|
+
valid, username, expires = cached
|
|
138
|
+
if now < expires:
|
|
139
|
+
# Refresh insertion order so an active key isn't evicted FIFO-style.
|
|
140
|
+
_auth_cache.move_to_end(key)
|
|
141
|
+
return valid, username
|
|
142
|
+
valid, username = validate_token(token)
|
|
143
|
+
# Negative results get a short TTL so a transient upstream 5xx doesn't
|
|
144
|
+
# lock the user out for 5 minutes; positive results cache fully.
|
|
145
|
+
ttl = _AUTH_CACHE_TTL if valid else _AUTH_NEG_TTL
|
|
146
|
+
with _auth_cache_lock:
|
|
147
|
+
_auth_cache[key] = (valid, username, now + ttl)
|
|
148
|
+
_auth_cache.move_to_end(key)
|
|
149
|
+
while len(_auth_cache) > _AUTH_CACHE_MAX:
|
|
150
|
+
_auth_cache.popitem(last=False)
|
|
83
151
|
return valid, username
|
|
84
152
|
|
|
85
153
|
|
|
@@ -119,29 +187,19 @@ def auth(token: str = Body("", embed=True)):
|
|
|
119
187
|
|
|
120
188
|
@app.post("/api/username")
|
|
121
189
|
def get_username_data(username: str = Body("", embed=True)):
|
|
122
|
-
"""Fetches public profile and repos for any GitHub username — no token required."""
|
|
123
190
|
if not username or not username.strip():
|
|
124
191
|
raise HTTPException(status_code=400, detail="Username is required.")
|
|
125
192
|
try:
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
# upstream outages.
|
|
130
|
-
if profile.get("login") == username.strip() and profile.get("html_url"):
|
|
131
|
-
pass
|
|
132
|
-
elif profile.get("login") == username.strip():
|
|
133
|
-
# Found but GitHub didn't include html_url — still treat as success.
|
|
134
|
-
pass
|
|
135
|
-
else:
|
|
136
|
-
raise HTTPException(status_code=502, detail="GitHub did not return a profile.")
|
|
137
|
-
repos = get_public_repos(username.strip())
|
|
193
|
+
clean = validate_github_username(username.strip())
|
|
194
|
+
profile = get_public_user(clean)
|
|
195
|
+
repos = get_public_repos(clean)
|
|
138
196
|
return {"profile": profile, "repos": repos}
|
|
139
197
|
except ValueError as exc:
|
|
140
198
|
raise HTTPException(status_code=404, detail=str(exc))
|
|
141
|
-
except
|
|
142
|
-
raise
|
|
199
|
+
except RuntimeError:
|
|
200
|
+
raise HTTPException(status_code=502, detail="GitHub is temporarily unavailable.")
|
|
143
201
|
except Exception as exc:
|
|
144
|
-
logger.warning(
|
|
202
|
+
logger.warning("Username fetch failed for %s: %s", username, exc)
|
|
145
203
|
raise HTTPException(status_code=500, detail="Failed to fetch GitHub data.")
|
|
146
204
|
|
|
147
205
|
|
|
@@ -189,6 +247,13 @@ def get_traffic(token: str = Body("", embed=True)):
|
|
|
189
247
|
if not df.empty:
|
|
190
248
|
df = df.drop_duplicates(subset=["date", "repository"], keep="last")
|
|
191
249
|
|
|
250
|
+
# Normalize `is_private` to a real bool. CSV round-trip + pd.concat
|
|
251
|
+
# widens the dtype to object (mix of bool, "True"/"False" strings, and
|
|
252
|
+
# NaN); without this, downstream code calling `bool("False")` would
|
|
253
|
+
# label every public repo as private.
|
|
254
|
+
if not df.empty and "is_private" in df.columns:
|
|
255
|
+
df["is_private"] = df["is_private"].map(_to_bool).astype(bool)
|
|
256
|
+
|
|
192
257
|
df = df.replace([float('inf'), float('-inf')], None).where(pd.notnull(df), None)
|
|
193
258
|
|
|
194
259
|
# Build a quick view-sum map to find the top 20 repos for deep fetching.
|
|
@@ -237,12 +302,20 @@ def get_star_history_endpoint(
|
|
|
237
302
|
# sense than 429 here so the SPA can distinguish "GitHub is throttling us"
|
|
238
303
|
# from "your request was malformed".
|
|
239
304
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
305
|
+
except ValueError as exc:
|
|
306
|
+
# fetch_star_history raises ValueError for empty owner / repo / trailing slash;
|
|
307
|
+
# returning 400 is more honest than letting Starlette surface a 500.
|
|
308
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
240
309
|
response.headers["Cache-Control"] = "public, max-age=86400"
|
|
241
310
|
return {"owner": owner, "repo": repo, "points": points}
|
|
242
311
|
|
|
243
312
|
|
|
244
313
|
@app.post("/api/upload-csv")
|
|
245
|
-
def upload_csv(
|
|
314
|
+
def upload_csv(
|
|
315
|
+
request: Request,
|
|
316
|
+
file: UploadFile = File(...),
|
|
317
|
+
_: None = Depends(_require_upload_access),
|
|
318
|
+
):
|
|
246
319
|
try:
|
|
247
320
|
data_dir = os.environ.get("GITLYTICS_DATA_DIR")
|
|
248
321
|
if data_dir:
|
|
@@ -277,7 +350,8 @@ def upload_csv(file: UploadFile = File(...)):
|
|
|
277
350
|
except HTTPException:
|
|
278
351
|
raise
|
|
279
352
|
except Exception as e:
|
|
280
|
-
|
|
353
|
+
logger.warning("CSV upload failed: %s", e)
|
|
354
|
+
raise HTTPException(status_code=400, detail="Failed to process CSV upload.")
|
|
281
355
|
|
|
282
356
|
|
|
283
357
|
# Static file serving
|
|
@@ -299,21 +373,27 @@ def serve_index():
|
|
|
299
373
|
def serve_spa_fallback(full_path: str):
|
|
300
374
|
"""SPA catch-all — returns index.html so React Router handles navigation."""
|
|
301
375
|
# Any unhandled /api/* path is a real 404, not a SPA route.
|
|
302
|
-
if full_path.startswith("api/"):
|
|
376
|
+
if full_path.startswith("api/") or full_path.startswith("assets/"):
|
|
303
377
|
return JSONResponse(status_code=404, content={"error": "Not found."})
|
|
304
378
|
|
|
305
|
-
|
|
306
|
-
|
|
379
|
+
# Confine the resolved path to frontend_dir — otherwise a request like
|
|
380
|
+
# `GET /..%2F..%2Fapp.py` would let `asset_file` escape the static
|
|
381
|
+
# directory and serve arbitrary files the worker can read.
|
|
382
|
+
frontend_root = frontend_dir.resolve()
|
|
383
|
+
try:
|
|
384
|
+
asset_file = (frontend_dir / full_path).resolve()
|
|
385
|
+
except (OSError, ValueError):
|
|
386
|
+
# `resolve()` can raise on broken symlinks or odd encodings.
|
|
387
|
+
return JSONResponse(status_code=404, content={"error": "Not found."})
|
|
388
|
+
is_inside = asset_file == frontend_root or asset_file.is_relative_to(frontend_root)
|
|
389
|
+
if is_inside and asset_file.is_file():
|
|
307
390
|
return FileResponse(asset_file)
|
|
308
391
|
|
|
309
392
|
index_file = frontend_dir / "index.html"
|
|
310
393
|
if index_file.exists():
|
|
311
394
|
return FileResponse(index_file)
|
|
312
395
|
|
|
313
|
-
return JSONResponse(
|
|
314
|
-
status_code=503,
|
|
315
|
-
content={"error": "Dashboard assets not found in the package installation."}
|
|
316
|
-
)
|
|
396
|
+
return JSONResponse(status_code=404, content={"error": "Not found."})
|
|
317
397
|
|
|
318
398
|
|
|
319
399
|
assets_dir = frontend_dir / "assets"
|
|
@@ -177,11 +177,15 @@ def run_sync_cycle(token: str, repo_names=None, data_dir="./data", output_mode="
|
|
|
177
177
|
|
|
178
178
|
final_rows.sort(key=lambda x: (str(x.get("date", "")), str(x.get("repository", ""))))
|
|
179
179
|
|
|
180
|
-
#
|
|
181
|
-
|
|
180
|
+
# Atomic write: dump to a sibling .tmp file then os.replace() it into
|
|
181
|
+
# place. A crash mid-write leaves the original CSV intact, so a single
|
|
182
|
+
# interrupted sync never corrupts the user's whole traffic history.
|
|
183
|
+
tmp_path = csv_path + ".tmp"
|
|
184
|
+
with open(tmp_path, "w", newline="", encoding="utf-8") as f:
|
|
182
185
|
writer = csv.DictWriter(f, fieldnames=existing_fields, extrasaction='ignore')
|
|
183
186
|
writer.writeheader()
|
|
184
187
|
writer.writerows(final_rows)
|
|
188
|
+
os.replace(tmp_path, csv_path)
|
|
185
189
|
|
|
186
190
|
logger.info(f"Successfully processed traffic data. Added {new_records_added} new daily records to {csv_path}")
|
|
187
191
|
|
|
@@ -234,19 +238,24 @@ def run_sync(token: str, repo_names=None, data_dir="./data", output_mode="monthl
|
|
|
234
238
|
# Always re-validate the token before fetching to catch expiry early
|
|
235
239
|
is_valid, msg = validate_token(token)
|
|
236
240
|
if not is_valid:
|
|
237
|
-
# Distinguish between a dead token (stop forever) and a network
|
|
241
|
+
# Distinguish between a dead token (stop forever) and a network
|
|
242
|
+
# blip (retry next cycle). Both 401 and 403 are terminal —
|
|
243
|
+
# 401 means the token is gone; 403 means the token is fine but
|
|
244
|
+
# lacks permissions, which is also a "never coming back" state.
|
|
238
245
|
msg_lower = msg.lower()
|
|
239
246
|
is_auth_failure = (
|
|
240
247
|
"401" in msg
|
|
248
|
+
or "403" in msg
|
|
241
249
|
or "authentication failed" in msg_lower
|
|
242
250
|
or "invalid token" in msg_lower
|
|
243
251
|
or "revoked" in msg_lower
|
|
244
252
|
or "bad credentials" in msg_lower
|
|
253
|
+
or "insufficient permissions" in msg_lower
|
|
245
254
|
)
|
|
246
255
|
if is_auth_failure:
|
|
247
256
|
logger.critical(
|
|
248
|
-
"FATAL: Token is expired, revoked, or
|
|
249
|
-
"Terminating daemon to prevent zombie process."
|
|
257
|
+
"FATAL: Token is expired, revoked, or lacks permissions "
|
|
258
|
+
"(401/403). Terminating daemon to prevent zombie process."
|
|
250
259
|
)
|
|
251
260
|
sys.exit(1)
|
|
252
261
|
else:
|
|
@@ -72,7 +72,12 @@ def main():
|
|
|
72
72
|
|
|
73
73
|
# ── DASHBOARD subcommand ──────────────────────────────────────────────────
|
|
74
74
|
dash_parser = subparsers.add_parser("dashboard", help="Serve the local React dashboard.")
|
|
75
|
-
dash_parser.add_argument("--host", default="127.0.0.1", help="Host IP.")
|
|
75
|
+
dash_parser.add_argument("--host", default="127.0.0.1", help="Host IP (default 127.0.0.1).")
|
|
76
|
+
dash_parser.add_argument(
|
|
77
|
+
"--public-bind",
|
|
78
|
+
action="store_true",
|
|
79
|
+
help="Allow binding to 0.0.0.0 (insecure without reverse-proxy auth).",
|
|
80
|
+
)
|
|
76
81
|
dash_parser.add_argument("--port", type=int, default=8000, help="Port to bind.")
|
|
77
82
|
dash_parser.add_argument("-t", "--token", help="GitHub token for headless TV display.")
|
|
78
83
|
dash_parser.add_argument("--data-dir", help="Inject historical CSV database.")
|
|
@@ -151,9 +156,16 @@ def main():
|
|
|
151
156
|
sys.exit(2)
|
|
152
157
|
|
|
153
158
|
elif args.command == "dashboard":
|
|
154
|
-
|
|
159
|
+
host = args.host
|
|
160
|
+
if host == "0.0.0.0" and not args.public_bind:
|
|
161
|
+
print("❌ Error: Use --public-bind to listen on 0.0.0.0 (default is 127.0.0.1).")
|
|
162
|
+
sys.exit(2)
|
|
163
|
+
if args.public_bind:
|
|
164
|
+
host = "0.0.0.0"
|
|
165
|
+
print("⚠️ Warning: dashboard bound to all interfaces — use only behind a trusted network or reverse proxy.")
|
|
166
|
+
print(f"\n[Gitlytics] Starting Gitlytics Dashboard on http://{host}:{args.port}\n")
|
|
155
167
|
serve_dashboard(
|
|
156
|
-
host=
|
|
168
|
+
host=host,
|
|
157
169
|
port=args.port,
|
|
158
170
|
token=token,
|
|
159
171
|
data_dir=args.data_dir
|
|
@@ -8,6 +8,7 @@ Handles fetching traffic and deep metadata from the GitHub API.
|
|
|
8
8
|
"""
|
|
9
9
|
import json
|
|
10
10
|
import logging
|
|
11
|
+
import re
|
|
11
12
|
|
|
12
13
|
import requests
|
|
13
14
|
import pandas as pd
|
|
@@ -17,16 +18,14 @@ from datetime import datetime, timezone, timedelta
|
|
|
17
18
|
logger = logging.getLogger(__name__)
|
|
18
19
|
|
|
19
20
|
BASE = "https://api.github.com"
|
|
21
|
+
GITHUB_USERNAME_RE = re.compile(r"^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,37}[a-zA-Z0-9])?$", re.IGNORECASE)
|
|
20
22
|
# Single source of truth for the GitHub media-type + API version headers.
|
|
21
23
|
_GITHUB_BASE_HEADERS = {
|
|
22
24
|
"Accept": "application/vnd.github+json",
|
|
23
25
|
"X-GitHub-Api-Version": "2022-11-28",
|
|
24
26
|
}
|
|
25
27
|
_PUBLIC_HEADERS = dict(_GITHUB_BASE_HEADERS)
|
|
26
|
-
|
|
27
|
-
_MAX_PR_PAGES = 10 # hard cap when walking /pulls
|
|
28
|
-
_MAX_COMMIT_PAGES = 50 # hard cap when counting /commits (50 * 100 = 5000)
|
|
29
|
-
_PER_PAGE = 100 # request size for paginated endpoints
|
|
28
|
+
_PER_PAGE = 100 # page size for paginated endpoints
|
|
30
29
|
|
|
31
30
|
|
|
32
31
|
class GitHubRateLimitError(Exception):
|
|
@@ -52,7 +51,8 @@ def validate_token(token: str) -> tuple[bool, str]:
|
|
|
52
51
|
except requests.exceptions.ConnectionError:
|
|
53
52
|
return False, "No internet connection."
|
|
54
53
|
except Exception as e:
|
|
55
|
-
|
|
54
|
+
logger.warning("Token validation error: %s", e)
|
|
55
|
+
return False, "Authentication failed."
|
|
56
56
|
|
|
57
57
|
if r.status_code == 200:
|
|
58
58
|
data = r.json()
|
|
@@ -84,8 +84,16 @@ def get_user_profile(token: str) -> dict:
|
|
|
84
84
|
return {"login": "", "name": "", "avatar_url": "", "followers": 0, "following": 0}
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
def validate_github_username(username: str) -> str:
|
|
88
|
+
u = (username or "").strip()
|
|
89
|
+
if not u or not GITHUB_USERNAME_RE.match(u):
|
|
90
|
+
raise ValueError("Invalid GitHub username.")
|
|
91
|
+
return u
|
|
92
|
+
|
|
93
|
+
|
|
87
94
|
def get_public_user(username: str) -> dict:
|
|
88
95
|
"""Fetches a public GitHub user profile — no PAT required."""
|
|
96
|
+
username = validate_github_username(username)
|
|
89
97
|
try:
|
|
90
98
|
r = requests.get(f"{BASE}/users/{username}", headers=_PUBLIC_HEADERS, timeout=10)
|
|
91
99
|
if r.status_code == 200:
|
|
@@ -106,15 +114,15 @@ def get_public_user(username: str) -> dict:
|
|
|
106
114
|
}
|
|
107
115
|
if r.status_code == 404:
|
|
108
116
|
raise ValueError(f"User '{username}' not found.")
|
|
109
|
-
# 5xx and other unexpected codes — log a warning and return a stub so the
|
|
110
|
-
# caller (the API endpoint) can return a 502 without leaking the failure.
|
|
111
117
|
logger.warning(f"Failed to fetch user {username}: HTTP {r.status_code}")
|
|
118
|
+
raise RuntimeError(f"GitHub returned HTTP {r.status_code}.")
|
|
112
119
|
except ValueError:
|
|
113
|
-
|
|
120
|
+
raise
|
|
121
|
+
except RuntimeError:
|
|
114
122
|
raise
|
|
115
123
|
except Exception as exc:
|
|
116
124
|
logger.warning(f"Could not fetch public user profile: {exc}")
|
|
117
|
-
|
|
125
|
+
raise RuntimeError("GitHub is unreachable.") from exc
|
|
118
126
|
|
|
119
127
|
|
|
120
128
|
def _normalise_repo(repo: dict) -> dict:
|
|
@@ -176,6 +184,7 @@ def _fetch_public_repos_by_stars(username: str, per_page: int) -> list:
|
|
|
176
184
|
|
|
177
185
|
|
|
178
186
|
def get_public_repos(username: str, max_repos: int = 50) -> list:
|
|
187
|
+
username = validate_github_username(username)
|
|
179
188
|
"""
|
|
180
189
|
Fetches up to max_repos best public repos for a username using two calls:
|
|
181
190
|
1. Most recently updated (catches active projects)
|
|
@@ -227,29 +236,33 @@ def _safe_get(url: str, headers: dict, params: dict = None) -> tuple:
|
|
|
227
236
|
return {}, -1
|
|
228
237
|
|
|
229
238
|
|
|
230
|
-
def
|
|
231
|
-
"""
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
239
|
+
def _count_via_link_header(url: str, headers: dict, params: dict = None) -> int:
|
|
240
|
+
"""Get item count in exactly 1 API call using the Link header pagination trick.
|
|
241
|
+
|
|
242
|
+
Requests per_page=1 so GitHub includes a Link header with the last page
|
|
243
|
+
number, which equals the total count. Falls back to len(response) when
|
|
244
|
+
the repository has only one page of results (no Link header present).
|
|
235
245
|
"""
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
246
|
+
p = dict(params or {})
|
|
247
|
+
p["per_page"] = 1
|
|
248
|
+
try:
|
|
249
|
+
r = requests.get(url, headers=headers, params=p, timeout=10)
|
|
250
|
+
if r.status_code != 200:
|
|
251
|
+
return 0
|
|
252
|
+
link = r.headers.get("Link", "")
|
|
253
|
+
match = re.search(r'[?&]page=(\d+)[^>]*>;\s*rel="last"', link)
|
|
254
|
+
if match:
|
|
255
|
+
return int(match.group(1))
|
|
256
|
+
# No Link header = all results fit on one page
|
|
257
|
+
data = r.json()
|
|
258
|
+
return len(data) if isinstance(data, list) else 0
|
|
259
|
+
except Exception as exc:
|
|
260
|
+
logger.warning("Link-header count failed for %s: %s", url, exc)
|
|
261
|
+
return 0
|
|
249
262
|
|
|
250
263
|
|
|
251
264
|
def get_deep_repo_stats(token: str, full_name: str) -> dict:
|
|
252
|
-
"""
|
|
265
|
+
"""Fetch commit count, open PRs, releases, and community health in exactly 4 API calls."""
|
|
253
266
|
h = make_headers(token)
|
|
254
267
|
stats = {
|
|
255
268
|
"total_commits": None,
|
|
@@ -262,60 +275,55 @@ def get_deep_repo_stats(token: str, full_name: str) -> dict:
|
|
|
262
275
|
"has_code_of_conduct": None,
|
|
263
276
|
}
|
|
264
277
|
|
|
265
|
-
# Commit activity
|
|
266
|
-
#
|
|
267
|
-
#
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
ca_url = f"{BASE}/repos/{full_name}/stats/commit_activity"
|
|
272
|
-
ca_data, status = _safe_get(ca_url, h)
|
|
273
|
-
if status == 202:
|
|
274
|
-
logger.info(f"Commit activity not ready yet (202) for {full_name}; will populate on next fetch.")
|
|
278
|
+
# Call 1: Commit activity (52-week window). GitHub computes this asynchronously
|
|
279
|
+
# and returns 202 while it's being prepared. When ready it gives us a weekly
|
|
280
|
+
# breakdown which also powers the CommitActivity chart in the dashboard.
|
|
281
|
+
ca_data, ca_status = _safe_get(f"{BASE}/repos/{full_name}/stats/commit_activity", h)
|
|
282
|
+
if ca_status == 202:
|
|
283
|
+
logger.info("Commit activity computing (202) for %s — will populate on next fetch.", full_name)
|
|
275
284
|
if isinstance(ca_data, list) and ca_data:
|
|
276
285
|
stats["total_commits"] = sum(week.get("total", 0) for week in ca_data)
|
|
277
286
|
else:
|
|
278
|
-
#
|
|
279
|
-
#
|
|
280
|
-
# always returns the real lifetime history for repos under ~10k commits.
|
|
281
|
-
commits_url = f"{BASE}/repos/{full_name}/commits"
|
|
287
|
+
# Call 1b (fallback): commit_activity not ready — use Link header trick to
|
|
288
|
+
# count commits in 1 request instead of walking up to 50 pages.
|
|
282
289
|
try:
|
|
283
|
-
stats["total_commits"] =
|
|
284
|
-
|
|
290
|
+
stats["total_commits"] = _count_via_link_header(
|
|
291
|
+
f"{BASE}/repos/{full_name}/commits", h
|
|
285
292
|
)
|
|
286
293
|
except Exception as exc:
|
|
287
|
-
logger.warning(
|
|
294
|
+
logger.warning("Could not fetch commit count for %s: %s", full_name, exc)
|
|
288
295
|
|
|
289
|
-
# Open PRs —
|
|
290
|
-
pr_url = f"{BASE}/repos/{full_name}/pulls"
|
|
296
|
+
# Call 2: Open PRs — Link header trick replaces up to 10 paginated calls.
|
|
291
297
|
try:
|
|
292
|
-
stats["open_prs"] =
|
|
298
|
+
stats["open_prs"] = _count_via_link_header(
|
|
299
|
+
f"{BASE}/repos/{full_name}/pulls", h, {"state": "open"}
|
|
300
|
+
)
|
|
293
301
|
except Exception as exc:
|
|
294
|
-
logger.warning(
|
|
302
|
+
logger.warning("Could not fetch open PRs for %s: %s", full_name, exc)
|
|
295
303
|
|
|
296
|
-
# Community health
|
|
304
|
+
# Call 3: Community health (README, license, contributing, CoC) — 1 call, unchanged.
|
|
297
305
|
cp_data, _ = _safe_get(f"{BASE}/repos/{full_name}/community/profile", h)
|
|
298
306
|
if isinstance(cp_data, dict) and "files" in cp_data:
|
|
299
|
-
files = cp_data
|
|
307
|
+
files = cp_data["files"]
|
|
300
308
|
stats["has_readme"] = bool(files.get("readme"))
|
|
301
309
|
stats["has_license"] = bool(files.get("license"))
|
|
302
310
|
stats["has_contributing"] = bool(files.get("contributing"))
|
|
303
311
|
stats["has_code_of_conduct"] = bool(files.get("code_of_conduct"))
|
|
304
312
|
|
|
305
|
-
# Releases —
|
|
306
|
-
#
|
|
313
|
+
# Call 4: Releases — Link header trick for count + per_page=1 returns the
|
|
314
|
+
# newest release (GitHub sorts newest-first) so we get last_release_at too.
|
|
307
315
|
try:
|
|
308
316
|
rel_url = f"{BASE}/repos/{full_name}/releases"
|
|
309
|
-
|
|
310
|
-
if
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
)
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
stats["last_release_at"] =
|
|
317
|
+
r = requests.get(rel_url, headers=h, params={"per_page": 1}, timeout=10)
|
|
318
|
+
if r.status_code == 200:
|
|
319
|
+
link = r.headers.get("Link", "")
|
|
320
|
+
match = re.search(r'[?&]page=(\d+)[^>]*>;\s*rel="last"', link)
|
|
321
|
+
stats["total_releases"] = int(match.group(1)) if match else len(r.json())
|
|
322
|
+
data = r.json()
|
|
323
|
+
if isinstance(data, list) and data:
|
|
324
|
+
stats["last_release_at"] = data[0].get("published_at") or data[0].get("created_at")
|
|
317
325
|
except Exception as exc:
|
|
318
|
-
logger.warning(
|
|
326
|
+
logger.warning("Could not fetch releases for %s: %s", full_name, exc)
|
|
319
327
|
|
|
320
328
|
return stats
|
|
321
329
|
|
|
@@ -408,6 +416,8 @@ def fetch_star_history(owner: str, repo: str, token: str | None = None) -> list[
|
|
|
408
416
|
if status in (403, 429):
|
|
409
417
|
# GitHub's rate-limit guard returns 403 with a 'rate limit' message in body.
|
|
410
418
|
raise GitHubRateLimitError("GitHub rate limit reached, try again later")
|
|
419
|
+
if status == 404:
|
|
420
|
+
raise StarHistoryFetchError("user or repo does not exist")
|
|
411
421
|
if status != 200 or not isinstance(meta, dict):
|
|
412
422
|
raise StarHistoryFetchError(f"Failed to fetch repo metadata (HTTP {status})")
|
|
413
423
|
total_stars = int(meta.get("stargazers_count", 0) or 0)
|