linearjeph 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- linearjeph/__init__.py +13 -0
- linearjeph/client.py +632 -0
- linearjeph/health.py +134 -0
- linearjeph/integrations/__init__.py +0 -0
- linearjeph/integrations/asgi.py +41 -0
- linearjeph/integrations/django.py +55 -0
- linearjeph/integrations/flask.py +38 -0
- linearjeph/prophecy/__init__.py +519 -0
- linearjeph/prophecy/__main__.py +67 -0
- linearjeph/prophecy/django.py +63 -0
- linearjeph/prophecy/middleware.py +151 -0
- linearjeph/prophecy/scanner.py +507 -0
- linearjeph/py.typed +0 -0
- linearjeph/telemetry.py +452 -0
- linearjeph-1.2.0.dist-info/METADATA +107 -0
- linearjeph-1.2.0.dist-info/RECORD +19 -0
- linearjeph-1.2.0.dist-info/WHEEL +5 -0
- linearjeph-1.2.0.dist-info/licenses/LICENSE +21 -0
- linearjeph-1.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Request tracking middlewares: WSGI, ASGI (FastAPI/Starlette), Flask. Records route template, method, status and
|
|
2
|
+
duration only. Never bodies, headers or query strings. Frameworks are imported lazily (the SDK has no hard dependencies)."""
|
|
3
|
+
import re
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
_ID_SEG = re.compile(r"^(\d+|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|[0-9a-fA-F]{16,}|(?=.*\d)[A-Za-z0-9_-]{20,})$")
|
|
7
|
+
_PARAM = re.compile(r"<(?:[^>:]+:)?([^>]+)>|\{([^}:]+)(?::[^}]*)?\}")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def normalize_path(path):
|
|
11
|
+
"""'/users/42/orders/9f8...' -> '/users/:id/orders/:id' (no query string, ever)."""
|
|
12
|
+
clean = str(path or "/").split("?")[0].split("#")[0]
|
|
13
|
+
out = "/".join(":id" if s and _ID_SEG.match(s) else s for s in clean.split("/"))
|
|
14
|
+
return out or "/"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def route_template(rule):
|
|
18
|
+
"""Framework route pattern -> '/users/:id' ('<int:id>' and '{id}' forms; regex groups reduced)."""
|
|
19
|
+
s = str(rule or "")
|
|
20
|
+
s = re.sub(r"\(\?P<(\w+)>[^)]*\)", r":\1", s)
|
|
21
|
+
s = _PARAM.sub(lambda m: ":" + (m.group(1) or m.group(2)), s)
|
|
22
|
+
s = re.sub(r"^\^|\$$", "", s)
|
|
23
|
+
if not s.startswith("/"):
|
|
24
|
+
s = "/" + s
|
|
25
|
+
return s
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _record(route, method, status, ms, uid=None):
|
|
29
|
+
from . import record_request
|
|
30
|
+
record_request(route, method, status, ms, uid)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class WSGIMiddleware(object):
|
|
34
|
+
"""WSGI: app.wsgi_app = WSGIMiddleware(app.wsgi_app). Apps can set environ['prophecy.route'] to give an exact template."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, app, ignore_paths=(), uid_getter=None):
|
|
37
|
+
self.app = app
|
|
38
|
+
self.ignore = tuple(ignore_paths or ())
|
|
39
|
+
self.uid_getter = uid_getter
|
|
40
|
+
|
|
41
|
+
def __call__(self, environ, start_response):
|
|
42
|
+
t0 = time.perf_counter()
|
|
43
|
+
status = {"code": 500}
|
|
44
|
+
|
|
45
|
+
def _start(s, headers, exc_info=None):
|
|
46
|
+
try:
|
|
47
|
+
status["code"] = int(str(s).split(" ", 1)[0])
|
|
48
|
+
except ValueError:
|
|
49
|
+
pass
|
|
50
|
+
return start_response(s, headers, exc_info) if exc_info else start_response(s, headers)
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
result = self.app(environ, _start)
|
|
54
|
+
except Exception:
|
|
55
|
+
self._done(environ, status["code"], t0)
|
|
56
|
+
raise
|
|
57
|
+
return _Closing(result, lambda: self._done(environ, status["code"], t0))
|
|
58
|
+
|
|
59
|
+
def _done(self, environ, code, t0):
|
|
60
|
+
try:
|
|
61
|
+
path = environ.get("PATH_INFO", "/")
|
|
62
|
+
if self.ignore and path.startswith(self.ignore):
|
|
63
|
+
return
|
|
64
|
+
route = environ.get("prophecy.route") or normalize_path(environ.get("SCRIPT_NAME", "") + path)
|
|
65
|
+
uid = None
|
|
66
|
+
if self.uid_getter:
|
|
67
|
+
try:
|
|
68
|
+
uid = self.uid_getter(environ)
|
|
69
|
+
except Exception: # noqa: BLE001
|
|
70
|
+
pass
|
|
71
|
+
_record(route, environ.get("REQUEST_METHOD", "GET"), code, (time.perf_counter() - t0) * 1000, uid)
|
|
72
|
+
except Exception: # noqa: BLE001 - never break a response
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class _Closing(object):
|
|
77
|
+
def __init__(self, result, done):
|
|
78
|
+
self._result, self._done, self._fired = result, done, False
|
|
79
|
+
|
|
80
|
+
def __iter__(self):
|
|
81
|
+
return iter(self._result)
|
|
82
|
+
|
|
83
|
+
def close(self):
|
|
84
|
+
try:
|
|
85
|
+
if hasattr(self._result, "close"):
|
|
86
|
+
self._result.close()
|
|
87
|
+
finally:
|
|
88
|
+
if not self._fired:
|
|
89
|
+
self._fired = True
|
|
90
|
+
self._done()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ASGIMiddleware(object):
|
|
94
|
+
"""ASGI (FastAPI, Starlette, Quart...): app.add_middleware(ASGIMiddleware). The route comes from scope['route']
|
|
95
|
+
(path_format) when the framework sets it, else the normalised path."""
|
|
96
|
+
|
|
97
|
+
def __init__(self, app, ignore_paths=()):
|
|
98
|
+
self.app = app
|
|
99
|
+
self.ignore = tuple(ignore_paths or ())
|
|
100
|
+
|
|
101
|
+
async def __call__(self, scope, receive, send):
|
|
102
|
+
if scope.get("type") != "http":
|
|
103
|
+
return await self.app(scope, receive, send)
|
|
104
|
+
t0 = time.perf_counter()
|
|
105
|
+
status = {"code": 500}
|
|
106
|
+
|
|
107
|
+
async def _send(msg):
|
|
108
|
+
if msg.get("type") == "http.response.start":
|
|
109
|
+
status["code"] = msg.get("status", 500)
|
|
110
|
+
await send(msg)
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
await self.app(scope, receive, _send)
|
|
114
|
+
finally:
|
|
115
|
+
try:
|
|
116
|
+
path = scope.get("path", "/")
|
|
117
|
+
if not (self.ignore and path.startswith(self.ignore)):
|
|
118
|
+
route = scope.get("route")
|
|
119
|
+
tpl = getattr(route, "path_format", None) or getattr(route, "path", None)
|
|
120
|
+
_record(route_template(tpl) if tpl else normalize_path(path), scope.get("method", "GET"), status["code"],
|
|
121
|
+
(time.perf_counter() - t0) * 1000)
|
|
122
|
+
except Exception: # noqa: BLE001
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
ProphecyASGIMiddleware = ASGIMiddleware
|
|
127
|
+
ProphecyWSGIMiddleware = WSGIMiddleware
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def init_flask(app, ignore_paths=(), uid_getter=None):
|
|
131
|
+
"""Flask: prophecy.init_flask(app) after prophecy.init(...). Uses request.url_rule for the route template."""
|
|
132
|
+
from flask import g, request # imported lazily: Flask is the customer's dependency
|
|
133
|
+
ignore = tuple(ignore_paths or ())
|
|
134
|
+
|
|
135
|
+
@app.before_request
|
|
136
|
+
def _prophecy_start():
|
|
137
|
+
g._prophecy_t0 = time.perf_counter()
|
|
138
|
+
|
|
139
|
+
@app.after_request
|
|
140
|
+
def _prophecy_end(resp):
|
|
141
|
+
try:
|
|
142
|
+
t0 = getattr(g, "_prophecy_t0", None)
|
|
143
|
+
if t0 is not None and not (ignore and request.path.startswith(ignore)):
|
|
144
|
+
rule = request.url_rule.rule if request.url_rule else None
|
|
145
|
+
uid = uid_getter(request) if uid_getter else None
|
|
146
|
+
_record(route_template(rule) if rule else normalize_path(request.path), request.method, resp.status_code,
|
|
147
|
+
(time.perf_counter() - t0) * 1000, uid)
|
|
148
|
+
except Exception: # noqa: BLE001
|
|
149
|
+
pass
|
|
150
|
+
return resp
|
|
151
|
+
return app
|
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""Prophecy codebase scanner. Reads files locally and returns a manifest (schema 1): names, counts and route templates
|
|
2
|
+
only. Never source text, never env values, never matched secrets (secrets are only counted).
|
|
3
|
+
|
|
4
|
+
Limits: skips node_modules/.git/vendor/venv/dist/build..., files > 1 MB, symlinks, at most 20,000 files.
|
|
5
|
+
Standard library only; same output schema as the Node SDK scanner.
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
|
|
12
|
+
MAX_FILES = 20000
|
|
13
|
+
MAX_BYTES = 1024 * 1024
|
|
14
|
+
CAPS = {"dependencies": 2000, "routes": 3000, "features": 200, "env_names": 500, "pricing_hints": 20}
|
|
15
|
+
SKIP_DIRS = {"node_modules", ".git", "vendor", "venv", ".venv", "env", "dist", "build", "__pycache__", ".next", ".nuxt",
|
|
16
|
+
".svelte-kit", "target", "coverage", ".tox", ".cache", "site-packages", ".idea", ".vscode", ".gradle", "Pods",
|
|
17
|
+
"bower_components", ".terraform", ".mypy_cache", ".pytest_cache", ".turbo", "out"}
|
|
18
|
+
KEEP_DOT_DIRS = {".github", ".circleci"}
|
|
19
|
+
CODE_EXT = {"js", "jsx", "ts", "tsx", "mjs", "cjs", "py", "go", "php", "rb", "java", "kt", "rs", "cs", "swift", "dart", "vue",
|
|
20
|
+
"svelte", "scala", "ex", "exs", "c", "cpp", "h", "sh", "sql", "html", "css", "scss"}
|
|
21
|
+
CONFIG_EXT = {"json", "yml", "yaml", "toml", "ini", "cfg", "xml", "tf", "properties", "gradle", "kts", "mod", "env", "md", "txt"}
|
|
22
|
+
LOCKFILES = {"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb", "poetry.lock", "Pipfile.lock", "uv.lock", "go.sum",
|
|
23
|
+
"composer.lock", "Gemfile.lock"}
|
|
24
|
+
JS_EXT = ("js", "jsx", "ts", "tsx", "mjs", "cjs")
|
|
25
|
+
SECRET_RES = [re.compile(p) for p in (
|
|
26
|
+
r"\bAKIA[0-9A-Z]{16}\b", r"\bsk_live_[0-9A-Za-z]{16,}", r"\bghp_[A-Za-z0-9]{30,}", r"\bgithub_pat_[A-Za-z0-9_]{40,}",
|
|
27
|
+
r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----", r"\bxox[baprs]-[A-Za-z0-9-]{10,}",
|
|
28
|
+
r"\bAIza[0-9A-Za-z_-]{35}", r"\bljp_sk_[A-Za-z0-9_-]{20,}")]
|
|
29
|
+
FRAMEWORK_DEPS = {
|
|
30
|
+
"express": "express", "fastify": "fastify", "koa": "koa", "next": "next", "@nestjs/core": "nestjs", "react": "react", "vue": "vue",
|
|
31
|
+
"nuxt": "nuxt", "svelte": "svelte", "@sveltejs/kit": "sveltekit", "@angular/core": "angular", "@hapi/hapi": "hapi",
|
|
32
|
+
"@remix-run/node": "remix", "astro": "astro", "django": "django", "flask": "flask", "fastapi": "fastapi", "starlette": "starlette",
|
|
33
|
+
"tornado": "tornado", "aiohttp": "aiohttp", "celery": "celery", "laravel/framework": "laravel",
|
|
34
|
+
"symfony/framework-bundle": "symfony", "rails": "rails", "sinatra": "sinatra", "github.com/gin-gonic/gin": "gin",
|
|
35
|
+
"github.com/go-chi/chi": "chi", "github.com/labstack/echo": "echo", "github.com/gofiber/fiber": "fiber",
|
|
36
|
+
"org.springframework.boot": "spring-boot", "flutter": "flutter"}
|
|
37
|
+
INTEGRATIONS = {
|
|
38
|
+
"stripe": (["stripe"], ["STRIPE"]), "paddle": (["paddle"], ["PADDLE"]), "paystack": (["paystack"], ["PAYSTACK"]),
|
|
39
|
+
"flutterwave": (["flutterwave"], ["FLUTTERWAVE", "FLW_"]), "bachs": (["bachs"], ["BACHS"]), "auth0": (["auth0"], ["AUTH0"]),
|
|
40
|
+
"clerk": (["@clerk", "clerk"], ["CLERK"]), "supabase": (["supabase"], ["SUPABASE"]), "firebase": (["firebase"], ["FIREBASE"]),
|
|
41
|
+
"sentry": (["sentry"], ["SENTRY"]), "segment": (["@segment", "analytics-node", "segment"], ["SEGMENT"]),
|
|
42
|
+
"mixpanel": (["mixpanel"], ["MIXPANEL"]), "posthog": (["posthog"], ["POSTHOG"]), "amplitude": (["amplitude"], ["AMPLITUDE"]),
|
|
43
|
+
"launchdarkly": (["launchdarkly"], ["LAUNCHDARKLY", "LD_SDK"]), "unleash": (["unleash"], ["UNLEASH"]),
|
|
44
|
+
"twilio": (["twilio"], ["TWILIO"]), "sendgrid": (["sendgrid"], ["SENDGRID"]), "resend": (["resend"], ["RESEND"]),
|
|
45
|
+
"mailgun": (["mailgun"], ["MAILGUN"]), "postmark": (["postmark"], ["POSTMARK"]), "openai": (["openai"], ["OPENAI"]),
|
|
46
|
+
"anthropic": (["anthropic"], ["ANTHROPIC"]),
|
|
47
|
+
"s3": (["@aws-sdk/client-s3", "aws-sdk", "boto3", "aws/aws-sdk-php", "aws-sdk-s3"], ["AWS_S3", "S3_", "AWS_ACCESS"]),
|
|
48
|
+
"cloudinary": (["cloudinary"], ["CLOUDINARY"]), "algolia": (["algolia"], ["ALGOLIA"]),
|
|
49
|
+
"redis": (["redis", "ioredis"], ["REDIS"]), "rabbitmq": (["amqplib", "pika"], ["RABBITMQ", "AMQP"]),
|
|
50
|
+
"kafka": (["kafkajs", "kafka-python", "confluent-kafka"], ["KAFKA"])}
|
|
51
|
+
HOSTS = {"vercel.json": "vercel", "netlify.toml": "netlify", "fly.toml": "fly", "render.yaml": "render", "Procfile": "heroku",
|
|
52
|
+
"app.yaml": "gcp-app-engine", "railway.json": "railway", "railway.toml": "railway", "wrangler.toml": "cloudflare"}
|
|
53
|
+
_M = "get|post|put|patch|delete|options|head|all"
|
|
54
|
+
_PLANS = "free|starter|basic|pro|premium|business|team|enterprise|growth|plus|standard|scale"
|
|
55
|
+
RE_EXPRESS = re.compile(r"\b[A-Za-z_$][\w$]*\.(" + _M + r")\(\s*([\"'`])(/[^\"'`\s]*)\2")
|
|
56
|
+
RE_PYROUTE = re.compile(r"@\w+(?:\.\w+)*\.(route|get|post|put|patch|delete)\(\s*(['\"])([^'\"]+)\2([^)]*)\)")
|
|
57
|
+
RE_DJANGO = re.compile(r"\b(re_path|path|url)\(\s*r?(['\"])([^'\"]*)\2")
|
|
58
|
+
RE_LARAVEL = re.compile(r"Route::(get|post|put|patch|delete|any|options)\(\s*(['\"])([^'\"]+)\2")
|
|
59
|
+
RE_RAILS_VERB = re.compile(r"^\s*(get|post|put|patch|delete)\s+(['\"])([^'\"]+)\2", re.M)
|
|
60
|
+
RE_RAILS_RES = re.compile(r"^\s*(resources|resource)\s+:(\w+)", re.M)
|
|
61
|
+
RE_GO_STD = re.compile(r"\.(?:HandleFunc|Handle)\(\s*\"([^\"]+)\"")
|
|
62
|
+
RE_GO_ROUTER = re.compile(r"\b\w+\.(Get|Post|Put|Patch|Delete|GET|POST|PUT|PATCH|DELETE)\(\s*\"(/[^\"]*)\"")
|
|
63
|
+
RE_FLAG = re.compile(r"\b(?:isEnabled|isFeatureEnabled|isFeatureFlagEnabled|getFeatureFlag|variation|boolVariation|useFlag|useFeatureFlag|"
|
|
64
|
+
r"flag_is_active|feature_enabled|feature_flag|is_enabled)\(\s*(?:[\w.]+\s*,\s*)?(['\"])([\w.:-]{1,80})\1")
|
|
65
|
+
RE_FLAG_PROP = re.compile(r"\bflags\.([A-Za-z_]\w{1,60})\b")
|
|
66
|
+
RE_ENV_JS = re.compile(r"process\.env\.([A-Z][A-Z0-9_]+)|process\.env\[\s*['\"]([A-Z][A-Z0-9_]+)['\"]\s*\]|import\.meta\.env\.([A-Z][A-Z0-9_]+)")
|
|
67
|
+
RE_ENV_PY = re.compile(r"os\.environ(?:\.get)?[\[(]\s*['\"]([A-Z][A-Z0-9_]+)['\"]|os\.getenv\(\s*['\"]([A-Z][A-Z0-9_]+)['\"]|"
|
|
68
|
+
r"\benv\(\s*['\"]([A-Z][A-Z0-9_]+)['\"]|\bENV\[\s*['\"]([A-Z][A-Z0-9_]+)['\"]|"
|
|
69
|
+
r"os\.Getenv\(\s*\"([A-Z][A-Z0-9_]+)\"|getenv\(\s*['\"]([A-Z][A-Z0-9_]+)['\"]")
|
|
70
|
+
RE_PLAN_PRICE = re.compile(r"\b(?:plan|tier|name|id|slug)['\"]?\s*[:=]\s*['\"]([A-Za-z][\w -]{1,30})['\"][\s\S]{0,120}?"
|
|
71
|
+
r"\b(?:price|amount|monthly\w*|cost)['\"]?\s*[:=]\s*(\d+(?:\.\d{1,2})?)", re.I)
|
|
72
|
+
RE_PLAN_KEY = re.compile(r"\b(" + _PLANS + r")['\"]?\s*[:=]\s*\{[^}]{0,120}?\b(?:price|amount|monthly\w*|cost)['\"]?\s*[:=]\s*(\d+(?:\.\d{1,2})?)", re.I)
|
|
73
|
+
RE_PLAN_CONST = re.compile(r"\b(" + _PLANS + r")_?(?:monthly_)?price\w*\s*[:=]\s*(\d+(?:\.\d{1,2})?)", re.I)
|
|
74
|
+
RE_CURRENCY = re.compile(r"\bcurrency['\"]?\s*[:=]\s*['\"]([A-Za-z]{3})['\"]", re.I)
|
|
75
|
+
RE_TEST_FILE = re.compile(r"\.(test|spec)\.[cm]?[jt]sx?$|^test_.*\.py$|_test\.(py|go|rb)$|Test\.php$|_spec\.rb$")
|
|
76
|
+
RE_TEST_DIR = re.compile(r"(^|/)(tests?|__tests__|spec)/")
|
|
77
|
+
RE_DEP_FILE = re.compile(r"^(package\.json|requirements.*\.txt|pyproject\.toml|Pipfile|go\.mod|composer\.json|Gemfile(\.lock)?|pom\.xml|build\.gradle(\.kts)?)$")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _glob_re(g):
|
|
81
|
+
s = g.lstrip("/").rstrip("/")
|
|
82
|
+
s = re.sub(r"[.+^${}()|\\]", lambda m: "\\" + m.group(0), s).replace("*", ".*").replace("?", ".")
|
|
83
|
+
return re.compile("^" + s + "$")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _read(path):
|
|
87
|
+
try:
|
|
88
|
+
with open(path, "r", encoding="latin-1") as f:
|
|
89
|
+
return f.read()
|
|
90
|
+
except OSError:
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _seg(s):
|
|
95
|
+
if s.startswith("[") and s.endswith("]"):
|
|
96
|
+
return ":" + (re.sub(r"[\[\].]", "", s) or "id")
|
|
97
|
+
return s
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def next_route(rel):
|
|
101
|
+
"""Next.js file path (relative, posix) -> (route, kind) or None."""
|
|
102
|
+
parts = rel.split("/")
|
|
103
|
+
fname = parts.pop()
|
|
104
|
+
m = re.match(r"^(route|page)\.(?:tsx|ts|jsx|js|mjs)$", fname)
|
|
105
|
+
if m and "app" in parts:
|
|
106
|
+
i = len(parts) - 1 - parts[::-1].index("app")
|
|
107
|
+
segs = [_seg(x) for x in parts[i + 1:] if not (x.startswith("(") and x.endswith(")")) and not x.startswith("@")]
|
|
108
|
+
return "/" + "/".join(segs), m.group(1)
|
|
109
|
+
pm = re.match(r"^(.+)\.(?:tsx|ts|jsx|js)$", fname)
|
|
110
|
+
if pm and "pages" in parts and not pm.group(1).startswith("_"):
|
|
111
|
+
i = len(parts) - 1 - parts[::-1].index("pages")
|
|
112
|
+
segs = [_seg(x) for x in parts[i + 1:]]
|
|
113
|
+
if pm.group(1) != "index":
|
|
114
|
+
segs.append(_seg(pm.group(1)))
|
|
115
|
+
return "/" + "/".join(segs), ("api" if segs and segs[0] == "api" else "page")
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _parse_deps(name, text, add, gem_lock):
|
|
120
|
+
if name == "package.json":
|
|
121
|
+
try:
|
|
122
|
+
j = json.loads(text)
|
|
123
|
+
except ValueError:
|
|
124
|
+
return
|
|
125
|
+
for k in ("dependencies", "devDependencies", "peerDependencies"):
|
|
126
|
+
for n, v in ((j or {}).get(k) or {}).items():
|
|
127
|
+
add("npm", n, v if isinstance(v, str) else "")
|
|
128
|
+
elif re.match(r"^requirements.*\.txt$", name):
|
|
129
|
+
for line in text.split("\n"):
|
|
130
|
+
t = line.strip()
|
|
131
|
+
if not t or t.startswith("-") or t.startswith("#"):
|
|
132
|
+
continue
|
|
133
|
+
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9_.-]*)(?:\[[^\]]*\])?\s*(?:(==|>=|~=|<=|!=|<|>)\s*([^\s;#,]+))?", t)
|
|
134
|
+
if m:
|
|
135
|
+
add("pypi", m.group(1), (m.group(3) if m.group(2) == "==" else (m.group(2) + m.group(3))) if m.group(3) else "")
|
|
136
|
+
elif name == "pyproject.toml":
|
|
137
|
+
arr = re.search(r"^\s*dependencies\s*=\s*\[([\s\S]*?)\]", text, re.M)
|
|
138
|
+
if arr:
|
|
139
|
+
for m in re.finditer(r"['\"]([A-Za-z0-9][A-Za-z0-9_.-]*)(?:\[[^\]]*\])?\s*([^'\"]*)['\"]", arr.group(1)):
|
|
140
|
+
add("pypi", m.group(1), re.sub(r"^==", "", m.group(2).strip()))
|
|
141
|
+
in_sec = False
|
|
142
|
+
for line in text.split("\n"):
|
|
143
|
+
h = re.match(r"^\s*\[([^\]]+)\]", line)
|
|
144
|
+
if h:
|
|
145
|
+
in_sec = bool(re.search(r"poetry\.(dev-)?dependencies$", h.group(1)) or "dependency-groups" in h.group(1))
|
|
146
|
+
continue
|
|
147
|
+
m = in_sec and re.match(r"^\s*([A-Za-z0-9][A-Za-z0-9_.-]*)\s*=\s*(?:['\"]([^'\"]*)['\"]|\{[^}]*version\s*=\s*['\"]([^'\"]*)['\"])", line)
|
|
148
|
+
if m and m.group(1).lower() != "python":
|
|
149
|
+
add("pypi", m.group(1), m.group(2) or m.group(3) or "")
|
|
150
|
+
elif name == "Pipfile":
|
|
151
|
+
in_sec = False
|
|
152
|
+
for line in text.split("\n"):
|
|
153
|
+
h = re.match(r"^\s*\[([^\]]+)\]", line)
|
|
154
|
+
if h:
|
|
155
|
+
in_sec = bool(re.match(r"^(dev-)?packages$", h.group(1)))
|
|
156
|
+
continue
|
|
157
|
+
m = in_sec and re.match(r"^\s*([A-Za-z0-9][A-Za-z0-9_.-]*)\s*=\s*(?:['\"]([^'\"]*)['\"]|\{[^}]*version\s*=\s*['\"]([^'\"]*)['\"])", line)
|
|
158
|
+
if m:
|
|
159
|
+
v = m.group(2) or m.group(3) or ""
|
|
160
|
+
add("pypi", m.group(1), "" if v == "*" else v)
|
|
161
|
+
elif name == "go.mod":
|
|
162
|
+
block = False
|
|
163
|
+
for line in text.split("\n"):
|
|
164
|
+
t = line.strip()
|
|
165
|
+
if re.match(r"^require\s*\($", t):
|
|
166
|
+
block = True
|
|
167
|
+
continue
|
|
168
|
+
if block and t == ")":
|
|
169
|
+
block = False
|
|
170
|
+
continue
|
|
171
|
+
m = re.match(r"^([\w./~-]+)\s+(v[\w.+-]+)", t) if block else re.match(r"^require\s+([\w./~-]+)\s+(v[\w.+-]+)", t)
|
|
172
|
+
if m:
|
|
173
|
+
add("go", m.group(1), m.group(2))
|
|
174
|
+
elif name == "composer.json":
|
|
175
|
+
try:
|
|
176
|
+
j = json.loads(text)
|
|
177
|
+
except ValueError:
|
|
178
|
+
return
|
|
179
|
+
for k in ("require", "require-dev"):
|
|
180
|
+
for n, v in ((j or {}).get(k) or {}).items():
|
|
181
|
+
if n != "php" and not n.startswith("ext-"):
|
|
182
|
+
add("composer", n, str(v))
|
|
183
|
+
elif name == "Gemfile":
|
|
184
|
+
for m in re.finditer(r"^\s*gem\s+['\"]([\w.-]+)['\"](?:\s*,\s*['\"]([^'\"]+)['\"])?", text, re.M):
|
|
185
|
+
add("gem", m.group(1), m.group(2) or "")
|
|
186
|
+
elif name == "Gemfile.lock":
|
|
187
|
+
for m in re.finditer(r"^ {4}([\w.-]+) \(([^)]+)\)$", text, re.M):
|
|
188
|
+
gem_lock[m.group(1)] = m.group(2)
|
|
189
|
+
elif name == "pom.xml":
|
|
190
|
+
for m in re.finditer(r"<dependency>([\s\S]*?)</dependency>", text):
|
|
191
|
+
g, a, v = (re.search(p, m.group(1)) for p in (r"<groupId>([^<]+)<", r"<artifactId>([^<]+)<", r"<version>([^<$]+)<"))
|
|
192
|
+
if g and a:
|
|
193
|
+
add("maven", g.group(1).strip() + ":" + a.group(1).strip(), v.group(1).strip() if v else "")
|
|
194
|
+
elif name in ("build.gradle", "build.gradle.kts"):
|
|
195
|
+
for m in re.finditer(r"\b(?:implementation|api|compile|testImplementation|runtimeOnly|compileOnly|kapt)\s*\(?\s*['\"]([^'\":]+:[^'\":]+)(?::[^'\"]*)?['\"]", text):
|
|
196
|
+
add("maven", m.group(1), "")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _strip_markdown(s):
|
|
200
|
+
s = re.sub(r"```[\s\S]*?```", " ", s)
|
|
201
|
+
s = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", s)
|
|
202
|
+
s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s)
|
|
203
|
+
s = re.sub(r"<[^>]+>", " ", s)
|
|
204
|
+
s = re.sub(r"^[#>\s*+-]+", "", s, flags=re.M)
|
|
205
|
+
s = re.sub(r"[*_`~|]", "", s)
|
|
206
|
+
return re.sub(r"\s+", " ", s).strip()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _redact(s):
|
|
210
|
+
for r in SECRET_RES:
|
|
211
|
+
s = r.sub("", s)
|
|
212
|
+
return s
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _first(m):
|
|
216
|
+
for g in m.groups():
|
|
217
|
+
if g:
|
|
218
|
+
return g
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def scan(root_dir=".", max_files=MAX_FILES):
|
|
223
|
+
"""Scan root_dir and return the manifest dict. Finishes in seconds on a large repo."""
|
|
224
|
+
root = os.path.abspath(str(root_dir or "."))
|
|
225
|
+
languages, deps, routes, route_keys, flags = {}, {}, [], set(), {}
|
|
226
|
+
env_names, mod_dirs, frameworks, ci, hosting, test_fw = set(), {}, set(), set(), set(), set()
|
|
227
|
+
pricing, pricing_keys, next_cands, gem_lock = [], set(), [], {}
|
|
228
|
+
infra = {"docker": False, "compose": False, "kubernetes": False, "terraform": False, "serverless": False}
|
|
229
|
+
sec = {"lockfile": False, "env_committed": False, "secrets_suspected": 0}
|
|
230
|
+
loc = files = test_files = 0
|
|
231
|
+
description = ""
|
|
232
|
+
|
|
233
|
+
def add_dep(eco, n, v):
|
|
234
|
+
k = (eco, n)
|
|
235
|
+
if len(deps) < CAPS["dependencies"] * 2 and k not in deps:
|
|
236
|
+
deps[k] = {"ecosystem": eco, "name": n[:200], "version": (v or "")[:100]}
|
|
237
|
+
|
|
238
|
+
def add_route(method, p, file):
|
|
239
|
+
if len(routes) >= CAPS["routes"]:
|
|
240
|
+
return
|
|
241
|
+
p = str(p)
|
|
242
|
+
if not p.startswith("/"):
|
|
243
|
+
p = "/" + p
|
|
244
|
+
k = (method, p, file)
|
|
245
|
+
if k in route_keys:
|
|
246
|
+
return
|
|
247
|
+
route_keys.add(k)
|
|
248
|
+
routes.append({"method": method, "path": p[:200], "file": file[:200]})
|
|
249
|
+
|
|
250
|
+
def add_env(n):
|
|
251
|
+
if n and len(env_names) < CAPS["env_names"]:
|
|
252
|
+
env_names.add(n)
|
|
253
|
+
|
|
254
|
+
gitignore = []
|
|
255
|
+
gi = _read(os.path.join(root, ".gitignore"))
|
|
256
|
+
if gi:
|
|
257
|
+
gitignore = [_glob_re(l.strip()) for l in gi.split("\n") if l.strip() and not l.strip().startswith(("#", "!"))]
|
|
258
|
+
|
|
259
|
+
stack = [""]
|
|
260
|
+
done = False
|
|
261
|
+
while stack and not done:
|
|
262
|
+
rel = stack.pop()
|
|
263
|
+
try:
|
|
264
|
+
entries = list(os.scandir(os.path.join(root, rel) if rel else root))
|
|
265
|
+
except OSError:
|
|
266
|
+
continue
|
|
267
|
+
for ent in entries:
|
|
268
|
+
name = ent.name
|
|
269
|
+
relp = rel + "/" + name if rel else name
|
|
270
|
+
try:
|
|
271
|
+
if ent.is_symlink():
|
|
272
|
+
continue # never follow symlinks
|
|
273
|
+
if ent.is_dir(follow_symlinks=False):
|
|
274
|
+
if name in SKIP_DIRS or (name.startswith(".") and name not in KEEP_DOT_DIRS):
|
|
275
|
+
continue
|
|
276
|
+
stack.append(relp)
|
|
277
|
+
continue
|
|
278
|
+
if not ent.is_file(follow_symlinks=False):
|
|
279
|
+
continue
|
|
280
|
+
size = ent.stat(follow_symlinks=False).st_size
|
|
281
|
+
except OSError:
|
|
282
|
+
continue
|
|
283
|
+
files += 1
|
|
284
|
+
if files > max_files:
|
|
285
|
+
files = max_files
|
|
286
|
+
done = True
|
|
287
|
+
break
|
|
288
|
+
full = os.path.join(root, relp)
|
|
289
|
+
if name in LOCKFILES:
|
|
290
|
+
sec["lockfile"] = True
|
|
291
|
+
if re.match(r"^\.env(\..+)?$", name) and not re.search(r"\.(example|sample|template|dist)$", name):
|
|
292
|
+
if not any(g.match(name) for g in gitignore):
|
|
293
|
+
sec["env_committed"] = True
|
|
294
|
+
continue # never read real env files
|
|
295
|
+
ext = name.rsplit(".", 1)[1].lower() if "." in name[1:] else ""
|
|
296
|
+
is_code = ext in CODE_EXT
|
|
297
|
+
is_cfg = ext in CONFIG_EXT or bool(re.match(r"^(Dockerfile|Procfile|Gemfile|Pipfile|Jenkinsfile)", name)) or name.startswith(".env.")
|
|
298
|
+
if name.startswith("Dockerfile"):
|
|
299
|
+
infra["docker"] = True
|
|
300
|
+
if re.match(r"^(docker-compose.*|compose)\.ya?ml$", name):
|
|
301
|
+
infra["compose"] = True
|
|
302
|
+
if ext == "tf":
|
|
303
|
+
infra["terraform"] = True
|
|
304
|
+
if re.match(r"^serverless\.(ya?ml|ts|js)$", name):
|
|
305
|
+
infra["serverless"] = True
|
|
306
|
+
if relp.startswith(".github/workflows/") and re.search(r"\.ya?ml$", name):
|
|
307
|
+
ci.add("github-actions")
|
|
308
|
+
for fname, label in ((".gitlab-ci.yml", "gitlab"), ("Jenkinsfile", "jenkins"), ("azure-pipelines.yml", "azure-pipelines"),
|
|
309
|
+
("bitbucket-pipelines.yml", "bitbucket")):
|
|
310
|
+
if name == fname:
|
|
311
|
+
ci.add(label)
|
|
312
|
+
if relp == ".circleci/config.yml":
|
|
313
|
+
ci.add("circleci")
|
|
314
|
+
if name in HOSTS:
|
|
315
|
+
hosting.add(HOSTS[name])
|
|
316
|
+
if not (is_code or is_cfg) or size > MAX_BYTES or name in LOCKFILES:
|
|
317
|
+
continue
|
|
318
|
+
if is_code:
|
|
319
|
+
languages[ext] = languages.get(ext, 0) + 1
|
|
320
|
+
text = _read(full)
|
|
321
|
+
if text is None:
|
|
322
|
+
continue
|
|
323
|
+
if is_code:
|
|
324
|
+
loc += text.count("\n") + (1 if text and not text.endswith("\n") else 0)
|
|
325
|
+
parts = relp.split("/")
|
|
326
|
+
if parts[0] in ("src", "apps", "packages", "modules", "features", "app", "lib", "services") and len(parts) > 2:
|
|
327
|
+
top = parts[0] + "/" + parts[1]
|
|
328
|
+
elif len(parts) > 1:
|
|
329
|
+
top = parts[0]
|
|
330
|
+
else:
|
|
331
|
+
top = None
|
|
332
|
+
if top and not top.startswith("."):
|
|
333
|
+
mod_dirs[top] = mod_dirs.get(top, 0) + 1
|
|
334
|
+
for r in SECRET_RES:
|
|
335
|
+
sec["secrets_suspected"] += len(r.findall(text)) # count only: matches are never stored
|
|
336
|
+
if RE_DEP_FILE.match(name):
|
|
337
|
+
_parse_deps(name, text, add_dep, gem_lock)
|
|
338
|
+
if re.match(r"^(\.env\.(example|sample|template|dist)|example\.env|env\.example)$", name):
|
|
339
|
+
for m in re.finditer(r"^\s*(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=", text, re.M):
|
|
340
|
+
add_env(m.group(1))
|
|
341
|
+
if ext in ("yml", "yaml") and re.search(r"^kind:\s*(Deployment|Service|Ingress|StatefulSet|DaemonSet|CronJob|Job|ConfigMap)\b", text, re.M) \
|
|
342
|
+
and re.search(r"^apiVersion:", text, re.M):
|
|
343
|
+
infra["kubernetes"] = True
|
|
344
|
+
if not is_code:
|
|
345
|
+
continue
|
|
346
|
+
is_test = bool(RE_TEST_FILE.search(name)) or (bool(RE_TEST_DIR.search(relp)) and ext in ("js", "jsx", "ts", "tsx", "py", "rb", "php", "go"))
|
|
347
|
+
if is_test:
|
|
348
|
+
test_files += 1
|
|
349
|
+
if ext == "go":
|
|
350
|
+
test_fw.add("go-test")
|
|
351
|
+
if re.search(r"^(import|from)\s+unittest", text, re.M):
|
|
352
|
+
test_fw.add("unittest")
|
|
353
|
+
if re.search(r"^\s*(import|from)\s+pytest|\bdef test_", text, re.M):
|
|
354
|
+
test_fw.add("pytest")
|
|
355
|
+
if re.search(r"\bfrom\s+['\"]vitest['\"]", text):
|
|
356
|
+
test_fw.add("vitest")
|
|
357
|
+
if "node:test" in text:
|
|
358
|
+
test_fw.add("node:test")
|
|
359
|
+
elif re.search(r"\bfrom\s+['\"]mocha['\"]", text):
|
|
360
|
+
test_fw.add("mocha")
|
|
361
|
+
continue
|
|
362
|
+
if ext in JS_EXT:
|
|
363
|
+
if re.search(r"(^|/)(app|pages)/", relp):
|
|
364
|
+
next_cands.append(relp)
|
|
365
|
+
for m in RE_EXPRESS.finditer(text):
|
|
366
|
+
add_route(m.group(1).upper(), m.group(3), relp)
|
|
367
|
+
for m in RE_ENV_JS.finditer(text):
|
|
368
|
+
add_env(_first(m))
|
|
369
|
+
elif ext == "py":
|
|
370
|
+
for m in RE_PYROUTE.finditer(text):
|
|
371
|
+
methods = [m.group(1).upper()]
|
|
372
|
+
if m.group(1) == "route":
|
|
373
|
+
mm = re.search(r"methods\s*=\s*[\[(]([^\])]*)[\])]", m.group(4) or "")
|
|
374
|
+
methods = [x.upper() for x in re.findall(r"['\"](\w+)['\"]", mm.group(1))] if mm else ["GET"]
|
|
375
|
+
if m.group(3).startswith("/"):
|
|
376
|
+
for me in methods:
|
|
377
|
+
add_route(me, m.group(3), relp)
|
|
378
|
+
if name == "urls.py":
|
|
379
|
+
for m in RE_DJANGO.finditer(text):
|
|
380
|
+
p = re.sub(r"^\^", "", m.group(3))
|
|
381
|
+
p = re.sub(r"\$$", "", p)
|
|
382
|
+
p = re.sub(r"\(\?P<(\w+)>[^)]*\)", r":\1", p)
|
|
383
|
+
p = re.sub(r"<(?:\w+:)?(\w+)>", r":\1", p)
|
|
384
|
+
add_route("ANY", p or "/", relp)
|
|
385
|
+
for m in RE_ENV_PY.finditer(text):
|
|
386
|
+
add_env(_first(m))
|
|
387
|
+
elif ext == "php":
|
|
388
|
+
for m in RE_LARAVEL.finditer(text):
|
|
389
|
+
add_route("ANY" if m.group(1) == "any" else m.group(1).upper(), re.sub(r"\{(\w+)\??\}", r":\1", m.group(3)), relp)
|
|
390
|
+
for m in RE_ENV_PY.finditer(text):
|
|
391
|
+
add_env(_first(m))
|
|
392
|
+
elif ext == "rb":
|
|
393
|
+
if name == "routes.rb":
|
|
394
|
+
for m in RE_RAILS_VERB.finditer(text):
|
|
395
|
+
add_route(m.group(1).upper(), m.group(3), relp)
|
|
396
|
+
for m in RE_RAILS_RES.finditer(text):
|
|
397
|
+
r = "/" + m.group(2)
|
|
398
|
+
if m.group(1) == "resources":
|
|
399
|
+
add_route("GET", r, relp)
|
|
400
|
+
add_route("POST", r, relp)
|
|
401
|
+
for v in ("GET", "PUT", "PATCH", "DELETE"):
|
|
402
|
+
add_route(v, r + "/:id", relp)
|
|
403
|
+
else:
|
|
404
|
+
for v in ("GET", "POST", "PUT", "PATCH", "DELETE"):
|
|
405
|
+
add_route(v, r, relp)
|
|
406
|
+
for m in RE_ENV_PY.finditer(text):
|
|
407
|
+
add_env(_first(m))
|
|
408
|
+
elif ext == "go":
|
|
409
|
+
for m in RE_GO_STD.finditer(text):
|
|
410
|
+
mm = re.match(r"^([A-Z]+)\s+(/.*)$", m.group(1))
|
|
411
|
+
if mm:
|
|
412
|
+
add_route(mm.group(1), mm.group(2), relp)
|
|
413
|
+
elif m.group(1).startswith("/"):
|
|
414
|
+
add_route("ANY", m.group(1), relp)
|
|
415
|
+
for m in RE_GO_ROUTER.finditer(text):
|
|
416
|
+
add_route(m.group(1).upper(), re.sub(r"\{(\w+)\}", r":\1", m.group(2)), relp)
|
|
417
|
+
for m in RE_ENV_PY.finditer(text):
|
|
418
|
+
add_env(_first(m))
|
|
419
|
+
for m in RE_FLAG.finditer(text):
|
|
420
|
+
if len(flags) < CAPS["features"]:
|
|
421
|
+
flags.setdefault(m.group(2), set()).add(relp)
|
|
422
|
+
for m in RE_FLAG_PROP.finditer(text):
|
|
423
|
+
if len(flags) < CAPS["features"]:
|
|
424
|
+
flags.setdefault(m.group(1), set()).add(relp)
|
|
425
|
+
if len(pricing) < CAPS["pricing_hints"] and not re.search(r"(^|/)(docs?|examples?)/", relp) and re.search(r"price|amount|monthly|cost", text, re.I):
|
|
426
|
+
for rx in (RE_PLAN_KEY, RE_PLAN_CONST, RE_PLAN_PRICE):
|
|
427
|
+
for m in rx.finditer(text):
|
|
428
|
+
if len(pricing) >= CAPS["pricing_hints"]:
|
|
429
|
+
break
|
|
430
|
+
plan, amount = m.group(1).strip().lower()[:40], float(m.group(2))
|
|
431
|
+
k = (plan, amount)
|
|
432
|
+
if not amount > 0 or k in pricing_keys:
|
|
433
|
+
continue
|
|
434
|
+
pricing_keys.add(k)
|
|
435
|
+
cur = RE_CURRENCY.search(text[max(0, m.start() - 200):m.end() + 200])
|
|
436
|
+
pricing.append({"plan": plan, "amount": int(amount) if amount == int(amount) else amount, "currency": cur.group(1).upper() if cur else ""})
|
|
437
|
+
|
|
438
|
+
for d in deps.values():
|
|
439
|
+
if d["ecosystem"] == "gem" and d["name"] in gem_lock:
|
|
440
|
+
d["version"] = gem_lock[d["name"]]
|
|
441
|
+
if gem_lock and not any(d["ecosystem"] == "gem" for d in deps.values()):
|
|
442
|
+
for n, v in gem_lock.items():
|
|
443
|
+
add_dep("gem", n, v)
|
|
444
|
+
dep_list = list(deps.values())[:CAPS["dependencies"]]
|
|
445
|
+
dep_names = [d["name"].lower() for d in dep_list]
|
|
446
|
+
|
|
447
|
+
def has_dep(x):
|
|
448
|
+
return any(n == x or n.startswith(x + "/") or n.startswith(x + ":") for n in dep_names)
|
|
449
|
+
|
|
450
|
+
for dep, label in FRAMEWORK_DEPS.items():
|
|
451
|
+
if has_dep(dep):
|
|
452
|
+
frameworks.add(label)
|
|
453
|
+
for n, v in (("jest", "jest"), ("vitest", "vitest"), ("mocha", "mocha"), ("pytest", "pytest"), ("phpunit/phpunit", "phpunit"),
|
|
454
|
+
("rspec", "rspec"), ("@playwright/test", "playwright"), ("cypress", "cypress")):
|
|
455
|
+
if has_dep(n) or has_dep(n + "-core"):
|
|
456
|
+
test_fw.add(v)
|
|
457
|
+
if "next" in frameworks:
|
|
458
|
+
for rel in next_cands:
|
|
459
|
+
r = next_route(rel)
|
|
460
|
+
if not r:
|
|
461
|
+
continue
|
|
462
|
+
route, kind = r
|
|
463
|
+
if kind == "route":
|
|
464
|
+
text = _read(os.path.join(root, rel)) or ""
|
|
465
|
+
ms = re.findall(r"export\s+(?:async\s+)?(?:function|const)\s+(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\b", text)
|
|
466
|
+
for m in (ms or ["GET"]):
|
|
467
|
+
add_route(m, route, rel)
|
|
468
|
+
else:
|
|
469
|
+
add_route("GET", route, rel)
|
|
470
|
+
env_list = sorted(env_names)
|
|
471
|
+
integrations = sorted(n for n, (dp, ev) in INTEGRATIONS.items()
|
|
472
|
+
if any(s in d for s in dp for d in dep_names) or any(p in e for p in ev for e in env_list))
|
|
473
|
+
features, groups = [], {}
|
|
474
|
+
for r in routes:
|
|
475
|
+
segs = [s for s in r["path"].split("/") if s and not re.match(r"^(api|v\d+|rest|public)$", s, re.I) and not s.startswith((":", "["))]
|
|
476
|
+
if not segs:
|
|
477
|
+
continue
|
|
478
|
+
g = re.sub(r"[^a-z0-9_-]", "", segs[0].lower())
|
|
479
|
+
if g:
|
|
480
|
+
arr = groups.setdefault(g, [])
|
|
481
|
+
if len(arr) < 5:
|
|
482
|
+
arr.append(r["path"])
|
|
483
|
+
for name, paths in groups.items():
|
|
484
|
+
if len(features) < CAPS["features"]:
|
|
485
|
+
features.append({"name": name[:80], "source": "route", "paths": paths})
|
|
486
|
+
for name, st in flags.items():
|
|
487
|
+
if len(features) < CAPS["features"]:
|
|
488
|
+
features.append({"name": name[:80], "source": "flag", "paths": sorted(st)[:5]})
|
|
489
|
+
for d, _n in sorted(mod_dirs.items(), key=lambda kv: -kv[1]):
|
|
490
|
+
if len(features) < CAPS["features"]:
|
|
491
|
+
features.append({"name": d.split("/")[-1][:80], "source": "dir", "paths": [d]})
|
|
492
|
+
for n in ("README.md", "readme.md", "Readme.md", "README", "README.rst", "README.txt"):
|
|
493
|
+
try:
|
|
494
|
+
with open(os.path.join(root, n), "rb") as f:
|
|
495
|
+
description = _redact(_strip_markdown(f.read(2048).decode("utf-8", "ignore")))
|
|
496
|
+
break
|
|
497
|
+
except OSError:
|
|
498
|
+
continue
|
|
499
|
+
infra_out = dict(infra)
|
|
500
|
+
infra_out["ci"] = sorted(ci)
|
|
501
|
+
infra_out["hosting_hints"] = sorted(hosting)
|
|
502
|
+
return {
|
|
503
|
+
"schema": 1, "scanned_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "root_name": os.path.basename(root),
|
|
504
|
+
"languages": languages, "loc": loc, "frameworks": sorted(frameworks), "dependencies": dep_list, "routes": routes[:CAPS["routes"]],
|
|
505
|
+
"features": features, "integrations": integrations, "infra": infra_out, "env_names": env_list[:CAPS["env_names"]],
|
|
506
|
+
"pricing_hints": pricing[:CAPS["pricing_hints"]], "description": description,
|
|
507
|
+
"tests": {"files": test_files, "frameworks": sorted(test_fw)}, "security": sec}
|
linearjeph/py.typed
ADDED
|
File without changes
|