linearjeph 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,151 @@
1
+ """Request tracking middlewares: WSGI, ASGI (FastAPI/Starlette), Flask. Records route template, method, status and
2
+ duration only. Never bodies, headers or query strings. Frameworks are imported lazily (the SDK has no hard dependencies)."""
3
+ import re
4
+ import time
5
+
6
+ _ID_SEG = re.compile(r"^(\d+|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|[0-9a-fA-F]{16,}|(?=.*\d)[A-Za-z0-9_-]{20,})$")
7
+ _PARAM = re.compile(r"<(?:[^>:]+:)?([^>]+)>|\{([^}:]+)(?::[^}]*)?\}")
8
+
9
+
10
+ def normalize_path(path):
11
+ """'/users/42/orders/9f8...' -> '/users/:id/orders/:id' (no query string, ever)."""
12
+ clean = str(path or "/").split("?")[0].split("#")[0]
13
+ out = "/".join(":id" if s and _ID_SEG.match(s) else s for s in clean.split("/"))
14
+ return out or "/"
15
+
16
+
17
+ def route_template(rule):
18
+ """Framework route pattern -> '/users/:id' ('<int:id>' and '{id}' forms; regex groups reduced)."""
19
+ s = str(rule or "")
20
+ s = re.sub(r"\(\?P<(\w+)>[^)]*\)", r":\1", s)
21
+ s = _PARAM.sub(lambda m: ":" + (m.group(1) or m.group(2)), s)
22
+ s = re.sub(r"^\^|\$$", "", s)
23
+ if not s.startswith("/"):
24
+ s = "/" + s
25
+ return s
26
+
27
+
28
+ def _record(route, method, status, ms, uid=None):
29
+ from . import record_request
30
+ record_request(route, method, status, ms, uid)
31
+
32
+
33
+ class WSGIMiddleware(object):
34
+ """WSGI: app.wsgi_app = WSGIMiddleware(app.wsgi_app). Apps can set environ['prophecy.route'] to give an exact template."""
35
+
36
+ def __init__(self, app, ignore_paths=(), uid_getter=None):
37
+ self.app = app
38
+ self.ignore = tuple(ignore_paths or ())
39
+ self.uid_getter = uid_getter
40
+
41
+ def __call__(self, environ, start_response):
42
+ t0 = time.perf_counter()
43
+ status = {"code": 500}
44
+
45
+ def _start(s, headers, exc_info=None):
46
+ try:
47
+ status["code"] = int(str(s).split(" ", 1)[0])
48
+ except ValueError:
49
+ pass
50
+ return start_response(s, headers, exc_info) if exc_info else start_response(s, headers)
51
+
52
+ try:
53
+ result = self.app(environ, _start)
54
+ except Exception:
55
+ self._done(environ, status["code"], t0)
56
+ raise
57
+ return _Closing(result, lambda: self._done(environ, status["code"], t0))
58
+
59
+ def _done(self, environ, code, t0):
60
+ try:
61
+ path = environ.get("PATH_INFO", "/")
62
+ if self.ignore and path.startswith(self.ignore):
63
+ return
64
+ route = environ.get("prophecy.route") or normalize_path(environ.get("SCRIPT_NAME", "") + path)
65
+ uid = None
66
+ if self.uid_getter:
67
+ try:
68
+ uid = self.uid_getter(environ)
69
+ except Exception: # noqa: BLE001
70
+ pass
71
+ _record(route, environ.get("REQUEST_METHOD", "GET"), code, (time.perf_counter() - t0) * 1000, uid)
72
+ except Exception: # noqa: BLE001 - never break a response
73
+ pass
74
+
75
+
76
+ class _Closing(object):
77
+ def __init__(self, result, done):
78
+ self._result, self._done, self._fired = result, done, False
79
+
80
+ def __iter__(self):
81
+ return iter(self._result)
82
+
83
+ def close(self):
84
+ try:
85
+ if hasattr(self._result, "close"):
86
+ self._result.close()
87
+ finally:
88
+ if not self._fired:
89
+ self._fired = True
90
+ self._done()
91
+
92
+
93
+ class ASGIMiddleware(object):
94
+ """ASGI (FastAPI, Starlette, Quart...): app.add_middleware(ASGIMiddleware). The route comes from scope['route']
95
+ (path_format) when the framework sets it, else the normalised path."""
96
+
97
+ def __init__(self, app, ignore_paths=()):
98
+ self.app = app
99
+ self.ignore = tuple(ignore_paths or ())
100
+
101
+ async def __call__(self, scope, receive, send):
102
+ if scope.get("type") != "http":
103
+ return await self.app(scope, receive, send)
104
+ t0 = time.perf_counter()
105
+ status = {"code": 500}
106
+
107
+ async def _send(msg):
108
+ if msg.get("type") == "http.response.start":
109
+ status["code"] = msg.get("status", 500)
110
+ await send(msg)
111
+
112
+ try:
113
+ await self.app(scope, receive, _send)
114
+ finally:
115
+ try:
116
+ path = scope.get("path", "/")
117
+ if not (self.ignore and path.startswith(self.ignore)):
118
+ route = scope.get("route")
119
+ tpl = getattr(route, "path_format", None) or getattr(route, "path", None)
120
+ _record(route_template(tpl) if tpl else normalize_path(path), scope.get("method", "GET"), status["code"],
121
+ (time.perf_counter() - t0) * 1000)
122
+ except Exception: # noqa: BLE001
123
+ pass
124
+
125
+
126
+ ProphecyASGIMiddleware = ASGIMiddleware
127
+ ProphecyWSGIMiddleware = WSGIMiddleware
128
+
129
+
130
+ def init_flask(app, ignore_paths=(), uid_getter=None):
131
+ """Flask: prophecy.init_flask(app) after prophecy.init(...). Uses request.url_rule for the route template."""
132
+ from flask import g, request # imported lazily: Flask is the customer's dependency
133
+ ignore = tuple(ignore_paths or ())
134
+
135
+ @app.before_request
136
+ def _prophecy_start():
137
+ g._prophecy_t0 = time.perf_counter()
138
+
139
+ @app.after_request
140
+ def _prophecy_end(resp):
141
+ try:
142
+ t0 = getattr(g, "_prophecy_t0", None)
143
+ if t0 is not None and not (ignore and request.path.startswith(ignore)):
144
+ rule = request.url_rule.rule if request.url_rule else None
145
+ uid = uid_getter(request) if uid_getter else None
146
+ _record(route_template(rule) if rule else normalize_path(request.path), request.method, resp.status_code,
147
+ (time.perf_counter() - t0) * 1000, uid)
148
+ except Exception: # noqa: BLE001
149
+ pass
150
+ return resp
151
+ return app
@@ -0,0 +1,507 @@
1
+ """Prophecy codebase scanner. Reads files locally and returns a manifest (schema 1): names, counts and route templates
2
+ only. Never source text, never env values, never matched secrets (secrets are only counted).
3
+
4
+ Limits: skips node_modules/.git/vendor/venv/dist/build..., files > 1 MB, symlinks, at most 20,000 files.
5
+ Standard library only; same output schema as the Node SDK scanner.
6
+ """
7
+ import json
8
+ import os
9
+ import re
10
+ from datetime import datetime, timezone
11
+
12
+ MAX_FILES = 20000
13
+ MAX_BYTES = 1024 * 1024
14
+ CAPS = {"dependencies": 2000, "routes": 3000, "features": 200, "env_names": 500, "pricing_hints": 20}
15
+ SKIP_DIRS = {"node_modules", ".git", "vendor", "venv", ".venv", "env", "dist", "build", "__pycache__", ".next", ".nuxt",
16
+ ".svelte-kit", "target", "coverage", ".tox", ".cache", "site-packages", ".idea", ".vscode", ".gradle", "Pods",
17
+ "bower_components", ".terraform", ".mypy_cache", ".pytest_cache", ".turbo", "out"}
18
+ KEEP_DOT_DIRS = {".github", ".circleci"}
19
+ CODE_EXT = {"js", "jsx", "ts", "tsx", "mjs", "cjs", "py", "go", "php", "rb", "java", "kt", "rs", "cs", "swift", "dart", "vue",
20
+ "svelte", "scala", "ex", "exs", "c", "cpp", "h", "sh", "sql", "html", "css", "scss"}
21
+ CONFIG_EXT = {"json", "yml", "yaml", "toml", "ini", "cfg", "xml", "tf", "properties", "gradle", "kts", "mod", "env", "md", "txt"}
22
+ LOCKFILES = {"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb", "poetry.lock", "Pipfile.lock", "uv.lock", "go.sum",
23
+ "composer.lock", "Gemfile.lock"}
24
+ JS_EXT = ("js", "jsx", "ts", "tsx", "mjs", "cjs")
25
+ SECRET_RES = [re.compile(p) for p in (
26
+ r"\bAKIA[0-9A-Z]{16}\b", r"\bsk_live_[0-9A-Za-z]{16,}", r"\bghp_[A-Za-z0-9]{30,}", r"\bgithub_pat_[A-Za-z0-9_]{40,}",
27
+ r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----", r"\bxox[baprs]-[A-Za-z0-9-]{10,}",
28
+ r"\bAIza[0-9A-Za-z_-]{35}", r"\bljp_sk_[A-Za-z0-9_-]{20,}")]
29
+ FRAMEWORK_DEPS = {
30
+ "express": "express", "fastify": "fastify", "koa": "koa", "next": "next", "@nestjs/core": "nestjs", "react": "react", "vue": "vue",
31
+ "nuxt": "nuxt", "svelte": "svelte", "@sveltejs/kit": "sveltekit", "@angular/core": "angular", "@hapi/hapi": "hapi",
32
+ "@remix-run/node": "remix", "astro": "astro", "django": "django", "flask": "flask", "fastapi": "fastapi", "starlette": "starlette",
33
+ "tornado": "tornado", "aiohttp": "aiohttp", "celery": "celery", "laravel/framework": "laravel",
34
+ "symfony/framework-bundle": "symfony", "rails": "rails", "sinatra": "sinatra", "github.com/gin-gonic/gin": "gin",
35
+ "github.com/go-chi/chi": "chi", "github.com/labstack/echo": "echo", "github.com/gofiber/fiber": "fiber",
36
+ "org.springframework.boot": "spring-boot", "flutter": "flutter"}
37
+ INTEGRATIONS = {
38
+ "stripe": (["stripe"], ["STRIPE"]), "paddle": (["paddle"], ["PADDLE"]), "paystack": (["paystack"], ["PAYSTACK"]),
39
+ "flutterwave": (["flutterwave"], ["FLUTTERWAVE", "FLW_"]), "bachs": (["bachs"], ["BACHS"]), "auth0": (["auth0"], ["AUTH0"]),
40
+ "clerk": (["@clerk", "clerk"], ["CLERK"]), "supabase": (["supabase"], ["SUPABASE"]), "firebase": (["firebase"], ["FIREBASE"]),
41
+ "sentry": (["sentry"], ["SENTRY"]), "segment": (["@segment", "analytics-node", "segment"], ["SEGMENT"]),
42
+ "mixpanel": (["mixpanel"], ["MIXPANEL"]), "posthog": (["posthog"], ["POSTHOG"]), "amplitude": (["amplitude"], ["AMPLITUDE"]),
43
+ "launchdarkly": (["launchdarkly"], ["LAUNCHDARKLY", "LD_SDK"]), "unleash": (["unleash"], ["UNLEASH"]),
44
+ "twilio": (["twilio"], ["TWILIO"]), "sendgrid": (["sendgrid"], ["SENDGRID"]), "resend": (["resend"], ["RESEND"]),
45
+ "mailgun": (["mailgun"], ["MAILGUN"]), "postmark": (["postmark"], ["POSTMARK"]), "openai": (["openai"], ["OPENAI"]),
46
+ "anthropic": (["anthropic"], ["ANTHROPIC"]),
47
+ "s3": (["@aws-sdk/client-s3", "aws-sdk", "boto3", "aws/aws-sdk-php", "aws-sdk-s3"], ["AWS_S3", "S3_", "AWS_ACCESS"]),
48
+ "cloudinary": (["cloudinary"], ["CLOUDINARY"]), "algolia": (["algolia"], ["ALGOLIA"]),
49
+ "redis": (["redis", "ioredis"], ["REDIS"]), "rabbitmq": (["amqplib", "pika"], ["RABBITMQ", "AMQP"]),
50
+ "kafka": (["kafkajs", "kafka-python", "confluent-kafka"], ["KAFKA"])}
51
+ HOSTS = {"vercel.json": "vercel", "netlify.toml": "netlify", "fly.toml": "fly", "render.yaml": "render", "Procfile": "heroku",
52
+ "app.yaml": "gcp-app-engine", "railway.json": "railway", "railway.toml": "railway", "wrangler.toml": "cloudflare"}
53
+ _M = "get|post|put|patch|delete|options|head|all"
54
+ _PLANS = "free|starter|basic|pro|premium|business|team|enterprise|growth|plus|standard|scale"
55
+ RE_EXPRESS = re.compile(r"\b[A-Za-z_$][\w$]*\.(" + _M + r")\(\s*([\"'`])(/[^\"'`\s]*)\2")
56
+ RE_PYROUTE = re.compile(r"@\w+(?:\.\w+)*\.(route|get|post|put|patch|delete)\(\s*(['\"])([^'\"]+)\2([^)]*)\)")
57
+ RE_DJANGO = re.compile(r"\b(re_path|path|url)\(\s*r?(['\"])([^'\"]*)\2")
58
+ RE_LARAVEL = re.compile(r"Route::(get|post|put|patch|delete|any|options)\(\s*(['\"])([^'\"]+)\2")
59
+ RE_RAILS_VERB = re.compile(r"^\s*(get|post|put|patch|delete)\s+(['\"])([^'\"]+)\2", re.M)
60
+ RE_RAILS_RES = re.compile(r"^\s*(resources|resource)\s+:(\w+)", re.M)
61
+ RE_GO_STD = re.compile(r"\.(?:HandleFunc|Handle)\(\s*\"([^\"]+)\"")
62
+ RE_GO_ROUTER = re.compile(r"\b\w+\.(Get|Post|Put|Patch|Delete|GET|POST|PUT|PATCH|DELETE)\(\s*\"(/[^\"]*)\"")
63
+ RE_FLAG = re.compile(r"\b(?:isEnabled|isFeatureEnabled|isFeatureFlagEnabled|getFeatureFlag|variation|boolVariation|useFlag|useFeatureFlag|"
64
+ r"flag_is_active|feature_enabled|feature_flag|is_enabled)\(\s*(?:[\w.]+\s*,\s*)?(['\"])([\w.:-]{1,80})\1")
65
+ RE_FLAG_PROP = re.compile(r"\bflags\.([A-Za-z_]\w{1,60})\b")
66
+ RE_ENV_JS = re.compile(r"process\.env\.([A-Z][A-Z0-9_]+)|process\.env\[\s*['\"]([A-Z][A-Z0-9_]+)['\"]\s*\]|import\.meta\.env\.([A-Z][A-Z0-9_]+)")
67
+ RE_ENV_PY = re.compile(r"os\.environ(?:\.get)?[\[(]\s*['\"]([A-Z][A-Z0-9_]+)['\"]|os\.getenv\(\s*['\"]([A-Z][A-Z0-9_]+)['\"]|"
68
+ r"\benv\(\s*['\"]([A-Z][A-Z0-9_]+)['\"]|\bENV\[\s*['\"]([A-Z][A-Z0-9_]+)['\"]|"
69
+ r"os\.Getenv\(\s*\"([A-Z][A-Z0-9_]+)\"|getenv\(\s*['\"]([A-Z][A-Z0-9_]+)['\"]")
70
+ RE_PLAN_PRICE = re.compile(r"\b(?:plan|tier|name|id|slug)['\"]?\s*[:=]\s*['\"]([A-Za-z][\w -]{1,30})['\"][\s\S]{0,120}?"
71
+ r"\b(?:price|amount|monthly\w*|cost)['\"]?\s*[:=]\s*(\d+(?:\.\d{1,2})?)", re.I)
72
+ RE_PLAN_KEY = re.compile(r"\b(" + _PLANS + r")['\"]?\s*[:=]\s*\{[^}]{0,120}?\b(?:price|amount|monthly\w*|cost)['\"]?\s*[:=]\s*(\d+(?:\.\d{1,2})?)", re.I)
73
+ RE_PLAN_CONST = re.compile(r"\b(" + _PLANS + r")_?(?:monthly_)?price\w*\s*[:=]\s*(\d+(?:\.\d{1,2})?)", re.I)
74
+ RE_CURRENCY = re.compile(r"\bcurrency['\"]?\s*[:=]\s*['\"]([A-Za-z]{3})['\"]", re.I)
75
+ RE_TEST_FILE = re.compile(r"\.(test|spec)\.[cm]?[jt]sx?$|^test_.*\.py$|_test\.(py|go|rb)$|Test\.php$|_spec\.rb$")
76
+ RE_TEST_DIR = re.compile(r"(^|/)(tests?|__tests__|spec)/")
77
+ RE_DEP_FILE = re.compile(r"^(package\.json|requirements.*\.txt|pyproject\.toml|Pipfile|go\.mod|composer\.json|Gemfile(\.lock)?|pom\.xml|build\.gradle(\.kts)?)$")
78
+
79
+
80
+ def _glob_re(g):
81
+ s = g.lstrip("/").rstrip("/")
82
+ s = re.sub(r"[.+^${}()|\\]", lambda m: "\\" + m.group(0), s).replace("*", ".*").replace("?", ".")
83
+ return re.compile("^" + s + "$")
84
+
85
+
86
+ def _read(path):
87
+ try:
88
+ with open(path, "r", encoding="latin-1") as f:
89
+ return f.read()
90
+ except OSError:
91
+ return None
92
+
93
+
94
+ def _seg(s):
95
+ if s.startswith("[") and s.endswith("]"):
96
+ return ":" + (re.sub(r"[\[\].]", "", s) or "id")
97
+ return s
98
+
99
+
100
+ def next_route(rel):
101
+ """Next.js file path (relative, posix) -> (route, kind) or None."""
102
+ parts = rel.split("/")
103
+ fname = parts.pop()
104
+ m = re.match(r"^(route|page)\.(?:tsx|ts|jsx|js|mjs)$", fname)
105
+ if m and "app" in parts:
106
+ i = len(parts) - 1 - parts[::-1].index("app")
107
+ segs = [_seg(x) for x in parts[i + 1:] if not (x.startswith("(") and x.endswith(")")) and not x.startswith("@")]
108
+ return "/" + "/".join(segs), m.group(1)
109
+ pm = re.match(r"^(.+)\.(?:tsx|ts|jsx|js)$", fname)
110
+ if pm and "pages" in parts and not pm.group(1).startswith("_"):
111
+ i = len(parts) - 1 - parts[::-1].index("pages")
112
+ segs = [_seg(x) for x in parts[i + 1:]]
113
+ if pm.group(1) != "index":
114
+ segs.append(_seg(pm.group(1)))
115
+ return "/" + "/".join(segs), ("api" if segs and segs[0] == "api" else "page")
116
+ return None
117
+
118
+
119
+ def _parse_deps(name, text, add, gem_lock):
120
+ if name == "package.json":
121
+ try:
122
+ j = json.loads(text)
123
+ except ValueError:
124
+ return
125
+ for k in ("dependencies", "devDependencies", "peerDependencies"):
126
+ for n, v in ((j or {}).get(k) or {}).items():
127
+ add("npm", n, v if isinstance(v, str) else "")
128
+ elif re.match(r"^requirements.*\.txt$", name):
129
+ for line in text.split("\n"):
130
+ t = line.strip()
131
+ if not t or t.startswith("-") or t.startswith("#"):
132
+ continue
133
+ m = re.match(r"^([A-Za-z0-9][A-Za-z0-9_.-]*)(?:\[[^\]]*\])?\s*(?:(==|>=|~=|<=|!=|<|>)\s*([^\s;#,]+))?", t)
134
+ if m:
135
+ add("pypi", m.group(1), (m.group(3) if m.group(2) == "==" else (m.group(2) + m.group(3))) if m.group(3) else "")
136
+ elif name == "pyproject.toml":
137
+ arr = re.search(r"^\s*dependencies\s*=\s*\[([\s\S]*?)\]", text, re.M)
138
+ if arr:
139
+ for m in re.finditer(r"['\"]([A-Za-z0-9][A-Za-z0-9_.-]*)(?:\[[^\]]*\])?\s*([^'\"]*)['\"]", arr.group(1)):
140
+ add("pypi", m.group(1), re.sub(r"^==", "", m.group(2).strip()))
141
+ in_sec = False
142
+ for line in text.split("\n"):
143
+ h = re.match(r"^\s*\[([^\]]+)\]", line)
144
+ if h:
145
+ in_sec = bool(re.search(r"poetry\.(dev-)?dependencies$", h.group(1)) or "dependency-groups" in h.group(1))
146
+ continue
147
+ m = in_sec and re.match(r"^\s*([A-Za-z0-9][A-Za-z0-9_.-]*)\s*=\s*(?:['\"]([^'\"]*)['\"]|\{[^}]*version\s*=\s*['\"]([^'\"]*)['\"])", line)
148
+ if m and m.group(1).lower() != "python":
149
+ add("pypi", m.group(1), m.group(2) or m.group(3) or "")
150
+ elif name == "Pipfile":
151
+ in_sec = False
152
+ for line in text.split("\n"):
153
+ h = re.match(r"^\s*\[([^\]]+)\]", line)
154
+ if h:
155
+ in_sec = bool(re.match(r"^(dev-)?packages$", h.group(1)))
156
+ continue
157
+ m = in_sec and re.match(r"^\s*([A-Za-z0-9][A-Za-z0-9_.-]*)\s*=\s*(?:['\"]([^'\"]*)['\"]|\{[^}]*version\s*=\s*['\"]([^'\"]*)['\"])", line)
158
+ if m:
159
+ v = m.group(2) or m.group(3) or ""
160
+ add("pypi", m.group(1), "" if v == "*" else v)
161
+ elif name == "go.mod":
162
+ block = False
163
+ for line in text.split("\n"):
164
+ t = line.strip()
165
+ if re.match(r"^require\s*\($", t):
166
+ block = True
167
+ continue
168
+ if block and t == ")":
169
+ block = False
170
+ continue
171
+ m = re.match(r"^([\w./~-]+)\s+(v[\w.+-]+)", t) if block else re.match(r"^require\s+([\w./~-]+)\s+(v[\w.+-]+)", t)
172
+ if m:
173
+ add("go", m.group(1), m.group(2))
174
+ elif name == "composer.json":
175
+ try:
176
+ j = json.loads(text)
177
+ except ValueError:
178
+ return
179
+ for k in ("require", "require-dev"):
180
+ for n, v in ((j or {}).get(k) or {}).items():
181
+ if n != "php" and not n.startswith("ext-"):
182
+ add("composer", n, str(v))
183
+ elif name == "Gemfile":
184
+ for m in re.finditer(r"^\s*gem\s+['\"]([\w.-]+)['\"](?:\s*,\s*['\"]([^'\"]+)['\"])?", text, re.M):
185
+ add("gem", m.group(1), m.group(2) or "")
186
+ elif name == "Gemfile.lock":
187
+ for m in re.finditer(r"^ {4}([\w.-]+) \(([^)]+)\)$", text, re.M):
188
+ gem_lock[m.group(1)] = m.group(2)
189
+ elif name == "pom.xml":
190
+ for m in re.finditer(r"<dependency>([\s\S]*?)</dependency>", text):
191
+ g, a, v = (re.search(p, m.group(1)) for p in (r"<groupId>([^<]+)<", r"<artifactId>([^<]+)<", r"<version>([^<$]+)<"))
192
+ if g and a:
193
+ add("maven", g.group(1).strip() + ":" + a.group(1).strip(), v.group(1).strip() if v else "")
194
+ elif name in ("build.gradle", "build.gradle.kts"):
195
+ for m in re.finditer(r"\b(?:implementation|api|compile|testImplementation|runtimeOnly|compileOnly|kapt)\s*\(?\s*['\"]([^'\":]+:[^'\":]+)(?::[^'\"]*)?['\"]", text):
196
+ add("maven", m.group(1), "")
197
+
198
+
199
+ def _strip_markdown(s):
200
+ s = re.sub(r"```[\s\S]*?```", " ", s)
201
+ s = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", s)
202
+ s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s)
203
+ s = re.sub(r"<[^>]+>", " ", s)
204
+ s = re.sub(r"^[#>\s*+-]+", "", s, flags=re.M)
205
+ s = re.sub(r"[*_`~|]", "", s)
206
+ return re.sub(r"\s+", " ", s).strip()
207
+
208
+
209
+ def _redact(s):
210
+ for r in SECRET_RES:
211
+ s = r.sub("", s)
212
+ return s
213
+
214
+
215
+ def _first(m):
216
+ for g in m.groups():
217
+ if g:
218
+ return g
219
+ return None
220
+
221
+
222
+ def scan(root_dir=".", max_files=MAX_FILES):
223
+ """Scan root_dir and return the manifest dict. Finishes in seconds on a large repo."""
224
+ root = os.path.abspath(str(root_dir or "."))
225
+ languages, deps, routes, route_keys, flags = {}, {}, [], set(), {}
226
+ env_names, mod_dirs, frameworks, ci, hosting, test_fw = set(), {}, set(), set(), set(), set()
227
+ pricing, pricing_keys, next_cands, gem_lock = [], set(), [], {}
228
+ infra = {"docker": False, "compose": False, "kubernetes": False, "terraform": False, "serverless": False}
229
+ sec = {"lockfile": False, "env_committed": False, "secrets_suspected": 0}
230
+ loc = files = test_files = 0
231
+ description = ""
232
+
233
+ def add_dep(eco, n, v):
234
+ k = (eco, n)
235
+ if len(deps) < CAPS["dependencies"] * 2 and k not in deps:
236
+ deps[k] = {"ecosystem": eco, "name": n[:200], "version": (v or "")[:100]}
237
+
238
+ def add_route(method, p, file):
239
+ if len(routes) >= CAPS["routes"]:
240
+ return
241
+ p = str(p)
242
+ if not p.startswith("/"):
243
+ p = "/" + p
244
+ k = (method, p, file)
245
+ if k in route_keys:
246
+ return
247
+ route_keys.add(k)
248
+ routes.append({"method": method, "path": p[:200], "file": file[:200]})
249
+
250
+ def add_env(n):
251
+ if n and len(env_names) < CAPS["env_names"]:
252
+ env_names.add(n)
253
+
254
+ gitignore = []
255
+ gi = _read(os.path.join(root, ".gitignore"))
256
+ if gi:
257
+ gitignore = [_glob_re(l.strip()) for l in gi.split("\n") if l.strip() and not l.strip().startswith(("#", "!"))]
258
+
259
+ stack = [""]
260
+ done = False
261
+ while stack and not done:
262
+ rel = stack.pop()
263
+ try:
264
+ entries = list(os.scandir(os.path.join(root, rel) if rel else root))
265
+ except OSError:
266
+ continue
267
+ for ent in entries:
268
+ name = ent.name
269
+ relp = rel + "/" + name if rel else name
270
+ try:
271
+ if ent.is_symlink():
272
+ continue # never follow symlinks
273
+ if ent.is_dir(follow_symlinks=False):
274
+ if name in SKIP_DIRS or (name.startswith(".") and name not in KEEP_DOT_DIRS):
275
+ continue
276
+ stack.append(relp)
277
+ continue
278
+ if not ent.is_file(follow_symlinks=False):
279
+ continue
280
+ size = ent.stat(follow_symlinks=False).st_size
281
+ except OSError:
282
+ continue
283
+ files += 1
284
+ if files > max_files:
285
+ files = max_files
286
+ done = True
287
+ break
288
+ full = os.path.join(root, relp)
289
+ if name in LOCKFILES:
290
+ sec["lockfile"] = True
291
+ if re.match(r"^\.env(\..+)?$", name) and not re.search(r"\.(example|sample|template|dist)$", name):
292
+ if not any(g.match(name) for g in gitignore):
293
+ sec["env_committed"] = True
294
+ continue # never read real env files
295
+ ext = name.rsplit(".", 1)[1].lower() if "." in name[1:] else ""
296
+ is_code = ext in CODE_EXT
297
+ is_cfg = ext in CONFIG_EXT or bool(re.match(r"^(Dockerfile|Procfile|Gemfile|Pipfile|Jenkinsfile)", name)) or name.startswith(".env.")
298
+ if name.startswith("Dockerfile"):
299
+ infra["docker"] = True
300
+ if re.match(r"^(docker-compose.*|compose)\.ya?ml$", name):
301
+ infra["compose"] = True
302
+ if ext == "tf":
303
+ infra["terraform"] = True
304
+ if re.match(r"^serverless\.(ya?ml|ts|js)$", name):
305
+ infra["serverless"] = True
306
+ if relp.startswith(".github/workflows/") and re.search(r"\.ya?ml$", name):
307
+ ci.add("github-actions")
308
+ for fname, label in ((".gitlab-ci.yml", "gitlab"), ("Jenkinsfile", "jenkins"), ("azure-pipelines.yml", "azure-pipelines"),
309
+ ("bitbucket-pipelines.yml", "bitbucket")):
310
+ if name == fname:
311
+ ci.add(label)
312
+ if relp == ".circleci/config.yml":
313
+ ci.add("circleci")
314
+ if name in HOSTS:
315
+ hosting.add(HOSTS[name])
316
+ if not (is_code or is_cfg) or size > MAX_BYTES or name in LOCKFILES:
317
+ continue
318
+ if is_code:
319
+ languages[ext] = languages.get(ext, 0) + 1
320
+ text = _read(full)
321
+ if text is None:
322
+ continue
323
+ if is_code:
324
+ loc += text.count("\n") + (1 if text and not text.endswith("\n") else 0)
325
+ parts = relp.split("/")
326
+ if parts[0] in ("src", "apps", "packages", "modules", "features", "app", "lib", "services") and len(parts) > 2:
327
+ top = parts[0] + "/" + parts[1]
328
+ elif len(parts) > 1:
329
+ top = parts[0]
330
+ else:
331
+ top = None
332
+ if top and not top.startswith("."):
333
+ mod_dirs[top] = mod_dirs.get(top, 0) + 1
334
+ for r in SECRET_RES:
335
+ sec["secrets_suspected"] += len(r.findall(text)) # count only: matches are never stored
336
+ if RE_DEP_FILE.match(name):
337
+ _parse_deps(name, text, add_dep, gem_lock)
338
+ if re.match(r"^(\.env\.(example|sample|template|dist)|example\.env|env\.example)$", name):
339
+ for m in re.finditer(r"^\s*(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=", text, re.M):
340
+ add_env(m.group(1))
341
+ if ext in ("yml", "yaml") and re.search(r"^kind:\s*(Deployment|Service|Ingress|StatefulSet|DaemonSet|CronJob|Job|ConfigMap)\b", text, re.M) \
342
+ and re.search(r"^apiVersion:", text, re.M):
343
+ infra["kubernetes"] = True
344
+ if not is_code:
345
+ continue
346
+ is_test = bool(RE_TEST_FILE.search(name)) or (bool(RE_TEST_DIR.search(relp)) and ext in ("js", "jsx", "ts", "tsx", "py", "rb", "php", "go"))
347
+ if is_test:
348
+ test_files += 1
349
+ if ext == "go":
350
+ test_fw.add("go-test")
351
+ if re.search(r"^(import|from)\s+unittest", text, re.M):
352
+ test_fw.add("unittest")
353
+ if re.search(r"^\s*(import|from)\s+pytest|\bdef test_", text, re.M):
354
+ test_fw.add("pytest")
355
+ if re.search(r"\bfrom\s+['\"]vitest['\"]", text):
356
+ test_fw.add("vitest")
357
+ if "node:test" in text:
358
+ test_fw.add("node:test")
359
+ elif re.search(r"\bfrom\s+['\"]mocha['\"]", text):
360
+ test_fw.add("mocha")
361
+ continue
362
+ if ext in JS_EXT:
363
+ if re.search(r"(^|/)(app|pages)/", relp):
364
+ next_cands.append(relp)
365
+ for m in RE_EXPRESS.finditer(text):
366
+ add_route(m.group(1).upper(), m.group(3), relp)
367
+ for m in RE_ENV_JS.finditer(text):
368
+ add_env(_first(m))
369
+ elif ext == "py":
370
+ for m in RE_PYROUTE.finditer(text):
371
+ methods = [m.group(1).upper()]
372
+ if m.group(1) == "route":
373
+ mm = re.search(r"methods\s*=\s*[\[(]([^\])]*)[\])]", m.group(4) or "")
374
+ methods = [x.upper() for x in re.findall(r"['\"](\w+)['\"]", mm.group(1))] if mm else ["GET"]
375
+ if m.group(3).startswith("/"):
376
+ for me in methods:
377
+ add_route(me, m.group(3), relp)
378
+ if name == "urls.py":
379
+ for m in RE_DJANGO.finditer(text):
380
+ p = re.sub(r"^\^", "", m.group(3))
381
+ p = re.sub(r"\$$", "", p)
382
+ p = re.sub(r"\(\?P<(\w+)>[^)]*\)", r":\1", p)
383
+ p = re.sub(r"<(?:\w+:)?(\w+)>", r":\1", p)
384
+ add_route("ANY", p or "/", relp)
385
+ for m in RE_ENV_PY.finditer(text):
386
+ add_env(_first(m))
387
+ elif ext == "php":
388
+ for m in RE_LARAVEL.finditer(text):
389
+ add_route("ANY" if m.group(1) == "any" else m.group(1).upper(), re.sub(r"\{(\w+)\??\}", r":\1", m.group(3)), relp)
390
+ for m in RE_ENV_PY.finditer(text):
391
+ add_env(_first(m))
392
+ elif ext == "rb":
393
+ if name == "routes.rb":
394
+ for m in RE_RAILS_VERB.finditer(text):
395
+ add_route(m.group(1).upper(), m.group(3), relp)
396
+ for m in RE_RAILS_RES.finditer(text):
397
+ r = "/" + m.group(2)
398
+ if m.group(1) == "resources":
399
+ add_route("GET", r, relp)
400
+ add_route("POST", r, relp)
401
+ for v in ("GET", "PUT", "PATCH", "DELETE"):
402
+ add_route(v, r + "/:id", relp)
403
+ else:
404
+ for v in ("GET", "POST", "PUT", "PATCH", "DELETE"):
405
+ add_route(v, r, relp)
406
+ for m in RE_ENV_PY.finditer(text):
407
+ add_env(_first(m))
408
+ elif ext == "go":
409
+ for m in RE_GO_STD.finditer(text):
410
+ mm = re.match(r"^([A-Z]+)\s+(/.*)$", m.group(1))
411
+ if mm:
412
+ add_route(mm.group(1), mm.group(2), relp)
413
+ elif m.group(1).startswith("/"):
414
+ add_route("ANY", m.group(1), relp)
415
+ for m in RE_GO_ROUTER.finditer(text):
416
+ add_route(m.group(1).upper(), re.sub(r"\{(\w+)\}", r":\1", m.group(2)), relp)
417
+ for m in RE_ENV_PY.finditer(text):
418
+ add_env(_first(m))
419
+ for m in RE_FLAG.finditer(text):
420
+ if len(flags) < CAPS["features"]:
421
+ flags.setdefault(m.group(2), set()).add(relp)
422
+ for m in RE_FLAG_PROP.finditer(text):
423
+ if len(flags) < CAPS["features"]:
424
+ flags.setdefault(m.group(1), set()).add(relp)
425
+ if len(pricing) < CAPS["pricing_hints"] and not re.search(r"(^|/)(docs?|examples?)/", relp) and re.search(r"price|amount|monthly|cost", text, re.I):
426
+ for rx in (RE_PLAN_KEY, RE_PLAN_CONST, RE_PLAN_PRICE):
427
+ for m in rx.finditer(text):
428
+ if len(pricing) >= CAPS["pricing_hints"]:
429
+ break
430
+ plan, amount = m.group(1).strip().lower()[:40], float(m.group(2))
431
+ k = (plan, amount)
432
+ if not amount > 0 or k in pricing_keys:
433
+ continue
434
+ pricing_keys.add(k)
435
+ cur = RE_CURRENCY.search(text[max(0, m.start() - 200):m.end() + 200])
436
+ pricing.append({"plan": plan, "amount": int(amount) if amount == int(amount) else amount, "currency": cur.group(1).upper() if cur else ""})
437
+
438
+ for d in deps.values():
439
+ if d["ecosystem"] == "gem" and d["name"] in gem_lock:
440
+ d["version"] = gem_lock[d["name"]]
441
+ if gem_lock and not any(d["ecosystem"] == "gem" for d in deps.values()):
442
+ for n, v in gem_lock.items():
443
+ add_dep("gem", n, v)
444
+ dep_list = list(deps.values())[:CAPS["dependencies"]]
445
+ dep_names = [d["name"].lower() for d in dep_list]
446
+
447
+ def has_dep(x):
448
+ return any(n == x or n.startswith(x + "/") or n.startswith(x + ":") for n in dep_names)
449
+
450
+ for dep, label in FRAMEWORK_DEPS.items():
451
+ if has_dep(dep):
452
+ frameworks.add(label)
453
+ for n, v in (("jest", "jest"), ("vitest", "vitest"), ("mocha", "mocha"), ("pytest", "pytest"), ("phpunit/phpunit", "phpunit"),
454
+ ("rspec", "rspec"), ("@playwright/test", "playwright"), ("cypress", "cypress")):
455
+ if has_dep(n) or has_dep(n + "-core"):
456
+ test_fw.add(v)
457
+ if "next" in frameworks:
458
+ for rel in next_cands:
459
+ r = next_route(rel)
460
+ if not r:
461
+ continue
462
+ route, kind = r
463
+ if kind == "route":
464
+ text = _read(os.path.join(root, rel)) or ""
465
+ ms = re.findall(r"export\s+(?:async\s+)?(?:function|const)\s+(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\b", text)
466
+ for m in (ms or ["GET"]):
467
+ add_route(m, route, rel)
468
+ else:
469
+ add_route("GET", route, rel)
470
+ env_list = sorted(env_names)
471
+ integrations = sorted(n for n, (dp, ev) in INTEGRATIONS.items()
472
+ if any(s in d for s in dp for d in dep_names) or any(p in e for p in ev for e in env_list))
473
+ features, groups = [], {}
474
+ for r in routes:
475
+ segs = [s for s in r["path"].split("/") if s and not re.match(r"^(api|v\d+|rest|public)$", s, re.I) and not s.startswith((":", "["))]
476
+ if not segs:
477
+ continue
478
+ g = re.sub(r"[^a-z0-9_-]", "", segs[0].lower())
479
+ if g:
480
+ arr = groups.setdefault(g, [])
481
+ if len(arr) < 5:
482
+ arr.append(r["path"])
483
+ for name, paths in groups.items():
484
+ if len(features) < CAPS["features"]:
485
+ features.append({"name": name[:80], "source": "route", "paths": paths})
486
+ for name, st in flags.items():
487
+ if len(features) < CAPS["features"]:
488
+ features.append({"name": name[:80], "source": "flag", "paths": sorted(st)[:5]})
489
+ for d, _n in sorted(mod_dirs.items(), key=lambda kv: -kv[1]):
490
+ if len(features) < CAPS["features"]:
491
+ features.append({"name": d.split("/")[-1][:80], "source": "dir", "paths": [d]})
492
+ for n in ("README.md", "readme.md", "Readme.md", "README", "README.rst", "README.txt"):
493
+ try:
494
+ with open(os.path.join(root, n), "rb") as f:
495
+ description = _redact(_strip_markdown(f.read(2048).decode("utf-8", "ignore")))
496
+ break
497
+ except OSError:
498
+ continue
499
+ infra_out = dict(infra)
500
+ infra_out["ci"] = sorted(ci)
501
+ infra_out["hosting_hints"] = sorted(hosting)
502
+ return {
503
+ "schema": 1, "scanned_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "root_name": os.path.basename(root),
504
+ "languages": languages, "loc": loc, "frameworks": sorted(frameworks), "dependencies": dep_list, "routes": routes[:CAPS["routes"]],
505
+ "features": features, "integrations": integrations, "infra": infra_out, "env_names": env_list[:CAPS["env_names"]],
506
+ "pricing_hints": pricing[:CAPS["pricing_hints"]], "description": description,
507
+ "tests": {"files": test_files, "frameworks": sorted(test_fw)}, "security": sec}
linearjeph/py.typed ADDED
File without changes