entrygraph 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. entrygraph/__init__.py +87 -0
  2. entrygraph/__main__.py +8 -0
  3. entrygraph/_version.py +24 -0
  4. entrygraph/api.py +549 -0
  5. entrygraph/cli/__init__.py +0 -0
  6. entrygraph/cli/main.py +387 -0
  7. entrygraph/cli/render.py +136 -0
  8. entrygraph/data/sinks/csharp.toml +106 -0
  9. entrygraph/data/sinks/go.toml +87 -0
  10. entrygraph/data/sinks/java.toml +92 -0
  11. entrygraph/data/sinks/javascript.toml +112 -0
  12. entrygraph/data/sinks/lib_javascript.toml +34 -0
  13. entrygraph/data/sinks/lib_python.toml +39 -0
  14. entrygraph/data/sinks/php.toml +125 -0
  15. entrygraph/data/sinks/python.toml +160 -0
  16. entrygraph/data/sinks/ruby.toml +102 -0
  17. entrygraph/data/sinks/rust.toml +68 -0
  18. entrygraph/db/__init__.py +0 -0
  19. entrygraph/db/engine.py +34 -0
  20. entrygraph/db/meta.py +70 -0
  21. entrygraph/db/models.py +176 -0
  22. entrygraph/db/queries.py +155 -0
  23. entrygraph/detect/__init__.py +0 -0
  24. entrygraph/detect/entrypoints/__init__.py +22 -0
  25. entrygraph/detect/entrypoints/base.py +124 -0
  26. entrygraph/detect/entrypoints/configs.py +139 -0
  27. entrygraph/detect/entrypoints/csharp.py +156 -0
  28. entrygraph/detect/entrypoints/golang.py +158 -0
  29. entrygraph/detect/entrypoints/java.py +187 -0
  30. entrygraph/detect/entrypoints/javascript.py +211 -0
  31. entrygraph/detect/entrypoints/php.py +133 -0
  32. entrygraph/detect/entrypoints/python.py +335 -0
  33. entrygraph/detect/entrypoints/ruby.py +147 -0
  34. entrygraph/detect/entrypoints/rust.py +153 -0
  35. entrygraph/detect/frameworks.py +369 -0
  36. entrygraph/detect/manifests.py +234 -0
  37. entrygraph/detect/taint.py +224 -0
  38. entrygraph/errors.py +27 -0
  39. entrygraph/extract/__init__.py +0 -0
  40. entrygraph/extract/base.py +51 -0
  41. entrygraph/extract/csharp.py +502 -0
  42. entrygraph/extract/golang.py +342 -0
  43. entrygraph/extract/ir.py +105 -0
  44. entrygraph/extract/java.py +329 -0
  45. entrygraph/extract/javascript.py +400 -0
  46. entrygraph/extract/php.py +426 -0
  47. entrygraph/extract/python.py +390 -0
  48. entrygraph/extract/registry.py +43 -0
  49. entrygraph/extract/ruby.py +321 -0
  50. entrygraph/extract/rust.py +482 -0
  51. entrygraph/fs/__init__.py +0 -0
  52. entrygraph/fs/hashing.py +78 -0
  53. entrygraph/fs/lang.py +134 -0
  54. entrygraph/fs/walker.py +167 -0
  55. entrygraph/graph/__init__.py +0 -0
  56. entrygraph/graph/adjacency.py +146 -0
  57. entrygraph/graph/cte.py +123 -0
  58. entrygraph/graph/scoring.py +101 -0
  59. entrygraph/kinds.py +51 -0
  60. entrygraph/parsing/__init__.py +0 -0
  61. entrygraph/parsing/parsers.py +49 -0
  62. entrygraph/parsing/queries.py +39 -0
  63. entrygraph/pipeline/__init__.py +0 -0
  64. entrygraph/pipeline/scanner.py +506 -0
  65. entrygraph/pipeline/worker.py +49 -0
  66. entrygraph/pipeline/writer.py +41 -0
  67. entrygraph/py.typed +0 -0
  68. entrygraph/queries/csharp/calls.scm +4 -0
  69. entrygraph/queries/csharp/definitions.scm +29 -0
  70. entrygraph/queries/csharp/imports.scm +4 -0
  71. entrygraph/queries/go/calls.scm +2 -0
  72. entrygraph/queries/go/definitions.scm +24 -0
  73. entrygraph/queries/go/imports.scm +1 -0
  74. entrygraph/queries/java/calls.scm +2 -0
  75. entrygraph/queries/java/definitions.scm +14 -0
  76. entrygraph/queries/java/imports.scm +2 -0
  77. entrygraph/queries/javascript/calls.scm +4 -0
  78. entrygraph/queries/javascript/definitions.scm +4 -0
  79. entrygraph/queries/javascript/imports.scm +6 -0
  80. entrygraph/queries/php/calls.scm +8 -0
  81. entrygraph/queries/php/definitions.scm +24 -0
  82. entrygraph/queries/php/imports.scm +1 -0
  83. entrygraph/queries/python/calls.scm +2 -0
  84. entrygraph/queries/python/definitions.scm +11 -0
  85. entrygraph/queries/python/imports.scm +2 -0
  86. entrygraph/queries/ruby/calls.scm +4 -0
  87. entrygraph/queries/ruby/definitions.scm +20 -0
  88. entrygraph/queries/ruby/imports.scm +7 -0
  89. entrygraph/queries/rust/calls.scm +5 -0
  90. entrygraph/queries/rust/definitions.scm +26 -0
  91. entrygraph/queries/rust/imports.scm +4 -0
  92. entrygraph/resolve/__init__.py +0 -0
  93. entrygraph/resolve/externals.py +61 -0
  94. entrygraph/resolve/hierarchy.py +152 -0
  95. entrygraph/resolve/resolver.py +275 -0
  96. entrygraph/resolve/symbol_table.py +48 -0
  97. entrygraph/results.py +138 -0
  98. entrygraph-0.1.0.dist-info/METADATA +204 -0
  99. entrygraph-0.1.0.dist-info/RECORD +102 -0
  100. entrygraph-0.1.0.dist-info/WHEEL +4 -0
  101. entrygraph-0.1.0.dist-info/entry_points.txt +2 -0
  102. entrygraph-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,369 @@
1
+ """Framework detection: manifest deps + code signals, noisy-or confidence.
2
+
3
+ confidence = 1 - prod(1 - w_i) over fired signals. A manifest dependency or an
4
+ import alone is strong evidence; file-presence signals corroborate.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import fnmatch
10
+ from dataclasses import dataclass
11
+ from typing import Literal
12
+
13
+ from entrygraph.detect.manifests import ManifestDeps
14
+
15
+ SignalKind = Literal["manifest_dep", "import", "file_presence", "symbol_name"]
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class FrameworkSignal:
20
+ kind: SignalKind
21
+ pattern: str # dep glob, import module, path glob, or symbol-name glob
22
+ weight: float
23
+
24
+
25
+ @dataclass(frozen=True, slots=True)
26
+ class FrameworkSpec:
27
+ name: str
28
+ language: str
29
+ signals: tuple[FrameworkSignal, ...]
30
+ threshold: float = 0.5
31
+
32
+
33
+ @dataclass(frozen=True, slots=True)
34
+ class DetectedFramework:
35
+ name: str
36
+ language: str
37
+ confidence: float
38
+ evidence: tuple[str, ...]
39
+
40
+ @property
41
+ def detected(self) -> bool:
42
+ return True
43
+
44
+
45
+ FRAMEWORKS: list[FrameworkSpec] = [
46
+ FrameworkSpec("flask", "python", (
47
+ FrameworkSignal("manifest_dep", "flask", 0.8),
48
+ FrameworkSignal("import", "flask", 0.7),
49
+ )),
50
+ FrameworkSpec("fastapi", "python", (
51
+ FrameworkSignal("manifest_dep", "fastapi", 0.8),
52
+ FrameworkSignal("import", "fastapi", 0.7),
53
+ )),
54
+ FrameworkSpec("django", "python", (
55
+ FrameworkSignal("manifest_dep", "django", 0.8),
56
+ FrameworkSignal("import", "django", 0.7),
57
+ FrameworkSignal("file_presence", "*manage.py", 0.3),
58
+ FrameworkSignal("file_presence", "*urls.py", 0.2),
59
+ )),
60
+ FrameworkSpec("click", "python", (
61
+ FrameworkSignal("manifest_dep", "click", 0.8),
62
+ FrameworkSignal("import", "click", 0.7),
63
+ )),
64
+ FrameworkSpec("typer", "python", (
65
+ FrameworkSignal("manifest_dep", "typer", 0.8),
66
+ FrameworkSignal("import", "typer", 0.7),
67
+ )),
68
+ FrameworkSpec("argparse", "python", (
69
+ FrameworkSignal("import", "argparse", 0.7),
70
+ )),
71
+ FrameworkSpec("celery", "python", (
72
+ FrameworkSignal("manifest_dep", "celery", 0.8),
73
+ FrameworkSignal("import", "celery", 0.7),
74
+ )),
75
+ FrameworkSpec("aws-lambda", "python", (
76
+ FrameworkSignal("file_presence", "serverless.y*ml", 0.5),
77
+ FrameworkSignal("file_presence", "template.y*ml", 0.3),
78
+ FrameworkSignal("symbol_name", "lambda_handler", 0.5),
79
+ )),
80
+ # JS/TS, Go, Java, Ruby specs land with their extractors (M8)
81
+ FrameworkSpec("express", "javascript", (
82
+ FrameworkSignal("manifest_dep", "express", 0.8),
83
+ FrameworkSignal("import", "express", 0.7),
84
+ )),
85
+ FrameworkSpec("fastify", "javascript", (
86
+ FrameworkSignal("manifest_dep", "fastify", 0.8),
87
+ FrameworkSignal("import", "fastify", 0.7),
88
+ )),
89
+ FrameworkSpec("nestjs", "javascript", (
90
+ FrameworkSignal("manifest_dep", "@nestjs/*", 0.8),
91
+ FrameworkSignal("import", "@nestjs/*", 0.7),
92
+ )),
93
+ FrameworkSpec("next", "javascript", (
94
+ FrameworkSignal("manifest_dep", "next", 0.8),
95
+ FrameworkSignal("file_presence", "next.config.*", 0.5),
96
+ )),
97
+ FrameworkSpec("react", "javascript", (
98
+ FrameworkSignal("manifest_dep", "react", 0.8),
99
+ FrameworkSignal("import", "react", 0.7),
100
+ )),
101
+ FrameworkSpec("gin", "go", (
102
+ FrameworkSignal("manifest_dep", "github.com/gin-gonic/gin", 0.8),
103
+ FrameworkSignal("import", "github.com/gin-gonic/gin", 0.7),
104
+ )),
105
+ FrameworkSpec("echo", "go", (
106
+ FrameworkSignal("manifest_dep", "github.com/labstack/echo*", 0.8),
107
+ FrameworkSignal("import", "github.com/labstack/echo*", 0.7),
108
+ )),
109
+ FrameworkSpec("cobra", "go", (
110
+ FrameworkSignal("manifest_dep", "github.com/spf13/cobra", 0.8),
111
+ FrameworkSignal("import", "github.com/spf13/cobra", 0.7),
112
+ )),
113
+ FrameworkSpec("net/http", "go", (
114
+ FrameworkSignal("import", "net/http", 0.7),
115
+ )),
116
+ FrameworkSpec("spring-boot", "java", (
117
+ FrameworkSignal("manifest_dep", "*spring-boot*", 0.8),
118
+ FrameworkSignal("import", "org.springframework*", 0.7),
119
+ )),
120
+ FrameworkSpec("jax-rs", "java", (
121
+ FrameworkSignal("import", "javax.ws.rs*", 0.7),
122
+ FrameworkSignal("import", "jakarta.ws.rs*", 0.7),
123
+ )),
124
+ FrameworkSpec("rails", "ruby", (
125
+ FrameworkSignal("manifest_dep", "rails", 0.8),
126
+ FrameworkSignal("file_presence", "config/routes.rb", 0.5),
127
+ )),
128
+ FrameworkSpec("sinatra", "ruby", (
129
+ FrameworkSignal("manifest_dep", "sinatra", 0.8),
130
+ FrameworkSignal("import", "sinatra", 0.7),
131
+ )),
132
+ FrameworkSpec("rake", "ruby", (
133
+ FrameworkSignal("file_presence", "Rakefile", 0.6),
134
+ )),
135
+ # --- C6: additional framework specs ---
136
+ # Python
137
+ FrameworkSpec("tornado", "python", (
138
+ FrameworkSignal("manifest_dep", "tornado", 0.8),
139
+ FrameworkSignal("import", "tornado", 0.7),
140
+ )),
141
+ FrameworkSpec("aiohttp", "python", (
142
+ FrameworkSignal("manifest_dep", "aiohttp", 0.8),
143
+ FrameworkSignal("import", "aiohttp", 0.7),
144
+ )),
145
+ FrameworkSpec("bottle", "python", (
146
+ FrameworkSignal("manifest_dep", "bottle", 0.8),
147
+ FrameworkSignal("import", "bottle", 0.7),
148
+ )),
149
+ FrameworkSpec("pyramid", "python", (
150
+ FrameworkSignal("manifest_dep", "pyramid", 0.8),
151
+ FrameworkSignal("import", "pyramid", 0.7),
152
+ )),
153
+ FrameworkSpec("sanic", "python", (
154
+ FrameworkSignal("manifest_dep", "sanic", 0.8),
155
+ FrameworkSignal("import", "sanic", 0.7),
156
+ )),
157
+ FrameworkSpec("dramatiq", "python", (
158
+ FrameworkSignal("manifest_dep", "dramatiq", 0.8),
159
+ FrameworkSignal("import", "dramatiq", 0.7),
160
+ )),
161
+ FrameworkSpec("rq", "python", (
162
+ FrameworkSignal("manifest_dep", "rq", 0.8),
163
+ FrameworkSignal("import", "rq", 0.7),
164
+ )),
165
+ FrameworkSpec("airflow", "python", (
166
+ FrameworkSignal("manifest_dep", "apache-airflow", 0.8),
167
+ FrameworkSignal("import", "airflow", 0.7),
168
+ FrameworkSignal("file_presence", "*dags/*", 0.2),
169
+ )),
170
+ # JS/TS
171
+ FrameworkSpec("koa", "javascript", (
172
+ FrameworkSignal("manifest_dep", "koa", 0.8),
173
+ FrameworkSignal("import", "koa", 0.7),
174
+ )),
175
+ FrameworkSpec("hapi", "javascript", (
176
+ FrameworkSignal("manifest_dep", "@hapi/hapi", 0.8),
177
+ FrameworkSignal("import", "@hapi/hapi", 0.7),
178
+ )),
179
+ FrameworkSpec("remix", "javascript", (
180
+ FrameworkSignal("manifest_dep", "@remix-run/*", 0.8),
181
+ FrameworkSignal("file_presence", "app/routes/*", 0.3),
182
+ )),
183
+ FrameworkSpec("hono", "javascript", (
184
+ FrameworkSignal("manifest_dep", "hono", 0.8),
185
+ FrameworkSignal("import", "hono", 0.7),
186
+ )),
187
+ FrameworkSpec("electron", "javascript", (
188
+ FrameworkSignal("manifest_dep", "electron", 0.8),
189
+ FrameworkSignal("import", "electron", 0.7),
190
+ )),
191
+ FrameworkSpec("socket.io", "javascript", (
192
+ FrameworkSignal("manifest_dep", "socket.io", 0.8),
193
+ FrameworkSignal("import", "socket.io", 0.7),
194
+ )),
195
+ # Go
196
+ FrameworkSpec("fiber", "go", (
197
+ FrameworkSignal("manifest_dep", "github.com/gofiber/fiber*", 0.8),
198
+ FrameworkSignal("import", "github.com/gofiber/fiber*", 0.7),
199
+ )),
200
+ FrameworkSpec("chi", "go", (
201
+ FrameworkSignal("manifest_dep", "github.com/go-chi/chi*", 0.8),
202
+ FrameworkSignal("import", "github.com/go-chi/chi*", 0.7),
203
+ )),
204
+ FrameworkSpec("gorilla-mux", "go", (
205
+ FrameworkSignal("manifest_dep", "github.com/gorilla/mux", 0.8),
206
+ FrameworkSignal("import", "github.com/gorilla/mux", 0.7),
207
+ )),
208
+ FrameworkSpec("grpc-go", "go", (
209
+ FrameworkSignal("manifest_dep", "google.golang.org/grpc", 0.8),
210
+ FrameworkSignal("import", "google.golang.org/grpc", 0.7),
211
+ )),
212
+ FrameworkSpec("urfave-cli", "go", (
213
+ FrameworkSignal("manifest_dep", "github.com/urfave/cli*", 0.8),
214
+ FrameworkSignal("import", "github.com/urfave/cli*", 0.7),
215
+ )),
216
+ # Java
217
+ FrameworkSpec("micronaut", "java", (
218
+ FrameworkSignal("manifest_dep", "*micronaut*", 0.8),
219
+ FrameworkSignal("import", "io.micronaut*", 0.7),
220
+ )),
221
+ FrameworkSpec("quarkus", "java", (
222
+ FrameworkSignal("manifest_dep", "*quarkus*", 0.8),
223
+ FrameworkSignal("import", "io.quarkus*", 0.7),
224
+ )),
225
+ FrameworkSpec("vertx", "java", (
226
+ FrameworkSignal("manifest_dep", "*vertx*", 0.8),
227
+ FrameworkSignal("import", "io.vertx*", 0.7),
228
+ )),
229
+ FrameworkSpec("servlet-api", "java", (
230
+ FrameworkSignal("import", "javax.servlet*", 0.7),
231
+ FrameworkSignal("import", "jakarta.servlet*", 0.7),
232
+ FrameworkSignal("file_presence", "*web.xml", 0.3),
233
+ )),
234
+ FrameworkSpec("dropwizard", "java", (
235
+ FrameworkSignal("manifest_dep", "*dropwizard*", 0.8),
236
+ FrameworkSignal("import", "io.dropwizard*", 0.7),
237
+ )),
238
+ # Ruby
239
+ FrameworkSpec("hanami", "ruby", (
240
+ FrameworkSignal("manifest_dep", "hanami", 0.8),
241
+ FrameworkSignal("import", "hanami", 0.7),
242
+ )),
243
+ FrameworkSpec("grape", "ruby", (
244
+ FrameworkSignal("manifest_dep", "grape", 0.8),
245
+ FrameworkSignal("import", "grape", 0.7),
246
+ )),
247
+ FrameworkSpec("sidekiq", "ruby", (
248
+ FrameworkSignal("manifest_dep", "sidekiq", 0.8),
249
+ FrameworkSignal("import", "sidekiq", 0.7),
250
+ )),
251
+ FrameworkSpec("rack", "ruby", (
252
+ FrameworkSignal("manifest_dep", "rack", 0.8),
253
+ FrameworkSignal("file_presence", "config.ru", 0.6),
254
+ )),
255
+ # --- C2/C3/C4: C#, PHP, Rust ---
256
+ FrameworkSpec("aspnetcore", "csharp", (
257
+ FrameworkSignal("manifest_dep", "microsoft.aspnetcore*", 0.8),
258
+ FrameworkSignal("import", "Microsoft.AspNetCore*", 0.7),
259
+ FrameworkSignal("file_presence", "*appsettings.json", 0.2),
260
+ )),
261
+ FrameworkSpec("ef-core", "csharp", (
262
+ FrameworkSignal("manifest_dep", "microsoft.entityframeworkcore*", 0.8),
263
+ FrameworkSignal("import", "Microsoft.EntityFrameworkCore*", 0.7),
264
+ )),
265
+ FrameworkSpec("laravel", "php", (
266
+ FrameworkSignal("manifest_dep", "laravel/framework", 0.8),
267
+ FrameworkSignal("file_presence", "artisan", 0.5),
268
+ FrameworkSignal("file_presence", "routes/web.php", 0.3),
269
+ )),
270
+ FrameworkSpec("symfony", "php", (
271
+ FrameworkSignal("manifest_dep", "symfony/framework-bundle", 0.8),
272
+ FrameworkSignal("manifest_dep", "symfony/*", 0.5),
273
+ FrameworkSignal("file_presence", "bin/console", 0.4),
274
+ )),
275
+ FrameworkSpec("wordpress", "php", (
276
+ FrameworkSignal("file_presence", "wp-config.php", 0.7),
277
+ FrameworkSignal("symbol_name", "wp_*", 0.2),
278
+ )),
279
+ FrameworkSpec("actix-web", "rust", (
280
+ FrameworkSignal("manifest_dep", "actix-web", 0.8),
281
+ FrameworkSignal("import", "actix_web", 0.7),
282
+ )),
283
+ FrameworkSpec("axum", "rust", (
284
+ FrameworkSignal("manifest_dep", "axum", 0.8),
285
+ FrameworkSignal("import", "axum", 0.7),
286
+ )),
287
+ FrameworkSpec("rocket", "rust", (
288
+ FrameworkSignal("manifest_dep", "rocket", 0.8),
289
+ FrameworkSignal("import", "rocket", 0.7),
290
+ )),
291
+ FrameworkSpec("clap", "rust", (
292
+ FrameworkSignal("manifest_dep", "clap", 0.8),
293
+ FrameworkSignal("import", "clap", 0.7),
294
+ )),
295
+ FrameworkSpec("tokio", "rust", (
296
+ FrameworkSignal("manifest_dep", "tokio", 0.8),
297
+ FrameworkSignal("import", "tokio", 0.7),
298
+ )),
299
+ ]
300
+
301
+
302
+ def detect_frameworks(
303
+ manifests: ManifestDeps,
304
+ import_signals: set[tuple[str, str]], # (language, module)
305
+ file_paths: list[str],
306
+ symbol_names: set[str] | None = None,
307
+ languages_present: set[str] | None = None,
308
+ ) -> list[DetectedFramework]:
309
+ symbol_names = symbol_names or set()
310
+ detected: list[DetectedFramework] = []
311
+ for spec in FRAMEWORKS:
312
+ if languages_present is not None and spec.language not in languages_present:
313
+ continue
314
+ fired: list[tuple[FrameworkSignal, str]] = []
315
+ for signal in spec.signals:
316
+ evidence = _check_signal(signal, spec, manifests, import_signals, file_paths, symbol_names)
317
+ if evidence:
318
+ fired.append((signal, evidence))
319
+ if not fired:
320
+ continue
321
+ confidence = 1.0
322
+ for signal, _ in fired:
323
+ confidence *= 1.0 - signal.weight
324
+ confidence = 1.0 - confidence
325
+ if confidence >= spec.threshold:
326
+ detected.append(
327
+ DetectedFramework(
328
+ name=spec.name,
329
+ language=spec.language,
330
+ confidence=round(confidence, 3),
331
+ evidence=tuple(e for _, e in fired),
332
+ )
333
+ )
334
+ detected.sort(key=lambda d: d.confidence, reverse=True)
335
+ return detected
336
+
337
+
338
+ def _check_signal(
339
+ signal: FrameworkSignal,
340
+ spec: FrameworkSpec,
341
+ manifests: ManifestDeps,
342
+ import_signals: set[tuple[str, str]],
343
+ file_paths: list[str],
344
+ symbol_names: set[str],
345
+ ) -> str | None:
346
+ if signal.kind == "manifest_dep":
347
+ for dep in manifests.for_language(spec.language):
348
+ if fnmatch.fnmatch(dep.lower(), signal.pattern.lower()):
349
+ return f"manifest dependency {dep!r}"
350
+ elif signal.kind == "import":
351
+ for lang, module in import_signals:
352
+ if _same_ecosystem(lang, spec.language) and fnmatch.fnmatch(module, signal.pattern):
353
+ return f"import of {module!r}"
354
+ elif signal.kind == "file_presence":
355
+ for path in file_paths:
356
+ if fnmatch.fnmatch(path, signal.pattern):
357
+ return f"file {path!r}"
358
+ elif signal.kind == "symbol_name":
359
+ for name in symbol_names:
360
+ if fnmatch.fnmatch(name, signal.pattern):
361
+ return f"symbol {name!r}"
362
+ return None
363
+
364
+
365
+ def _same_ecosystem(lang: str, spec_lang: str) -> bool:
366
+ js = {"javascript", "typescript", "tsx"}
367
+ if spec_lang == "javascript":
368
+ return lang in js
369
+ return lang == spec_lang
@@ -0,0 +1,234 @@
1
+ """Dependency-manifest parsers — stdlib + regex only.
2
+
3
+ Each parser takes file text and returns a set of normalized dependency names.
4
+ ``parse_manifests(root)`` walks the well-known manifest locations and merges
5
+ results per ecosystem.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import re
12
+ import tomllib
13
+ import xml.etree.ElementTree as ET
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+
17
+ _REQ_LINE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
18
+ _GO_REQUIRE_BLOCK = re.compile(r"require\s*\(([^)]*)\)", re.DOTALL)
19
+ _GO_REQUIRE_LINE = re.compile(r"^\s*require\s+(\S+)", re.MULTILINE)
20
+ _GO_MODULE_LINE = re.compile(r"^\s*(\S+)\s+v[\w.\-+]+", re.MULTILINE)
21
+ _GRADLE_DEP = re.compile(
22
+ r"""(?:implementation|api|compile|runtimeOnly|compileOnly|testImplementation)\s*[\(\s]\s*['"]([^'"]+)['"]"""
23
+ )
24
+ _GEMFILE_GEM = re.compile(r"""^\s*gem\s+['"]([^'"]+)['"]""", re.MULTILINE)
25
+
26
+
27
+ def parse_requirements_txt(text: str) -> set[str]:
28
+ deps: set[str] = set()
29
+ for line in text.splitlines():
30
+ line = line.strip()
31
+ if not line or line.startswith(("#", "-")):
32
+ continue
33
+ match = _REQ_LINE.match(line)
34
+ if match:
35
+ deps.add(match.group(1).lower().replace("_", "-"))
36
+ return deps
37
+
38
+
39
+ def parse_pyproject_toml(text: str) -> set[str]:
40
+ try:
41
+ data = tomllib.loads(text)
42
+ except tomllib.TOMLDecodeError:
43
+ return set()
44
+ deps: set[str] = set()
45
+ project = data.get("project", {})
46
+ raw: list[str] = list(project.get("dependencies", []) or [])
47
+ for extra_deps in (project.get("optional-dependencies", {}) or {}).values():
48
+ raw.extend(extra_deps or [])
49
+ for spec in raw:
50
+ match = _REQ_LINE.match(spec)
51
+ if match:
52
+ deps.add(match.group(1).lower().replace("_", "-"))
53
+ poetry = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) or {}
54
+ deps.update(k.lower().replace("_", "-") for k in poetry if k.lower() != "python")
55
+ return deps
56
+
57
+
58
+ def parse_package_json(text: str) -> set[str]:
59
+ try:
60
+ data = json.loads(text)
61
+ except (json.JSONDecodeError, TypeError):
62
+ return set()
63
+ if not isinstance(data, dict):
64
+ return set()
65
+ deps: set[str] = set()
66
+ for key in ("dependencies", "devDependencies", "peerDependencies"):
67
+ section = data.get(key)
68
+ if isinstance(section, dict):
69
+ deps.update(section)
70
+ return deps
71
+
72
+
73
+ def parse_go_mod(text: str) -> set[str]:
74
+ deps: set[str] = set()
75
+ for block in _GO_REQUIRE_BLOCK.findall(text):
76
+ deps.update(_GO_MODULE_LINE.findall(block))
77
+ for single in _GO_REQUIRE_LINE.findall(text):
78
+ if single != "(":
79
+ deps.add(single)
80
+ return deps
81
+
82
+
83
+ def parse_pom_xml(text: str) -> set[str]:
84
+ try:
85
+ root = ET.fromstring(text)
86
+ except ET.ParseError:
87
+ return set()
88
+ ns = {"m": root.tag.split("}")[0].strip("{")} if root.tag.startswith("{") else {}
89
+ prefix = "m:" if ns else ""
90
+ deps: set[str] = set()
91
+ for dep in root.iter():
92
+ if dep.tag.endswith("dependency"):
93
+ group = dep.find(f"{prefix}groupId", ns)
94
+ artifact = dep.find(f"{prefix}artifactId", ns)
95
+ if group is not None and artifact is not None:
96
+ deps.add(f"{group.text}:{artifact.text}")
97
+ return deps
98
+
99
+
100
+ def parse_build_gradle(text: str) -> set[str]:
101
+ return {m for m in _GRADLE_DEP.findall(text)}
102
+
103
+
104
+ def parse_gemfile(text: str) -> set[str]:
105
+ return set(_GEMFILE_GEM.findall(text))
106
+
107
+
108
+ def parse_csproj(text: str) -> set[str]:
109
+ """NuGet <PackageReference Include="..."> (and <Reference> fallback)."""
110
+ try:
111
+ root = ET.fromstring(text)
112
+ except ET.ParseError:
113
+ return set()
114
+ deps: set[str] = set()
115
+ for el in root.iter():
116
+ tag = el.tag.split("}")[-1]
117
+ if tag in ("PackageReference", "Reference"):
118
+ include = el.get("Include")
119
+ if include:
120
+ deps.add(include.split(",", 1)[0].strip().lower())
121
+ return deps
122
+
123
+
124
+ def parse_packages_config(text: str) -> set[str]:
125
+ """Legacy NuGet packages.config: <package id="..." />."""
126
+ try:
127
+ root = ET.fromstring(text)
128
+ except ET.ParseError:
129
+ return set()
130
+ return {
131
+ el.get("id").lower()
132
+ for el in root.iter()
133
+ if el.tag.split("}")[-1] == "package" and el.get("id")
134
+ }
135
+
136
+
137
+ def parse_composer_json(text: str) -> set[str]:
138
+ try:
139
+ data = json.loads(text)
140
+ except (json.JSONDecodeError, TypeError):
141
+ return set()
142
+ if not isinstance(data, dict):
143
+ return set()
144
+ deps: set[str] = set()
145
+ for key in ("require", "require-dev"):
146
+ section = data.get(key)
147
+ if isinstance(section, dict):
148
+ for name in section:
149
+ low = name.lower()
150
+ if low == "php" or low.startswith("ext-"):
151
+ continue
152
+ deps.add(low)
153
+ return deps
154
+
155
+
156
+ def parse_cargo_toml(text: str) -> set[str]:
157
+ try:
158
+ data = tomllib.loads(text)
159
+ except tomllib.TOMLDecodeError:
160
+ return set()
161
+ deps: set[str] = set()
162
+
163
+ def collect(section) -> None:
164
+ if isinstance(section, dict):
165
+ deps.update(k.lower().replace("_", "-") for k in section)
166
+
167
+ for key in ("dependencies", "dev-dependencies", "build-dependencies"):
168
+ collect(data.get(key))
169
+ collect(data.get("workspace", {}).get("dependencies"))
170
+ for target in (data.get("target", {}) or {}).values():
171
+ if isinstance(target, dict):
172
+ collect(target.get("dependencies"))
173
+ return deps
174
+
175
+
176
+ @dataclass(slots=True)
177
+ class ManifestDeps:
178
+ """Dependencies per ecosystem, plus which manifest files provided them."""
179
+
180
+ python: set[str] = field(default_factory=set)
181
+ javascript: set[str] = field(default_factory=set)
182
+ go: set[str] = field(default_factory=set)
183
+ java: set[str] = field(default_factory=set)
184
+ ruby: set[str] = field(default_factory=set)
185
+ csharp: set[str] = field(default_factory=set)
186
+ php: set[str] = field(default_factory=set)
187
+ rust: set[str] = field(default_factory=set)
188
+ sources: list[str] = field(default_factory=list) # repo-relative manifest paths
189
+
190
+ def for_language(self, language: str) -> set[str]:
191
+ if language in ("typescript", "tsx"):
192
+ language = "javascript"
193
+ return getattr(self, language, set())
194
+
195
+
196
+ _MANIFEST_SPECS: list[tuple[str, str, object]] = [ # (glob, ecosystem, parser fn)
197
+ ("requirements*.txt", "python", parse_requirements_txt),
198
+ ("pyproject.toml", "python", parse_pyproject_toml),
199
+ ("package.json", "javascript", parse_package_json),
200
+ ("go.mod", "go", parse_go_mod),
201
+ ("pom.xml", "java", parse_pom_xml),
202
+ ("build.gradle", "java", parse_build_gradle),
203
+ ("build.gradle.kts", "java", parse_build_gradle),
204
+ ("Gemfile", "ruby", parse_gemfile),
205
+ ("*.csproj", "csharp", parse_csproj),
206
+ ("packages.config", "csharp", parse_packages_config),
207
+ ("composer.json", "php", parse_composer_json),
208
+ ("Cargo.toml", "rust", parse_cargo_toml),
209
+ ]
210
+
211
+ _MANIFEST_SEARCH_DEPTH = 3 # root plus a couple of levels for monorepos
212
+
213
+
214
+ def parse_manifests(root: str | Path) -> ManifestDeps:
215
+ root = Path(root)
216
+ result = ManifestDeps()
217
+ for pattern, ecosystem, parser in _MANIFEST_SPECS:
218
+ for depth in range(_MANIFEST_SEARCH_DEPTH):
219
+ glob = "/".join(["*"] * depth + [pattern]) if depth else pattern
220
+ for manifest in root.glob(glob):
221
+ if not manifest.is_file():
222
+ continue
223
+ if any(part in ("node_modules", "vendor", ".venv", "venv", "target", "packages")
224
+ for part in manifest.parts):
225
+ continue
226
+ try:
227
+ text = manifest.read_text(encoding="utf-8", errors="replace")
228
+ except OSError:
229
+ continue
230
+ deps = parser(text)
231
+ if deps:
232
+ getattr(result, ecosystem).update(deps)
233
+ result.sources.append(manifest.relative_to(root).as_posix())
234
+ return result