woolly 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- woolly/cache.py +21 -6
- woolly/commands/check.py +264 -82
- woolly/http.py +22 -3
- woolly/languages/base.py +187 -10
- woolly/languages/python.py +221 -8
- woolly/languages/rust.py +59 -1
- woolly/reporters/__init__.py +15 -1
- woolly/reporters/base.py +25 -0
- woolly/reporters/json.py +102 -3
- woolly/reporters/markdown.py +83 -9
- woolly/reporters/stdout.py +171 -38
- woolly/reporters/template.py +245 -0
- {woolly-0.3.0.dist-info → woolly-0.5.0.dist-info}/METADATA +3 -1
- woolly-0.5.0.dist-info/RECORD +26 -0
- woolly-0.3.0.dist-info/RECORD +0 -25
- {woolly-0.3.0.dist-info → woolly-0.5.0.dist-info}/WHEEL +0 -0
- {woolly-0.3.0.dist-info → woolly-0.5.0.dist-info}/entry_points.txt +0 -0
- {woolly-0.3.0.dist-info → woolly-0.5.0.dist-info}/licenses/LICENSE +0 -0
woolly/languages/base.py
CHANGED
|
@@ -30,7 +30,10 @@ from typing import Literal, Optional
|
|
|
30
30
|
from pydantic import BaseModel, Field
|
|
31
31
|
|
|
32
32
|
from woolly.cache import FEDORA_CACHE_TTL, read_cache, write_cache
|
|
33
|
-
from woolly.debug import log_cache_hit, log_cache_miss, log_command_output
|
|
33
|
+
from woolly.debug import log, log_cache_hit, log_cache_miss, log_command_output
|
|
34
|
+
|
|
35
|
+
# Default timeout (seconds) for dnf repoquery subprocess calls.
|
|
36
|
+
_DNF_TIMEOUT = 60
|
|
34
37
|
|
|
35
38
|
|
|
36
39
|
class PackageInfo(BaseModel):
|
|
@@ -41,6 +44,7 @@ class PackageInfo(BaseModel):
|
|
|
41
44
|
description: Optional[str] = None
|
|
42
45
|
homepage: Optional[str] = None
|
|
43
46
|
repository: Optional[str] = None
|
|
47
|
+
license: Optional[str] = None
|
|
44
48
|
|
|
45
49
|
|
|
46
50
|
class Dependency(BaseModel):
|
|
@@ -50,6 +54,14 @@ class Dependency(BaseModel):
|
|
|
50
54
|
version_requirement: str
|
|
51
55
|
optional: bool = False
|
|
52
56
|
kind: Literal["normal", "dev", "build"] = "normal"
|
|
57
|
+
group: Optional[str] = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class FeatureInfo(BaseModel):
|
|
61
|
+
"""Information about a feature flag (Rust) or extra (Python)."""
|
|
62
|
+
|
|
63
|
+
name: str
|
|
64
|
+
dependencies: list[str] = Field(default_factory=list)
|
|
53
65
|
|
|
54
66
|
|
|
55
67
|
class FedoraPackageStatus(BaseModel):
|
|
@@ -82,6 +94,10 @@ class LanguageProvider(ABC):
|
|
|
82
94
|
fedora_provides_prefix: str
|
|
83
95
|
cache_namespace: str
|
|
84
96
|
|
|
97
|
+
# Optional Fedora targeting attributes (set at runtime)
|
|
98
|
+
fedora_release: Optional[str] = None
|
|
99
|
+
fedora_repos: Optional[list[str]] = None
|
|
100
|
+
|
|
85
101
|
# ----------------------------------------------------------------
|
|
86
102
|
# Abstract methods - MUST be implemented by subclasses
|
|
87
103
|
# ----------------------------------------------------------------
|
|
@@ -113,6 +129,20 @@ class LanguageProvider(ABC):
|
|
|
113
129
|
"""
|
|
114
130
|
pass
|
|
115
131
|
|
|
132
|
+
@abstractmethod
|
|
133
|
+
def fetch_features(self, package_name: str, version: str) -> list[FeatureInfo]:
|
|
134
|
+
"""
|
|
135
|
+
Fetch feature flags (Rust) or extras (Python) for a specific package version.
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
package_name: The name of the package.
|
|
139
|
+
version: The specific version to get features for.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
List of FeatureInfo objects.
|
|
143
|
+
"""
|
|
144
|
+
pass
|
|
145
|
+
|
|
116
146
|
# ----------------------------------------------------------------
|
|
117
147
|
# Concrete methods - shared implementation for all providers
|
|
118
148
|
# ----------------------------------------------------------------
|
|
@@ -164,6 +194,92 @@ class LanguageProvider(ABC):
|
|
|
164
194
|
if d.kind == "normal" and (include_optional or not d.optional)
|
|
165
195
|
]
|
|
166
196
|
|
|
197
|
+
def get_dev_dependencies(
|
|
198
|
+
self,
|
|
199
|
+
package_name: str,
|
|
200
|
+
version: Optional[str] = None,
|
|
201
|
+
) -> list[Dependency]:
|
|
202
|
+
"""
|
|
203
|
+
Get dev dependencies for a package.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
package_name: The name of the package.
|
|
207
|
+
version: Specific version, or None for latest.
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
List of Dependency objects with kind='dev'.
|
|
211
|
+
"""
|
|
212
|
+
if version is None:
|
|
213
|
+
version = self.get_latest_version(package_name)
|
|
214
|
+
if version is None:
|
|
215
|
+
return []
|
|
216
|
+
|
|
217
|
+
deps = self.fetch_dependencies(package_name, version)
|
|
218
|
+
return [d for d in deps if d.kind == "dev"]
|
|
219
|
+
|
|
220
|
+
def get_build_dependencies(
|
|
221
|
+
self,
|
|
222
|
+
package_name: str,
|
|
223
|
+
version: Optional[str] = None,
|
|
224
|
+
) -> list[Dependency]:
|
|
225
|
+
"""
|
|
226
|
+
Get build dependencies for a package.
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
package_name: The name of the package.
|
|
230
|
+
version: Specific version, or None for latest.
|
|
231
|
+
|
|
232
|
+
Returns:
|
|
233
|
+
List of Dependency objects with kind='build'.
|
|
234
|
+
"""
|
|
235
|
+
if version is None:
|
|
236
|
+
version = self.get_latest_version(package_name)
|
|
237
|
+
if version is None:
|
|
238
|
+
return []
|
|
239
|
+
|
|
240
|
+
deps = self.fetch_dependencies(package_name, version)
|
|
241
|
+
return [d for d in deps if d.kind == "build"]
|
|
242
|
+
|
|
243
|
+
def get_all_dependencies(
|
|
244
|
+
self,
|
|
245
|
+
package_name: str,
|
|
246
|
+
version: Optional[str] = None,
|
|
247
|
+
include_optional: bool = False,
|
|
248
|
+
) -> tuple[list[tuple[str, str, bool]], list[Dependency], list[Dependency]]:
|
|
249
|
+
"""
|
|
250
|
+
Fetch all dependencies in a single call and partition by kind.
|
|
251
|
+
|
|
252
|
+
This avoids repeated ``fetch_dependencies`` (and its cache reads)
|
|
253
|
+
when the caller needs normal, dev, and build dependencies for the
|
|
254
|
+
same package/version.
|
|
255
|
+
|
|
256
|
+
Args:
|
|
257
|
+
package_name: The name of the package.
|
|
258
|
+
version: Specific version, or None for latest.
|
|
259
|
+
include_optional: If True, include optional normal dependencies.
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
Tuple of (normal_deps, dev_deps, build_deps) where
|
|
263
|
+
normal_deps is a list of ``(name, version_requirement, is_optional)``
|
|
264
|
+
tuples, and dev/build_deps are lists of :class:`Dependency`.
|
|
265
|
+
"""
|
|
266
|
+
if version is None:
|
|
267
|
+
version = self.get_latest_version(package_name)
|
|
268
|
+
if version is None:
|
|
269
|
+
return ([], [], [])
|
|
270
|
+
|
|
271
|
+
deps = self.fetch_dependencies(package_name, version)
|
|
272
|
+
|
|
273
|
+
normal = [
|
|
274
|
+
(d.name, d.version_requirement, d.optional)
|
|
275
|
+
for d in deps
|
|
276
|
+
if d.kind == "normal" and (include_optional or not d.optional)
|
|
277
|
+
]
|
|
278
|
+
dev = [d for d in deps if d.kind == "dev"]
|
|
279
|
+
build = [d for d in deps if d.kind == "build"]
|
|
280
|
+
|
|
281
|
+
return (normal, dev, build)
|
|
282
|
+
|
|
167
283
|
def get_fedora_provides_pattern(self, package_name: str) -> str:
|
|
168
284
|
"""
|
|
169
285
|
Get the Fedora provides pattern for this package.
|
|
@@ -214,6 +330,44 @@ class LanguageProvider(ABC):
|
|
|
214
330
|
# Fedora repository query methods - shared implementation
|
|
215
331
|
# ----------------------------------------------------------------
|
|
216
332
|
|
|
333
|
+
def _fedora_cache_suffix(self) -> str:
|
|
334
|
+
"""
|
|
335
|
+
Build a cache-key suffix that incorporates the targeted
|
|
336
|
+
Fedora release and repo selection so that results for different
|
|
337
|
+
targets are cached independently.
|
|
338
|
+
|
|
339
|
+
Returns:
|
|
340
|
+
A string suffix (may be empty when no targeting is set).
|
|
341
|
+
"""
|
|
342
|
+
parts: list[str] = []
|
|
343
|
+
if self.fedora_release:
|
|
344
|
+
parts.append(f"rel={self.fedora_release}")
|
|
345
|
+
if self.fedora_repos:
|
|
346
|
+
parts.append(f"repos={','.join(sorted(self.fedora_repos))}")
|
|
347
|
+
return ":".join(parts)
|
|
348
|
+
|
|
349
|
+
def _build_dnf_repoquery_cmd(self, extra_args: list[str]) -> list[str]:
|
|
350
|
+
"""
|
|
351
|
+
Build the base ``dnf repoquery`` command, injecting
|
|
352
|
+
``--releasever`` and ``--repo`` flags when Fedora targeting
|
|
353
|
+
attributes are set.
|
|
354
|
+
|
|
355
|
+
Args:
|
|
356
|
+
extra_args: Additional arguments appended after the base
|
|
357
|
+
command (e.g. ``["--whatprovides", pattern]``).
|
|
358
|
+
|
|
359
|
+
Returns:
|
|
360
|
+
Full command list ready for :func:`subprocess.check_output`.
|
|
361
|
+
"""
|
|
362
|
+
cmd = ["dnf", "repoquery"]
|
|
363
|
+
if self.fedora_release:
|
|
364
|
+
cmd.append(f"--releasever={self.fedora_release}")
|
|
365
|
+
if self.fedora_repos:
|
|
366
|
+
for repo in self.fedora_repos:
|
|
367
|
+
cmd.extend(["--repo", repo])
|
|
368
|
+
cmd.extend(extra_args)
|
|
369
|
+
return cmd
|
|
370
|
+
|
|
217
371
|
def _repoquery_package(
|
|
218
372
|
self, package_name: str
|
|
219
373
|
) -> tuple[bool, list[str], list[str]]:
|
|
@@ -226,7 +380,10 @@ class LanguageProvider(ABC):
|
|
|
226
380
|
Returns:
|
|
227
381
|
Tuple of (is_packaged, versions_list, package_names)
|
|
228
382
|
"""
|
|
383
|
+
suffix = self._fedora_cache_suffix()
|
|
229
384
|
cache_key = f"repoquery:{self.name}:{package_name}"
|
|
385
|
+
if suffix:
|
|
386
|
+
cache_key += f":{suffix}"
|
|
230
387
|
cached = read_cache("fedora", cache_key, FEDORA_CACHE_TTL)
|
|
231
388
|
if cached is not None:
|
|
232
389
|
log_cache_hit("fedora", cache_key)
|
|
@@ -234,14 +391,14 @@ class LanguageProvider(ABC):
|
|
|
234
391
|
|
|
235
392
|
log_cache_miss("fedora", cache_key)
|
|
236
393
|
provide_pattern = self.get_fedora_provides_pattern(package_name)
|
|
237
|
-
cmd =
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
394
|
+
cmd = self._build_dnf_repoquery_cmd(
|
|
395
|
+
[
|
|
396
|
+
"--whatprovides",
|
|
397
|
+
provide_pattern,
|
|
398
|
+
"--queryformat",
|
|
399
|
+
"%{NAME}|%{VERSION}",
|
|
400
|
+
]
|
|
401
|
+
)
|
|
245
402
|
|
|
246
403
|
try:
|
|
247
404
|
out = (
|
|
@@ -249,6 +406,7 @@ class LanguageProvider(ABC):
|
|
|
249
406
|
cmd,
|
|
250
407
|
stdin=subprocess.DEVNULL,
|
|
251
408
|
stderr=subprocess.DEVNULL,
|
|
409
|
+
timeout=_DNF_TIMEOUT,
|
|
252
410
|
)
|
|
253
411
|
.decode()
|
|
254
412
|
.strip()
|
|
@@ -272,6 +430,11 @@ class LanguageProvider(ABC):
|
|
|
272
430
|
result = (True, sorted(versions), sorted(packages))
|
|
273
431
|
write_cache("fedora", cache_key, [result[0], result[1], result[2]])
|
|
274
432
|
return result
|
|
433
|
+
except subprocess.TimeoutExpired:
|
|
434
|
+
log(" ".join(cmd), level="warning", reason="timeout")
|
|
435
|
+
result = (False, [], [])
|
|
436
|
+
write_cache("fedora", cache_key, list(result))
|
|
437
|
+
return result
|
|
275
438
|
except subprocess.CalledProcessError as e:
|
|
276
439
|
log_command_output(" ".join(cmd), "", exit_code=e.returncode)
|
|
277
440
|
result = (False, [], [])
|
|
@@ -288,7 +451,10 @@ class LanguageProvider(ABC):
|
|
|
288
451
|
Returns:
|
|
289
452
|
List of version strings provided by Fedora packages.
|
|
290
453
|
"""
|
|
454
|
+
suffix = self._fedora_cache_suffix()
|
|
291
455
|
cache_key = f"provides:{self.name}:{package_name}"
|
|
456
|
+
if suffix:
|
|
457
|
+
cache_key += f":{suffix}"
|
|
292
458
|
cached = read_cache("fedora", cache_key, FEDORA_CACHE_TTL)
|
|
293
459
|
if cached is not None:
|
|
294
460
|
log_cache_hit("fedora", cache_key)
|
|
@@ -297,7 +463,13 @@ class LanguageProvider(ABC):
|
|
|
297
463
|
log_cache_miss("fedora", cache_key)
|
|
298
464
|
provide_pattern = self.get_fedora_provides_pattern(package_name)
|
|
299
465
|
normalized = self.normalize_package_name(package_name)
|
|
300
|
-
cmd =
|
|
466
|
+
cmd = self._build_dnf_repoquery_cmd(
|
|
467
|
+
[
|
|
468
|
+
"--provides",
|
|
469
|
+
"--whatprovides",
|
|
470
|
+
provide_pattern,
|
|
471
|
+
]
|
|
472
|
+
)
|
|
301
473
|
|
|
302
474
|
try:
|
|
303
475
|
out = (
|
|
@@ -305,6 +477,7 @@ class LanguageProvider(ABC):
|
|
|
305
477
|
cmd,
|
|
306
478
|
stdin=subprocess.DEVNULL,
|
|
307
479
|
stderr=subprocess.DEVNULL,
|
|
480
|
+
timeout=_DNF_TIMEOUT,
|
|
308
481
|
)
|
|
309
482
|
.decode()
|
|
310
483
|
.strip()
|
|
@@ -329,6 +502,10 @@ class LanguageProvider(ABC):
|
|
|
329
502
|
result = sorted(versions)
|
|
330
503
|
write_cache("fedora", cache_key, result)
|
|
331
504
|
return result
|
|
505
|
+
except subprocess.TimeoutExpired:
|
|
506
|
+
log(" ".join(cmd), level="warning", reason="timeout")
|
|
507
|
+
write_cache("fedora", cache_key, [])
|
|
508
|
+
return []
|
|
332
509
|
except subprocess.CalledProcessError as e:
|
|
333
510
|
log_command_output(" ".join(cmd), "", exit_code=e.returncode)
|
|
334
511
|
write_cache("fedora", cache_key, [])
|
woolly/languages/python.py
CHANGED
|
@@ -16,7 +16,7 @@ from woolly.debug import (
|
|
|
16
16
|
log_cache_hit,
|
|
17
17
|
log_cache_miss,
|
|
18
18
|
)
|
|
19
|
-
from woolly.languages.base import Dependency, LanguageProvider, PackageInfo
|
|
19
|
+
from woolly.languages.base import Dependency, FeatureInfo, LanguageProvider, PackageInfo
|
|
20
20
|
|
|
21
21
|
PYPI_API = "https://pypi.org/pypi"
|
|
22
22
|
|
|
@@ -44,6 +44,7 @@ class PythonProvider(LanguageProvider):
|
|
|
44
44
|
description=cached["info"].get("summary"),
|
|
45
45
|
homepage=cached["info"].get("home_page"),
|
|
46
46
|
repository=cached["info"].get("project_url"),
|
|
47
|
+
license=self._extract_license(cached["info"]),
|
|
47
48
|
)
|
|
48
49
|
|
|
49
50
|
log_cache_miss(self.cache_namespace, cache_key)
|
|
@@ -69,8 +70,46 @@ class PythonProvider(LanguageProvider):
|
|
|
69
70
|
description=data["info"].get("summary"),
|
|
70
71
|
homepage=data["info"].get("home_page"),
|
|
71
72
|
repository=data["info"].get("project_url"),
|
|
73
|
+
license=self._extract_license(data["info"]),
|
|
72
74
|
)
|
|
73
75
|
|
|
76
|
+
def _fetch_version_data(self, package_name: str, version: str) -> Optional[dict]:
|
|
77
|
+
"""
|
|
78
|
+
Fetch the raw PyPI JSON response for a specific package version.
|
|
79
|
+
|
|
80
|
+
The result is cached under a ``version_data`` key so that both
|
|
81
|
+
:meth:`fetch_dependencies` and :meth:`fetch_features` can share
|
|
82
|
+
the same HTTP response without duplicating the request.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
package_name: The name of the package.
|
|
86
|
+
version: The specific version to fetch.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
Parsed JSON dict, or None on failure.
|
|
90
|
+
"""
|
|
91
|
+
cache_key = f"version_data:{package_name}:{version}"
|
|
92
|
+
cached = read_cache(self.cache_namespace, cache_key, DEFAULT_CACHE_TTL)
|
|
93
|
+
if cached is not None:
|
|
94
|
+
log_cache_hit(self.cache_namespace, cache_key)
|
|
95
|
+
if cached is False: # Explicit "not found" cache
|
|
96
|
+
return None
|
|
97
|
+
return cached
|
|
98
|
+
|
|
99
|
+
log_cache_miss(self.cache_namespace, cache_key)
|
|
100
|
+
url = f"{PYPI_API}/{package_name}/{version}/json"
|
|
101
|
+
log_api_request("GET", url)
|
|
102
|
+
r = http.get(url)
|
|
103
|
+
log_api_response(r.status_code, r.text[:500] if r.text else None)
|
|
104
|
+
|
|
105
|
+
if r.status_code != 200:
|
|
106
|
+
write_cache(self.cache_namespace, cache_key, False)
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
data = r.json()
|
|
110
|
+
write_cache(self.cache_namespace, cache_key, data)
|
|
111
|
+
return data
|
|
112
|
+
|
|
74
113
|
def fetch_dependencies(self, package_name: str, version: str) -> list[Dependency]:
|
|
75
114
|
"""
|
|
76
115
|
Fetch dependencies for a specific package version.
|
|
@@ -87,21 +126,17 @@ class PythonProvider(LanguageProvider):
|
|
|
87
126
|
version_requirement=d["version_requirement"],
|
|
88
127
|
optional=d.get("optional", False),
|
|
89
128
|
kind=d.get("kind", "normal"),
|
|
129
|
+
group=d.get("group"),
|
|
90
130
|
)
|
|
91
131
|
for d in cached
|
|
92
132
|
]
|
|
93
133
|
|
|
94
134
|
log_cache_miss(self.cache_namespace, cache_key)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
r = http.get(url)
|
|
98
|
-
log_api_response(r.status_code, r.text[:500] if r.text else None)
|
|
99
|
-
|
|
100
|
-
if r.status_code != 200:
|
|
135
|
+
data = self._fetch_version_data(package_name, version)
|
|
136
|
+
if data is None:
|
|
101
137
|
write_cache(self.cache_namespace, cache_key, [])
|
|
102
138
|
return []
|
|
103
139
|
|
|
104
|
-
data = r.json()
|
|
105
140
|
requires_dist = data["info"].get("requires_dist") or []
|
|
106
141
|
|
|
107
142
|
deps = []
|
|
@@ -117,6 +152,7 @@ class PythonProvider(LanguageProvider):
|
|
|
117
152
|
"version_requirement": d.version_requirement,
|
|
118
153
|
"optional": d.optional,
|
|
119
154
|
"kind": d.kind,
|
|
155
|
+
"group": d.group,
|
|
120
156
|
}
|
|
121
157
|
for d in deps
|
|
122
158
|
]
|
|
@@ -124,6 +160,180 @@ class PythonProvider(LanguageProvider):
|
|
|
124
160
|
|
|
125
161
|
return deps
|
|
126
162
|
|
|
163
|
+
def fetch_features(self, package_name: str, version: str) -> list[FeatureInfo]:
|
|
164
|
+
"""
|
|
165
|
+
Fetch extras (groups) for a specific Python package version.
|
|
166
|
+
|
|
167
|
+
PyPI provides extras via `provides_extra` and links dependencies
|
|
168
|
+
to extras via `requires_dist` markers.
|
|
169
|
+
"""
|
|
170
|
+
cache_key = f"features:{package_name}:{version}"
|
|
171
|
+
cached = read_cache(self.cache_namespace, cache_key, DEFAULT_CACHE_TTL)
|
|
172
|
+
if cached is not None:
|
|
173
|
+
log_cache_hit(self.cache_namespace, cache_key)
|
|
174
|
+
return [
|
|
175
|
+
FeatureInfo(name=f["name"], dependencies=f["dependencies"])
|
|
176
|
+
for f in cached
|
|
177
|
+
]
|
|
178
|
+
|
|
179
|
+
log_cache_miss(self.cache_namespace, cache_key)
|
|
180
|
+
data = self._fetch_version_data(package_name, version)
|
|
181
|
+
if data is None:
|
|
182
|
+
write_cache(self.cache_namespace, cache_key, [])
|
|
183
|
+
return []
|
|
184
|
+
|
|
185
|
+
provides_extra = data["info"].get("provides_extra") or []
|
|
186
|
+
requires_dist = data["info"].get("requires_dist") or []
|
|
187
|
+
|
|
188
|
+
# Build a mapping of extra -> dependencies
|
|
189
|
+
extras_map: dict[str, list[str]] = {extra: [] for extra in provides_extra}
|
|
190
|
+
|
|
191
|
+
for req in requires_dist:
|
|
192
|
+
extra_name = self._extract_extra_name(req)
|
|
193
|
+
if extra_name and extra_name in extras_map:
|
|
194
|
+
# Extract just the package name from the requirement
|
|
195
|
+
parsed = self._parse_requirement(req)
|
|
196
|
+
if parsed:
|
|
197
|
+
extras_map[extra_name].append(parsed.name)
|
|
198
|
+
|
|
199
|
+
features = [
|
|
200
|
+
FeatureInfo(name=name, dependencies=sorted(deps))
|
|
201
|
+
for name, deps in sorted(extras_map.items())
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
# Cache as dicts
|
|
205
|
+
cache_data = [
|
|
206
|
+
{"name": f.name, "dependencies": f.dependencies} for f in features
|
|
207
|
+
]
|
|
208
|
+
write_cache(self.cache_namespace, cache_key, cache_data)
|
|
209
|
+
|
|
210
|
+
return features
|
|
211
|
+
|
|
212
|
+
def _extract_extra_name(self, req_string: str) -> Optional[str]:
|
|
213
|
+
"""
|
|
214
|
+
Extract the extra name from a PEP 508 requirement string.
|
|
215
|
+
|
|
216
|
+
Examples:
|
|
217
|
+
"PySocks>=1.5.6; extra == 'socks'" -> "socks"
|
|
218
|
+
"requests>=2.20.0" -> None
|
|
219
|
+
"""
|
|
220
|
+
match = re.search(r"""extra\s*==\s*['"]([\w-]+)['"]""", req_string)
|
|
221
|
+
if match:
|
|
222
|
+
return match.group(1)
|
|
223
|
+
return None
|
|
224
|
+
|
|
225
|
+
# Well-known classifier suffix -> short SPDX-like name
|
|
226
|
+
_CLASSIFIER_LICENSE_MAP: dict[str, str] = {
|
|
227
|
+
"MIT License": "MIT",
|
|
228
|
+
"BSD License": "BSD",
|
|
229
|
+
"ISC License (ISCL)": "ISC",
|
|
230
|
+
"Apache Software License": "Apache-2.0",
|
|
231
|
+
"GNU General Public License v2 (GPLv2)": "GPL-2.0",
|
|
232
|
+
"GNU General Public License v2 or later (GPLv2+)": "GPL-2.0-or-later",
|
|
233
|
+
"GNU General Public License v3 (GPLv3)": "GPL-3.0",
|
|
234
|
+
"GNU General Public License v3 or later (GPLv3+)": "GPL-3.0-or-later",
|
|
235
|
+
"GNU Lesser General Public License v2 (LGPLv2)": "LGPL-2.0",
|
|
236
|
+
"GNU Lesser General Public License v2 or later (LGPLv2+)": "LGPL-2.0-or-later",
|
|
237
|
+
"GNU Lesser General Public License v3 (LGPLv3)": "LGPL-3.0",
|
|
238
|
+
"GNU Lesser General Public License v3 or later (LGPLv3+)": "LGPL-3.0-or-later",
|
|
239
|
+
"GNU Affero General Public License v3": "AGPL-3.0",
|
|
240
|
+
"GNU Affero General Public License v3 or later (AGPLv3+)": "AGPL-3.0-or-later",
|
|
241
|
+
"Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0",
|
|
242
|
+
"Eclipse Public License 1.0 (EPL-1.0)": "EPL-1.0",
|
|
243
|
+
"Eclipse Public License 2.0 (EPL-2.0)": "EPL-2.0",
|
|
244
|
+
"The Unlicense (Unlicense)": "Unlicense",
|
|
245
|
+
"Public Domain": "Public Domain",
|
|
246
|
+
"Python Software Foundation License": "PSF",
|
|
247
|
+
"Zope Public License": "ZPL",
|
|
248
|
+
"Academic Free License (AFL)": "AFL",
|
|
249
|
+
"Artistic License": "Artistic",
|
|
250
|
+
"Boost Software License 1.0 (BSL-1.0)": "BSL-1.0",
|
|
251
|
+
"European Union Public Licence 1.1 (EUPL 1.1)": "EUPL-1.1",
|
|
252
|
+
"European Union Public Licence 1.2 (EUPL 1.2)": "EUPL-1.2",
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
@staticmethod
|
|
256
|
+
def _looks_like_license_name(value: str) -> bool:
|
|
257
|
+
"""
|
|
258
|
+
Heuristic check: does this look like a short license name/identifier
|
|
259
|
+
rather than the full license text?
|
|
260
|
+
|
|
261
|
+
A short license name is typically under 100 characters, fits on a
|
|
262
|
+
single line, and does not contain common full-text markers like
|
|
263
|
+
"Permission is hereby granted" or "THE SOFTWARE IS PROVIDED".
|
|
264
|
+
"""
|
|
265
|
+
if len(value) > 100:
|
|
266
|
+
return False
|
|
267
|
+
if "\n" in value:
|
|
268
|
+
return False
|
|
269
|
+
# Common full-text markers
|
|
270
|
+
full_text_markers = [
|
|
271
|
+
"permission is hereby granted",
|
|
272
|
+
"the software is provided",
|
|
273
|
+
"redistribution and use",
|
|
274
|
+
"licensed under the",
|
|
275
|
+
"this software",
|
|
276
|
+
"copyright (c)",
|
|
277
|
+
"all rights reserved",
|
|
278
|
+
'provided "as is"',
|
|
279
|
+
"provided 'as is'",
|
|
280
|
+
]
|
|
281
|
+
lower = value.lower()
|
|
282
|
+
return not any(marker in lower for marker in full_text_markers)
|
|
283
|
+
|
|
284
|
+
def _license_from_classifiers(self, info: dict) -> Optional[str]:
|
|
285
|
+
"""
|
|
286
|
+
Try to derive a short license name from the trove classifiers.
|
|
287
|
+
|
|
288
|
+
PyPI classifiers follow the pattern::
|
|
289
|
+
|
|
290
|
+
License :: OSI Approved :: MIT License
|
|
291
|
+
|
|
292
|
+
Returns the first match mapped to a short identifier, or the
|
|
293
|
+
classifier suffix as-is if no mapping exists.
|
|
294
|
+
"""
|
|
295
|
+
classifiers = info.get("classifiers") or []
|
|
296
|
+
for clf in classifiers:
|
|
297
|
+
if clf.startswith("License :: OSI Approved :: "):
|
|
298
|
+
suffix = clf.split(" :: ")[-1]
|
|
299
|
+
return self._CLASSIFIER_LICENSE_MAP.get(suffix, suffix)
|
|
300
|
+
if clf.startswith("License :: "):
|
|
301
|
+
suffix = clf.split(" :: ")[-1]
|
|
302
|
+
return self._CLASSIFIER_LICENSE_MAP.get(suffix, suffix)
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
def _extract_license(self, info: dict) -> Optional[str]:
|
|
306
|
+
"""
|
|
307
|
+
Extract a *short* license identifier from PyPI package info.
|
|
308
|
+
|
|
309
|
+
Resolution order:
|
|
310
|
+
|
|
311
|
+
1. ``license_expression`` (PEP 639) – always a proper SPDX expression.
|
|
312
|
+
2. ``license`` field, **only** when it looks like a short name
|
|
313
|
+
(not the full license text that older packages sometimes embed).
|
|
314
|
+
3. Trove classifiers (``License :: …``).
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
info: The ``info`` dict from the PyPI API response.
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
Short license string, or None if unavailable.
|
|
321
|
+
"""
|
|
322
|
+
# 1. PEP 639 license expression – best source
|
|
323
|
+
license_expr = info.get("license_expression")
|
|
324
|
+
if license_expr and license_expr.strip():
|
|
325
|
+
return license_expr.strip()
|
|
326
|
+
|
|
327
|
+
# 2. license field, only if it looks like a short identifier
|
|
328
|
+
license_val = info.get("license")
|
|
329
|
+
if license_val and license_val.strip():
|
|
330
|
+
cleaned = license_val.strip()
|
|
331
|
+
if self._looks_like_license_name(cleaned):
|
|
332
|
+
return cleaned
|
|
333
|
+
|
|
334
|
+
# 3. Fall back to classifiers
|
|
335
|
+
return self._license_from_classifiers(info)
|
|
336
|
+
|
|
127
337
|
def _parse_requirement(self, req_string: str) -> Optional[Dependency]:
|
|
128
338
|
"""
|
|
129
339
|
Parse a PEP 508 requirement string.
|
|
@@ -136,11 +346,13 @@ class PythonProvider(LanguageProvider):
|
|
|
136
346
|
# Check if this is an optional/extra dependency
|
|
137
347
|
is_optional = False
|
|
138
348
|
kind = "normal"
|
|
349
|
+
group = None
|
|
139
350
|
|
|
140
351
|
if "extra ==" in req_string or "extra==" in req_string:
|
|
141
352
|
is_optional = True
|
|
142
353
|
# Keep kind as "normal" so optional dependencies can be included
|
|
143
354
|
# when --optional flag is used
|
|
355
|
+
group = self._extract_extra_name(req_string)
|
|
144
356
|
|
|
145
357
|
# Extract the package name and version requirement
|
|
146
358
|
# Handle environment markers (everything after ';')
|
|
@@ -166,6 +378,7 @@ class PythonProvider(LanguageProvider):
|
|
|
166
378
|
version_requirement=version_req or "*",
|
|
167
379
|
optional=is_optional,
|
|
168
380
|
kind=kind,
|
|
381
|
+
group=group,
|
|
169
382
|
)
|
|
170
383
|
|
|
171
384
|
def normalize_package_name(self, package_name: str) -> str:
|
woolly/languages/rust.py
CHANGED
|
@@ -15,7 +15,7 @@ from woolly.debug import (
|
|
|
15
15
|
log_cache_hit,
|
|
16
16
|
log_cache_miss,
|
|
17
17
|
)
|
|
18
|
-
from woolly.languages.base import Dependency, LanguageProvider, PackageInfo
|
|
18
|
+
from woolly.languages.base import Dependency, FeatureInfo, LanguageProvider, PackageInfo
|
|
19
19
|
|
|
20
20
|
CRATES_API = "https://crates.io/api/v1/crates"
|
|
21
21
|
|
|
@@ -29,6 +29,25 @@ class RustProvider(LanguageProvider):
|
|
|
29
29
|
fedora_provides_prefix = "crate"
|
|
30
30
|
cache_namespace = "crates"
|
|
31
31
|
|
|
32
|
+
@staticmethod
|
|
33
|
+
def _extract_license(data: dict) -> Optional[str]:
|
|
34
|
+
"""Extract license from crates.io API response.
|
|
35
|
+
|
|
36
|
+
Some crates have ``license: null`` at the crate level but include the
|
|
37
|
+
license in the ``versions`` array. Fall back to the latest version's
|
|
38
|
+
license when the crate-level field is missing.
|
|
39
|
+
"""
|
|
40
|
+
license_val = data.get("crate", {}).get("license")
|
|
41
|
+
if license_val:
|
|
42
|
+
return license_val
|
|
43
|
+
|
|
44
|
+
# Fall back to the latest version's license (versions are newest-first)
|
|
45
|
+
versions = data.get("versions") or []
|
|
46
|
+
if versions:
|
|
47
|
+
return versions[0].get("license")
|
|
48
|
+
|
|
49
|
+
return None
|
|
50
|
+
|
|
32
51
|
def fetch_package_info(self, package_name: str) -> Optional[PackageInfo]:
|
|
33
52
|
"""Fetch crate information from crates.io."""
|
|
34
53
|
cache_key = f"info:{package_name}"
|
|
@@ -43,6 +62,7 @@ class RustProvider(LanguageProvider):
|
|
|
43
62
|
description=cached["crate"].get("description"),
|
|
44
63
|
homepage=cached["crate"].get("homepage"),
|
|
45
64
|
repository=cached["crate"].get("repository"),
|
|
65
|
+
license=self._extract_license(cached),
|
|
46
66
|
)
|
|
47
67
|
|
|
48
68
|
log_cache_miss(self.cache_namespace, cache_key)
|
|
@@ -68,6 +88,7 @@ class RustProvider(LanguageProvider):
|
|
|
68
88
|
description=data["crate"].get("description"),
|
|
69
89
|
homepage=data["crate"].get("homepage"),
|
|
70
90
|
repository=data["crate"].get("repository"),
|
|
91
|
+
license=self._extract_license(data),
|
|
71
92
|
)
|
|
72
93
|
|
|
73
94
|
def fetch_dependencies(self, package_name: str, version: str) -> list[Dependency]:
|
|
@@ -110,6 +131,43 @@ class RustProvider(LanguageProvider):
|
|
|
110
131
|
for d in deps
|
|
111
132
|
]
|
|
112
133
|
|
|
134
|
+
def fetch_features(self, package_name: str, version: str) -> list[FeatureInfo]:
|
|
135
|
+
"""Fetch feature flags for a specific crate version from crates.io."""
|
|
136
|
+
cache_key = f"features:{package_name}:{version}"
|
|
137
|
+
cached = read_cache(self.cache_namespace, cache_key, DEFAULT_CACHE_TTL)
|
|
138
|
+
if cached is not None:
|
|
139
|
+
log_cache_hit(self.cache_namespace, cache_key)
|
|
140
|
+
return [
|
|
141
|
+
FeatureInfo(name=f["name"], dependencies=f["dependencies"])
|
|
142
|
+
for f in cached
|
|
143
|
+
]
|
|
144
|
+
|
|
145
|
+
log_cache_miss(self.cache_namespace, cache_key)
|
|
146
|
+
url = f"{CRATES_API}/{package_name}/{version}"
|
|
147
|
+
log_api_request("GET", url)
|
|
148
|
+
r = http.get(url)
|
|
149
|
+
log_api_response(r.status_code, r.text[:500] if r.text else None)
|
|
150
|
+
|
|
151
|
+
if r.status_code != 200:
|
|
152
|
+
write_cache(self.cache_namespace, cache_key, [])
|
|
153
|
+
return []
|
|
154
|
+
|
|
155
|
+
data = r.json()
|
|
156
|
+
features_dict = data.get("version", {}).get("features", {})
|
|
157
|
+
|
|
158
|
+
features = [
|
|
159
|
+
FeatureInfo(name=name, dependencies=deps)
|
|
160
|
+
for name, deps in sorted(features_dict.items())
|
|
161
|
+
]
|
|
162
|
+
|
|
163
|
+
# Cache as dicts
|
|
164
|
+
cache_data = [
|
|
165
|
+
{"name": f.name, "dependencies": f.dependencies} for f in features
|
|
166
|
+
]
|
|
167
|
+
write_cache(self.cache_namespace, cache_key, cache_data)
|
|
168
|
+
|
|
169
|
+
return features
|
|
170
|
+
|
|
113
171
|
def get_alternative_names(self, package_name: str) -> list[str]:
|
|
114
172
|
"""
|
|
115
173
|
Get alternative names to try for crate lookup.
|