dockerls 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. dockerls/__init__.py +31 -0
  2. dockerls/application/__init__.py +0 -0
  3. dockerls/application/dto/__init__.py +3 -0
  4. dockerls/application/dto/analysis.py +257 -0
  5. dockerls/application/services/__init__.py +0 -0
  6. dockerls/application/services/alternatives_lookup.py +167 -0
  7. dockerls/application/services/composite_repository.py +106 -0
  8. dockerls/application/services/cross_validation.py +233 -0
  9. dockerls/application/services/ecosystems.py +350 -0
  10. dockerls/application/services/fallback_scanner.py +97 -0
  11. dockerls/application/services/hardening_analysis.py +174 -0
  12. dockerls/application/services/migration.py +297 -0
  13. dockerls/application/services/progress.py +56 -0
  14. dockerls/application/services/remediation.py +321 -0
  15. dockerls/application/services/scan_history_store.py +88 -0
  16. dockerls/application/services/scanner_factory.py +88 -0
  17. dockerls/application/services/source_registry.py +151 -0
  18. dockerls/application/services/tag_history_store.py +76 -0
  19. dockerls/application/services/teardown.py +50 -0
  20. dockerls/application/services/verdict.py +298 -0
  21. dockerls/application/services/version_discovery.py +108 -0
  22. dockerls/application/use_cases/__init__.py +0 -0
  23. dockerls/application/use_cases/analyze_dockerfile.py +103 -0
  24. dockerls/application/use_cases/analyze_image.py +173 -0
  25. dockerls/application/use_cases/build_image.py +1795 -0
  26. dockerls/application/use_cases/compare_images.py +91 -0
  27. dockerls/application/use_cases/fleet_scan.py +240 -0
  28. dockerls/application/use_cases/recommend_images.py +1078 -0
  29. dockerls/application/use_cases/registry_audit.py +133 -0
  30. dockerls/application/use_cases/search_images.py +23 -0
  31. dockerls/application/use_cases/upgrade_base.py +167 -0
  32. dockerls/cache/__init__.py +0 -0
  33. dockerls/cache/sqlite_cache.py +184 -0
  34. dockerls/cli/__init__.py +0 -0
  35. dockerls/cli/analysis_baseline.py +98 -0
  36. dockerls/cli/app.py +294 -0
  37. dockerls/cli/commands/__init__.py +0 -0
  38. dockerls/cli/commands/advisor.py +262 -0
  39. dockerls/cli/commands/alternatives.py +291 -0
  40. dockerls/cli/commands/analyze.py +429 -0
  41. dockerls/cli/commands/analyze_dockerfile.py +104 -0
  42. dockerls/cli/commands/base_cmd.py +244 -0
  43. dockerls/cli/commands/base_image.py +551 -0
  44. dockerls/cli/commands/build.py +1300 -0
  45. dockerls/cli/commands/cache_cmd.py +104 -0
  46. dockerls/cli/commands/compare.py +177 -0
  47. dockerls/cli/commands/controls.py +110 -0
  48. dockerls/cli/commands/doctor.py +566 -0
  49. dockerls/cli/commands/export.py +81 -0
  50. dockerls/cli/commands/fleet.py +159 -0
  51. dockerls/cli/commands/health.py +84 -0
  52. dockerls/cli/commands/login.py +53 -0
  53. dockerls/cli/commands/policy_cmd.py +111 -0
  54. dockerls/cli/commands/provenance_cmd.py +162 -0
  55. dockerls/cli/commands/recommend.py +761 -0
  56. dockerls/cli/commands/registry_audit_cmd.py +103 -0
  57. dockerls/cli/commands/sbom.py +144 -0
  58. dockerls/cli/commands/search.py +86 -0
  59. dockerls/cli/commands/verify.py +115 -0
  60. dockerls/cli/commands/version.py +12 -0
  61. dockerls/cli/commands/vex_cmd.py +117 -0
  62. dockerls/cli/dependencies.py +530 -0
  63. dockerls/cli/image_names.py +79 -0
  64. dockerls/cli/options.py +42 -0
  65. dockerls/cli/progress.py +145 -0
  66. dockerls/cli/publish_prompt.py +123 -0
  67. dockerls/cli/rendering.py +214 -0
  68. dockerls/cli/runtime.py +65 -0
  69. dockerls/cli/scan_failure.py +71 -0
  70. dockerls/cli/text.py +39 -0
  71. dockerls/cli/validators.py +35 -0
  72. dockerls/cli/vulnerability_view.py +154 -0
  73. dockerls/domain/__init__.py +0 -0
  74. dockerls/domain/entities/__init__.py +75 -0
  75. dockerls/domain/entities/declared_metadata.py +147 -0
  76. dockerls/domain/entities/dockerfile_analysis.py +318 -0
  77. dockerls/domain/entities/image.py +109 -0
  78. dockerls/domain/entities/image_facts.py +137 -0
  79. dockerls/domain/entities/recommendation.py +33 -0
  80. dockerls/domain/entities/scan_result.py +129 -0
  81. dockerls/domain/entities/vulnerability.py +232 -0
  82. dockerls/domain/interfaces/__init__.py +17 -0
  83. dockerls/domain/interfaces/cache_store.py +18 -0
  84. dockerls/domain/interfaces/dockerfile_validator.py +99 -0
  85. dockerls/domain/interfaces/eol_checker.py +11 -0
  86. dockerls/domain/interfaces/image_repository.py +15 -0
  87. dockerls/domain/interfaces/scanner.py +15 -0
  88. dockerls/domain/security_controls.py +362 -0
  89. dockerls/domain/value_objects/__init__.py +49 -0
  90. dockerls/domain/value_objects/attack_surface.py +198 -0
  91. dockerls/domain/value_objects/base_recipe.py +600 -0
  92. dockerls/domain/value_objects/base_upgrade.py +292 -0
  93. dockerls/domain/value_objects/build_labels.py +99 -0
  94. dockerls/domain/value_objects/build_policy.py +412 -0
  95. dockerls/domain/value_objects/confidence.py +156 -0
  96. dockerls/domain/value_objects/fleet.py +174 -0
  97. dockerls/domain/value_objects/gate.py +327 -0
  98. dockerls/domain/value_objects/hardening.py +303 -0
  99. dockerls/domain/value_objects/image_reference.py +90 -0
  100. dockerls/domain/value_objects/inheritance.py +352 -0
  101. dockerls/domain/value_objects/network_policy.py +280 -0
  102. dockerls/domain/value_objects/production_readiness.py +145 -0
  103. dockerls/domain/value_objects/provenance.py +211 -0
  104. dockerls/domain/value_objects/recipe_diff.py +188 -0
  105. dockerls/domain/value_objects/registry_audit.py +195 -0
  106. dockerls/domain/value_objects/registry_target.py +225 -0
  107. dockerls/domain/value_objects/remediation_score.py +62 -0
  108. dockerls/domain/value_objects/scan_history.py +183 -0
  109. dockerls/domain/value_objects/scan_plan.py +193 -0
  110. dockerls/domain/value_objects/scanner_db.py +143 -0
  111. dockerls/domain/value_objects/security_score.py +160 -0
  112. dockerls/domain/value_objects/security_tier.py +122 -0
  113. dockerls/domain/value_objects/tag_history.py +180 -0
  114. dockerls/domain/value_objects/tool_release.py +253 -0
  115. dockerls/domain/value_objects/tristate.py +47 -0
  116. dockerls/domain/value_objects/vex.py +249 -0
  117. dockerls/exit_codes.py +23 -0
  118. dockerls/exporters/__init__.py +0 -0
  119. dockerls/exporters/base.py +17 -0
  120. dockerls/exporters/csv_exporter.py +85 -0
  121. dockerls/exporters/factory.py +31 -0
  122. dockerls/exporters/html_exporter.py +105 -0
  123. dockerls/exporters/json_exporter.py +19 -0
  124. dockerls/exporters/markdown_exporter.py +84 -0
  125. dockerls/exporters/sarif_exporter.py +245 -0
  126. dockerls/infrastructure/__init__.py +0 -0
  127. dockerls/infrastructure/config/__init__.py +0 -0
  128. dockerls/infrastructure/config/policy_file.py +165 -0
  129. dockerls/infrastructure/config/settings.py +197 -0
  130. dockerls/infrastructure/database/__init__.py +0 -0
  131. dockerls/infrastructure/database/models.py +78 -0
  132. dockerls/infrastructure/dockerfile_validator.py +1899 -0
  133. dockerls/infrastructure/evidence.py +99 -0
  134. dockerls/infrastructure/hashing.py +165 -0
  135. dockerls/infrastructure/logging/__init__.py +0 -0
  136. dockerls/infrastructure/logging/setup.py +98 -0
  137. dockerls/infrastructure/network/__init__.py +0 -0
  138. dockerls/infrastructure/network/guarded_client.py +107 -0
  139. dockerls/infrastructure/network/host_guard.py +117 -0
  140. dockerls/infrastructure/redaction.py +135 -0
  141. dockerls/infrastructure/templates/hardening/alpine.dockerfile +44 -0
  142. dockerls/infrastructure/templates/hardening/debian.dockerfile +45 -0
  143. dockerls/infrastructure/templates/hardening/distroless.dockerfile +34 -0
  144. dockerls/infrastructure/templates/hardening/go-alpine.dockerfile +52 -0
  145. dockerls/infrastructure/templates/hardening/go-debian.dockerfile +54 -0
  146. dockerls/infrastructure/templates/hardening/go-distroless.dockerfile +43 -0
  147. dockerls/infrastructure/templates/hardening/go-scratch.dockerfile +48 -0
  148. dockerls/infrastructure/templates/hardening/go.dockerfile +50 -0
  149. dockerls/infrastructure/templates/hardening/gradle-alpine.dockerfile +51 -0
  150. dockerls/infrastructure/templates/hardening/gradle.dockerfile +52 -0
  151. dockerls/infrastructure/templates/hardening/java-alpine.dockerfile +50 -0
  152. dockerls/infrastructure/templates/hardening/java-debian.dockerfile +50 -0
  153. dockerls/infrastructure/templates/hardening/java-distroless.dockerfile +39 -0
  154. dockerls/infrastructure/templates/hardening/java-ubuntu.dockerfile +54 -0
  155. dockerls/infrastructure/templates/hardening/java.dockerfile +60 -0
  156. dockerls/infrastructure/templates/hardening/maven-alpine.dockerfile +56 -0
  157. dockerls/infrastructure/templates/hardening/maven.dockerfile +57 -0
  158. dockerls/infrastructure/templates/hardening/node-alpine.dockerfile +47 -0
  159. dockerls/infrastructure/templates/hardening/node-debian.dockerfile +54 -0
  160. dockerls/infrastructure/templates/hardening/node-distroless.dockerfile +46 -0
  161. dockerls/infrastructure/templates/hardening/node-ubuntu.dockerfile +63 -0
  162. dockerls/infrastructure/templates/hardening/node.dockerfile +61 -0
  163. dockerls/infrastructure/templates/hardening/php-alpine.dockerfile +45 -0
  164. dockerls/infrastructure/templates/hardening/php-debian.dockerfile +45 -0
  165. dockerls/infrastructure/templates/hardening/php-ubuntu.dockerfile +49 -0
  166. dockerls/infrastructure/templates/hardening/php.dockerfile +44 -0
  167. dockerls/infrastructure/templates/hardening/python-alpine.dockerfile +51 -0
  168. dockerls/infrastructure/templates/hardening/python-debian.dockerfile +54 -0
  169. dockerls/infrastructure/templates/hardening/python-distroless.dockerfile +51 -0
  170. dockerls/infrastructure/templates/hardening/python-ubuntu.dockerfile +60 -0
  171. dockerls/infrastructure/templates/hardening/python.dockerfile +58 -0
  172. dockerls/infrastructure/templates/hardening/ruby-alpine.dockerfile +48 -0
  173. dockerls/infrastructure/templates/hardening/ruby-debian.dockerfile +50 -0
  174. dockerls/infrastructure/templates/hardening/rust-alpine.dockerfile +50 -0
  175. dockerls/infrastructure/templates/hardening/rust-debian.dockerfile +48 -0
  176. dockerls/infrastructure/templates/hardening/rust-scratch.dockerfile +44 -0
  177. dockerls/infrastructure/templates/hardening/rust.dockerfile +54 -0
  178. dockerls/infrastructure/templates/hardening/ubuntu.dockerfile +49 -0
  179. dockerls/infrastructure/toolchain/__init__.py +0 -0
  180. dockerls/infrastructure/toolchain/db_metadata.py +115 -0
  181. dockerls/infrastructure/toolchain/installer.py +435 -0
  182. dockerls/integrations/__init__.py +0 -0
  183. dockerls/integrations/dhi/__init__.py +0 -0
  184. dockerls/integrations/dhi/catalog.py +457 -0
  185. dockerls/integrations/dhi/definition.py +151 -0
  186. dockerls/integrations/dhi/repository.py +238 -0
  187. dockerls/integrations/dockerhub/__init__.py +0 -0
  188. dockerls/integrations/dockerhub/client.py +318 -0
  189. dockerls/integrations/dockerhub/urls.py +75 -0
  190. dockerls/integrations/endoflife/__init__.py +0 -0
  191. dockerls/integrations/endoflife/checker.py +216 -0
  192. dockerls/integrations/engine/__init__.py +0 -0
  193. dockerls/integrations/engine/batch.py +197 -0
  194. dockerls/integrations/engine/client.py +330 -0
  195. dockerls/integrations/engine/locator.py +96 -0
  196. dockerls/integrations/exploitdb/__init__.py +0 -0
  197. dockerls/integrations/exploitdb/client.py +271 -0
  198. dockerls/integrations/grype/__init__.py +0 -0
  199. dockerls/integrations/grype/scanner.py +336 -0
  200. dockerls/integrations/registry/__init__.py +0 -0
  201. dockerls/integrations/registry/hardened.py +284 -0
  202. dockerls/integrations/registry/inspector.py +420 -0
  203. dockerls/integrations/registry/oci.py +259 -0
  204. dockerls/integrations/registry/private.py +79 -0
  205. dockerls/integrations/registry/urls.py +36 -0
  206. dockerls/integrations/scan_errors.py +67 -0
  207. dockerls/integrations/scan_target.py +57 -0
  208. dockerls/integrations/signing/__init__.py +0 -0
  209. dockerls/integrations/signing/cosign.py +484 -0
  210. dockerls/integrations/threat_intel/__init__.py +0 -0
  211. dockerls/integrations/threat_intel/client.py +290 -0
  212. dockerls/integrations/trivy/__init__.py +0 -0
  213. dockerls/integrations/trivy/cache_pool.py +176 -0
  214. dockerls/integrations/trivy/scanner.py +447 -0
  215. dockerls/utils/__init__.py +0 -0
  216. dockerls/utils/auth.py +108 -0
  217. dockerls/utils/executables.py +39 -0
  218. dockerls/utils/ignore_file.py +130 -0
  219. dockerls/utils/rate_limit.py +130 -0
  220. dockerls/utils/resources.py +183 -0
  221. dockerls/utils/retry.py +31 -0
  222. dockerls/utils/safe_yaml.py +166 -0
  223. dockerls/utils/subprocess_runner.py +216 -0
  224. dockerls/utils/validation.py +72 -0
  225. dockerls-1.0.0.dist-info/METADATA +563 -0
  226. dockerls-1.0.0.dist-info/RECORD +230 -0
  227. dockerls-1.0.0.dist-info/WHEEL +5 -0
  228. dockerls-1.0.0.dist-info/entry_points.txt +2 -0
  229. dockerls-1.0.0.dist-info/licenses/LICENSE +21 -0
  230. dockerls-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,130 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from datetime import date
5
+ from pathlib import Path
6
+
7
+ import yaml
8
+ from loguru import logger
9
+ from pydantic import BaseModel, field_validator
10
+
11
+ from dockerls.domain.value_objects.vex import VexJustification, parse_justification
12
+
13
+ DEFAULT_IGNORE_FILENAME = ".dockerls-ignore.yaml"
14
+
15
+ #: A forma mínima de um identificador de aviso. Larga o bastante para
16
+ #: `CVE-2024-1234`, `GHSA-xxxx-yyyy-zzzz`, `DSA-5555-1`, `RUSTSEC-2024-0001`
17
+ #: e `ALAS2-2024-2500`; estreita o bastante para recusar o vazio, que casa
18
+ #: com todo achado que o scanner reportou sem identificador.
19
+ _ADVISORY_ID = re.compile(r"^[A-Z][A-Z0-9]*(?:[-_.][A-Z0-9]+)+$")
20
+
21
+
22
+ class IgnoreRule(BaseModel):
23
+ cve: str
24
+ justification: str = ""
25
+ expires: date | None = None
26
+ #: A justificativa **do padrão VEX**, quando quem escreveu a regra
27
+ #: souber declarar uma. Opcional de propósito: `not_affected` é uma
28
+ #: afirmação técnica com vocabulário fechado, e a maioria das isenções
29
+ #: é decisão de risco ("aceito até o Q3"), que não é nenhuma delas.
30
+ #:
31
+ #: Sem ela, o documento OpenVEX sai como `affected` com a justificativa
32
+ #: em texto -- o consumidor vê a exceção e o motivo, sem receber uma
33
+ #: alegação técnica que ninguém fez. Ver `domain/value_objects/vex.py`.
34
+ vex_justification: str = ""
35
+
36
+ @field_validator("vex_justification")
37
+ @classmethod
38
+ def _known_vex_justification(cls, v: str) -> str:
39
+ """Recusa um valor que não existe no padrão.
40
+
41
+ Aceitá-lo produziria um documento VEX que outra ferramenta rejeita
42
+ na leitura -- e o erro apareceria longe daqui, sem dizer que veio
43
+ de uma linha do arquivo de isenções.
44
+ """
45
+ text = v.strip().lower()
46
+ if not text:
47
+ return ""
48
+ if parse_justification(text) is None:
49
+ allowed = ", ".join(str(j) for j in VexJustification)
50
+ raise ValueError(f"unknown VEX justification {v!r}; expected one of: {allowed}")
51
+ return text
52
+
53
+ @field_validator("cve")
54
+ @classmethod
55
+ def _upper_cve(cls, v: str) -> str:
56
+ """Normaliza o identificador, e recusa um que não identifica nada.
57
+
58
+ Um `cve: ""` passava por aqui e entrava no conjunto de isenções como
59
+ a string vazia. As isenções são aplicadas com
60
+ `vuln.cve_id.upper() not in ignored`, e um scanner **deixa
61
+ `cve_id` vazio** quando o aviso não tem identificador publicado --
62
+ o Trivy faz exatamente isso, como o exportador SARIF documenta. Uma
63
+ única linha em branco no arquivo de isenções apagava portanto
64
+ *todos* os achados sem identificador do relatório, e com eles do
65
+ score, do tier e do veredito de produção: vulnerabilidades reais
66
+ somem e a imagem sobe de nota.
67
+
68
+ A forma exigida é frouxa de propósito -- `GHSA-...`, `DSA-...`,
69
+ `RUSTSEC-...` e `ALAS-...` são identificadores legítimos e não
70
+ parecem com `CVE-`. O que ela recusa é o que não identifica: vazio,
71
+ espaço em branco, e qualquer coisa curta demais para ser um
72
+ identificador de aviso.
73
+ """
74
+ text = v.strip().upper()
75
+ if not _ADVISORY_ID.match(text):
76
+ raise ValueError(
77
+ f"{v!r} does not name an advisory. An exemption has to say which "
78
+ "vulnerability it exempts: a blank id matches every finding the "
79
+ "scanner reported without one, and silently drops them from the scan"
80
+ )
81
+ return text
82
+
83
+ @property
84
+ def is_expired(self) -> bool:
85
+ if self.expires is None:
86
+ return False
87
+ return date.today() > self.expires
88
+
89
+
90
+ def load_ignore_rules(path: Path | None = None) -> list[IgnoreRule]:
91
+ """Load CVE ignore rules from a `.dockerls-ignore.yaml` file. Expired
92
+ rules are dropped (a vulnerability whose exemption lapsed is no longer
93
+ ignored). Missing or malformed files degrade to "no rules" rather than
94
+ failing the scan."""
95
+ target = path or Path.cwd() / DEFAULT_IGNORE_FILENAME
96
+ if not target.exists():
97
+ return []
98
+
99
+ try:
100
+ raw = yaml.safe_load(target.read_text(encoding="utf-8")) or {}
101
+ except yaml.YAMLError as e:
102
+ logger.warning(f"Could not parse {target}: {e}")
103
+ return []
104
+
105
+ entries = raw.get("ignores", []) if isinstance(raw, dict) else []
106
+ rules: list[IgnoreRule] = []
107
+ for entry in entries:
108
+ try:
109
+ rule = IgnoreRule.model_validate(entry)
110
+ except Exception as e:
111
+ logger.warning(f"Skipping invalid ignore rule {entry}: {e}")
112
+ continue
113
+ if rule.is_expired:
114
+ logger.info(f"Ignore rule for {rule.cve} expired on {rule.expires}, no longer applied")
115
+ continue
116
+ rules.append(rule)
117
+ return rules
118
+
119
+
120
+ def active_ignored_cve_ids(rules: list[IgnoreRule]) -> set[str]:
121
+ """Identificadores isentos e ainda válidos.
122
+
123
+ A string vazia é filtrada de novo aqui, e não só no validador: este
124
+ conjunto é comparado contra `vuln.cve_id`, que é vazio nos achados sem
125
+ identificador publicado, e um `""` que escapasse -- por uma `IgnoreRule`
126
+ construída em código, sem passar pelo arquivo -- apagaria todos eles do
127
+ relatório sem deixar rastro. Duas guardas para um erro que some em
128
+ silêncio é o preço certo.
129
+ """
130
+ return {r.cve for r in rules if not r.is_expired and r.cve.strip()}
@@ -0,0 +1,130 @@
1
+ """Client-side rate limiting and circuit breaking for external APIs.
2
+
3
+ Every provider DockerLs talks to has a budget: the GitHub REST API allows 60
4
+ requests an hour to an anonymous client, Docker Hub throttles pulls, and a
5
+ registry that is having a bad day answers 5xx to everything. Two failure
6
+ modes follow from that, and both are the caller's fault rather than the
7
+ API's:
8
+
9
+ * a burst that spends the whole budget in the first second of a run, so the
10
+ rest of the run gets 403s that look like the data does not exist;
11
+ * a retry loop that keeps hammering an endpoint which is already failing,
12
+ turning a slow dependency into a hung command.
13
+
14
+ The token bucket addresses the first: requests are paced, and a caller that
15
+ asks faster than the budget allows waits rather than being refused. The
16
+ circuit breaker addresses the second: after a run of consecutive failures
17
+ the breaker opens and calls fail *immediately* for a cooldown, so an
18
+ unreachable provider costs one timeout instead of one per candidate. It
19
+ closes again after a single successful probe.
20
+
21
+ Neither ever turns a failure into a success. An open breaker raises, and the
22
+ caller reports the source as unavailable -- which is the honest outcome, and
23
+ the one the rest of this codebase is built to represent.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import asyncio
29
+ import random
30
+ import time
31
+ from dataclasses import dataclass, field
32
+
33
+ #: Jitter applied to the wait a caller computes, as a fraction of that wait.
34
+ #: Without it, N workers that all hit the limit at once wake up at the same
35
+ #: instant and collide again -- a thundering herd of their own making.
36
+ JITTER_FRACTION = 0.1
37
+
38
+
39
+ class RateLimiter:
40
+ """Token bucket: at most `rate` operations per `period` seconds, on average.
41
+
42
+ Bursts up to `burst` are allowed, because the common shape here is a
43
+ handful of parallel lookups followed by a long quiet stretch, and pacing
44
+ those to a strict interval would only add latency for no protection.
45
+ """
46
+
47
+ def __init__(self, rate: int, period: float = 1.0, burst: int | None = None):
48
+ if rate < 1:
49
+ raise ValueError("rate must be at least 1")
50
+ if period <= 0:
51
+ raise ValueError("period must be positive")
52
+ self._rate = rate
53
+ self._period = period
54
+ self._capacity = float(burst if burst is not None else rate)
55
+ self._tokens = self._capacity
56
+ self._updated = time.monotonic()
57
+ self._lock = asyncio.Lock()
58
+
59
+ async def acquire(self) -> None:
60
+ """Wait until a token is available, then consume it."""
61
+ while True:
62
+ async with self._lock:
63
+ now = time.monotonic()
64
+ elapsed = now - self._updated
65
+ self._updated = now
66
+ self._tokens = min(
67
+ self._capacity, self._tokens + elapsed * (self._rate / self._period)
68
+ )
69
+ if self._tokens >= 1.0:
70
+ self._tokens -= 1.0
71
+ return
72
+ deficit = 1.0 - self._tokens
73
+ wait = deficit * (self._period / self._rate)
74
+ # Slept outside the lock so waiting callers do not serialize on
75
+ # each other's sleeps.
76
+ # Timing jitter, not a secret: this decides how long a waiting
77
+ # caller sleeps so N workers do not wake in lockstep. A CSPRNG
78
+ # here would buy nothing -- there is nothing to predict and
79
+ # nothing to forge.
80
+ jitter = random.random() # nosec B311 # noqa: S311 - scheduling, not crypto
81
+ await asyncio.sleep(wait * (1.0 + jitter * JITTER_FRACTION))
82
+
83
+
84
+ class CircuitOpenError(RuntimeError):
85
+ """The breaker is open: the provider is failing and is not being called."""
86
+
87
+
88
+ @dataclass
89
+ class CircuitBreaker:
90
+ """Stop calling a provider that is consistently failing.
91
+
92
+ State is intentionally minimal -- consecutive failures and the time the
93
+ breaker opened. There is no half-open request accounting: the first call
94
+ after the cooldown is the probe, and its outcome decides whether the
95
+ breaker closes or re-opens.
96
+ """
97
+
98
+ #: Consecutive failures that trip the breaker.
99
+ threshold: int = 5
100
+ #: Seconds the breaker stays open before allowing a probe.
101
+ cooldown: float = 60.0
102
+ _failures: int = field(default=0, init=False)
103
+ _opened_at: float = field(default=0.0, init=False)
104
+
105
+ @property
106
+ def is_open(self) -> bool:
107
+ if self._failures < self.threshold:
108
+ return False
109
+ return (time.monotonic() - self._opened_at) < self.cooldown
110
+
111
+ def check(self, provider: str) -> None:
112
+ """Raise `CircuitOpenError` when the provider is being skipped."""
113
+ if self.is_open:
114
+ remaining = self.cooldown - (time.monotonic() - self._opened_at)
115
+ raise CircuitOpenError(
116
+ f"{provider} is unavailable after {self._failures} consecutive "
117
+ f"failures; retrying in {max(0.0, remaining):.0f}s"
118
+ )
119
+
120
+ def record_success(self) -> None:
121
+ self._failures = 0
122
+ self._opened_at = 0.0
123
+
124
+ def record_failure(self) -> None:
125
+ self._failures += 1
126
+ if self._failures >= self.threshold:
127
+ # Re-stamped on every failure past the threshold so a provider
128
+ # that keeps failing keeps the breaker open, rather than
129
+ # re-opening the floodgates one cooldown after the first trip.
130
+ self._opened_at = time.monotonic()
@@ -0,0 +1,183 @@
1
+ """How much of this machine DockerLs may actually use.
2
+
3
+ Concurrency here is not the usual "how many coroutines" question. Each
4
+ worker slot holds a **scanner process**: `trivy image` reads a multi-hundred
5
+ megabyte vulnerability database, unpacks image layers and matches packages,
6
+ using a full core and hundreds of megabytes of RSS while it does. Ten of
7
+ them at once on a two-core CI runner does not go ten times faster -- it goes
8
+ slower, evicts the page cache, and can get the whole job OOM-killed.
9
+
10
+ The default was a flat `workers = 10`, chosen with no reference to the
11
+ machine. This module replaces the guess with a measurement, and it is
12
+ deliberately **container-aware**: inside Docker or Kubernetes,
13
+ `os.cpu_count()` reports the host's cores while the cgroup allows a
14
+ fraction of one. A tool whose whole purpose is analysing containers is
15
+ routinely run inside one, so reading the host's numbers is not a small
16
+ inaccuracy -- it is the common case, and it is where the oversubscription
17
+ hurts most.
18
+
19
+ Nothing here is a hard limit on what the operator may ask for. An explicit
20
+ `--workers` is still honoured: the point is that the *default* should fit
21
+ the machine it lands on, and that asking for more than the machine has
22
+ should say so out loud.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import os
28
+ from pathlib import Path
29
+
30
+ from loguru import logger
31
+
32
+ #: Memory one scanner process should be assumed to need. Trivy's resident
33
+ #: set while matching a large image sits in the hundreds of megabytes; this
34
+ #: is the conservative end, because the cost of underestimating it is an
35
+ #: OOM kill and the cost of overestimating it is one fewer parallel scan.
36
+ SCANNER_MEMORY_BYTES = 768 * 1024 * 1024
37
+
38
+ #: Never derive fewer than this. One worker is always possible, and a
39
+ #: machine that reports something absurd should still make progress.
40
+ MIN_RECOMMENDED = 1
41
+
42
+ #: Never derive more than this, however large the machine. Past this point
43
+ #: the bottleneck is the registry and the disk, not the CPU, and every extra
44
+ #: process is another few hundred megabytes for no throughput.
45
+ MAX_RECOMMENDED = 16
46
+
47
+ _CGROUP_V2_CPU = Path("/sys/fs/cgroup/cpu.max")
48
+ _CGROUP_V2_MEMORY = Path("/sys/fs/cgroup/memory.max")
49
+ _CGROUP_V1_QUOTA = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
50
+ _CGROUP_V1_PERIOD = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
51
+ _CGROUP_V1_MEMORY = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
52
+ _MEMINFO = Path("/proc/meminfo")
53
+
54
+
55
+ def cpu_capacity() -> float:
56
+ """Cores this process may actually use, honouring cgroup quota.
57
+
58
+ Three sources, narrowest wins: the cgroup CPU quota (what a container
59
+ is allowed), the CPU affinity mask (what a taskset or a scheduler
60
+ policy permits), and the machine's core count. A quota of 0.5 cores
61
+ returns 0.5, not 1 -- the caller decides how to round, and rounding up
62
+ to 1 at the bottom is the caller's floor, not a fact about the machine.
63
+ """
64
+ candidates: list[float] = []
65
+
66
+ try:
67
+ candidates.append(float(len(os.sched_getaffinity(0))))
68
+ except (AttributeError, OSError):
69
+ count = os.cpu_count()
70
+ if count:
71
+ candidates.append(float(count))
72
+
73
+ quota = _cgroup_cpu_quota()
74
+ if quota is not None:
75
+ candidates.append(quota)
76
+
77
+ return min(candidates) if candidates else 1.0
78
+
79
+
80
+ def available_memory_bytes() -> int | None:
81
+ """Memory this process can count on, or None when it cannot be read.
82
+
83
+ `MemAvailable` rather than `MemFree`: the kernel's own estimate of what
84
+ can be allocated without swapping, which is the number that decides
85
+ whether starting another scanner is survivable. A cgroup limit, when
86
+ present and lower, wins -- that is the ceiling the OOM killer uses.
87
+ """
88
+ values = [v for v in (_meminfo_available(), _cgroup_memory_limit()) if v is not None]
89
+ return min(values) if values else None
90
+
91
+
92
+ def recommended_workers() -> int:
93
+ """How many scanner processes this machine should run at once.
94
+
95
+ The minimum of what the CPU allows and what memory allows, clamped.
96
+ Both terms matter and they fail differently: too many for the CPU makes
97
+ the run slower, too many for the memory makes it die.
98
+ """
99
+ by_cpu = max(MIN_RECOMMENDED, int(cpu_capacity()))
100
+
101
+ memory = available_memory_bytes()
102
+ by_memory = (
103
+ max(MIN_RECOMMENDED, int(memory // SCANNER_MEMORY_BYTES))
104
+ if memory is not None
105
+ else MAX_RECOMMENDED
106
+ )
107
+
108
+ return max(MIN_RECOMMENDED, min(by_cpu, by_memory, MAX_RECOMMENDED))
109
+
110
+
111
+ def describe_capacity() -> str:
112
+ """One line for the logs, so a slow run can be explained afterwards."""
113
+ memory = available_memory_bytes()
114
+ memory_text = f"{memory / (1024**3):.1f} GiB available" if memory else "memory unknown"
115
+ return f"{cpu_capacity():.2g} usable CPU(s), {memory_text}"
116
+
117
+
118
+ def _cgroup_cpu_quota() -> float | None:
119
+ """Cores permitted by the cgroup, v2 first then v1, or None if unlimited."""
120
+ text = _read(_CGROUP_V2_CPU)
121
+ if text:
122
+ parts = text.split()
123
+ if len(parts) == 2 and parts[0] != "max":
124
+ quota, period = _to_float(parts[0]), _to_float(parts[1])
125
+ if quota and period and period > 0:
126
+ return quota / period
127
+ return None
128
+
129
+ quota = _to_float(_read(_CGROUP_V1_QUOTA))
130
+ period = _to_float(_read(_CGROUP_V1_PERIOD))
131
+ if quota is not None and quota > 0 and period:
132
+ return quota / period
133
+ return None
134
+
135
+
136
+ def _cgroup_memory_limit() -> int | None:
137
+ for path in (_CGROUP_V2_MEMORY, _CGROUP_V1_MEMORY):
138
+ text = _read(path)
139
+ if not text or text == "max":
140
+ continue
141
+ value = _to_float(text)
142
+ # cgroup v1 writes an enormous sentinel to mean "no limit"; anything
143
+ # at that scale is not a real container budget.
144
+ if value is not None and 0 < value < (1 << 62):
145
+ return int(value)
146
+ return None
147
+
148
+
149
+ def _meminfo_available() -> int | None:
150
+ text = _read(_MEMINFO)
151
+ if not text:
152
+ return None
153
+ for line in text.splitlines():
154
+ if line.startswith("MemAvailable:"):
155
+ parts = line.split()
156
+ if len(parts) >= 2:
157
+ value = _to_float(parts[1])
158
+ if value is not None:
159
+ return int(value * 1024) # reported in kB
160
+ return None
161
+
162
+
163
+ def _read(path: Path) -> str:
164
+ """Read a small pseudo-file, treating every failure as "not present".
165
+
166
+ These paths are absent on macOS, absent outside containers, and
167
+ unreadable under some sandboxes. None of that is an error worth
168
+ surfacing: the caller simply falls back to the next source.
169
+ """
170
+ try:
171
+ return path.read_text(encoding="utf-8").strip()
172
+ except OSError:
173
+ return ""
174
+
175
+
176
+ def _to_float(text: str | None) -> float | None:
177
+ if not text:
178
+ return None
179
+ try:
180
+ return float(text)
181
+ except ValueError:
182
+ logger.debug(f"Unexpected value in a resource limit file: {text!r}")
183
+ return None
@@ -0,0 +1,31 @@
1
+ from __future__ import annotations
2
+
3
+ from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential
4
+
5
+ DEFAULT_MAX_ATTEMPTS = 3
6
+ DEFAULT_BACKOFF_BASE = 2.0
7
+ MAX_BACKOFF_SECONDS = 10.0
8
+
9
+
10
+ def retry_policy(
11
+ max_attempts: int = DEFAULT_MAX_ATTEMPTS,
12
+ backoff_base: float = DEFAULT_BACKOFF_BASE,
13
+ ) -> AsyncRetrying:
14
+ """Build a retry policy from configuration.
15
+
16
+ A `@retry(...)` decorator is evaluated once at import time, which is why
17
+ `retry_max_attempts` and `retry_backoff_base` could be declared as
18
+ settings and documented while having no possible effect. Building the
19
+ policy per call is what lets configuration actually reach it.
20
+
21
+ `reraise=True` so callers still see the original exception rather than
22
+ tenacity's RetryError wrapper -- the existing error handling in the
23
+ clients catches httpx errors by type.
24
+ """
25
+ return AsyncRetrying(
26
+ stop=stop_after_attempt(max(1, max_attempts)),
27
+ wait=wait_exponential(
28
+ multiplier=1, exp_base=max(1.1, backoff_base), max=MAX_BACKOFF_SECONDS
29
+ ),
30
+ reraise=True,
31
+ )
@@ -0,0 +1,166 @@
1
+ """YAML parsing for documents fetched from the internet.
2
+
3
+ Every YAML this tool reads comes from somewhere it does not control -- the
4
+ Docker Hardened Images catalogue is fetched over HTTPS from GitHub, and a
5
+ proxy, a compromised mirror or a malicious fork can put anything in the
6
+ response body. `yaml.load` would be remote code execution outright; even
7
+ `yaml.safe_load` is only safe against *construction*, not against
8
+ resource exhaustion: a few hundred bytes of nested aliases expand into
9
+ gigabytes (the "billion laughs" / YAML bomb), and a deeply nested document
10
+ can exhaust the C stack during composition.
11
+
12
+ So parsing here is guarded on three axes before the loader ever runs:
13
+
14
+ * **size** -- the document is refused above a byte budget, so a hostile or
15
+ misconfigured endpoint cannot stream an unbounded body into memory;
16
+ * **expansion size** -- anchors are legitimate in these definitions, so
17
+ they are not banned; instead the document is *composed* into a node graph
18
+ (where an alias is a shared reference, and therefore costs nothing) and
19
+ the size it would expand to is computed over that graph before anything
20
+ is constructed. A counting heuristic is not enough here and this is worth
21
+ stating plainly: nine levels of nine-fold aliasing is only ~72 aliases --
22
+ comfortably under any sane per-document count -- and expands to 387
23
+ million nodes. What has to be bounded is the product, not the tally;
24
+ * **nesting depth** -- measured on the parsed result, and refused above a
25
+ bound no legitimate image definition comes close to.
26
+
27
+ A refusal raises `UnsafeYAMLError`, which callers treat as "this definition
28
+ could not be read" -- never as "this definition is empty".
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ from typing import Any
34
+
35
+ import yaml
36
+
37
+ #: 4 MiB. The largest definition in the DHI catalogue is under 100 KiB, so
38
+ #: this leaves two orders of magnitude of headroom while still bounding the
39
+ #: memory a single hostile response can cost.
40
+ MAX_DOCUMENT_BYTES = 4 * 1024 * 1024
41
+
42
+ #: Nodes the document may expand to. Measured on the composed graph before
43
+ #: construction, so a bomb is refused in linear time instead of being
44
+ #: discovered when memory runs out. Real definitions compose a few thousand
45
+ #: nodes; this leaves two orders of magnitude of headroom.
46
+ MAX_EXPANDED_NODES = 1_000_000
47
+
48
+ #: Composition depth of the parsed structure. Image definitions nest about
49
+ #: six levels; anything past this is a stack-exhaustion attempt.
50
+ MAX_DEPTH = 32
51
+
52
+
53
+ class UnsafeYAMLError(ValueError):
54
+ """A document was refused before or after parsing, on a safety bound."""
55
+
56
+
57
+ def safe_load_yaml(raw: str | bytes, *, origin: str = "<yaml>") -> Any:
58
+ """Parse `raw` with `yaml.safe_load` behind explicit resource bounds.
59
+
60
+ Returns whatever the document contained (usually a dict). Raises
61
+ `UnsafeYAMLError` when a bound is exceeded or the document is malformed
62
+ -- the caller decides what an unreadable definition means, and in this
63
+ codebase it always means "unknown", never "clean".
64
+ """
65
+ text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
66
+
67
+ encoded_size = len(text.encode("utf-8", errors="replace"))
68
+ if encoded_size > MAX_DOCUMENT_BYTES:
69
+ raise UnsafeYAMLError(
70
+ f"{origin}: document is {encoded_size} bytes, over the {MAX_DOCUMENT_BYTES} limit"
71
+ )
72
+
73
+ # Compose, measure, then construct -- in that order. Composition builds
74
+ # a graph in which an alias is one edge rather than a copy, so it is
75
+ # bounded by the document's own length; construction is what multiplies,
76
+ # and it does not happen until the expansion has been shown to be safe.
77
+ # SafeLoader is used throughout: it constructs only plain scalars, lists
78
+ # and dicts, never arbitrary objects from `!!python/object` tags.
79
+ loader = yaml.SafeLoader(text)
80
+ try:
81
+ try:
82
+ node = loader.get_single_node()
83
+ except yaml.YAMLError as e:
84
+ raise UnsafeYAMLError(f"{origin}: malformed YAML ({e})") from e
85
+ except RecursionError as e:
86
+ raise UnsafeYAMLError(f"{origin}: document nesting exhausted the parser") from e
87
+
88
+ if node is None:
89
+ return None
90
+
91
+ expanded = _expanded_size(node, {})
92
+ if expanded > MAX_EXPANDED_NODES:
93
+ raise UnsafeYAMLError(
94
+ f"{origin}: document expands to at least {expanded} nodes, over the "
95
+ f"{MAX_EXPANDED_NODES} limit (alias-expansion bomb)"
96
+ )
97
+
98
+ try:
99
+ data = loader.construct_document(node)
100
+ except yaml.YAMLError as e:
101
+ raise UnsafeYAMLError(f"{origin}: malformed YAML ({e})") from e
102
+ except RecursionError as e:
103
+ raise UnsafeYAMLError(f"{origin}: document nesting exhausted the parser") from e
104
+ finally:
105
+ loader.dispose()
106
+
107
+ depth = _depth(data)
108
+ if depth > MAX_DEPTH:
109
+ raise UnsafeYAMLError(f"{origin}: nesting depth {depth} exceeds the {MAX_DEPTH} limit")
110
+ return data
111
+
112
+
113
+ def _expanded_size(node: yaml.Node, memo: dict[int, int]) -> int:
114
+ """How many nodes `node` would become once every alias is expanded.
115
+
116
+ The composed graph shares anchored nodes, so each one is measured once
117
+ and memoised by identity; a node referenced nine times contributes its
118
+ size nine times to its parent's total. That is exactly the quantity a
119
+ YAML bomb maximises, and computing it costs one walk of the graph.
120
+
121
+ Totals are clamped to the limit as they accumulate. Without the clamp
122
+ the arithmetic itself becomes the denial of service: the classic bomb's
123
+ true size is a 400-million-digit intermediate nobody needs to compute in
124
+ order to know it is too big.
125
+ """
126
+ cached = memo.get(id(node))
127
+ if cached is not None:
128
+ return cached
129
+
130
+ if isinstance(node, yaml.ScalarNode):
131
+ memo[id(node)] = 1
132
+ return 1
133
+
134
+ total = 1
135
+ children: list[yaml.Node] = []
136
+ if isinstance(node, yaml.SequenceNode):
137
+ children = list(node.value)
138
+ elif isinstance(node, yaml.MappingNode):
139
+ children = [child for pair in node.value for child in pair]
140
+
141
+ for child in children:
142
+ total += _expanded_size(child, memo)
143
+ if total > MAX_EXPANDED_NODES:
144
+ total = MAX_EXPANDED_NODES + 1
145
+ break
146
+
147
+ memo[id(node)] = total
148
+ return total
149
+
150
+
151
+ def _depth(value: Any, level: int = 0) -> int:
152
+ """Nesting depth of an already-parsed structure.
153
+
154
+ Iterative rather than recursive on the hot path would be safer still,
155
+ but the loader has already bounded what can reach here: `MAX_DEPTH` is
156
+ checked against a structure `yaml.SafeLoader` managed to build, and the
157
+ loader raises `RecursionError` (handled above) well before Python's own
158
+ limit on anything deeper.
159
+ """
160
+ if level > MAX_DEPTH:
161
+ return level
162
+ if isinstance(value, dict):
163
+ return max((_depth(v, level + 1) for v in value.values()), default=level)
164
+ if isinstance(value, list | tuple):
165
+ return max((_depth(v, level + 1) for v in value), default=level)
166
+ return level