dockerls 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dockerls/__init__.py +31 -0
- dockerls/application/__init__.py +0 -0
- dockerls/application/dto/__init__.py +3 -0
- dockerls/application/dto/analysis.py +257 -0
- dockerls/application/services/__init__.py +0 -0
- dockerls/application/services/alternatives_lookup.py +167 -0
- dockerls/application/services/composite_repository.py +106 -0
- dockerls/application/services/cross_validation.py +233 -0
- dockerls/application/services/ecosystems.py +350 -0
- dockerls/application/services/fallback_scanner.py +97 -0
- dockerls/application/services/hardening_analysis.py +174 -0
- dockerls/application/services/migration.py +297 -0
- dockerls/application/services/progress.py +56 -0
- dockerls/application/services/remediation.py +321 -0
- dockerls/application/services/scan_history_store.py +88 -0
- dockerls/application/services/scanner_factory.py +88 -0
- dockerls/application/services/source_registry.py +151 -0
- dockerls/application/services/tag_history_store.py +76 -0
- dockerls/application/services/teardown.py +50 -0
- dockerls/application/services/verdict.py +298 -0
- dockerls/application/services/version_discovery.py +108 -0
- dockerls/application/use_cases/__init__.py +0 -0
- dockerls/application/use_cases/analyze_dockerfile.py +103 -0
- dockerls/application/use_cases/analyze_image.py +173 -0
- dockerls/application/use_cases/build_image.py +1795 -0
- dockerls/application/use_cases/compare_images.py +91 -0
- dockerls/application/use_cases/fleet_scan.py +240 -0
- dockerls/application/use_cases/recommend_images.py +1078 -0
- dockerls/application/use_cases/registry_audit.py +133 -0
- dockerls/application/use_cases/search_images.py +23 -0
- dockerls/application/use_cases/upgrade_base.py +167 -0
- dockerls/cache/__init__.py +0 -0
- dockerls/cache/sqlite_cache.py +184 -0
- dockerls/cli/__init__.py +0 -0
- dockerls/cli/analysis_baseline.py +98 -0
- dockerls/cli/app.py +294 -0
- dockerls/cli/commands/__init__.py +0 -0
- dockerls/cli/commands/advisor.py +262 -0
- dockerls/cli/commands/alternatives.py +291 -0
- dockerls/cli/commands/analyze.py +429 -0
- dockerls/cli/commands/analyze_dockerfile.py +104 -0
- dockerls/cli/commands/base_cmd.py +244 -0
- dockerls/cli/commands/base_image.py +551 -0
- dockerls/cli/commands/build.py +1300 -0
- dockerls/cli/commands/cache_cmd.py +104 -0
- dockerls/cli/commands/compare.py +177 -0
- dockerls/cli/commands/controls.py +110 -0
- dockerls/cli/commands/doctor.py +566 -0
- dockerls/cli/commands/export.py +81 -0
- dockerls/cli/commands/fleet.py +159 -0
- dockerls/cli/commands/health.py +84 -0
- dockerls/cli/commands/login.py +53 -0
- dockerls/cli/commands/policy_cmd.py +111 -0
- dockerls/cli/commands/provenance_cmd.py +162 -0
- dockerls/cli/commands/recommend.py +761 -0
- dockerls/cli/commands/registry_audit_cmd.py +103 -0
- dockerls/cli/commands/sbom.py +144 -0
- dockerls/cli/commands/search.py +86 -0
- dockerls/cli/commands/verify.py +115 -0
- dockerls/cli/commands/version.py +12 -0
- dockerls/cli/commands/vex_cmd.py +117 -0
- dockerls/cli/dependencies.py +530 -0
- dockerls/cli/image_names.py +79 -0
- dockerls/cli/options.py +42 -0
- dockerls/cli/progress.py +145 -0
- dockerls/cli/publish_prompt.py +123 -0
- dockerls/cli/rendering.py +214 -0
- dockerls/cli/runtime.py +65 -0
- dockerls/cli/scan_failure.py +71 -0
- dockerls/cli/text.py +39 -0
- dockerls/cli/validators.py +35 -0
- dockerls/cli/vulnerability_view.py +154 -0
- dockerls/domain/__init__.py +0 -0
- dockerls/domain/entities/__init__.py +75 -0
- dockerls/domain/entities/declared_metadata.py +147 -0
- dockerls/domain/entities/dockerfile_analysis.py +318 -0
- dockerls/domain/entities/image.py +109 -0
- dockerls/domain/entities/image_facts.py +137 -0
- dockerls/domain/entities/recommendation.py +33 -0
- dockerls/domain/entities/scan_result.py +129 -0
- dockerls/domain/entities/vulnerability.py +232 -0
- dockerls/domain/interfaces/__init__.py +17 -0
- dockerls/domain/interfaces/cache_store.py +18 -0
- dockerls/domain/interfaces/dockerfile_validator.py +99 -0
- dockerls/domain/interfaces/eol_checker.py +11 -0
- dockerls/domain/interfaces/image_repository.py +15 -0
- dockerls/domain/interfaces/scanner.py +15 -0
- dockerls/domain/security_controls.py +362 -0
- dockerls/domain/value_objects/__init__.py +49 -0
- dockerls/domain/value_objects/attack_surface.py +198 -0
- dockerls/domain/value_objects/base_recipe.py +600 -0
- dockerls/domain/value_objects/base_upgrade.py +292 -0
- dockerls/domain/value_objects/build_labels.py +99 -0
- dockerls/domain/value_objects/build_policy.py +412 -0
- dockerls/domain/value_objects/confidence.py +156 -0
- dockerls/domain/value_objects/fleet.py +174 -0
- dockerls/domain/value_objects/gate.py +327 -0
- dockerls/domain/value_objects/hardening.py +303 -0
- dockerls/domain/value_objects/image_reference.py +90 -0
- dockerls/domain/value_objects/inheritance.py +352 -0
- dockerls/domain/value_objects/network_policy.py +280 -0
- dockerls/domain/value_objects/production_readiness.py +145 -0
- dockerls/domain/value_objects/provenance.py +211 -0
- dockerls/domain/value_objects/recipe_diff.py +188 -0
- dockerls/domain/value_objects/registry_audit.py +195 -0
- dockerls/domain/value_objects/registry_target.py +225 -0
- dockerls/domain/value_objects/remediation_score.py +62 -0
- dockerls/domain/value_objects/scan_history.py +183 -0
- dockerls/domain/value_objects/scan_plan.py +193 -0
- dockerls/domain/value_objects/scanner_db.py +143 -0
- dockerls/domain/value_objects/security_score.py +160 -0
- dockerls/domain/value_objects/security_tier.py +122 -0
- dockerls/domain/value_objects/tag_history.py +180 -0
- dockerls/domain/value_objects/tool_release.py +253 -0
- dockerls/domain/value_objects/tristate.py +47 -0
- dockerls/domain/value_objects/vex.py +249 -0
- dockerls/exit_codes.py +23 -0
- dockerls/exporters/__init__.py +0 -0
- dockerls/exporters/base.py +17 -0
- dockerls/exporters/csv_exporter.py +85 -0
- dockerls/exporters/factory.py +31 -0
- dockerls/exporters/html_exporter.py +105 -0
- dockerls/exporters/json_exporter.py +19 -0
- dockerls/exporters/markdown_exporter.py +84 -0
- dockerls/exporters/sarif_exporter.py +245 -0
- dockerls/infrastructure/__init__.py +0 -0
- dockerls/infrastructure/config/__init__.py +0 -0
- dockerls/infrastructure/config/policy_file.py +165 -0
- dockerls/infrastructure/config/settings.py +197 -0
- dockerls/infrastructure/database/__init__.py +0 -0
- dockerls/infrastructure/database/models.py +78 -0
- dockerls/infrastructure/dockerfile_validator.py +1899 -0
- dockerls/infrastructure/evidence.py +99 -0
- dockerls/infrastructure/hashing.py +165 -0
- dockerls/infrastructure/logging/__init__.py +0 -0
- dockerls/infrastructure/logging/setup.py +98 -0
- dockerls/infrastructure/network/__init__.py +0 -0
- dockerls/infrastructure/network/guarded_client.py +107 -0
- dockerls/infrastructure/network/host_guard.py +117 -0
- dockerls/infrastructure/redaction.py +135 -0
- dockerls/infrastructure/templates/hardening/alpine.dockerfile +44 -0
- dockerls/infrastructure/templates/hardening/debian.dockerfile +45 -0
- dockerls/infrastructure/templates/hardening/distroless.dockerfile +34 -0
- dockerls/infrastructure/templates/hardening/go-alpine.dockerfile +52 -0
- dockerls/infrastructure/templates/hardening/go-debian.dockerfile +54 -0
- dockerls/infrastructure/templates/hardening/go-distroless.dockerfile +43 -0
- dockerls/infrastructure/templates/hardening/go-scratch.dockerfile +48 -0
- dockerls/infrastructure/templates/hardening/go.dockerfile +50 -0
- dockerls/infrastructure/templates/hardening/gradle-alpine.dockerfile +51 -0
- dockerls/infrastructure/templates/hardening/gradle.dockerfile +52 -0
- dockerls/infrastructure/templates/hardening/java-alpine.dockerfile +50 -0
- dockerls/infrastructure/templates/hardening/java-debian.dockerfile +50 -0
- dockerls/infrastructure/templates/hardening/java-distroless.dockerfile +39 -0
- dockerls/infrastructure/templates/hardening/java-ubuntu.dockerfile +54 -0
- dockerls/infrastructure/templates/hardening/java.dockerfile +60 -0
- dockerls/infrastructure/templates/hardening/maven-alpine.dockerfile +56 -0
- dockerls/infrastructure/templates/hardening/maven.dockerfile +57 -0
- dockerls/infrastructure/templates/hardening/node-alpine.dockerfile +47 -0
- dockerls/infrastructure/templates/hardening/node-debian.dockerfile +54 -0
- dockerls/infrastructure/templates/hardening/node-distroless.dockerfile +46 -0
- dockerls/infrastructure/templates/hardening/node-ubuntu.dockerfile +63 -0
- dockerls/infrastructure/templates/hardening/node.dockerfile +61 -0
- dockerls/infrastructure/templates/hardening/php-alpine.dockerfile +45 -0
- dockerls/infrastructure/templates/hardening/php-debian.dockerfile +45 -0
- dockerls/infrastructure/templates/hardening/php-ubuntu.dockerfile +49 -0
- dockerls/infrastructure/templates/hardening/php.dockerfile +44 -0
- dockerls/infrastructure/templates/hardening/python-alpine.dockerfile +51 -0
- dockerls/infrastructure/templates/hardening/python-debian.dockerfile +54 -0
- dockerls/infrastructure/templates/hardening/python-distroless.dockerfile +51 -0
- dockerls/infrastructure/templates/hardening/python-ubuntu.dockerfile +60 -0
- dockerls/infrastructure/templates/hardening/python.dockerfile +58 -0
- dockerls/infrastructure/templates/hardening/ruby-alpine.dockerfile +48 -0
- dockerls/infrastructure/templates/hardening/ruby-debian.dockerfile +50 -0
- dockerls/infrastructure/templates/hardening/rust-alpine.dockerfile +50 -0
- dockerls/infrastructure/templates/hardening/rust-debian.dockerfile +48 -0
- dockerls/infrastructure/templates/hardening/rust-scratch.dockerfile +44 -0
- dockerls/infrastructure/templates/hardening/rust.dockerfile +54 -0
- dockerls/infrastructure/templates/hardening/ubuntu.dockerfile +49 -0
- dockerls/infrastructure/toolchain/__init__.py +0 -0
- dockerls/infrastructure/toolchain/db_metadata.py +115 -0
- dockerls/infrastructure/toolchain/installer.py +435 -0
- dockerls/integrations/__init__.py +0 -0
- dockerls/integrations/dhi/__init__.py +0 -0
- dockerls/integrations/dhi/catalog.py +457 -0
- dockerls/integrations/dhi/definition.py +151 -0
- dockerls/integrations/dhi/repository.py +238 -0
- dockerls/integrations/dockerhub/__init__.py +0 -0
- dockerls/integrations/dockerhub/client.py +318 -0
- dockerls/integrations/dockerhub/urls.py +75 -0
- dockerls/integrations/endoflife/__init__.py +0 -0
- dockerls/integrations/endoflife/checker.py +216 -0
- dockerls/integrations/engine/__init__.py +0 -0
- dockerls/integrations/engine/batch.py +197 -0
- dockerls/integrations/engine/client.py +330 -0
- dockerls/integrations/engine/locator.py +96 -0
- dockerls/integrations/exploitdb/__init__.py +0 -0
- dockerls/integrations/exploitdb/client.py +271 -0
- dockerls/integrations/grype/__init__.py +0 -0
- dockerls/integrations/grype/scanner.py +336 -0
- dockerls/integrations/registry/__init__.py +0 -0
- dockerls/integrations/registry/hardened.py +284 -0
- dockerls/integrations/registry/inspector.py +420 -0
- dockerls/integrations/registry/oci.py +259 -0
- dockerls/integrations/registry/private.py +79 -0
- dockerls/integrations/registry/urls.py +36 -0
- dockerls/integrations/scan_errors.py +67 -0
- dockerls/integrations/scan_target.py +57 -0
- dockerls/integrations/signing/__init__.py +0 -0
- dockerls/integrations/signing/cosign.py +484 -0
- dockerls/integrations/threat_intel/__init__.py +0 -0
- dockerls/integrations/threat_intel/client.py +290 -0
- dockerls/integrations/trivy/__init__.py +0 -0
- dockerls/integrations/trivy/cache_pool.py +176 -0
- dockerls/integrations/trivy/scanner.py +447 -0
- dockerls/utils/__init__.py +0 -0
- dockerls/utils/auth.py +108 -0
- dockerls/utils/executables.py +39 -0
- dockerls/utils/ignore_file.py +130 -0
- dockerls/utils/rate_limit.py +130 -0
- dockerls/utils/resources.py +183 -0
- dockerls/utils/retry.py +31 -0
- dockerls/utils/safe_yaml.py +166 -0
- dockerls/utils/subprocess_runner.py +216 -0
- dockerls/utils/validation.py +72 -0
- dockerls-1.0.0.dist-info/METADATA +563 -0
- dockerls-1.0.0.dist-info/RECORD +230 -0
- dockerls-1.0.0.dist-info/WHEEL +5 -0
- dockerls-1.0.0.dist-info/entry_points.txt +2 -0
- dockerls-1.0.0.dist-info/licenses/LICENSE +21 -0
- dockerls-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from datetime import date
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import yaml
|
|
8
|
+
from loguru import logger
|
|
9
|
+
from pydantic import BaseModel, field_validator
|
|
10
|
+
|
|
11
|
+
from dockerls.domain.value_objects.vex import VexJustification, parse_justification
|
|
12
|
+
|
|
13
|
+
DEFAULT_IGNORE_FILENAME = ".dockerls-ignore.yaml"
|
|
14
|
+
|
|
15
|
+
#: A forma mínima de um identificador de aviso. Larga o bastante para
|
|
16
|
+
#: `CVE-2024-1234`, `GHSA-xxxx-yyyy-zzzz`, `DSA-5555-1`, `RUSTSEC-2024-0001`
|
|
17
|
+
#: e `ALAS2-2024-2500`; estreita o bastante para recusar o vazio, que casa
|
|
18
|
+
#: com todo achado que o scanner reportou sem identificador.
|
|
19
|
+
_ADVISORY_ID = re.compile(r"^[A-Z][A-Z0-9]*(?:[-_.][A-Z0-9]+)+$")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class IgnoreRule(BaseModel):
|
|
23
|
+
cve: str
|
|
24
|
+
justification: str = ""
|
|
25
|
+
expires: date | None = None
|
|
26
|
+
#: A justificativa **do padrão VEX**, quando quem escreveu a regra
|
|
27
|
+
#: souber declarar uma. Opcional de propósito: `not_affected` é uma
|
|
28
|
+
#: afirmação técnica com vocabulário fechado, e a maioria das isenções
|
|
29
|
+
#: é decisão de risco ("aceito até o Q3"), que não é nenhuma delas.
|
|
30
|
+
#:
|
|
31
|
+
#: Sem ela, o documento OpenVEX sai como `affected` com a justificativa
|
|
32
|
+
#: em texto -- o consumidor vê a exceção e o motivo, sem receber uma
|
|
33
|
+
#: alegação técnica que ninguém fez. Ver `domain/value_objects/vex.py`.
|
|
34
|
+
vex_justification: str = ""
|
|
35
|
+
|
|
36
|
+
@field_validator("vex_justification")
|
|
37
|
+
@classmethod
|
|
38
|
+
def _known_vex_justification(cls, v: str) -> str:
|
|
39
|
+
"""Recusa um valor que não existe no padrão.
|
|
40
|
+
|
|
41
|
+
Aceitá-lo produziria um documento VEX que outra ferramenta rejeita
|
|
42
|
+
na leitura -- e o erro apareceria longe daqui, sem dizer que veio
|
|
43
|
+
de uma linha do arquivo de isenções.
|
|
44
|
+
"""
|
|
45
|
+
text = v.strip().lower()
|
|
46
|
+
if not text:
|
|
47
|
+
return ""
|
|
48
|
+
if parse_justification(text) is None:
|
|
49
|
+
allowed = ", ".join(str(j) for j in VexJustification)
|
|
50
|
+
raise ValueError(f"unknown VEX justification {v!r}; expected one of: {allowed}")
|
|
51
|
+
return text
|
|
52
|
+
|
|
53
|
+
@field_validator("cve")
|
|
54
|
+
@classmethod
|
|
55
|
+
def _upper_cve(cls, v: str) -> str:
|
|
56
|
+
"""Normaliza o identificador, e recusa um que não identifica nada.
|
|
57
|
+
|
|
58
|
+
Um `cve: ""` passava por aqui e entrava no conjunto de isenções como
|
|
59
|
+
a string vazia. As isenções são aplicadas com
|
|
60
|
+
`vuln.cve_id.upper() not in ignored`, e um scanner **deixa
|
|
61
|
+
`cve_id` vazio** quando o aviso não tem identificador publicado --
|
|
62
|
+
o Trivy faz exatamente isso, como o exportador SARIF documenta. Uma
|
|
63
|
+
única linha em branco no arquivo de isenções apagava portanto
|
|
64
|
+
*todos* os achados sem identificador do relatório, e com eles do
|
|
65
|
+
score, do tier e do veredito de produção: vulnerabilidades reais
|
|
66
|
+
somem e a imagem sobe de nota.
|
|
67
|
+
|
|
68
|
+
A forma exigida é frouxa de propósito -- `GHSA-...`, `DSA-...`,
|
|
69
|
+
`RUSTSEC-...` e `ALAS-...` são identificadores legítimos e não
|
|
70
|
+
parecem com `CVE-`. O que ela recusa é o que não identifica: vazio,
|
|
71
|
+
espaço em branco, e qualquer coisa curta demais para ser um
|
|
72
|
+
identificador de aviso.
|
|
73
|
+
"""
|
|
74
|
+
text = v.strip().upper()
|
|
75
|
+
if not _ADVISORY_ID.match(text):
|
|
76
|
+
raise ValueError(
|
|
77
|
+
f"{v!r} does not name an advisory. An exemption has to say which "
|
|
78
|
+
"vulnerability it exempts: a blank id matches every finding the "
|
|
79
|
+
"scanner reported without one, and silently drops them from the scan"
|
|
80
|
+
)
|
|
81
|
+
return text
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def is_expired(self) -> bool:
|
|
85
|
+
if self.expires is None:
|
|
86
|
+
return False
|
|
87
|
+
return date.today() > self.expires
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def load_ignore_rules(path: Path | None = None) -> list[IgnoreRule]:
|
|
91
|
+
"""Load CVE ignore rules from a `.dockerls-ignore.yaml` file. Expired
|
|
92
|
+
rules are dropped (a vulnerability whose exemption lapsed is no longer
|
|
93
|
+
ignored). Missing or malformed files degrade to "no rules" rather than
|
|
94
|
+
failing the scan."""
|
|
95
|
+
target = path or Path.cwd() / DEFAULT_IGNORE_FILENAME
|
|
96
|
+
if not target.exists():
|
|
97
|
+
return []
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
raw = yaml.safe_load(target.read_text(encoding="utf-8")) or {}
|
|
101
|
+
except yaml.YAMLError as e:
|
|
102
|
+
logger.warning(f"Could not parse {target}: {e}")
|
|
103
|
+
return []
|
|
104
|
+
|
|
105
|
+
entries = raw.get("ignores", []) if isinstance(raw, dict) else []
|
|
106
|
+
rules: list[IgnoreRule] = []
|
|
107
|
+
for entry in entries:
|
|
108
|
+
try:
|
|
109
|
+
rule = IgnoreRule.model_validate(entry)
|
|
110
|
+
except Exception as e:
|
|
111
|
+
logger.warning(f"Skipping invalid ignore rule {entry}: {e}")
|
|
112
|
+
continue
|
|
113
|
+
if rule.is_expired:
|
|
114
|
+
logger.info(f"Ignore rule for {rule.cve} expired on {rule.expires}, no longer applied")
|
|
115
|
+
continue
|
|
116
|
+
rules.append(rule)
|
|
117
|
+
return rules
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def active_ignored_cve_ids(rules: list[IgnoreRule]) -> set[str]:
|
|
121
|
+
"""Identificadores isentos e ainda válidos.
|
|
122
|
+
|
|
123
|
+
A string vazia é filtrada de novo aqui, e não só no validador: este
|
|
124
|
+
conjunto é comparado contra `vuln.cve_id`, que é vazio nos achados sem
|
|
125
|
+
identificador publicado, e um `""` que escapasse -- por uma `IgnoreRule`
|
|
126
|
+
construída em código, sem passar pelo arquivo -- apagaria todos eles do
|
|
127
|
+
relatório sem deixar rastro. Duas guardas para um erro que some em
|
|
128
|
+
silêncio é o preço certo.
|
|
129
|
+
"""
|
|
130
|
+
return {r.cve for r in rules if not r.is_expired and r.cve.strip()}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Client-side rate limiting and circuit breaking for external APIs.
|
|
2
|
+
|
|
3
|
+
Every provider DockerLs talks to has a budget: the GitHub REST API allows 60
|
|
4
|
+
requests an hour to an anonymous client, Docker Hub throttles pulls, and a
|
|
5
|
+
registry that is having a bad day answers 5xx to everything. Two failure
|
|
6
|
+
modes follow from that, and both are the caller's fault rather than the
|
|
7
|
+
API's:
|
|
8
|
+
|
|
9
|
+
* a burst that spends the whole budget in the first second of a run, so the
|
|
10
|
+
rest of the run gets 403s that look like the data does not exist;
|
|
11
|
+
* a retry loop that keeps hammering an endpoint which is already failing,
|
|
12
|
+
turning a slow dependency into a hung command.
|
|
13
|
+
|
|
14
|
+
The token bucket addresses the first: requests are paced, and a caller that
|
|
15
|
+
asks faster than the budget allows waits rather than being refused. The
|
|
16
|
+
circuit breaker addresses the second: after a run of consecutive failures
|
|
17
|
+
the breaker opens and calls fail *immediately* for a cooldown, so an
|
|
18
|
+
unreachable provider costs one timeout instead of one per candidate. It
|
|
19
|
+
closes again after a single successful probe.
|
|
20
|
+
|
|
21
|
+
Neither ever turns a failure into a success. An open breaker raises, and the
|
|
22
|
+
caller reports the source as unavailable -- which is the honest outcome, and
|
|
23
|
+
the one the rest of this codebase is built to represent.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import asyncio
|
|
29
|
+
import random
|
|
30
|
+
import time
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
|
|
33
|
+
#: Jitter applied to the wait a caller computes, as a fraction of that wait.
|
|
34
|
+
#: Without it, N workers that all hit the limit at once wake up at the same
|
|
35
|
+
#: instant and collide again -- a thundering herd of their own making.
|
|
36
|
+
JITTER_FRACTION = 0.1
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class RateLimiter:
|
|
40
|
+
"""Token bucket: at most `rate` operations per `period` seconds, on average.
|
|
41
|
+
|
|
42
|
+
Bursts up to `burst` are allowed, because the common shape here is a
|
|
43
|
+
handful of parallel lookups followed by a long quiet stretch, and pacing
|
|
44
|
+
those to a strict interval would only add latency for no protection.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, rate: int, period: float = 1.0, burst: int | None = None):
|
|
48
|
+
if rate < 1:
|
|
49
|
+
raise ValueError("rate must be at least 1")
|
|
50
|
+
if period <= 0:
|
|
51
|
+
raise ValueError("period must be positive")
|
|
52
|
+
self._rate = rate
|
|
53
|
+
self._period = period
|
|
54
|
+
self._capacity = float(burst if burst is not None else rate)
|
|
55
|
+
self._tokens = self._capacity
|
|
56
|
+
self._updated = time.monotonic()
|
|
57
|
+
self._lock = asyncio.Lock()
|
|
58
|
+
|
|
59
|
+
async def acquire(self) -> None:
|
|
60
|
+
"""Wait until a token is available, then consume it."""
|
|
61
|
+
while True:
|
|
62
|
+
async with self._lock:
|
|
63
|
+
now = time.monotonic()
|
|
64
|
+
elapsed = now - self._updated
|
|
65
|
+
self._updated = now
|
|
66
|
+
self._tokens = min(
|
|
67
|
+
self._capacity, self._tokens + elapsed * (self._rate / self._period)
|
|
68
|
+
)
|
|
69
|
+
if self._tokens >= 1.0:
|
|
70
|
+
self._tokens -= 1.0
|
|
71
|
+
return
|
|
72
|
+
deficit = 1.0 - self._tokens
|
|
73
|
+
wait = deficit * (self._period / self._rate)
|
|
74
|
+
# Slept outside the lock so waiting callers do not serialize on
|
|
75
|
+
# each other's sleeps.
|
|
76
|
+
# Timing jitter, not a secret: this decides how long a waiting
|
|
77
|
+
# caller sleeps so N workers do not wake in lockstep. A CSPRNG
|
|
78
|
+
# here would buy nothing -- there is nothing to predict and
|
|
79
|
+
# nothing to forge.
|
|
80
|
+
jitter = random.random() # nosec B311 # noqa: S311 - scheduling, not crypto
|
|
81
|
+
await asyncio.sleep(wait * (1.0 + jitter * JITTER_FRACTION))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class CircuitOpenError(RuntimeError):
|
|
85
|
+
"""The breaker is open: the provider is failing and is not being called."""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class CircuitBreaker:
|
|
90
|
+
"""Stop calling a provider that is consistently failing.
|
|
91
|
+
|
|
92
|
+
State is intentionally minimal -- consecutive failures and the time the
|
|
93
|
+
breaker opened. There is no half-open request accounting: the first call
|
|
94
|
+
after the cooldown is the probe, and its outcome decides whether the
|
|
95
|
+
breaker closes or re-opens.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
#: Consecutive failures that trip the breaker.
|
|
99
|
+
threshold: int = 5
|
|
100
|
+
#: Seconds the breaker stays open before allowing a probe.
|
|
101
|
+
cooldown: float = 60.0
|
|
102
|
+
_failures: int = field(default=0, init=False)
|
|
103
|
+
_opened_at: float = field(default=0.0, init=False)
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def is_open(self) -> bool:
|
|
107
|
+
if self._failures < self.threshold:
|
|
108
|
+
return False
|
|
109
|
+
return (time.monotonic() - self._opened_at) < self.cooldown
|
|
110
|
+
|
|
111
|
+
def check(self, provider: str) -> None:
|
|
112
|
+
"""Raise `CircuitOpenError` when the provider is being skipped."""
|
|
113
|
+
if self.is_open:
|
|
114
|
+
remaining = self.cooldown - (time.monotonic() - self._opened_at)
|
|
115
|
+
raise CircuitOpenError(
|
|
116
|
+
f"{provider} is unavailable after {self._failures} consecutive "
|
|
117
|
+
f"failures; retrying in {max(0.0, remaining):.0f}s"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def record_success(self) -> None:
|
|
121
|
+
self._failures = 0
|
|
122
|
+
self._opened_at = 0.0
|
|
123
|
+
|
|
124
|
+
def record_failure(self) -> None:
|
|
125
|
+
self._failures += 1
|
|
126
|
+
if self._failures >= self.threshold:
|
|
127
|
+
# Re-stamped on every failure past the threshold so a provider
|
|
128
|
+
# that keeps failing keeps the breaker open, rather than
|
|
129
|
+
# re-opening the floodgates one cooldown after the first trip.
|
|
130
|
+
self._opened_at = time.monotonic()
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""How much of this machine DockerLs may actually use.
|
|
2
|
+
|
|
3
|
+
Concurrency here is not the usual "how many coroutines" question. Each
|
|
4
|
+
worker slot holds a **scanner process**: `trivy image` reads a multi-hundred
|
|
5
|
+
megabyte vulnerability database, unpacks image layers and matches packages,
|
|
6
|
+
using a full core and hundreds of megabytes of RSS while it does. Ten of
|
|
7
|
+
them at once on a two-core CI runner does not go ten times faster -- it goes
|
|
8
|
+
slower, evicts the page cache, and can get the whole job OOM-killed.
|
|
9
|
+
|
|
10
|
+
The default was a flat `workers = 10`, chosen with no reference to the
|
|
11
|
+
machine. This module replaces the guess with a measurement, and it is
|
|
12
|
+
deliberately **container-aware**: inside Docker or Kubernetes,
|
|
13
|
+
`os.cpu_count()` reports the host's cores while the cgroup allows a
|
|
14
|
+
fraction of one. A tool whose whole purpose is analysing containers is
|
|
15
|
+
routinely run inside one, so reading the host's numbers is not a small
|
|
16
|
+
inaccuracy -- it is the common case, and it is where the oversubscription
|
|
17
|
+
hurts most.
|
|
18
|
+
|
|
19
|
+
Nothing here is a hard limit on what the operator may ask for. An explicit
|
|
20
|
+
`--workers` is still honoured: the point is that the *default* should fit
|
|
21
|
+
the machine it lands on, and that asking for more than the machine has
|
|
22
|
+
should say so out loud.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from loguru import logger
|
|
31
|
+
|
|
32
|
+
#: Memory one scanner process should be assumed to need. Trivy's resident
|
|
33
|
+
#: set while matching a large image sits in the hundreds of megabytes; this
|
|
34
|
+
#: is the conservative end, because the cost of underestimating it is an
|
|
35
|
+
#: OOM kill and the cost of overestimating it is one fewer parallel scan.
|
|
36
|
+
SCANNER_MEMORY_BYTES = 768 * 1024 * 1024
|
|
37
|
+
|
|
38
|
+
#: Never derive fewer than this. One worker is always possible, and a
|
|
39
|
+
#: machine that reports something absurd should still make progress.
|
|
40
|
+
MIN_RECOMMENDED = 1
|
|
41
|
+
|
|
42
|
+
#: Never derive more than this, however large the machine. Past this point
|
|
43
|
+
#: the bottleneck is the registry and the disk, not the CPU, and every extra
|
|
44
|
+
#: process is another few hundred megabytes for no throughput.
|
|
45
|
+
MAX_RECOMMENDED = 16
|
|
46
|
+
|
|
47
|
+
_CGROUP_V2_CPU = Path("/sys/fs/cgroup/cpu.max")
|
|
48
|
+
_CGROUP_V2_MEMORY = Path("/sys/fs/cgroup/memory.max")
|
|
49
|
+
_CGROUP_V1_QUOTA = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
|
|
50
|
+
_CGROUP_V1_PERIOD = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
|
|
51
|
+
_CGROUP_V1_MEMORY = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
|
52
|
+
_MEMINFO = Path("/proc/meminfo")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def cpu_capacity() -> float:
|
|
56
|
+
"""Cores this process may actually use, honouring cgroup quota.
|
|
57
|
+
|
|
58
|
+
Three sources, narrowest wins: the cgroup CPU quota (what a container
|
|
59
|
+
is allowed), the CPU affinity mask (what a taskset or a scheduler
|
|
60
|
+
policy permits), and the machine's core count. A quota of 0.5 cores
|
|
61
|
+
returns 0.5, not 1 -- the caller decides how to round, and rounding up
|
|
62
|
+
to 1 at the bottom is the caller's floor, not a fact about the machine.
|
|
63
|
+
"""
|
|
64
|
+
candidates: list[float] = []
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
candidates.append(float(len(os.sched_getaffinity(0))))
|
|
68
|
+
except (AttributeError, OSError):
|
|
69
|
+
count = os.cpu_count()
|
|
70
|
+
if count:
|
|
71
|
+
candidates.append(float(count))
|
|
72
|
+
|
|
73
|
+
quota = _cgroup_cpu_quota()
|
|
74
|
+
if quota is not None:
|
|
75
|
+
candidates.append(quota)
|
|
76
|
+
|
|
77
|
+
return min(candidates) if candidates else 1.0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def available_memory_bytes() -> int | None:
|
|
81
|
+
"""Memory this process can count on, or None when it cannot be read.
|
|
82
|
+
|
|
83
|
+
`MemAvailable` rather than `MemFree`: the kernel's own estimate of what
|
|
84
|
+
can be allocated without swapping, which is the number that decides
|
|
85
|
+
whether starting another scanner is survivable. A cgroup limit, when
|
|
86
|
+
present and lower, wins -- that is the ceiling the OOM killer uses.
|
|
87
|
+
"""
|
|
88
|
+
values = [v for v in (_meminfo_available(), _cgroup_memory_limit()) if v is not None]
|
|
89
|
+
return min(values) if values else None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def recommended_workers() -> int:
|
|
93
|
+
"""How many scanner processes this machine should run at once.
|
|
94
|
+
|
|
95
|
+
The minimum of what the CPU allows and what memory allows, clamped.
|
|
96
|
+
Both terms matter and they fail differently: too many for the CPU makes
|
|
97
|
+
the run slower, too many for the memory makes it die.
|
|
98
|
+
"""
|
|
99
|
+
by_cpu = max(MIN_RECOMMENDED, int(cpu_capacity()))
|
|
100
|
+
|
|
101
|
+
memory = available_memory_bytes()
|
|
102
|
+
by_memory = (
|
|
103
|
+
max(MIN_RECOMMENDED, int(memory // SCANNER_MEMORY_BYTES))
|
|
104
|
+
if memory is not None
|
|
105
|
+
else MAX_RECOMMENDED
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
return max(MIN_RECOMMENDED, min(by_cpu, by_memory, MAX_RECOMMENDED))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def describe_capacity() -> str:
|
|
112
|
+
"""One line for the logs, so a slow run can be explained afterwards."""
|
|
113
|
+
memory = available_memory_bytes()
|
|
114
|
+
memory_text = f"{memory / (1024**3):.1f} GiB available" if memory else "memory unknown"
|
|
115
|
+
return f"{cpu_capacity():.2g} usable CPU(s), {memory_text}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _cgroup_cpu_quota() -> float | None:
|
|
119
|
+
"""Cores permitted by the cgroup, v2 first then v1, or None if unlimited."""
|
|
120
|
+
text = _read(_CGROUP_V2_CPU)
|
|
121
|
+
if text:
|
|
122
|
+
parts = text.split()
|
|
123
|
+
if len(parts) == 2 and parts[0] != "max":
|
|
124
|
+
quota, period = _to_float(parts[0]), _to_float(parts[1])
|
|
125
|
+
if quota and period and period > 0:
|
|
126
|
+
return quota / period
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
quota = _to_float(_read(_CGROUP_V1_QUOTA))
|
|
130
|
+
period = _to_float(_read(_CGROUP_V1_PERIOD))
|
|
131
|
+
if quota is not None and quota > 0 and period:
|
|
132
|
+
return quota / period
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _cgroup_memory_limit() -> int | None:
|
|
137
|
+
for path in (_CGROUP_V2_MEMORY, _CGROUP_V1_MEMORY):
|
|
138
|
+
text = _read(path)
|
|
139
|
+
if not text or text == "max":
|
|
140
|
+
continue
|
|
141
|
+
value = _to_float(text)
|
|
142
|
+
# cgroup v1 writes an enormous sentinel to mean "no limit"; anything
|
|
143
|
+
# at that scale is not a real container budget.
|
|
144
|
+
if value is not None and 0 < value < (1 << 62):
|
|
145
|
+
return int(value)
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _meminfo_available() -> int | None:
|
|
150
|
+
text = _read(_MEMINFO)
|
|
151
|
+
if not text:
|
|
152
|
+
return None
|
|
153
|
+
for line in text.splitlines():
|
|
154
|
+
if line.startswith("MemAvailable:"):
|
|
155
|
+
parts = line.split()
|
|
156
|
+
if len(parts) >= 2:
|
|
157
|
+
value = _to_float(parts[1])
|
|
158
|
+
if value is not None:
|
|
159
|
+
return int(value * 1024) # reported in kB
|
|
160
|
+
return None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _read(path: Path) -> str:
|
|
164
|
+
"""Read a small pseudo-file, treating every failure as "not present".
|
|
165
|
+
|
|
166
|
+
These paths are absent on macOS, absent outside containers, and
|
|
167
|
+
unreadable under some sandboxes. None of that is an error worth
|
|
168
|
+
surfacing: the caller simply falls back to the next source.
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
return path.read_text(encoding="utf-8").strip()
|
|
172
|
+
except OSError:
|
|
173
|
+
return ""
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _to_float(text: str | None) -> float | None:
|
|
177
|
+
if not text:
|
|
178
|
+
return None
|
|
179
|
+
try:
|
|
180
|
+
return float(text)
|
|
181
|
+
except ValueError:
|
|
182
|
+
logger.debug(f"Unexpected value in a resource limit file: {text!r}")
|
|
183
|
+
return None
|
dockerls/utils/retry.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential
|
|
4
|
+
|
|
5
|
+
DEFAULT_MAX_ATTEMPTS = 3
|
|
6
|
+
DEFAULT_BACKOFF_BASE = 2.0
|
|
7
|
+
MAX_BACKOFF_SECONDS = 10.0
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def retry_policy(
|
|
11
|
+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
|
12
|
+
backoff_base: float = DEFAULT_BACKOFF_BASE,
|
|
13
|
+
) -> AsyncRetrying:
|
|
14
|
+
"""Build a retry policy from configuration.
|
|
15
|
+
|
|
16
|
+
A `@retry(...)` decorator is evaluated once at import time, which is why
|
|
17
|
+
`retry_max_attempts` and `retry_backoff_base` could be declared as
|
|
18
|
+
settings and documented while having no possible effect. Building the
|
|
19
|
+
policy per call is what lets configuration actually reach it.
|
|
20
|
+
|
|
21
|
+
`reraise=True` so callers still see the original exception rather than
|
|
22
|
+
tenacity's RetryError wrapper -- the existing error handling in the
|
|
23
|
+
clients catches httpx errors by type.
|
|
24
|
+
"""
|
|
25
|
+
return AsyncRetrying(
|
|
26
|
+
stop=stop_after_attempt(max(1, max_attempts)),
|
|
27
|
+
wait=wait_exponential(
|
|
28
|
+
multiplier=1, exp_base=max(1.1, backoff_base), max=MAX_BACKOFF_SECONDS
|
|
29
|
+
),
|
|
30
|
+
reraise=True,
|
|
31
|
+
)
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""YAML parsing for documents fetched from the internet.
|
|
2
|
+
|
|
3
|
+
Every YAML this tool reads comes from somewhere it does not control -- the
|
|
4
|
+
Docker Hardened Images catalogue is fetched over HTTPS from GitHub, and a
|
|
5
|
+
proxy, a compromised mirror or a malicious fork can put anything in the
|
|
6
|
+
response body. `yaml.load` would be remote code execution outright; even
|
|
7
|
+
`yaml.safe_load` is only safe against *construction*, not against
|
|
8
|
+
resource exhaustion: a few hundred bytes of nested aliases expand into
|
|
9
|
+
gigabytes (the "billion laughs" / YAML bomb), and a deeply nested document
|
|
10
|
+
can exhaust the C stack during composition.
|
|
11
|
+
|
|
12
|
+
So parsing here is guarded on three axes before the loader ever runs:
|
|
13
|
+
|
|
14
|
+
* **size** -- the document is refused above a byte budget, so a hostile or
|
|
15
|
+
misconfigured endpoint cannot stream an unbounded body into memory;
|
|
16
|
+
* **expansion size** -- anchors are legitimate in these definitions, so
|
|
17
|
+
they are not banned; instead the document is *composed* into a node graph
|
|
18
|
+
(where an alias is a shared reference, and therefore costs nothing) and
|
|
19
|
+
the size it would expand to is computed over that graph before anything
|
|
20
|
+
is constructed. A counting heuristic is not enough here and this is worth
|
|
21
|
+
stating plainly: nine levels of nine-fold aliasing is only ~72 aliases --
|
|
22
|
+
comfortably under any sane per-document count -- and expands to 387
|
|
23
|
+
million nodes. What has to be bounded is the product, not the tally;
|
|
24
|
+
* **nesting depth** -- measured on the parsed result, and refused above a
|
|
25
|
+
bound no legitimate image definition comes close to.
|
|
26
|
+
|
|
27
|
+
A refusal raises `UnsafeYAMLError`, which callers treat as "this definition
|
|
28
|
+
could not be read" -- never as "this definition is empty".
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
import yaml
|
|
36
|
+
|
|
37
|
+
#: 4 MiB. The largest definition in the DHI catalogue is under 100 KiB, so
|
|
38
|
+
#: this leaves two orders of magnitude of headroom while still bounding the
|
|
39
|
+
#: memory a single hostile response can cost.
|
|
40
|
+
MAX_DOCUMENT_BYTES = 4 * 1024 * 1024
|
|
41
|
+
|
|
42
|
+
#: Nodes the document may expand to. Measured on the composed graph before
|
|
43
|
+
#: construction, so a bomb is refused in linear time instead of being
|
|
44
|
+
#: discovered when memory runs out. Real definitions compose a few thousand
|
|
45
|
+
#: nodes; this leaves two orders of magnitude of headroom.
|
|
46
|
+
MAX_EXPANDED_NODES = 1_000_000
|
|
47
|
+
|
|
48
|
+
#: Composition depth of the parsed structure. Image definitions nest about
|
|
49
|
+
#: six levels; anything past this is a stack-exhaustion attempt.
|
|
50
|
+
MAX_DEPTH = 32
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class UnsafeYAMLError(ValueError):
|
|
54
|
+
"""A document was refused before or after parsing, on a safety bound."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def safe_load_yaml(raw: str | bytes, *, origin: str = "<yaml>") -> Any:
|
|
58
|
+
"""Parse `raw` with `yaml.safe_load` behind explicit resource bounds.
|
|
59
|
+
|
|
60
|
+
Returns whatever the document contained (usually a dict). Raises
|
|
61
|
+
`UnsafeYAMLError` when a bound is exceeded or the document is malformed
|
|
62
|
+
-- the caller decides what an unreadable definition means, and in this
|
|
63
|
+
codebase it always means "unknown", never "clean".
|
|
64
|
+
"""
|
|
65
|
+
text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
66
|
+
|
|
67
|
+
encoded_size = len(text.encode("utf-8", errors="replace"))
|
|
68
|
+
if encoded_size > MAX_DOCUMENT_BYTES:
|
|
69
|
+
raise UnsafeYAMLError(
|
|
70
|
+
f"{origin}: document is {encoded_size} bytes, over the {MAX_DOCUMENT_BYTES} limit"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# Compose, measure, then construct -- in that order. Composition builds
|
|
74
|
+
# a graph in which an alias is one edge rather than a copy, so it is
|
|
75
|
+
# bounded by the document's own length; construction is what multiplies,
|
|
76
|
+
# and it does not happen until the expansion has been shown to be safe.
|
|
77
|
+
# SafeLoader is used throughout: it constructs only plain scalars, lists
|
|
78
|
+
# and dicts, never arbitrary objects from `!!python/object` tags.
|
|
79
|
+
loader = yaml.SafeLoader(text)
|
|
80
|
+
try:
|
|
81
|
+
try:
|
|
82
|
+
node = loader.get_single_node()
|
|
83
|
+
except yaml.YAMLError as e:
|
|
84
|
+
raise UnsafeYAMLError(f"{origin}: malformed YAML ({e})") from e
|
|
85
|
+
except RecursionError as e:
|
|
86
|
+
raise UnsafeYAMLError(f"{origin}: document nesting exhausted the parser") from e
|
|
87
|
+
|
|
88
|
+
if node is None:
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
expanded = _expanded_size(node, {})
|
|
92
|
+
if expanded > MAX_EXPANDED_NODES:
|
|
93
|
+
raise UnsafeYAMLError(
|
|
94
|
+
f"{origin}: document expands to at least {expanded} nodes, over the "
|
|
95
|
+
f"{MAX_EXPANDED_NODES} limit (alias-expansion bomb)"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
data = loader.construct_document(node)
|
|
100
|
+
except yaml.YAMLError as e:
|
|
101
|
+
raise UnsafeYAMLError(f"{origin}: malformed YAML ({e})") from e
|
|
102
|
+
except RecursionError as e:
|
|
103
|
+
raise UnsafeYAMLError(f"{origin}: document nesting exhausted the parser") from e
|
|
104
|
+
finally:
|
|
105
|
+
loader.dispose()
|
|
106
|
+
|
|
107
|
+
depth = _depth(data)
|
|
108
|
+
if depth > MAX_DEPTH:
|
|
109
|
+
raise UnsafeYAMLError(f"{origin}: nesting depth {depth} exceeds the {MAX_DEPTH} limit")
|
|
110
|
+
return data
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _expanded_size(node: yaml.Node, memo: dict[int, int]) -> int:
|
|
114
|
+
"""How many nodes `node` would become once every alias is expanded.
|
|
115
|
+
|
|
116
|
+
The composed graph shares anchored nodes, so each one is measured once
|
|
117
|
+
and memoised by identity; a node referenced nine times contributes its
|
|
118
|
+
size nine times to its parent's total. That is exactly the quantity a
|
|
119
|
+
YAML bomb maximises, and computing it costs one walk of the graph.
|
|
120
|
+
|
|
121
|
+
Totals are clamped to the limit as they accumulate. Without the clamp
|
|
122
|
+
the arithmetic itself becomes the denial of service: the classic bomb's
|
|
123
|
+
true size is a 400-million-digit intermediate nobody needs to compute in
|
|
124
|
+
order to know it is too big.
|
|
125
|
+
"""
|
|
126
|
+
cached = memo.get(id(node))
|
|
127
|
+
if cached is not None:
|
|
128
|
+
return cached
|
|
129
|
+
|
|
130
|
+
if isinstance(node, yaml.ScalarNode):
|
|
131
|
+
memo[id(node)] = 1
|
|
132
|
+
return 1
|
|
133
|
+
|
|
134
|
+
total = 1
|
|
135
|
+
children: list[yaml.Node] = []
|
|
136
|
+
if isinstance(node, yaml.SequenceNode):
|
|
137
|
+
children = list(node.value)
|
|
138
|
+
elif isinstance(node, yaml.MappingNode):
|
|
139
|
+
children = [child for pair in node.value for child in pair]
|
|
140
|
+
|
|
141
|
+
for child in children:
|
|
142
|
+
total += _expanded_size(child, memo)
|
|
143
|
+
if total > MAX_EXPANDED_NODES:
|
|
144
|
+
total = MAX_EXPANDED_NODES + 1
|
|
145
|
+
break
|
|
146
|
+
|
|
147
|
+
memo[id(node)] = total
|
|
148
|
+
return total
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _depth(value: Any, level: int = 0) -> int:
|
|
152
|
+
"""Nesting depth of an already-parsed structure.
|
|
153
|
+
|
|
154
|
+
Iterative rather than recursive on the hot path would be safer still,
|
|
155
|
+
but the loader has already bounded what can reach here: `MAX_DEPTH` is
|
|
156
|
+
checked against a structure `yaml.SafeLoader` managed to build, and the
|
|
157
|
+
loader raises `RecursionError` (handled above) well before Python's own
|
|
158
|
+
limit on anything deeper.
|
|
159
|
+
"""
|
|
160
|
+
if level > MAX_DEPTH:
|
|
161
|
+
return level
|
|
162
|
+
if isinstance(value, dict):
|
|
163
|
+
return max((_depth(v, level + 1) for v in value.values()), default=level)
|
|
164
|
+
if isinstance(value, list | tuple):
|
|
165
|
+
return max((_depth(v, level + 1) for v in value), default=level)
|
|
166
|
+
return level
|