console-error-scanner 1.16.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- console_error_scanner/__init__.py +5 -0
- console_error_scanner/__main__.py +370 -0
- console_error_scanner/app.py +1870 -0
- console_error_scanner/app.tcss +91 -0
- console_error_scanner/i18n.py +74 -0
- console_error_scanner/locale/de.json +417 -0
- console_error_scanner/locale/en.json +417 -0
- console_error_scanner/models/__init__.py +1 -0
- console_error_scanner/models/history.py +220 -0
- console_error_scanner/models/robots.py +154 -0
- console_error_scanner/models/scan_result.py +265 -0
- console_error_scanner/models/settings.py +242 -0
- console_error_scanner/models/sitemap.py +347 -0
- console_error_scanner/models/whitelist.py +226 -0
- console_error_scanner/screens/__init__.py +1 -0
- console_error_scanner/screens/diet_advisor.py +128 -0
- console_error_scanner/screens/error_detail.py +164 -0
- console_error_scanner/screens/history.py +187 -0
- console_error_scanner/screens/scan_summary.py +234 -0
- console_error_scanner/screens/settings.py +292 -0
- console_error_scanner/screens/top_errors.py +304 -0
- console_error_scanner/screens/whitelist.py +154 -0
- console_error_scanner/services/__init__.py +1 -0
- console_error_scanner/services/image_clipboard.py +104 -0
- console_error_scanner/services/preview_service.py +291 -0
- console_error_scanner/services/rate_limit.py +56 -0
- console_error_scanner/services/reporter.py +561 -0
- console_error_scanner/services/scanner.py +965 -0
- console_error_scanner/services/site_score.py +151 -0
- console_error_scanner/widgets/__init__.py +1 -0
- console_error_scanner/widgets/bar_chart.py +60 -0
- console_error_scanner/widgets/preview_panel.py +356 -0
- console_error_scanner/widgets/results_table.py +594 -0
- console_error_scanner/widgets/stats_panel.py +387 -0
- console_error_scanner/widgets/summary_panel.py +176 -0
- console_error_scanner-1.16.3.dist-info/METADATA +509 -0
- console_error_scanner-1.16.3.dist-info/RECORD +41 -0
- console_error_scanner-1.16.3.dist-info/WHEEL +5 -0
- console_error_scanner-1.16.3.dist-info/entry_points.txt +2 -0
- console_error_scanner-1.16.3.dist-info/licenses/LICENSE +202 -0
- console_error_scanner-1.16.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""robots.txt-Auswertung: prueft, ob eine Seite abgerufen werden darf.
|
|
2
|
+
|
|
3
|
+
Geprueft werden nur die SEITEN aus der Sitemap. Ressourcen, die eine Seite
|
|
4
|
+
nachlaedt (Bilder, Skripte), werden bewusst nicht geprueft: sie liegen oft auf
|
|
5
|
+
einer CDN-Domain mit eigener robots.txt, und wer die Seite ausliefern darf,
|
|
6
|
+
liefert ihre Ressourcen ohnehin mit aus.
|
|
7
|
+
|
|
8
|
+
Ausgewertet wird nur der `User-agent: *`-Block; der Scanner tritt nicht unter
|
|
9
|
+
einem eigenen Bot-Namen auf.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from urllib.parse import urlparse, urlunparse
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _compile_pattern(pattern: str) -> re.Pattern[str]:
|
|
21
|
+
"""Uebersetzt ein robots-Pfadmuster in eine Regex.
|
|
22
|
+
|
|
23
|
+
Der Standard (RFC 9309) kennt zwei Sonderzeichen: `*` fuer eine beliebige
|
|
24
|
+
Zeichenfolge und `$` am Ende fuer "Pfad endet hier". Ohne diese Behandlung
|
|
25
|
+
greifen reale Regeln wie `Disallow: /*CR-Dokumentation.pdf$` nie, weil kein
|
|
26
|
+
Pfad buchstaeblich mit `/*CR-` beginnt.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
pattern:
|
|
30
|
+
Pfadmuster aus einer Disallow-/Allow-Zeile.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
Kompilierte Regex, die am Pfadanfang ankert.
|
|
34
|
+
"""
|
|
35
|
+
anchored_end = pattern.endswith("$")
|
|
36
|
+
raw = pattern[:-1] if anchored_end else pattern
|
|
37
|
+
body = "".join(".*" if char == "*" else re.escape(char) for char in raw)
|
|
38
|
+
return re.compile(f"^{body}$" if anchored_end else f"^{body}")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class RobotsChecker:
|
|
42
|
+
"""Laedt und parst die robots.txt einer Domain (Disallow/Allow)."""
|
|
43
|
+
|
|
44
|
+
def __init__(self) -> None:
|
|
45
|
+
# (Musterlaenge, kompiliertes Muster, erlaubt) - die Laenge entscheidet,
|
|
46
|
+
# welche Regel bei mehreren Treffern gewinnt (spezifischste zuerst).
|
|
47
|
+
self._rules: list[tuple[int, re.Pattern[str], bool]] = []
|
|
48
|
+
self._loaded = False
|
|
49
|
+
|
|
50
|
+
async def load(
|
|
51
|
+
self,
|
|
52
|
+
base_url: str,
|
|
53
|
+
cookies: list[dict[str, str]] | None = None,
|
|
54
|
+
proxy: str = "",
|
|
55
|
+
client: httpx.AsyncClient | None = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
"""Laedt die robots.txt zur angegebenen URL.
|
|
58
|
+
|
|
59
|
+
Nicht erreichbar oder kein 200 = keine Regeln = alles erlaubt. Das ist
|
|
60
|
+
bewusst die freundliche Auslegung: ein fehlendes robots.txt ist keine
|
|
61
|
+
Sperre.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
base_url:
|
|
65
|
+
Beliebige URL der Zieldomain; daraus wird /robots.txt abgeleitet.
|
|
66
|
+
cookies:
|
|
67
|
+
Optionale Cookies (z.B. fuer geschuetzte Testsysteme).
|
|
68
|
+
proxy:
|
|
69
|
+
Optionale Proxy-URL (Corporate-Proxy/Zscaler).
|
|
70
|
+
client:
|
|
71
|
+
Optionaler, bereits konfigurierter Client - sonst wird einer
|
|
72
|
+
geoeffnet. Dient vor allem der Testbarkeit ohne Netzwerk.
|
|
73
|
+
"""
|
|
74
|
+
parsed = urlparse(base_url)
|
|
75
|
+
robots_url = urlunparse((parsed.scheme, parsed.netloc, "/robots.txt", "", "", ""))
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
if client is not None:
|
|
79
|
+
response = await client.get(robots_url)
|
|
80
|
+
if response.status_code == 200:
|
|
81
|
+
self._parse(response.text)
|
|
82
|
+
else:
|
|
83
|
+
jar = httpx.Cookies()
|
|
84
|
+
for cookie in cookies or []:
|
|
85
|
+
jar.set(cookie["name"], cookie["value"])
|
|
86
|
+
async with httpx.AsyncClient(
|
|
87
|
+
timeout=10.0,
|
|
88
|
+
follow_redirects=True,
|
|
89
|
+
verify=False,
|
|
90
|
+
cookies=jar,
|
|
91
|
+
proxy=proxy.strip() or None,
|
|
92
|
+
) as own_client:
|
|
93
|
+
response = await own_client.get(robots_url)
|
|
94
|
+
if response.status_code == 200:
|
|
95
|
+
self._parse(response.text)
|
|
96
|
+
except Exception: # noqa: BLE001 - robots.txt ist optional
|
|
97
|
+
pass
|
|
98
|
+
self._loaded = True
|
|
99
|
+
|
|
100
|
+
def _parse(self, text: str) -> None:
|
|
101
|
+
"""Parst Disallow/Allow aus dem `User-agent: *`-Block."""
|
|
102
|
+
in_wildcard_block = False
|
|
103
|
+
|
|
104
|
+
for raw_line in text.splitlines():
|
|
105
|
+
line = raw_line.strip()
|
|
106
|
+
if "#" in line:
|
|
107
|
+
line = line[: line.index("#")].strip()
|
|
108
|
+
if not line:
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
lower = line.lower()
|
|
112
|
+
if lower.startswith("user-agent:"):
|
|
113
|
+
in_wildcard_block = line[len("user-agent:") :].strip() == "*"
|
|
114
|
+
continue
|
|
115
|
+
if not in_wildcard_block:
|
|
116
|
+
continue
|
|
117
|
+
|
|
118
|
+
if lower.startswith("disallow:"):
|
|
119
|
+
path = line[len("disallow:") :].strip()
|
|
120
|
+
if path: # "Disallow:" ohne Pfad heisst: alles erlaubt
|
|
121
|
+
self._rules.append((len(path), _compile_pattern(path), False))
|
|
122
|
+
elif lower.startswith("allow:"):
|
|
123
|
+
path = line[len("allow:") :].strip()
|
|
124
|
+
if path:
|
|
125
|
+
self._rules.append((len(path), _compile_pattern(path), True))
|
|
126
|
+
|
|
127
|
+
def is_allowed(self, url: str) -> bool:
|
|
128
|
+
"""Prueft, ob die URL abgerufen werden darf.
|
|
129
|
+
|
|
130
|
+
Bei mehreren passenden Regeln gewinnt die laengste (spezifischste); bei
|
|
131
|
+
gleicher Laenge gewinnt Allow - so schreibt es RFC 9309 vor.
|
|
132
|
+
"""
|
|
133
|
+
if not self._rules:
|
|
134
|
+
return True
|
|
135
|
+
|
|
136
|
+
parsed = urlparse(url)
|
|
137
|
+
path = parsed.path or "/"
|
|
138
|
+
if parsed.query:
|
|
139
|
+
path = f"{path}?{parsed.query}"
|
|
140
|
+
|
|
141
|
+
best_length = -1
|
|
142
|
+
allowed = True
|
|
143
|
+
for length, pattern, rule_allows in self._rules:
|
|
144
|
+
if not pattern.search(path):
|
|
145
|
+
continue
|
|
146
|
+
if length > best_length or (length == best_length and rule_allows):
|
|
147
|
+
best_length = length
|
|
148
|
+
allowed = rule_allows
|
|
149
|
+
return allowed
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def is_loaded(self) -> bool:
|
|
153
|
+
"""Gibt zurueck, ob ein Ladeversuch stattgefunden hat."""
|
|
154
|
+
return self._loaded
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"""Datenmodelle fuer Scan-Ergebnisse."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ErrorType(Enum):
|
|
10
|
+
"""Art des Fehlers."""
|
|
11
|
+
|
|
12
|
+
CONSOLE_ERROR = "console_error"
|
|
13
|
+
CONSOLE_WARNING = "console_warning"
|
|
14
|
+
HTTP_404 = "http_404"
|
|
15
|
+
HTTP_4XX = "http_4xx"
|
|
16
|
+
HTTP_5XX = "http_5xx"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PageStatus(Enum):
|
|
20
|
+
"""Status einer gescannten Seite."""
|
|
21
|
+
|
|
22
|
+
PENDING = "pending"
|
|
23
|
+
SCANNING = "scanning"
|
|
24
|
+
OK = "ok"
|
|
25
|
+
WARNING = "warning"
|
|
26
|
+
ERROR = "error"
|
|
27
|
+
TIMEOUT = "timeout"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class PageError:
|
|
32
|
+
"""Ein einzelner Fehler auf einer Seite."""
|
|
33
|
+
|
|
34
|
+
error_type: ErrorType
|
|
35
|
+
message: str
|
|
36
|
+
source: str = ""
|
|
37
|
+
line_number: int = 0
|
|
38
|
+
whitelisted: bool = False
|
|
39
|
+
|
|
40
|
+
def to_dict(self) -> dict:
|
|
41
|
+
"""Konvertiert den Fehler in ein Dictionary."""
|
|
42
|
+
return {
|
|
43
|
+
"error_type": self.error_type.value,
|
|
44
|
+
"message": self.message,
|
|
45
|
+
"source": self.source,
|
|
46
|
+
"line_number": self.line_number,
|
|
47
|
+
"whitelisted": self.whitelisted,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class ResourceSize:
|
|
53
|
+
"""Eine einzelne geladene Ressource mit ihrer Transfergroesse.
|
|
54
|
+
|
|
55
|
+
Dient dem "Diaet-Ratgeber" (groesste Brocken einer Seite).
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
url: str
|
|
59
|
+
size_bytes: int
|
|
60
|
+
resource_type: str = ""
|
|
61
|
+
|
|
62
|
+
def to_dict(self) -> dict:
|
|
63
|
+
"""Konvertiert die Ressource in ein Dictionary."""
|
|
64
|
+
return {
|
|
65
|
+
"url": self.url,
|
|
66
|
+
"size_bytes": self.size_bytes,
|
|
67
|
+
"resource_type": self.resource_type,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class ScanResult:
|
|
73
|
+
"""Ergebnis des Scans einer einzelnen Seite."""
|
|
74
|
+
|
|
75
|
+
url: str
|
|
76
|
+
status: PageStatus = PageStatus.PENDING
|
|
77
|
+
http_status_code: int = 0
|
|
78
|
+
# Ladezeit: Browser-Load-Event (Navigation Timing 'loadEventEnd'), Fallback
|
|
79
|
+
# auf die Wall-Clock-Messung goto->networkidle. Siehe Scanner.
|
|
80
|
+
load_time_ms: int = 0
|
|
81
|
+
# DOMContentLoaded-Zeit (Navigation Timing 'domContentLoadedEventEnd'), ms.
|
|
82
|
+
dom_content_loaded_ms: int = 0
|
|
83
|
+
# Anzahl ALLER Requests der Seite (wie Edge-Netzwerkmonitor).
|
|
84
|
+
request_count: int = 0
|
|
85
|
+
page_size_bytes: int = 0
|
|
86
|
+
errors: list[PageError] = field(default_factory=list)
|
|
87
|
+
retry_count: int = 0
|
|
88
|
+
# HTTP-Response-Daten der Hauptseite (Document)
|
|
89
|
+
response_headers: dict[str, str] = field(default_factory=dict)
|
|
90
|
+
content_type: str = ""
|
|
91
|
+
last_modified: str = ""
|
|
92
|
+
# Groesste Einzelressourcen (same-host), absteigend - fuer den Diaet-Ratgeber.
|
|
93
|
+
resource_sizes: list[ResourceSize] = field(default_factory=list)
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def console_error_count(self) -> int:
|
|
97
|
+
"""Anzahl der Console-Errors (ohne whitelisted)."""
|
|
98
|
+
return sum(1 for e in self.errors if e.error_type == ErrorType.CONSOLE_ERROR and not e.whitelisted)
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def console_warning_count(self) -> int:
|
|
102
|
+
"""Anzahl der Console-Warnings (ohne whitelisted)."""
|
|
103
|
+
return sum(1 for e in self.errors if e.error_type == ErrorType.CONSOLE_WARNING and not e.whitelisted)
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def http_404_count(self) -> int:
|
|
107
|
+
"""Anzahl der HTTP 404 Fehler (ohne whitelisted)."""
|
|
108
|
+
return sum(1 for e in self.errors if e.error_type == ErrorType.HTTP_404 and not e.whitelisted)
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def http_4xx_count(self) -> int:
|
|
112
|
+
"""Anzahl der HTTP 4xx Fehler ohne 404 (ohne whitelisted)."""
|
|
113
|
+
return sum(1 for e in self.errors if e.error_type == ErrorType.HTTP_4XX and not e.whitelisted)
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def http_5xx_count(self) -> int:
|
|
117
|
+
"""Anzahl der HTTP 5xx Fehler (ohne whitelisted)."""
|
|
118
|
+
return sum(1 for e in self.errors if e.error_type == ErrorType.HTTP_5XX and not e.whitelisted)
|
|
119
|
+
|
|
120
|
+
# Fehlertypen die als "echter Fehler" (ERR) zaehlen
|
|
121
|
+
_ERROR_TYPES = {ErrorType.CONSOLE_ERROR, ErrorType.HTTP_404, ErrorType.HTTP_4XX, ErrorType.HTTP_5XX}
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def has_errors(self) -> bool:
|
|
125
|
+
"""Hat die Seite nicht-whitelisted echte Fehler (ohne Warnings)?"""
|
|
126
|
+
return any(e.error_type in self._ERROR_TYPES and not e.whitelisted for e in self.errors)
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def has_issues(self) -> bool:
|
|
130
|
+
"""Hat die Seite irgendwelche nicht-whitelisted Eintraege (inkl. Warnings)?"""
|
|
131
|
+
return any(not e.whitelisted for e in self.errors)
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def ignored_count(self) -> int:
|
|
135
|
+
"""Anzahl der per Whitelist ignorierten Fehler."""
|
|
136
|
+
return sum(1 for e in self.errors if e.whitelisted)
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def has_only_ignored_errors(self) -> bool:
|
|
140
|
+
"""Hat die Seite nur whitelisted Fehler (keine echten Issues)?"""
|
|
141
|
+
return len(self.errors) > 0 and not self.has_issues
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def status_icon(self) -> str:
|
|
145
|
+
"""Icon fuer den aktuellen Status."""
|
|
146
|
+
if self.has_only_ignored_errors:
|
|
147
|
+
return "IGN"
|
|
148
|
+
icons = {
|
|
149
|
+
PageStatus.PENDING: "...",
|
|
150
|
+
PageStatus.SCANNING: ">>>",
|
|
151
|
+
PageStatus.OK: "OK",
|
|
152
|
+
PageStatus.WARNING: "WARN",
|
|
153
|
+
PageStatus.ERROR: "ERR",
|
|
154
|
+
PageStatus.TIMEOUT: "T/O",
|
|
155
|
+
}
|
|
156
|
+
return icons.get(self.status, "?")
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def total_error_count(self) -> int:
|
|
160
|
+
"""Gesamtanzahl aller nicht-whitelisted Fehler."""
|
|
161
|
+
return sum(1 for e in self.errors if not e.whitelisted)
|
|
162
|
+
|
|
163
|
+
def to_dict(self) -> dict:
|
|
164
|
+
"""Konvertiert das Ergebnis in ein Dictionary."""
|
|
165
|
+
active_errors = [e for e in self.errors if not e.whitelisted]
|
|
166
|
+
ignored_errors = [e for e in self.errors if e.whitelisted]
|
|
167
|
+
return {
|
|
168
|
+
"url": self.url,
|
|
169
|
+
"status": self.status.value,
|
|
170
|
+
"http_status_code": self.http_status_code,
|
|
171
|
+
"load_time_ms": self.load_time_ms,
|
|
172
|
+
"dom_content_loaded_ms": self.dom_content_loaded_ms,
|
|
173
|
+
"request_count": self.request_count,
|
|
174
|
+
"page_size_bytes": self.page_size_bytes,
|
|
175
|
+
"total_errors": self.total_error_count,
|
|
176
|
+
"console_errors": self.console_error_count,
|
|
177
|
+
"console_warnings": self.console_warning_count,
|
|
178
|
+
"http_404_errors": self.http_404_count,
|
|
179
|
+
"http_4xx_errors": self.http_4xx_count,
|
|
180
|
+
"http_5xx_errors": self.http_5xx_count,
|
|
181
|
+
"ignored_count": self.ignored_count,
|
|
182
|
+
"retry_count": self.retry_count,
|
|
183
|
+
"errors": [e.to_dict() for e in active_errors],
|
|
184
|
+
"ignored_errors": [e.to_dict() for e in ignored_errors],
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@dataclass
|
|
189
|
+
class ScanSummary:
|
|
190
|
+
"""Gesamtzusammenfassung eines Scans."""
|
|
191
|
+
|
|
192
|
+
sitemap_url: str = ""
|
|
193
|
+
total_urls: int = 0
|
|
194
|
+
scanned_urls: int = 0
|
|
195
|
+
urls_with_errors: int = 0
|
|
196
|
+
total_console_errors: int = 0
|
|
197
|
+
total_console_warnings: int = 0
|
|
198
|
+
total_http_404: int = 0
|
|
199
|
+
total_http_4xx: int = 0
|
|
200
|
+
total_http_5xx: int = 0
|
|
201
|
+
total_timeouts: int = 0
|
|
202
|
+
total_ignored: int = 0
|
|
203
|
+
scan_duration_ms: int = 0
|
|
204
|
+
|
|
205
|
+
@staticmethod
|
|
206
|
+
def from_results(sitemap_url: str, results: list[ScanResult], duration_ms: int = 0) -> ScanSummary:
|
|
207
|
+
"""Erstellt eine Zusammenfassung aus den Scan-Ergebnissen."""
|
|
208
|
+
summary = ScanSummary(sitemap_url=sitemap_url)
|
|
209
|
+
summary.total_urls = len(results)
|
|
210
|
+
summary.scan_duration_ms = duration_ms
|
|
211
|
+
|
|
212
|
+
for result in results:
|
|
213
|
+
if result.status in (PageStatus.OK, PageStatus.WARNING, PageStatus.ERROR):
|
|
214
|
+
summary.scanned_urls += 1
|
|
215
|
+
if result.has_errors:
|
|
216
|
+
summary.urls_with_errors += 1
|
|
217
|
+
if result.status == PageStatus.TIMEOUT:
|
|
218
|
+
summary.total_timeouts += 1
|
|
219
|
+
summary.total_console_errors += result.console_error_count
|
|
220
|
+
summary.total_console_warnings += result.console_warning_count
|
|
221
|
+
summary.total_http_404 += result.http_404_count
|
|
222
|
+
summary.total_http_4xx += result.http_4xx_count
|
|
223
|
+
summary.total_http_5xx += result.http_5xx_count
|
|
224
|
+
summary.total_ignored += result.ignored_count
|
|
225
|
+
|
|
226
|
+
return summary
|
|
227
|
+
|
|
228
|
+
def to_dict(self) -> dict:
|
|
229
|
+
"""Konvertiert die Zusammenfassung in ein Dictionary."""
|
|
230
|
+
return {
|
|
231
|
+
"sitemap_url": self.sitemap_url,
|
|
232
|
+
"total_urls": self.total_urls,
|
|
233
|
+
"scanned_urls": self.scanned_urls,
|
|
234
|
+
"urls_with_errors": self.urls_with_errors,
|
|
235
|
+
"total_console_errors": self.total_console_errors,
|
|
236
|
+
"total_console_warnings": self.total_console_warnings,
|
|
237
|
+
"total_http_404": self.total_http_404,
|
|
238
|
+
"total_http_4xx": self.total_http_4xx,
|
|
239
|
+
"total_http_5xx": self.total_http_5xx,
|
|
240
|
+
"total_timeouts": self.total_timeouts,
|
|
241
|
+
"total_ignored": self.total_ignored,
|
|
242
|
+
"scan_duration_ms": self.scan_duration_ms,
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def format_page_size(size_bytes: int) -> str:
|
|
247
|
+
"""Formatiert Bytes in lesbare Groesse (KB oder MB).
|
|
248
|
+
|
|
249
|
+
Unter 1 MB: "245 KB", ab 1 MB: "1.3 MB".
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
size_bytes: Groesse in Bytes.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
Formatierter String.
|
|
256
|
+
"""
|
|
257
|
+
if size_bytes <= 0:
|
|
258
|
+
return "-"
|
|
259
|
+
kb = size_bytes / 1024
|
|
260
|
+
if kb < 1:
|
|
261
|
+
return "< 1 KB"
|
|
262
|
+
if kb < 1024:
|
|
263
|
+
return f"{kb:.0f} KB"
|
|
264
|
+
mb = kb / 1024
|
|
265
|
+
return f"{mb:.1f} MB"
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Persistierte Einstellungen fuer Console Error Scanner.
|
|
2
|
+
|
|
3
|
+
Speichert und laedt Benutzereinstellungen aus
|
|
4
|
+
~/.console-error-scanner/settings.json.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from ..i18n import detect_language
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# textual-themes 0.5 hat 25 Themes umbenannt (trademark-safety pass).
|
|
20
|
+
# Settings-Files aelterer Versionen koennen alte Slugs gespeichert haben -
|
|
21
|
+
# die werden beim Laden transparent gemappt.
|
|
22
|
+
_LEGACY_THEME_MAP: dict[str, str] = {
|
|
23
|
+
"c64": "brotkasten",
|
|
24
|
+
"amiga": "boing",
|
|
25
|
+
"atari-st": "gemstone",
|
|
26
|
+
"ibm-terminal": "classic-terminal",
|
|
27
|
+
"nextstep": "next",
|
|
28
|
+
"beos": "bebox",
|
|
29
|
+
"ubuntu": "bunty",
|
|
30
|
+
"macos": "cupertino",
|
|
31
|
+
"windows-xp": "luna",
|
|
32
|
+
"msdos": "commandr",
|
|
33
|
+
"solaris-cde": "motif",
|
|
34
|
+
"os2-warp": "warp",
|
|
35
|
+
"opensuse": "geeko",
|
|
36
|
+
"linux-mint": "minty",
|
|
37
|
+
"red-hat": "crimson",
|
|
38
|
+
"raspberry-pi": "razzy",
|
|
39
|
+
"freebsd": "beastie",
|
|
40
|
+
"tudor": "fifty-eight",
|
|
41
|
+
"goldfinger": "goldfinder",
|
|
42
|
+
"hulk": "hulkula",
|
|
43
|
+
"batman": "flughund",
|
|
44
|
+
"gameboy": "brick",
|
|
45
|
+
"pan-am": "clipper",
|
|
46
|
+
"miami-vice": "miami",
|
|
47
|
+
"martini-racing": "racing",
|
|
48
|
+
"superman": "metropolis",
|
|
49
|
+
"spiderman": "spiderized",
|
|
50
|
+
"gulf-racing": "textual-dark",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
SETTINGS_DIR = Path.home() / ".console-error-scanner"
|
|
55
|
+
SETTINGS_FILE = SETTINGS_DIR / "settings.json"
|
|
56
|
+
|
|
57
|
+
# Absturzberichte landen neben den Einstellungen. Angehaengt, nicht ersetzt -
|
|
58
|
+
# so bleibt auch ein frueherer Absturz nachvollziehbar.
|
|
59
|
+
CRASH_LOG_NAME = "last-crash.txt"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class Settings:
|
|
64
|
+
"""Persistierte Benutzereinstellungen.
|
|
65
|
+
|
|
66
|
+
Attributes:
|
|
67
|
+
theme:
|
|
68
|
+
Name des Textual-Themes.
|
|
69
|
+
language:
|
|
70
|
+
UI-Sprache (de/en).
|
|
71
|
+
accept_consent:
|
|
72
|
+
Consent-Akzeptierung aktiv.
|
|
73
|
+
trigger_lazy_load:
|
|
74
|
+
Seite scrollen fuer Lazy-Loading.
|
|
75
|
+
concurrency:
|
|
76
|
+
Parallele Browser-Tabs.
|
|
77
|
+
timeout:
|
|
78
|
+
Timeout pro Seite in Sekunden.
|
|
79
|
+
console_level:
|
|
80
|
+
Console-Level (error/warn/all).
|
|
81
|
+
user_agent:
|
|
82
|
+
Custom User-Agent.
|
|
83
|
+
cookies:
|
|
84
|
+
Roh-Cookies-String "name=value; name2=value2".
|
|
85
|
+
whitelist_path:
|
|
86
|
+
Pfad zur whitelist.json (leer = keine Whitelist).
|
|
87
|
+
size_warn_mb:
|
|
88
|
+
Schwellwert in MB: ueberschreitet die Seitengroesse diesen Wert,
|
|
89
|
+
wird die URL in der Liste rot hervorgehoben. 0 = aus.
|
|
90
|
+
score_error_weight:
|
|
91
|
+
Gewicht der Fehlerquote im Site-Score (in Prozent, 0-100). Das
|
|
92
|
+
Gewicht der Seitengroesse ist der Rest (100 - score_error_weight).
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
theme: str = "textual-dark"
|
|
96
|
+
# Erststart: Sprache aus der Systemumgebung, danach aus der Datei.
|
|
97
|
+
language: str = field(default_factory=lambda: detect_language())
|
|
98
|
+
accept_consent: bool = True
|
|
99
|
+
trigger_lazy_load: bool = True
|
|
100
|
+
respect_robots: bool = True
|
|
101
|
+
concurrency: int = 8
|
|
102
|
+
# Aufrufe pro Minute; 0 = ungebremst. Voreingestellt gedrosselt, weil
|
|
103
|
+
# jede Seite durch einen echten Browser laeuft und entsprechend wiegt.
|
|
104
|
+
rate_limit_enabled: bool = True
|
|
105
|
+
rate_per_minute: int = 60
|
|
106
|
+
timeout: int = 60
|
|
107
|
+
console_level: str = "warn"
|
|
108
|
+
user_agent: str = ""
|
|
109
|
+
cookies: str = ""
|
|
110
|
+
whitelist_path: str = ""
|
|
111
|
+
no_headless: bool = False
|
|
112
|
+
show_preview: bool = False
|
|
113
|
+
size_warn_mb: int = 10
|
|
114
|
+
score_error_weight: int = 60
|
|
115
|
+
proxy_url: str = ""
|
|
116
|
+
# JIRA-Exportformat: "markdown" (Jira Cloud) oder "wiki" (Server/DC).
|
|
117
|
+
jira_format: str = "markdown"
|
|
118
|
+
|
|
119
|
+
SETTINGS_DIR = SETTINGS_DIR
|
|
120
|
+
SETTINGS_FILE = SETTINGS_FILE
|
|
121
|
+
|
|
122
|
+
def to_dict(self) -> dict:
|
|
123
|
+
"""Konvertiert die Einstellungen in ein Dictionary fuer JSON."""
|
|
124
|
+
return {
|
|
125
|
+
"theme": self.theme,
|
|
126
|
+
"language": self.language,
|
|
127
|
+
"accept_consent": self.accept_consent,
|
|
128
|
+
"trigger_lazy_load": self.trigger_lazy_load,
|
|
129
|
+
"respect_robots": self.respect_robots,
|
|
130
|
+
"concurrency": self.concurrency,
|
|
131
|
+
"rate_limit_enabled": self.rate_limit_enabled,
|
|
132
|
+
"rate_per_minute": self.rate_per_minute,
|
|
133
|
+
"timeout": self.timeout,
|
|
134
|
+
"console_level": self.console_level,
|
|
135
|
+
"user_agent": self.user_agent,
|
|
136
|
+
"cookies": self.cookies,
|
|
137
|
+
"whitelist_path": self.whitelist_path,
|
|
138
|
+
"no_headless": self.no_headless,
|
|
139
|
+
"show_preview": self.show_preview,
|
|
140
|
+
"size_warn_mb": self.size_warn_mb,
|
|
141
|
+
"score_error_weight": self.score_error_weight,
|
|
142
|
+
"proxy_url": self.proxy_url,
|
|
143
|
+
"jira_format": self.jira_format,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
@staticmethod
|
|
147
|
+
def load() -> Settings:
|
|
148
|
+
"""Laedt die Einstellungen aus der JSON-Datei.
|
|
149
|
+
|
|
150
|
+
Gibt Default-Einstellungen zurueck bei Fehler oder fehlender Datei.
|
|
151
|
+
Migriert dabei alte Theme-Slugs aus textual-themes < 0.5 auf
|
|
152
|
+
ihre aktuellen Namen und persistiert die Migration.
|
|
153
|
+
"""
|
|
154
|
+
if not SETTINGS_FILE.is_file():
|
|
155
|
+
return Settings()
|
|
156
|
+
|
|
157
|
+
try:
|
|
158
|
+
raw = SETTINGS_FILE.read_text(encoding="utf-8")
|
|
159
|
+
data = json.loads(raw)
|
|
160
|
+
if not isinstance(data, dict):
|
|
161
|
+
return Settings()
|
|
162
|
+
settings = Settings(
|
|
163
|
+
theme=data.get("theme", "textual-dark"),
|
|
164
|
+
language=data.get("language", detect_language()),
|
|
165
|
+
accept_consent=data.get("accept_consent", True),
|
|
166
|
+
trigger_lazy_load=data.get("trigger_lazy_load", True),
|
|
167
|
+
respect_robots=data.get("respect_robots", True),
|
|
168
|
+
concurrency=int(data.get("concurrency", 8)),
|
|
169
|
+
rate_limit_enabled=bool(data.get("rate_limit_enabled", True)),
|
|
170
|
+
rate_per_minute=int(data.get("rate_per_minute", 60)),
|
|
171
|
+
timeout=int(data.get("timeout", 60)),
|
|
172
|
+
console_level=str(data.get("console_level", "warn")),
|
|
173
|
+
user_agent=str(data.get("user_agent", "")),
|
|
174
|
+
cookies=str(data.get("cookies", "")),
|
|
175
|
+
whitelist_path=str(data.get("whitelist_path", "")),
|
|
176
|
+
no_headless=bool(data.get("no_headless", False)),
|
|
177
|
+
show_preview=bool(data.get("show_preview", False)),
|
|
178
|
+
size_warn_mb=int(data.get("size_warn_mb", 10)),
|
|
179
|
+
score_error_weight=int(data.get("score_error_weight", 60)),
|
|
180
|
+
proxy_url=str(data.get("proxy_url", "")),
|
|
181
|
+
jira_format=("wiki" if str(data.get("jira_format", "markdown")).lower() == "wiki" else "markdown"),
|
|
182
|
+
)
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
logger.warning("Settings konnten nicht geladen werden: %s", exc)
|
|
185
|
+
return Settings()
|
|
186
|
+
|
|
187
|
+
# Legacy-Theme-Slug migrieren
|
|
188
|
+
if settings.theme in _LEGACY_THEME_MAP:
|
|
189
|
+
settings.theme = _LEGACY_THEME_MAP[settings.theme]
|
|
190
|
+
settings.save()
|
|
191
|
+
|
|
192
|
+
return settings
|
|
193
|
+
|
|
194
|
+
def save(self) -> None:
|
|
195
|
+
"""Speichert die Einstellungen in die JSON-Datei."""
|
|
196
|
+
try:
|
|
197
|
+
SETTINGS_DIR.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
SETTINGS_FILE.write_text(
|
|
199
|
+
json.dumps(self.to_dict(), indent=2, ensure_ascii=False),
|
|
200
|
+
encoding="utf-8",
|
|
201
|
+
)
|
|
202
|
+
except Exception as exc:
|
|
203
|
+
logger.warning("Settings konnten nicht gespeichert werden: %s", exc)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def parse_cookies(raw: str) -> list[dict[str, str]]:
|
|
207
|
+
"""Parst einen Cookie-String 'name=value; name2=value2' zu einer Liste.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
raw: Cookie-String (Trenner: ';' oder ','; Whitespace tolerant).
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
Liste von {"name": ..., "value": ...}-Dicts. Eintraege ohne '='
|
|
214
|
+
werden uebersprungen.
|
|
215
|
+
"""
|
|
216
|
+
if not raw or not raw.strip():
|
|
217
|
+
return []
|
|
218
|
+
parts: list[str] = []
|
|
219
|
+
for chunk in raw.split(";"):
|
|
220
|
+
parts.extend(p.strip() for p in chunk.split(",") if p.strip())
|
|
221
|
+
cookies: list[dict[str, str]] = []
|
|
222
|
+
for entry in parts:
|
|
223
|
+
if "=" not in entry:
|
|
224
|
+
continue
|
|
225
|
+
name, value = entry.split("=", 1)
|
|
226
|
+
name = name.strip()
|
|
227
|
+
value = value.strip()
|
|
228
|
+
if name:
|
|
229
|
+
cookies.append({"name": name, "value": value})
|
|
230
|
+
return cookies
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def cookies_to_string(cookies: list[dict[str, str]]) -> str:
|
|
234
|
+
"""Wandelt eine Cookie-Liste zurueck in den persistierbaren String.
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
cookies: Liste von {"name": ..., "value": ...}-Dicts.
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
String der Form 'name=value; name2=value2'.
|
|
241
|
+
"""
|
|
242
|
+
return "; ".join(f"{c.get('name', '').strip()}={c.get('value', '').strip()}" for c in cookies if c.get("name"))
|