grison 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- grison/__init__.py +9 -0
- grison/cli.py +304 -0
- grison/markdown/__init__.py +33 -0
- grison/markdown/converter.py +303 -0
- grison/markdown/document.py +124 -0
- grison/markdown/mapping.py +125 -0
- grison/model/__init__.py +39 -0
- grison/model/cvss.py +172 -0
- grison/model/cwe.py +46 -0
- grison/model/data/cwe.json +1452 -0
- grison/model/enums.py +78 -0
- grison/model/finding.py +153 -0
- grison/ports.py +39 -0
- grison/remote/__init__.py +2 -0
- grison/remote/bookstack.py +93 -0
- grison/remote/bootstrap.py +70 -0
- grison/remote/bsmap.py +105 -0
- grison/remote/creds.py +90 -0
- grison/remote/ghostwriter.py +270 -0
- grison/remote/gwmap.py +174 -0
- grison/remote/methodology.py +340 -0
- grison/remote/snapshot.py +129 -0
- grison/remote/sync.py +657 -0
- grison/scanners/__init__.py +57 -0
- grison/scanners/acunetix.py +128 -0
- grison/scanners/base.py +56 -0
- grison/scanners/burp.py +99 -0
- grison/scanners/detect.py +82 -0
- grison/scanners/ir/__init__.py +14 -0
- grison/scanners/ir/finding.py +31 -0
- grison/scanners/ir/severity.py +75 -0
- grison/scanners/nessus.py +125 -0
- grison/scanners/nmap.py +142 -0
- grison/scanners/openvas.py +128 -0
- grison/scanners/qualys.py +127 -0
- grison/scanners/sslyze.py +404 -0
- grison/scanners/zap.py +155 -0
- grison/sinks/__init__.py +8 -0
- grison/sinks/file_sink.py +104 -0
- grison/sinks/pipeline.py +91 -0
- grison/validate.py +66 -0
- grison/workspace.py +43 -0
- grison-0.1.0.dist-info/METADATA +133 -0
- grison-0.1.0.dist-info/RECORD +47 -0
- grison-0.1.0.dist-info/WHEEL +4 -0
- grison-0.1.0.dist-info/entry_points.txt +2 -0
- grison-0.1.0.dist-info/licenses/LICENSE +21 -0
grison/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""grison — a markdown hub between security scanners and Valiente's infra.
|
|
2
|
+
|
|
3
|
+
Everything is a *source*, *transform*, or *sink* of markdown. See the module
|
|
4
|
+
layout (ports & adapters): ``model`` (schema), ``scanners`` (source adapters),
|
|
5
|
+
``markdown`` (serialization + HTML⇄md converter), ``sinks`` (file sink),
|
|
6
|
+
``remote`` (Ghostwriter + BookStack), and ``cli``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
grison/cli.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"""grison CLI — three verbs: ``parse``, ``status``, ``sync``.
|
|
2
|
+
|
|
3
|
+
The path names the backend (``findings/`` ⇄ Ghostwriter, ``methodology/`` ⇄
|
|
4
|
+
BookStack); location decides identity; the first ``sync`` bootstraps the workspace.
|
|
5
|
+
``parse`` and ``status`` are offline; ``sync`` reconciles findings with Ghostwriter
|
|
6
|
+
and methodology with BookStack (push/pull/collision derived per record).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import fcntl
|
|
12
|
+
from collections.abc import Iterator
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Annotated
|
|
16
|
+
|
|
17
|
+
import typer
|
|
18
|
+
|
|
19
|
+
from grison.model import FindingType
|
|
20
|
+
from grison.remote.bookstack import BookStackClient
|
|
21
|
+
from grison.remote.bootstrap import bootstrap_workspace
|
|
22
|
+
from grison.remote.creds import MissingCreds
|
|
23
|
+
from grison.remote.creds import load as load_creds
|
|
24
|
+
from grison.remote.ghostwriter import GhostwriterClient
|
|
25
|
+
from grison.remote.methodology import MethResult, sync_methodology
|
|
26
|
+
from grison.remote.sync import SyncResult
|
|
27
|
+
from grison.remote.sync import sync as run_sync
|
|
28
|
+
from grison.sinks import ParseSummary, run_parse
|
|
29
|
+
from grison.validate import validate_file
|
|
30
|
+
from grison.workspace import bootstrap_tree, inbox_dir
|
|
31
|
+
|
|
32
|
+
app = typer.Typer(
|
|
33
|
+
name="grison",
|
|
34
|
+
help="A markdown hub between security scanners and Valiente's Ghostwriter + BookStack.",
|
|
35
|
+
no_args_is_help=True,
|
|
36
|
+
add_completion=False,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@app.callback()
|
|
41
|
+
def _root() -> None:
|
|
42
|
+
"""grison — parse scanner artifacts to markdown, then status/sync with the remotes."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@app.command()
|
|
46
|
+
def parse(
|
|
47
|
+
paths: Annotated[list[Path], typer.Argument(help="Scanner export file(s) or dir(s).")],
|
|
48
|
+
scanner: Annotated[
|
|
49
|
+
str | None,
|
|
50
|
+
typer.Option("--scanner", help="Force a scanner type instead of auto-detecting."),
|
|
51
|
+
] = None,
|
|
52
|
+
out: Annotated[
|
|
53
|
+
Path | None,
|
|
54
|
+
typer.Option("-o", "--out", help="Output dir (default: findings/inbox/)."),
|
|
55
|
+
] = None,
|
|
56
|
+
finding_type: Annotated[
|
|
57
|
+
FindingType | None,
|
|
58
|
+
typer.Option("--finding-type", help="Override the per-scanner finding-type default."),
|
|
59
|
+
] = None,
|
|
60
|
+
min_severity: Annotated[
|
|
61
|
+
str | None,
|
|
62
|
+
typer.Option("--min-severity", help="Keep only e.g. 'high,critical' or 'medium-critical'."),
|
|
63
|
+
] = None,
|
|
64
|
+
dry_run: Annotated[bool, typer.Option("--dry-run", help="Preview without writing.")] = False,
|
|
65
|
+
) -> None:
|
|
66
|
+
"""Turn scanner export(s) into markdown findings in findings/inbox/ (offline)."""
|
|
67
|
+
if out is None:
|
|
68
|
+
bootstrap_tree(Path.cwd()) # the binary scaffolds; no init
|
|
69
|
+
out_dir = inbox_dir(Path.cwd())
|
|
70
|
+
else:
|
|
71
|
+
out_dir = out
|
|
72
|
+
summary = run_parse(
|
|
73
|
+
paths,
|
|
74
|
+
out_dir,
|
|
75
|
+
scanner=scanner,
|
|
76
|
+
finding_type=finding_type,
|
|
77
|
+
min_severity=min_severity,
|
|
78
|
+
dry_run=dry_run,
|
|
79
|
+
)
|
|
80
|
+
_print_parse_summary(summary, out_dir, dry_run=dry_run)
|
|
81
|
+
if summary.errors:
|
|
82
|
+
raise typer.Exit(code=1)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@app.command()
|
|
86
|
+
def status(
|
|
87
|
+
paths: Annotated[list[Path], typer.Argument(help="Finding markdown file(s) or dir(s).")],
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Report per-record validity (schema / enum / CVSS / CWE / GW whitelist)."""
|
|
90
|
+
files = _resolve_md(paths)
|
|
91
|
+
if not files:
|
|
92
|
+
typer.secho("no markdown files found", fg=typer.colors.YELLOW)
|
|
93
|
+
raise typer.Exit(code=0)
|
|
94
|
+
|
|
95
|
+
invalid = 0
|
|
96
|
+
for f in files:
|
|
97
|
+
errors = validate_file(f)
|
|
98
|
+
if errors:
|
|
99
|
+
invalid += 1
|
|
100
|
+
typer.secho(f"INVALID {f}", fg=typer.colors.RED)
|
|
101
|
+
for e in errors:
|
|
102
|
+
typer.echo(f" - {e}")
|
|
103
|
+
else:
|
|
104
|
+
typer.secho(f"valid {f}", fg=typer.colors.GREEN)
|
|
105
|
+
|
|
106
|
+
valid = len(files) - invalid
|
|
107
|
+
typer.echo("")
|
|
108
|
+
fg = typer.colors.RED if invalid else typer.colors.GREEN
|
|
109
|
+
typer.secho(f"{valid} valid, {invalid} invalid", fg=fg)
|
|
110
|
+
if invalid:
|
|
111
|
+
raise typer.Exit(code=1)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@app.command()
|
|
115
|
+
def sync(
|
|
116
|
+
dry_run: Annotated[
|
|
117
|
+
bool, typer.Option("--dry-run", help="Preview the plan, write nothing (== status).")
|
|
118
|
+
] = False,
|
|
119
|
+
force_local: Annotated[
|
|
120
|
+
Path | None,
|
|
121
|
+
typer.Option("--force-local", help="Resolve a file's collision by taking local (push)."),
|
|
122
|
+
] = None,
|
|
123
|
+
force_remote: Annotated[
|
|
124
|
+
Path | None,
|
|
125
|
+
typer.Option("--force-remote", help="Resolve a file's collision by taking remote (pull)."),
|
|
126
|
+
] = None,
|
|
127
|
+
) -> None:
|
|
128
|
+
"""Reconcile the workspace with Ghostwriter — push/pull/collision derived per record.
|
|
129
|
+
|
|
130
|
+
Bootstraps on first run. Direction isn't chosen: a locally-edited record pushes, a
|
|
131
|
+
remote-changed one pulls, and a record changed on both sides is surfaced (never
|
|
132
|
+
overwritten). Every remote write is snapshot-backed.
|
|
133
|
+
"""
|
|
134
|
+
root = Path.cwd()
|
|
135
|
+
boot = bootstrap_workspace(root)
|
|
136
|
+
creds = load_creds(root)
|
|
137
|
+
try:
|
|
138
|
+
creds.require_ghostwriter()
|
|
139
|
+
except MissingCreds as e:
|
|
140
|
+
if boot.env_created:
|
|
141
|
+
typer.secho(f"Scaffolded workspace + wrote {boot.env_path}", fg=typer.colors.GREEN)
|
|
142
|
+
typer.secho(str(e), fg=typer.colors.YELLOW)
|
|
143
|
+
raise typer.Exit(code=1) from None
|
|
144
|
+
|
|
145
|
+
fl = {force_local.resolve()} if force_local else set()
|
|
146
|
+
fr = {force_remote.resolve()} if force_remote else set()
|
|
147
|
+
with _workspace_lock(root): # one sync at a time per workspace (GW has no compare-and-swap)
|
|
148
|
+
with GhostwriterClient(creds) as client:
|
|
149
|
+
result = run_sync(root, client, dry_run=dry_run, force_local=fl, force_remote=fr)
|
|
150
|
+
_print_sync_summary(result, dry_run=dry_run)
|
|
151
|
+
bad = bool(
|
|
152
|
+
result.collisions or result.invalid or result.mass_change_blocked or result.errors
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
if creds.bs_url and creds.bs_token_id and creds.bs_token_secret:
|
|
156
|
+
with BookStackClient(creds) as bs:
|
|
157
|
+
m = sync_methodology(root, bs, dry_run=dry_run, force_local=fl, force_remote=fr)
|
|
158
|
+
_print_meth_summary(m, dry_run=dry_run)
|
|
159
|
+
bad = bad or bool(
|
|
160
|
+
m.collisions or m.invalid or m.drift or m.artifacts
|
|
161
|
+
or m.mass_change_blocked or m.errors
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
if bad:
|
|
165
|
+
raise typer.Exit(code=1)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@contextmanager
|
|
169
|
+
def _workspace_lock(root: Path) -> Iterator[None]:
|
|
170
|
+
"""Serialize sync runs per workspace via an exclusive flock on .grison/lock."""
|
|
171
|
+
lock_path = root / ".grison" / "lock"
|
|
172
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
173
|
+
fh = lock_path.open("w", encoding="utf-8")
|
|
174
|
+
try:
|
|
175
|
+
try:
|
|
176
|
+
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
177
|
+
except BlockingIOError:
|
|
178
|
+
typer.secho(
|
|
179
|
+
"another grison sync is already running in this workspace (.grison/lock held)",
|
|
180
|
+
fg=typer.colors.RED,
|
|
181
|
+
)
|
|
182
|
+
raise typer.Exit(code=1) from None
|
|
183
|
+
yield
|
|
184
|
+
finally:
|
|
185
|
+
fcntl.flock(fh, fcntl.LOCK_UN)
|
|
186
|
+
fh.close()
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _print_meth_summary(m: MethResult, *, dry_run: bool) -> None:
|
|
190
|
+
tense = "would " if dry_run else ""
|
|
191
|
+
typer.secho(
|
|
192
|
+
f"methodology: {tense}pull {len(m.pulled)}, {tense}push {len(m.pushed)}, "
|
|
193
|
+
f"{tense}create {len(m.created)} ({len(m.unchanged)} clean, {len(m.repaired)} repaired)",
|
|
194
|
+
fg=typer.colors.GREEN,
|
|
195
|
+
)
|
|
196
|
+
if m.snapshot_dir:
|
|
197
|
+
typer.echo(f"snapshot: {m.snapshot_dir}")
|
|
198
|
+
if m.mass_change_blocked:
|
|
199
|
+
typer.secho(
|
|
200
|
+
"MASS-CHANGE GUARD tripped on methodology — writes withheld.", fg=typer.colors.RED
|
|
201
|
+
)
|
|
202
|
+
for p, why in m.drift:
|
|
203
|
+
typer.secho(f"structure-drift {p}: {why}", fg=typer.colors.RED)
|
|
204
|
+
for p, what in m.artifacts:
|
|
205
|
+
typer.secho(f"artifact {p}: {what}", fg=typer.colors.RED)
|
|
206
|
+
if m.collisions:
|
|
207
|
+
typer.secho(
|
|
208
|
+
f"{len(m.collisions)} collision(s) — hand-merge then --force-*:", fg=typer.colors.RED
|
|
209
|
+
)
|
|
210
|
+
for p in m.collisions:
|
|
211
|
+
typer.echo(f" ! {p}")
|
|
212
|
+
for p in m.invalid:
|
|
213
|
+
typer.secho(f"broken link {p}", fg=typer.colors.RED)
|
|
214
|
+
for p, reason in m.skipped:
|
|
215
|
+
typer.secho(f"skipped {p}: {reason}", fg=typer.colors.YELLOW)
|
|
216
|
+
for e in m.errors:
|
|
217
|
+
typer.secho(f" error: {e}", fg=typer.colors.RED)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _print_sync_summary(result: SyncResult, *, dry_run: bool) -> None:
|
|
221
|
+
tense = "would " if dry_run else ""
|
|
222
|
+
ev = ""
|
|
223
|
+
if result.evidence_up or result.evidence_down or result.evidence_deleted:
|
|
224
|
+
ev = f" [evidence ↑{result.evidence_up} ↓{result.evidence_down}"
|
|
225
|
+
if result.evidence_deleted:
|
|
226
|
+
ev += f" ✕{result.evidence_deleted}"
|
|
227
|
+
ev += "]"
|
|
228
|
+
typer.secho(
|
|
229
|
+
f"{tense}pull {len(result.pulled)}, {tense}push {len(result.pushed)}, "
|
|
230
|
+
f"{tense}insert {len(result.inserted)} ({len(result.unchanged)} clean, "
|
|
231
|
+
f"{len(result.repaired)} repaired){ev}",
|
|
232
|
+
fg=typer.colors.GREEN,
|
|
233
|
+
)
|
|
234
|
+
if result.snapshot_dir:
|
|
235
|
+
typer.echo(f"snapshot: {result.snapshot_dir}")
|
|
236
|
+
if result.mass_change_blocked:
|
|
237
|
+
typer.secho(
|
|
238
|
+
"MASS-CHANGE GUARD tripped — remote writes withheld. Re-run a narrower path "
|
|
239
|
+
"or confirm with a targeted sync.",
|
|
240
|
+
fg=typer.colors.RED,
|
|
241
|
+
)
|
|
242
|
+
if result.collisions:
|
|
243
|
+
typer.secho(
|
|
244
|
+
f"{len(result.collisions)} collision(s) — hand-merge then --force-local/-remote:",
|
|
245
|
+
fg=typer.colors.RED,
|
|
246
|
+
)
|
|
247
|
+
for p in result.collisions:
|
|
248
|
+
typer.echo(f" ! {p} (remote at {p.with_suffix('.remote.md').name})")
|
|
249
|
+
if result.invalid:
|
|
250
|
+
typer.secho(f"{len(result.invalid)} broken link(s) (id set, no sync base) — re-link with "
|
|
251
|
+
"--force-remote/--force-local:", fg=typer.colors.RED)
|
|
252
|
+
for p in result.invalid:
|
|
253
|
+
typer.echo(f" ? {p}")
|
|
254
|
+
for p, reason in result.skipped:
|
|
255
|
+
typer.secho(f"skipped {p}: {reason}", fg=typer.colors.YELLOW)
|
|
256
|
+
for e in result.errors:
|
|
257
|
+
typer.secho(f" error: {e}", fg=typer.colors.RED)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _resolve_md(paths: list[Path]) -> list[Path]:
|
|
261
|
+
files: list[Path] = []
|
|
262
|
+
for p in paths:
|
|
263
|
+
if p.is_dir():
|
|
264
|
+
files.extend(sorted(p.glob("*.md")))
|
|
265
|
+
elif p.is_file():
|
|
266
|
+
files.append(p)
|
|
267
|
+
return files
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _print_parse_summary(summary: ParseSummary, out_dir: Path, *, dry_run: bool) -> None:
|
|
271
|
+
n_files = sum(summary.files_parsed.values())
|
|
272
|
+
by_scanner = ", ".join(f"{k}: {v}" for k, v in sorted(summary.files_parsed.items()))
|
|
273
|
+
typer.secho(
|
|
274
|
+
f"Parsed {len(summary.findings)} finding(s) from {n_files} file(s)"
|
|
275
|
+
+ (f" ({by_scanner})" if by_scanner else ""),
|
|
276
|
+
fg=typer.colors.GREEN,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
sink = summary.sink
|
|
280
|
+
if sink is not None:
|
|
281
|
+
verb = "Would write" if dry_run else "Wrote"
|
|
282
|
+
typer.echo(f"{verb} {len(sink.written)} → {out_dir} ({len(sink.unchanged)} unchanged)")
|
|
283
|
+
|
|
284
|
+
for path, reason in summary.skipped_files:
|
|
285
|
+
typer.secho(f"skipped {path.name}: {reason}", fg=typer.colors.YELLOW)
|
|
286
|
+
|
|
287
|
+
if summary.warnings:
|
|
288
|
+
typer.secho(f"{len(summary.warnings)} warning(s):", fg=typer.colors.YELLOW)
|
|
289
|
+
for w in summary.warnings:
|
|
290
|
+
typer.echo(f" - {w}")
|
|
291
|
+
|
|
292
|
+
if summary.errors:
|
|
293
|
+
typer.secho(f"{len(summary.errors)} finding(s) failed validation:", fg=typer.colors.RED)
|
|
294
|
+
for e in summary.errors:
|
|
295
|
+
typer.echo(f" - {e}")
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def main() -> None:
|
|
299
|
+
"""Console-script entry point (``grison``)."""
|
|
300
|
+
app()
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
if __name__ == "__main__":
|
|
304
|
+
main()
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Markdown layer: the HTML⇄markdown converter, Finding⇄document serialization,
|
|
2
|
+
and scanner-IR → house-schema mapping.
|
|
3
|
+
|
|
4
|
+
The GW field vocabulary is tiny and closed; the converter fails loudly on anything
|
|
5
|
+
outside it. A Finding's prose fields are markdown; ``##`` section headers are grison
|
|
6
|
+
structure that map to Ghostwriter's separate fields.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from grison.markdown.converter import ConverterError, html_to_md, md_to_html
|
|
12
|
+
from grison.markdown.document import (
|
|
13
|
+
DocumentError,
|
|
14
|
+
finding_to_markdown,
|
|
15
|
+
markdown_to_finding,
|
|
16
|
+
)
|
|
17
|
+
from grison.markdown.mapping import (
|
|
18
|
+
MappingResult,
|
|
19
|
+
default_finding_type,
|
|
20
|
+
ir_to_finding,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"ConverterError",
|
|
25
|
+
"DocumentError",
|
|
26
|
+
"MappingResult",
|
|
27
|
+
"default_finding_type",
|
|
28
|
+
"finding_to_markdown",
|
|
29
|
+
"html_to_md",
|
|
30
|
+
"ir_to_finding",
|
|
31
|
+
"markdown_to_finding",
|
|
32
|
+
"md_to_html",
|
|
33
|
+
]
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""Bespoke HTML<->markdown converter for the tiny closed vocabulary Ghostwriter's
|
|
2
|
+
rich-text fields accept.
|
|
3
|
+
|
|
4
|
+
Ghostwriter finding fields render a small, fixed subset of HTML (paragraphs,
|
|
5
|
+
unordered lists, bold/code/em/links/hard-breaks, plus TinyMCE's cosmetic
|
|
6
|
+
``<span>`` highlight wrapper). grison round-trips those fields against local
|
|
7
|
+
markdown, so this module hand-rolls both directions instead of depending on a
|
|
8
|
+
general-purpose HTML/markdown library: anything outside the whitelist below
|
|
9
|
+
must fail loudly (:class:`ConverterError`) rather than degrade silently or
|
|
10
|
+
get dropped on the floor.
|
|
11
|
+
|
|
12
|
+
Whitelist (both directions):
|
|
13
|
+
block: ``<p>`` <-> paragraph, ``<ul><li>`` <-> ``- `` list item
|
|
14
|
+
inline: ``<strong>`` <-> ``**bold**``, ``<code>`` <-> `` `code` ``,
|
|
15
|
+
``<em>`` <-> ``*em*``/``_em_``, ``<a href>`` <-> ``[text](url)``,
|
|
16
|
+
``<br>`` <-> a hard line break inside a paragraph
|
|
17
|
+
``<span>`` is unwrapped (kept, tag dropped) rather than rejected, since
|
|
18
|
+
TinyMCE wraps highlighted text in it.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from html.parser import HTMLParser
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ConverterError(ValueError):
|
|
29
|
+
"""Raised when HTML or markdown outside the tiny closed GW vocabulary is seen."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
_BLOCK_TAGS = {"p", "ul", "li"}
|
|
33
|
+
_INLINE_TAGS = {"strong", "code", "em", "a", "br"}
|
|
34
|
+
_UNWRAP_TAGS = {"span"}
|
|
35
|
+
_ALLOWED_TAGS = _BLOCK_TAGS | _INLINE_TAGS | _UNWRAP_TAGS
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _esc(text: str) -> str:
|
|
39
|
+
"""HTML-escape text/attribute content. ``"`` is escaped too so a URL containing a
|
|
40
|
+
quote can't break out of the ``href="…"`` attribute and inject markup."""
|
|
41
|
+
return (
|
|
42
|
+
text.replace("&", "&")
|
|
43
|
+
.replace("<", "<")
|
|
44
|
+
.replace(">", ">")
|
|
45
|
+
.replace('"', """)
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# --- html -> markdown -------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class _Node:
|
|
54
|
+
tag: str
|
|
55
|
+
attrs: dict[str, str] = field(default_factory=dict)
|
|
56
|
+
children: list[_Node | str] = field(default_factory=list)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _TreeBuilder(HTMLParser):
|
|
60
|
+
"""Builds a tiny tree from an HTML fragment, rejecting non-whitelisted tags."""
|
|
61
|
+
|
|
62
|
+
def __init__(self) -> None:
|
|
63
|
+
super().__init__(convert_charrefs=True)
|
|
64
|
+
self.root = _Node("root")
|
|
65
|
+
self.stack: list[_Node] = [self.root]
|
|
66
|
+
|
|
67
|
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
68
|
+
self._open(tag, attrs)
|
|
69
|
+
|
|
70
|
+
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
71
|
+
self._open(tag, attrs) # self-closing form, e.g. <br/>
|
|
72
|
+
|
|
73
|
+
def _open(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
74
|
+
if tag not in _ALLOWED_TAGS:
|
|
75
|
+
raise ConverterError(f"unsupported HTML tag: <{tag}>")
|
|
76
|
+
if tag == "br":
|
|
77
|
+
self.stack[-1].children.append(_Node("br"))
|
|
78
|
+
return
|
|
79
|
+
node_attrs: dict[str, str] = {}
|
|
80
|
+
if tag == "a":
|
|
81
|
+
for name, value in attrs:
|
|
82
|
+
if name == "href":
|
|
83
|
+
node_attrs["href"] = value or ""
|
|
84
|
+
node = _Node(tag, node_attrs)
|
|
85
|
+
self.stack[-1].children.append(node)
|
|
86
|
+
self.stack.append(node)
|
|
87
|
+
|
|
88
|
+
def handle_endtag(self, tag: str) -> None:
|
|
89
|
+
if tag == "br":
|
|
90
|
+
return
|
|
91
|
+
if tag not in _ALLOWED_TAGS:
|
|
92
|
+
raise ConverterError(f"unsupported HTML tag: </{tag}>")
|
|
93
|
+
if len(self.stack) <= 1 or self.stack[-1].tag != tag:
|
|
94
|
+
raise ConverterError(f"mismatched closing tag: </{tag}>")
|
|
95
|
+
self.stack.pop()
|
|
96
|
+
|
|
97
|
+
def handle_data(self, data: str) -> None:
|
|
98
|
+
self.stack[-1].children.append(data)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def html_to_md(html: str) -> str:
|
|
102
|
+
"""Convert a GW rich-text HTML fragment to markdown."""
|
|
103
|
+
builder = _TreeBuilder()
|
|
104
|
+
builder.feed(html)
|
|
105
|
+
builder.close()
|
|
106
|
+
if len(builder.stack) != 1:
|
|
107
|
+
raise ConverterError(f"unclosed HTML tag: <{builder.stack[-1].tag}>")
|
|
108
|
+
blocks = _group_top_level(builder.root.children)
|
|
109
|
+
return "\n\n".join(_render_block(block) for block in blocks)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _group_top_level(children: list[_Node | str]) -> list[_Node]:
|
|
113
|
+
"""Split top-level children into p/ul blocks, wrapping stray inline content
|
|
114
|
+
in an implicit paragraph and dropping insignificant top-level whitespace."""
|
|
115
|
+
blocks: list[_Node] = []
|
|
116
|
+
buffer: list[_Node | str] = []
|
|
117
|
+
|
|
118
|
+
def flush() -> None:
|
|
119
|
+
if buffer:
|
|
120
|
+
blocks.append(_Node("p", children=list(buffer)))
|
|
121
|
+
buffer.clear()
|
|
122
|
+
|
|
123
|
+
for child in children:
|
|
124
|
+
if isinstance(child, str) and child.strip() == "":
|
|
125
|
+
continue
|
|
126
|
+
if isinstance(child, _Node) and child.tag in ("p", "ul"):
|
|
127
|
+
flush()
|
|
128
|
+
blocks.append(child)
|
|
129
|
+
else:
|
|
130
|
+
buffer.append(child)
|
|
131
|
+
flush()
|
|
132
|
+
return blocks
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _render_block(node: _Node) -> str:
|
|
136
|
+
if node.tag == "p":
|
|
137
|
+
return _render_inline(node.children)
|
|
138
|
+
if node.tag == "ul":
|
|
139
|
+
lines = []
|
|
140
|
+
for li in node.children:
|
|
141
|
+
if isinstance(li, str):
|
|
142
|
+
if li.strip() == "":
|
|
143
|
+
continue
|
|
144
|
+
raise ConverterError("stray text directly inside <ul> (expected <li>)")
|
|
145
|
+
if li.tag != "li":
|
|
146
|
+
raise ConverterError(f"unsupported <ul> child: <{li.tag}>")
|
|
147
|
+
lines.append("- " + _render_li(li))
|
|
148
|
+
return "\n".join(lines)
|
|
149
|
+
raise ConverterError(f"unsupported block-level tag: <{node.tag}>")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _render_li(li: _Node) -> str:
|
|
153
|
+
"""Render a list item, unwrapping the ``<p>`` GW wraps item content in and
|
|
154
|
+
flattening any nested ``<ul>`` into sibling items.
|
|
155
|
+
|
|
156
|
+
The corpus impact/mitigation/references fields are ``<ul><li><p>…</p></li></ul>``;
|
|
157
|
+
a bare ``<li>`` of inline content is rendered directly (preserving inline spacing).
|
|
158
|
+
Nested lists (``<li>…<ul>…</ul></li>``) are flattened — markdown here is
|
|
159
|
+
intentionally single-level, and flattening round-trips stably.
|
|
160
|
+
"""
|
|
161
|
+
if not any(isinstance(c, _Node) and c.tag in ("p", "ul") for c in li.children):
|
|
162
|
+
return _render_inline(li.children)
|
|
163
|
+
inline_parts: list[str] = []
|
|
164
|
+
nested: list[str] = []
|
|
165
|
+
for child in li.children:
|
|
166
|
+
if isinstance(child, _Node) and child.tag == "ul":
|
|
167
|
+
nested.extend(_render_block(child).split("\n")) # already "- …" lines
|
|
168
|
+
elif isinstance(child, _Node) and child.tag == "p":
|
|
169
|
+
rendered = _render_inline(child.children)
|
|
170
|
+
if rendered:
|
|
171
|
+
inline_parts.append(rendered)
|
|
172
|
+
elif isinstance(child, str):
|
|
173
|
+
if child.strip():
|
|
174
|
+
inline_parts.append(child.strip())
|
|
175
|
+
else:
|
|
176
|
+
rendered = _render_inline([child])
|
|
177
|
+
if rendered:
|
|
178
|
+
inline_parts.append(rendered)
|
|
179
|
+
head = " ".join(inline_parts)
|
|
180
|
+
lines = ([head] if head else []) + nested
|
|
181
|
+
return "\n".join(lines)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _render_inline(nodes: list[_Node | str]) -> str:
|
|
185
|
+
parts = []
|
|
186
|
+
for n in nodes:
|
|
187
|
+
if isinstance(n, str):
|
|
188
|
+
parts.append(n)
|
|
189
|
+
elif n.tag == "br":
|
|
190
|
+
parts.append("\n")
|
|
191
|
+
elif n.tag == "strong":
|
|
192
|
+
parts.append(f"**{_render_inline(n.children)}**")
|
|
193
|
+
elif n.tag == "em":
|
|
194
|
+
parts.append(f"*{_render_inline(n.children)}*")
|
|
195
|
+
elif n.tag == "code":
|
|
196
|
+
parts.append(f"`{_render_code_text(n.children)}`")
|
|
197
|
+
elif n.tag == "a":
|
|
198
|
+
href = n.attrs.get("href", "")
|
|
199
|
+
parts.append(f"[{_render_inline(n.children)}]({href})")
|
|
200
|
+
elif n.tag == "span":
|
|
201
|
+
parts.append(_render_inline(n.children))
|
|
202
|
+
else:
|
|
203
|
+
raise ConverterError(f"unsupported tag in inline content: <{n.tag}>")
|
|
204
|
+
return "".join(parts)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _render_code_text(nodes: list[_Node | str]) -> str:
|
|
208
|
+
"""<code> content is never inline-parsed, so just flatten its text (unwrapping
|
|
209
|
+
any cosmetic <span>, but rejecting any other nested tag)."""
|
|
210
|
+
parts = []
|
|
211
|
+
for n in nodes:
|
|
212
|
+
if isinstance(n, str):
|
|
213
|
+
parts.append(n)
|
|
214
|
+
elif n.tag == "span":
|
|
215
|
+
parts.append(_render_code_text(n.children))
|
|
216
|
+
else:
|
|
217
|
+
raise ConverterError(f"unsupported nested tag inside <code>: <{n.tag}>")
|
|
218
|
+
return "".join(parts)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# --- markdown -> html --------------------------------------------------------
|
|
222
|
+
|
|
223
|
+
_HEADING_RE = re.compile(r"^#{1,6}\s")
|
|
224
|
+
_ORDERED_RE = re.compile(r"^\d+\.\s")
|
|
225
|
+
_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
|
|
226
|
+
_BLOCKQUOTE_RE = re.compile(r"^>\s?")
|
|
227
|
+
_TABLE_SEP_RE = re.compile(r"\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?")
|
|
228
|
+
_SETEXT_EQ_RE = re.compile(r"=+")
|
|
229
|
+
_RULE_RE = re.compile(r"-{2,}")
|
|
230
|
+
|
|
231
|
+
_TOKEN_RE = re.compile(
|
|
232
|
+
r"`(?P<code>[^`]*)`"
|
|
233
|
+
r"|\*\*(?P<strong>.+?)\*\*"
|
|
234
|
+
r"|\[(?P<link_text>[^\]]*)\]\((?P<link_url>[^)]*)\)"
|
|
235
|
+
r"|\*(?P<em1>.+?)\*"
|
|
236
|
+
r"|_(?P<em2>.+?)_"
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def md_to_html(md: str) -> str:
|
|
241
|
+
"""Convert markdown (the tiny closed GW subset) to an HTML fragment."""
|
|
242
|
+
blocks = re.split(r"\n\s*\n", md)
|
|
243
|
+
return "\n\n".join(_render_md_block(block) for block in blocks)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _render_md_block(block: str) -> str:
|
|
247
|
+
lines = block.split("\n")
|
|
248
|
+
for line in lines:
|
|
249
|
+
_check_line_whitelist(line)
|
|
250
|
+
if lines and all(_is_list_line(line) for line in lines):
|
|
251
|
+
items = "".join(f"<li>{_inline_to_html(line[2:])}</li>" for line in lines)
|
|
252
|
+
return f"<ul>{items}</ul>"
|
|
253
|
+
rendered = [_inline_to_html(line) for line in lines]
|
|
254
|
+
return f"<p>{'<br>'.join(rendered)}</p>"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _is_list_line(line: str) -> bool:
|
|
258
|
+
return line.startswith("- ") or line.startswith("* ")
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _check_line_whitelist(line: str) -> None:
|
|
262
|
+
stripped = line.strip()
|
|
263
|
+
if _HEADING_RE.match(line):
|
|
264
|
+
raise ConverterError(f"unsupported markdown: ATX heading ({line!r})")
|
|
265
|
+
if _ORDERED_RE.match(line):
|
|
266
|
+
raise ConverterError(f"unsupported markdown: ordered list ({line!r})")
|
|
267
|
+
if _IMAGE_RE.search(line):
|
|
268
|
+
raise ConverterError(f"unsupported markdown: image ({line!r})")
|
|
269
|
+
if stripped.startswith("```"):
|
|
270
|
+
raise ConverterError(f"unsupported markdown: fenced code block ({line!r})")
|
|
271
|
+
if _BLOCKQUOTE_RE.match(line):
|
|
272
|
+
raise ConverterError(f"unsupported markdown: blockquote ({line!r})")
|
|
273
|
+
# A markdown table is identified by its separator row (---|---); a bare pipe is
|
|
274
|
+
# not — shell commands inside `code` legitimately contain ` | ` (e.g. `a | nc`).
|
|
275
|
+
if _TABLE_SEP_RE.fullmatch(stripped):
|
|
276
|
+
raise ConverterError(f"unsupported markdown: table ({line!r})")
|
|
277
|
+
if _SETEXT_EQ_RE.fullmatch(stripped):
|
|
278
|
+
raise ConverterError(f"unsupported markdown: setext heading underline ({line!r})")
|
|
279
|
+
if _RULE_RE.fullmatch(stripped):
|
|
280
|
+
raise ConverterError(f"unsupported markdown: setext heading underline or rule ({line!r})")
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _inline_to_html(text: str) -> str:
|
|
284
|
+
out = []
|
|
285
|
+
pos = 0
|
|
286
|
+
for m in _TOKEN_RE.finditer(text):
|
|
287
|
+
if m.start() > pos:
|
|
288
|
+
out.append(_esc(text[pos : m.start()]))
|
|
289
|
+
if m.group("code") is not None:
|
|
290
|
+
out.append(f"<code>{_esc(m.group('code'))}</code>")
|
|
291
|
+
elif m.group("strong") is not None:
|
|
292
|
+
out.append(f"<strong>{_esc(m.group('strong'))}</strong>")
|
|
293
|
+
elif m.group("link_text") is not None:
|
|
294
|
+
href = _esc(m.group("link_url"))
|
|
295
|
+
link_text = _esc(m.group("link_text"))
|
|
296
|
+
out.append(f'<a href="{href}" target="_blank" rel="noopener">{link_text}</a>')
|
|
297
|
+
elif m.group("em1") is not None:
|
|
298
|
+
out.append(f"<em>{_esc(m.group('em1'))}</em>")
|
|
299
|
+
else:
|
|
300
|
+
out.append(f"<em>{_esc(m.group('em2'))}</em>")
|
|
301
|
+
pos = m.end()
|
|
302
|
+
out.append(_esc(text[pos:]))
|
|
303
|
+
return "".join(out)
|