scanfuse 0.2.0__py3-none-any.whl

scanfuse/utils.py

@@ -0,0 +1,184 @@
+import re
+
+from scanfuse.models import Evidence, Finding, Host, SourceRecord
+
+# Names that must not be used as a file name on Windows, plus the files scanfuse writes itself.
+RESERVED_SLUGS = frozenset(
+    {"index", "report", "con", "prn", "aux", "nul"}
+    | {f"com{i}" for i in range(1, 10)}
+    | {f"lpt{i}" for i in range(1, 10)}
+)
+
+SEVERITY_ORDER = {"Info": 0, "Low": 1, "Medium": 2, "High": 3, "Critical": 4}
+
+# Identifiers that name the same weakness in every tool. Only these (together with title and
+# port) decide whether two findings are the same. Tool ids like 'nessus-plugin:123' do not.
+_GLOBAL_ID = re.compile(r"^(CVE-\d{4}-\d{4,}|GHSA(-[0-9a-z]{4}){3})$", re.IGNORECASE)
+
+
+def slugify(text: str) -> str:
+    """
+    Convert text to a safe, lower-cased filename slug.
+    Example: 'SQL Injection (Time-based)' -> 'sql-injection-time-based'
+    """
+    text = text.lower()
+    text = re.sub(r'[^a-z0-9]+', '-', text)
+    result = text[:100].strip('-')
+    return result if result else "unknown-finding"
+
+
+def unique_slug(title: str, seen: set[str]) -> str:
+    """Return a slug that is not in `seen` and is not a reserved file name. Adds it to `seen`."""
+    base = slugify(title)
+    slug = base
+    counter = 1
+    while slug in seen or slug in RESERVED_SLUGS:
+        slug = f"{base}-{counter}"
+        counter += 1
+    seen.add(slug)
+    return slug
+
+
+def _global_ids(f: Finding) -> frozenset[str]:
+    return frozenset(i.upper() for i in f.identifiers if _GLOBAL_ID.match(i))
+
+
+def _local_ids(f: Finding) -> dict[str, frozenset[str]]:
+    """Tool specific ids grouped by prefix, e.g. {'nessus-plugin': {'12345'}}."""
+    grouped: dict[str, set[str]] = {}
+    for ident in f.identifiers:
+        if ":" in ident:
+            prefix, _, value = ident.partition(":")
+            grouped.setdefault(prefix, set()).add(value)
+    return {k: frozenset(v) for k, v in grouped.items()}
+
+
+def _identity(f: Finding) -> tuple[str, frozenset[str], str | None]:
+    return (f.title.strip().casefold(), _global_ids(f), f.port)
+
+
+def _compatible(group_ids: dict[str, frozenset[str]], f: Finding) -> bool:
+    """Two findings of the same tool with different tool ids are different findings."""
+    for prefix, values in _local_ids(f).items():
+        if prefix in group_ids and group_ids[prefix] != values:
+            return False
+    return True
+
+
+def _record(f: Finding) -> SourceRecord:
+    return SourceRecord(
+        source_tool=f.source_tool, title=f.title, severity=f.severity,
+        cvss_score=f.cvss_score, cvss_vector=f.cvss_vector, description=f.description,
+        remediation=f.remediation, identifiers=list(f.identifiers), port=f.port,
+    )
+
+
+def _merge_os(existing: str | None, new: str | None) -> str | None:
+    if not existing:
+        return new
+    if not new or new == existing or new in existing.split(" / "):
+        return existing
+    return f"{existing} / {new}"
+
+
+def deduplicate_findings(findings: list[Finding]) -> list[Finding]:
+    """
+    Merge findings that are the same finding reported more than once.
+
+    Two findings are the same if their titles are equal (ignoring case and surrounding space),
+    they carry the same CVE/GHSA ids, they have the same port, and they do not carry different
+    ids of the same tool. Similar but not equal titles are never merged.
+
+    A merged finding keeps every host and every piece of evidence. Evidence is tagged with the
+    tool that reported it. The description, remediation, severity and CVSS data of every
+    original finding stay available in `sources`. The input objects are not modified. The order
+    follows the first occurrence of each finding.
+    """
+    # Each group: [merged finding, tool ids, originals]
+    groups: list[list] = []
+    by_identity: dict[tuple[str, frozenset[str], str | None], list[int]] = {}
+
+    for f in findings:
+        key = _identity(f)
+        target = None
+        for idx in by_identity.get(key, ()):
+            if _compatible(groups[idx][1], f):
+                target = groups[idx]
+                break
+        if target is None:
+            by_identity.setdefault(key, []).append(len(groups))
+            groups.append([f, dict(_local_ids(f)), [f]])
+        else:
+            target[2].append(f)
+            for prefix, values in _local_ids(f).items():
+                target[1].setdefault(prefix, values)
+
+    result: list[Finding] = []
+    for first, _ids, originals in groups:
+        if len(originals) == 1:
+            result.append(first)
+        else:
+            result.append(_merge_group(originals))
+    return result
+
+
+def _merge_group(originals: list[Finding]) -> Finding:
+    base = originals[0].model_copy(deep=True)
+    base.sources = [_record(f) for f in originals]
+
+    tools: list[str] = []
+    for f in originals:
+        for tool in f.source_tool.split(","):
+            tool = tool.strip()
+            if tool and tool not in tools:
+                tools.append(tool)
+    base.source_tool = ", ".join(tools)
+
+    host_index: dict[tuple[str, str | None], Host] = {}
+    base.hosts = []
+    evidence_index: dict[tuple[str, str], Evidence] = {}
+    base.evidence = []
+    identifiers: list[str] = []
+
+    for f in originals:
+        for host in f.hosts:
+            key = (host.ip, host.hostname)
+            known = host_index.get(key)
+            if known is None:
+                copy = host.model_copy()
+                host_index[key] = copy
+                base.hosts.append(copy)
+            else:
+                known.os = _merge_os(known.os, host.os)
+        for ev in f.evidence:
+            known_ev = evidence_index.get((ev.type, ev.data))
+            tool = ev.source or f.source_tool
+            if known_ev is None:
+                tagged = ev.model_copy(update={"source": tool})
+                evidence_index[(ev.type, ev.data)] = tagged
+                base.evidence.append(tagged)
+            else:
+                # Same evidence from another tool: keep one copy and list every tool.
+                tools_seen = (known_ev.source or "").split(", ")
+                if tool not in tools_seen:
+                    known_ev.source = ", ".join([t for t in tools_seen if t] + [tool])
+        for ident in f.identifiers:
+            if ident not in identifiers:
+                identifiers.append(ident)
+    base.identifiers = identifiers
+    base.references = list(dict.fromkeys(url for f in originals for url in f.references))
+
+    # Summary values: highest severity, highest CVSS score with its own vector.
+    base.severity = max((f.severity for f in originals), key=lambda s: SEVERITY_ORDER.get(s, 0))
+    scored = [f for f in originals if f.cvss_score is not None]
+    if scored:
+        best = max(scored, key=lambda f: f.cvss_score or 0.0)
+        base.cvss_score = best.cvss_score
+        base.cvss_vector = best.cvss_vector
+    else:
+        base.cvss_score = None
+        base.cvss_vector = next((f.cvss_vector for f in originals if f.cvss_vector), None)
+    # Text: first non-empty value; every original text stays in `sources`.
+    base.description = next((f.description for f in originals if f.description), base.description)
+    base.remediation = next((f.remediation for f in originals if f.remediation), None)
+    return base

scanfuse-0.2.0.dist-info/METADATA

@@ -0,0 +1,208 @@
+Metadata-Version: 2.5
+Name: scanfuse
+Version: 0.2.0
+Summary: Fast, local-first pentest finding extractor & markdown report engine.
+Project-URL: Repository, https://github.com/gencay-e/scanfuse
+Project-URL: Changelog, https://github.com/gencay-e/scanfuse/blob/main/CHANGELOG.md
+Author: Gencay
+License-Expression: MIT
+License-File: LICENSE
+Keywords: burp,nessus,nmap,nuclei,pentest,report,security,trivy,zap
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Information Technology
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Security
+Requires-Python: >=3.12
+Requires-Dist: defusedxml<1,>=0.7.1
+Requires-Dist: jinja2<4,>=3.1.3
+Requires-Dist: markdown<4,>=3.11
+Requires-Dist: nh3<1,>=0.3.7
+Requires-Dist: pydantic<3,>=2.7.0
+Requires-Dist: pyyaml<7,>=6.0.1
+Requires-Dist: rich<16,>=13.7.0
+Requires-Dist: typer<1,>=0.12.0
+Provides-Extra: dev
+Requires-Dist: hypothesis>=6.100.0; extra == 'dev'
+Requires-Dist: mypy>=1.9.0; extra == 'dev'
+Requires-Dist: pyinstaller>=6.5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.3.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scanfuse
+
+A local command-line tool that reads pentest scanner reports and writes one report from them.
+
+<img alt="Scan files from Nmap, Nuclei, ZAP, Burp, Nessus and Trivy go into scanfuse, which writes index.md, report.html, report.json and report.csv" src="docs/images/scanfuse-flow.svg" width="900">
+
+It imports Nmap, Nuclei, ZAP, Burp, Nessus and Trivy files and writes Markdown (with an index and a standalone HTML page), JSON or CSV. It runs offline and sends nothing anywhere.
+
+Version 0.2.0, a pre-release. See [CHANGELOG.md](CHANGELOG.md).
+
+## Example
+
+<img alt="The summary of a report" src="docs/images/report-summary.png" width="900">
+
+<img alt="A finding in the report" src="docs/images/report-finding.png" width="900">
+
+`report.html` follows the light or dark setting of your system; the screenshots show the dark one.
+
+The report above was made from six real scan files (Nmap, ZAP, Trivy, Burp, Nessus). The input files, the
+command and the full report are in [examples/](examples/).
+
+## Install
+
+You need Python 3.12 or newer.
+
+```bash
+pipx install scanfuse                                   # from PyPI
+pipx install git+https://github.com/gencay-e/scanfuse   # latest state from GitHub
+```
+
+`pip install scanfuse` works as well. From a checkout of this repository: `pipx install .`
+
+Ready-made programs for Linux, Windows and macOS (no Python needed) are on the [Releases page](https://github.com/gencay-e/scanfuse/releases), with a `SHA256SUMS` file. To build one yourself, see [Build a standalone program](#build-a-standalone-program).
+
+## Use
+
+```bash
+scanfuse scan.xml -o report
+scanfuse nmap.xml nuclei.json zap.xml -o report
+scanfuse scan.json -f trivy --format-out json -o report
+scanfuse scan.xml --min-severity High --host 192.0.2.1 -o report
+cat scan.json | scanfuse -o report
+```
+
+`scanfuse` takes one or more files, or `-` for standard input. It finds the format by itself. Use `-f` to choose one: `nmap`, `nuclei`, `zap`, `burp`, `nessus`, `trivy`.
+
+| Option | Meaning |
+|---|---|
+| `-o`, `--output` | Output directory. Default `report`. |
+| `--format-out` | `markdown` (default), `json` or `csv`. `markdown` writes one file per finding, `index.md` and `report.html`. |
+| `-s`, `--min-severity` | `Critical`, `High`, `Medium`, `Low` or `Info`. Case does not matter. |
+| `--host` | Only findings on this host. Must match the IP or host name exactly, ignoring case. `*` and `?` are wildcards, for example `10.0.0.*`. |
+| `-t`, `--template-dir` | Folder with your own `finding.md.j2` and `index.md.j2`. The folder must exist. Missing files fall back to the built-in templates. |
+| `--replace` | Overwrite an existing report. See below. |
+| `--csv-excel` | CSV with a byte order mark and `;` as separator, which German-language Excel expects. |
+| `--version` | Print the version. |
+
+### Existing reports
+
+`scanfuse` never overwrites files by accident. If the output directory already contains a file that the new report would write, it stops with exit code 2 and changes nothing. Add `--replace` to overwrite the report. With `--replace`, files of the earlier report that are no longer part of it are removed. Files that scanfuse did not write are left alone.
+
+The report is built in a temporary folder first. If moving the files fails, scanfuse tries to restore the old report. If restoration also fails, the error gives the location of the retained backup. A process or system crash during replacement can still leave a mixed report.
+
+### Incomplete runs
+
+If one input file cannot be read, scanfuse still writes the findings from the other files and exits with code 1. The report says so:
+
+* `report.json`: `"complete": false` and the list of `errors`.
+* `index.md` and `report.html`: an **INCOMPLETE** notice with the errors.
+* `report.csv`: the column `Import Status` says `incomplete (...)` in every row.
+
+An invalid `.scanfuse.yaml` counts as an error too. The report is then incomplete, because ignore rules were not applied.
+
+### Exit codes
+
+| Code | Meaning |
+|---|---|
+| 0 | Complete. |
+| 1 | Incomplete or failed: an input could not be read, `.scanfuse.yaml` is invalid, nothing could be written, or no input was given. |
+| 2 | Invalid option, or the output files already exist (use `--replace`). |
+
+## Merging duplicates
+
+Findings from several files are merged when they are the same finding:
+
+* the titles are equal, ignoring case and surrounding spaces,
+* they carry the same CVE or GHSA ids,
+* they have the same port, and
+* the same tool does not give them different tool ids (for example two Nessus plugin ids).
+
+Titles that are only similar are not merged. A merged finding keeps all hosts and all evidence. Evidence shows which tool reported it. What each tool wrote (description, remediation, severity, CVSS score and vector, ids) is listed under **Reported by** in the Markdown and in `sources` in the JSON. The summary severity is the highest of all reports. The summary CVSS score is the highest score, together with its own vector.
+
+## Output
+
+### JSON
+
+`report.json` is an object:
+
+```json
+{
+  "schema_version": 2,
+  "complete": true,
+  "errors": [],
+  "findings": [ { "title": "...", "severity": "High", "identifiers": ["CVE-2024-0001"], "port": "443/tcp", "sources": [] } ]
+}
+```
+
+### CSV
+
+Columns: `Title`, `Severity`, `CVSS`, `Hosts`, `Source Tool`, `Description`, `Remediation`, `Evidence`, `CVSS Vector`, `Port`, `Identifiers`, `References`, `Other Reports`, `Import Status`. `Hosts` shows `ip (host name)` when a host name is known.
+
+Cells that start with `=`, `+`, `-` or `@` get a leading `'` to reduce the risk of spreadsheet formulas. This also covers look-alike characters such as a full-width `＝` and a leading no-break space. These cases were tested by reading the files back.
+
+In Excel, import `--csv-excel` output with **Data > From Text/CSV** and use the semicolon delimiter. In a Windows Excel test, this preserved a multiline evidence field; the older Text Import Wizard split it across rows. Formula handling and LibreOffice remain unverified in those applications.
+
+### Scanner text in the report
+
+Scanner output is untrusted. scanfuse treats it like this:
+
+* Titles, host names, ids and tool names are shown as plain text. They cannot add links, headings or HTML.
+* Descriptions and remediation text keep Markdown formatting, including links. Raw HTML is escaped, and the text cannot create a heading. A link in a scanner description is shown as the scanner wrote it, so check it before you click.
+* Evidence is shown unchanged in a code block.
+* The HTML page removes scripts, images, frames and `javascript:` links.
+* XML with entity declarations is rejected.
+
+## Configuration: `.scanfuse.yaml`
+
+`scanfuse` reads `.scanfuse.yaml` from the current directory.
+
+```yaml
+ignore_titles:
+  - "Wappalyzer"
+  - "Header Missing"
+```
+
+Findings whose title contains one of these texts (ignoring case) are left out. `ignore_titles` must be a list of strings. A single string is read as a list with one entry. Anything else, a file that is not a mapping, or a YAML syntax error is reported as an error and the rules are not applied.
+
+## Build a standalone program
+
+```bash
+uv sync --extra dev
+uv run python build.py
+```
+
+The program is written to `dist/`. To check it against the test suite:
+
+```bash
+SCANFUSE_EXE=dist/scanfuse uv run pytest tests/test_packaged_cli.py     # Linux, macOS
+$env:SCANFUSE_EXE = (Resolve-Path .\dist\scanfuse.exe).Path             # Windows PowerShell
+```
+
+A program built on Linux depends on the C library of the system it was built on. Build on the oldest system you want to support.
+
+## Development
+
+```bash
+uv sync --locked --extra dev
+uv run ruff check .
+uv run mypy scanfuse
+uv run pytest tests/
+```
+
+## Limits
+
+* scanfuse was tested with real output from nmap (7.94, 7.12, 6.40), ZAP 1.4, Burp, Nessus and Trivy (three file layouts), taken from the public [DefectDojo sample-scan-files](https://github.com/DefectDojo/sample-scan-files). Those files are old. Newer versions of the tools may add fields that scanfuse does not read yet. Nuclei has no real sample file in the tests. Open an issue with a cleaned sample if a file fails.
+* Format detection looks at the start of the file. Use `-f` if it guesses wrong.
+* The HTML page and the CSV file were checked by reading the files. They were not opened by hand in a browser, Excel or LibreOffice.
+* 100,000 results were processed completely in all three output formats on one Linux machine (4 CPUs, 16 GB). JSON took 14 seconds, CSV 10 seconds, Markdown with HTML 124 seconds, and the peak memory was between 0.7 and 1.0 GB. The HTML page for that many findings is large. You can repeat the check with `python tools/large_check.py 100000`.
+* Standalone programs are built and tested in CI on Ubuntu 22.04 (x86_64), Windows (x86_64) and macOS (arm64). Other Linux distributions, Linux on ARM and macOS on Intel were not tested.
+
+## License
+
+MIT, see [LICENSE](LICENSE).

scanfuse-0.2.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+scanfuse/__init__.py,sha256=dJrjg3JsK3QAihGLinGdgf_A4LpgeYy0u74MdkNnj5s,84
+scanfuse/cli.py,sha256=riaKbAExNhAHwEW_wJMq2upl_Fl2U6DTjHq93TjpREg,9153
+scanfuse/engine.py,sha256=fYkn8ZaGhZrTuMg-V2x97vDFVWCkCK9ezS-6dqYAnOI,23445
+scanfuse/html_report.py,sha256=fQNMk7_oiCYUktjjj5gRTDPkQZpiCr8NPZmDMM-HlJA,694
+scanfuse/models.py,sha256=aDLdP-H7riqZx0rJBPlvLzVxIBxbqUShQCMyIzIQ6NE,1993
+scanfuse/utils.py,sha256=qBDUbto7kqEm1vsVhsjTvS_tAsb6N7ltU3BKo-or1-I,7209
+scanfuse/parsers/__init__.py,sha256=L3_KOxgzaJQPN9a4HRkgIimGVpL_s8LLuy7zQ-ZbuKI,53
+scanfuse/parsers/base.py,sha256=meJhEc-4VzyFVvHGWqeia2rEza7_ojb5_aBdbK_Fnmo,4654
+scanfuse/parsers/burp.py,sha256=E7Pt33uoNDjsc23lvgO1xxqrjmnZKAgU3Clu9r72Aps,6124
+scanfuse/parsers/nessus.py,sha256=H8V4Htplzq4J9eBG4z5yd3pGgCKl3KJSxUThk0DKN_Q,5733
+scanfuse/parsers/nmap.py,sha256=trEXWUkvTxYpG_8KqT45tLYVi5MvomdZlS95hdO-GXY,5830
+scanfuse/parsers/nuclei.py,sha256=weukJn1Bl605MjLjnhPwv91l-ToCNXnL42weYqBFBxE,5425
+scanfuse/parsers/trivy.py,sha256=0-In3j1sd6e-pC2Hvn8TRisDTU1NauLlt4x9ZbMxHBA,8370
+scanfuse/parsers/zap.py,sha256=nqkHeL1UlHdhB2brSxQYBOHkkQoH9MnG2qSQskJ8pn8,5333
+scanfuse/templates/finding.md.j2,sha256=Ki8T5Nmq24mT6ekUXRyNpVU5XJVlr4OTN_70JJYkLpU,1647
+scanfuse/templates/index.md.j2,sha256=tCjvKRhYuv9oSowEbr5pf_Z3BkJDr00R8vjD1WE0DoQ,1446
+scanfuse-0.2.0.dist-info/METADATA,sha256=3LWLGFcgYtR0hMx-cJz54ggKNVDuICe8S_SyQnp5Rzc,10266
+scanfuse-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+scanfuse-0.2.0.dist-info/entry_points.txt,sha256=31QZsLdhrPN7xa0cezdGDmxtXTvUK6C1xws66xp1OSA,46
+scanfuse-0.2.0.dist-info/licenses/LICENSE,sha256=yXy4VpeLDAfRK0Bucm6wn3hjAAUmMcEjhbNjfH6ujbI,1063
+scanfuse-0.2.0.dist-info/RECORD,,

scanfuse-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

scanfuse-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+scanfuse = scanfuse.cli:app

scanfuse-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Gencay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.