xpl-intel 1.2.0__py3-none-any.whl

ingestion/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "1.2.0"
+PRODUCT = "XPL-Intel"

ingestion/cli.py

@@ -0,0 +1,162 @@
+"""XPL-Intel CLI — CVE ingestion pipeline."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from datetime import datetime, timezone
+
+import argcomplete
+
+from ingestion import PRODUCT, __version__
+from schemas.advisory_v1 import Advisory
+
+ALL_SOURCES = ["kev", "nvd", "watchtowr", "nuclei", "exploitdb", "vulncheck", "ghsa"]
+OUTPUT_FORMATS = ["json", "sqlite", "csv"]
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="xpl-intel",
+        description=f"{PRODUCT} v{__version__} — CVE ingestion pipeline",
+    )
+    parser.add_argument("--version", action="version", version=f"{PRODUCT} {__version__}")
+
+    sub = parser.add_subparsers(dest="command")
+
+    ingest_p = sub.add_parser("ingest", help="Run ingestion from sources")
+    ingest_p.add_argument("--sources", "-s", nargs="+",
+                          choices=[*ALL_SOURCES, "all"],
+                          default=["all"])
+    ingest_p.add_argument("--nvd-key", help="NVD API key (optional)")
+    ingest_p.add_argument("--nvd-days", type=int, default=7, help="NVD lookback days")
+    ingest_p.add_argument("--nuclei-dir", default="nuclei-templates", help="Nuclei templates dir")
+    ingest_p.add_argument("--exploitdb-csv", help="ExploitDB CSV path")
+    ingest_p.add_argument("--vulncheck-token", help="VulnCheck API token")
+    ingest_p.add_argument("--github-token", help="GitHub token for GHSA")
+    ingest_p.add_argument("--ghsa-severity", default="critical,high",
+                          help="GHSA severity filter (default: critical,high)")
+    ingest_p.add_argument("--ghsa-days", type=int, default=7, help="GHSA lookback days")
+    ingest_p.add_argument("--output", "-o", default="output", help="Output directory")
+    ingest_p.add_argument("--format", "-f", nargs="+",
+                          choices=OUTPUT_FORMATS, default=["json"],
+                          help="Output formats (default: json)")
+
+    sub.add_parser("sources", help="List available sources")
+
+    argcomplete.autocomplete(parser)
+    args = parser.parse_args(argv)
+    if not args.command:
+        parser.print_help()
+        return 0
+
+    if args.command == "sources":
+        _print_sources()
+        return 0
+
+    return _run_ingest(args)
+
+
+def _print_sources() -> None:
+    sources = [
+        ("kev", "CISA KEV catalog (CSV daily)", "https://www.cisa.gov/known-exploited-vulnerabilities-catalog"),
+        ("nvd", "NVD API v2 (delta weekly)", "https://nvd.nist.gov/"),
+        ("watchtowr", "watchTowr Labs RSS", "https://labs.watchtowr.com/"),
+        ("nuclei", "Nuclei templates (local)", "https://github.com/projectdiscovery/nuclei-templates"),
+        ("exploitdb", "ExploitDB CSV (searchsploit)", "https://www.exploit-db.com/"),
+        ("vulncheck", "VulnCheck KEV enriched", "https://vulncheck.com/"),
+        ("ghsa", "GitHub Advisory Database", "https://github.com/advisories"),
+    ]
+    print(f"{PRODUCT} v{__version__} — Available sources:\n")
+    for name, desc, url in sources:
+        print(f"  {name:12s} {desc}")
+        print(f"  {'':12s} {url}")
+        print()
+
+
+def _run_ingest(args: argparse.Namespace) -> int:
+    from ingestion.dedup import deduplicate
+
+    sources_to_run: list[str] = args.sources
+    if "all" in sources_to_run:
+        sources_to_run = list(ALL_SOURCES)
+
+    all_advisories: list[Advisory] = []
+
+    for source in sources_to_run:
+        try:
+            advisories = _fetch_source(source, args)
+            print(f"[+] {source}: {len(advisories)} advisories")
+            all_advisories.extend(advisories)
+        except Exception as exc:
+            print(f"[-] {source}: {exc}", file=sys.stderr)
+
+    if not all_advisories:
+        print("[*] No advisories ingested")
+        return 0
+
+    deduped = deduplicate(all_advisories)
+    print(f"[+] Deduplicated: {len(all_advisories)} → {len(deduped)}")
+
+    os.makedirs(args.output, exist_ok=True)
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    formats: list[str] = args.format
+    for fmt in formats:
+        _write_output(deduped, args.output, ts, fmt)
+
+    return 0
+
+
+def _write_output(advisories: list[Advisory], output_dir: str, ts: str, fmt: str) -> None:
+    if fmt == "json":
+        from ingestion.output.json_writer import write_json
+        path = write_json(advisories, os.path.join(output_dir, f"advisories-{ts}.json"))
+    elif fmt == "sqlite":
+        from ingestion.output.sqlite_writer import write_sqlite
+        path = write_sqlite(advisories, os.path.join(output_dir, f"advisories-{ts}.db"))
+    elif fmt == "csv":
+        from ingestion.output.csv_writer import write_csv
+        path = write_csv(advisories, os.path.join(output_dir, f"advisories-{ts}.csv"))
+    else:
+        print(f"[-] Unknown format: {fmt}", file=sys.stderr)
+        return
+    print(f"[+] Output ({fmt}): {path} ({len(advisories)} advisories)")
+
+
+def _fetch_source(source: str, args: argparse.Namespace) -> list[Advisory]:
+    if source == "kev":
+        from ingestion.sources.cisa_kev import fetch as fetch_kev
+        return fetch_kev()
+    if source == "nvd":
+        from ingestion.sources.nvd_api import fetch as fetch_nvd
+        return fetch_nvd(days_back=args.nvd_days, api_key=args.nvd_key)
+    if source == "watchtowr":
+        from ingestion.sources.watchtowr_rss import fetch as fetch_watchtowr
+        return fetch_watchtowr()
+    if source == "nuclei":
+        from ingestion.sources.nuclei_templates import fetch as fetch_nuclei
+        return fetch_nuclei(templates_dir=args.nuclei_dir)
+    if source == "exploitdb":
+        from ingestion.sources.exploitdb import fetch as fetch_exploitdb
+        return fetch_exploitdb(csv_path=args.exploitdb_csv)
+    if source == "vulncheck":
+        from ingestion.sources.vulncheck_kev import fetch as fetch_vulncheck
+        token = args.vulncheck_token or os.environ.get("VULNCHECK_TOKEN", "")
+        if not token:
+            raise ValueError("VulnCheck API token required (--vulncheck-token or VULNCHECK_TOKEN env)")
+        return fetch_vulncheck(api_token=token)
+    if source == "ghsa":
+        from ingestion.sources.github_advisory import fetch as fetch_ghsa
+        token = args.github_token or os.environ.get("GITHUB_TOKEN")
+        return fetch_ghsa(
+            days_back=args.ghsa_days,
+            github_token=token,
+            severity=args.ghsa_severity,
+        )
+    return []
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ingestion/dedup.py

@@ -0,0 +1,49 @@
+"""Deduplication engine — merge advisories by CVE+CNA, keep richest data."""
+
+from __future__ import annotations
+
+from schemas.advisory_v1 import Advisory
+
+
+def deduplicate(advisories: list[Advisory]) -> list[Advisory]:
+    seen: dict[str, Advisory] = {}
+
+    for adv in advisories:
+        key = adv.cve_id
+        if key in seen:
+            seen[key] = _merge(seen[key], adv)
+        else:
+            seen[key] = adv
+
+    return sorted(seen.values(), key=lambda a: a.cve_id)
+
+
+def _merge(existing: Advisory, new: Advisory) -> Advisory:
+    if not existing.description and new.description:
+        existing.description = new.description
+    if new.cvss_score is not None and (existing.cvss_score is None or new.cvss_score > existing.cvss_score):
+        existing.cvss_score = new.cvss_score
+        existing.cvss_vector = new.cvss_vector or existing.cvss_vector
+    if new.epss_score is not None:
+        existing.epss_score = new.epss_score
+        existing.epss_percentile = new.epss_percentile
+    if new.kev:
+        existing.kev = True
+        existing.kev_date_added = new.kev_date_added or existing.kev_date_added
+        existing.kev_due_date = new.kev_due_date or existing.kev_due_date
+    if not existing.vendor and new.vendor:
+        existing.vendor = new.vendor
+    if not existing.product and new.product:
+        existing.product = new.product
+    if new.cwe and not existing.cwe:
+        existing.cwe = new.cwe
+
+    existing.references = list(dict.fromkeys(existing.references + new.references))[:20]
+    existing.poc_urls = list(dict.fromkeys(existing.poc_urls + new.poc_urls))[:10]
+    existing.tags = list(dict.fromkeys(existing.tags + new.tags))[:20]
+    existing.mitre_techniques = list(dict.fromkeys(existing.mitre_techniques + new.mitre_techniques))
+
+    sources = f"{existing.source},{new.source}"
+    existing.source = ",".join(dict.fromkeys(sources.split(",")))
+
+    return existing

ingestion/output/__init__.py

@@ -0,0 +1,9 @@
+"""Output writers for xpl-intel advisories."""
+
+from __future__ import annotations
+
+from ingestion.output.csv_writer import write_csv
+from ingestion.output.json_writer import write_json
+from ingestion.output.sqlite_writer import write_sqlite
+
+__all__ = ["write_csv", "write_json", "write_sqlite"]

ingestion/output/csv_writer.py

@@ -0,0 +1,36 @@
+"""CSV output writer for xpl-intel advisories."""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+from schemas.advisory_v1 import Advisory
+
+CSV_FIELDS = [
+    "cve_id", "title", "description", "cvss_score", "cvss_vector",
+    "epss_score", "epss_percentile", "kev", "kev_date_added", "kev_due_date",
+    "vendor", "product", "affected_versions", "fixed_versions", "patch_url",
+    "poc_urls", "references", "cwe", "source", "source_url",
+    "status", "ingested_at", "mitre_techniques", "tags",
+]
+
+
+def write_csv(advisories: list[Advisory], path: str | Path) -> Path:
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(out, "w", newline="", encoding="utf-8") as fh:
+        writer = csv.DictWriter(fh, fieldnames=CSV_FIELDS)
+        writer.writeheader()
+        for adv in advisories:
+            row = adv.to_dict()
+            row.pop("schema", None)
+            for key in ("affected_versions", "fixed_versions", "poc_urls",
+                        "references", "mitre_techniques", "tags"):
+                if isinstance(row.get(key), list):
+                    row[key] = json.dumps(row[key])
+            writer.writerow({k: row.get(k, "") for k in CSV_FIELDS})
+
+    return out

ingestion/output/json_writer.py

@@ -0,0 +1,17 @@
+"""JSON output writer."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from schemas.advisory_v1 import Advisory
+
+
+def write_json(advisories: list[Advisory], path: str | Path) -> Path:
+    out = Path(path)
+    data: list[dict[str, Any]] = [a.to_dict() for a in advisories]
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8")
+    return out

ingestion/output/sqlite_writer.py

@@ -0,0 +1,116 @@
+"""SQLite output writer for xpl-intel advisories."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+
+from schemas.advisory_v1 import Advisory
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS advisories (
+    cve_id TEXT PRIMARY KEY,
+    title TEXT NOT NULL,
+    description TEXT DEFAULT '',
+    cvss_score REAL,
+    cvss_vector TEXT DEFAULT '',
+    epss_score REAL,
+    epss_percentile REAL,
+    kev INTEGER DEFAULT 0,
+    kev_date_added TEXT,
+    kev_due_date TEXT,
+    vendor TEXT DEFAULT '',
+    product TEXT DEFAULT '',
+    affected_versions TEXT DEFAULT '[]',
+    fixed_versions TEXT DEFAULT '[]',
+    patch_url TEXT DEFAULT '',
+    poc_urls TEXT DEFAULT '[]',
+    "references" TEXT DEFAULT '[]',
+    cwe TEXT DEFAULT '',
+    source TEXT DEFAULT '',
+    source_url TEXT DEFAULT '',
+    status TEXT DEFAULT 'unverified',
+    ingested_at TEXT DEFAULT '',
+    mitre_techniques TEXT DEFAULT '[]',
+    tags TEXT DEFAULT '[]'
+);
+CREATE INDEX IF NOT EXISTS idx_advisories_vendor ON advisories(vendor);
+CREATE INDEX IF NOT EXISTS idx_advisories_cvss ON advisories(cvss_score);
+CREATE INDEX IF NOT EXISTS idx_advisories_kev ON advisories(kev);
+CREATE INDEX IF NOT EXISTS idx_advisories_source ON advisories(source);
+"""
+
+UPSERT_SQL = """
+INSERT INTO advisories (
+    cve_id, title, description, cvss_score, cvss_vector,
+    epss_score, epss_percentile, kev, kev_date_added, kev_due_date,
+    vendor, product, affected_versions, fixed_versions, patch_url,
+    poc_urls, "references", cwe, source, source_url,
+    status, ingested_at, mitre_techniques, tags
+) VALUES (
+    ?, ?, ?, ?, ?,
+    ?, ?, ?, ?, ?,
+    ?, ?, ?, ?, ?,
+    ?, ?, ?, ?, ?,
+    ?, ?, ?, ?
+) ON CONFLICT(cve_id) DO UPDATE SET
+    title = CASE WHEN excluded.title != '' THEN excluded.title ELSE advisories.title END,
+    description = CASE WHEN excluded.description != '' THEN excluded.description ELSE advisories.description END,
+    cvss_score = CASE WHEN excluded.cvss_score IS NOT NULL THEN excluded.cvss_score ELSE advisories.cvss_score END,
+    cvss_vector = CASE WHEN excluded.cvss_vector != '' THEN excluded.cvss_vector ELSE advisories.cvss_vector END,
+    kev = MAX(advisories.kev, excluded.kev),
+    source = advisories.source || ',' || excluded.source,
+    ingested_at = excluded.ingested_at
+"""
+
+
+def write_sqlite(advisories: list[Advisory], path: str | Path) -> Path:
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+
+    conn = sqlite3.connect(str(out))
+    try:
+        conn.executescript(SCHEMA_SQL)
+        rows = [_advisory_to_row(a) for a in advisories]
+        conn.executemany(UPSERT_SQL, rows)
+        conn.commit()
+    finally:
+        conn.close()
+
+    return out
+
+
+def _advisory_to_row(adv: Advisory) -> tuple[
+    str, str, str, float | None, str,
+    float | None, float | None, int, str | None, str | None,
+    str, str, str, str, str,
+    str, str, str, str, str,
+    str, str, str, str,
+]:
+    return (
+        adv.cve_id,
+        adv.title,
+        adv.description,
+        adv.cvss_score,
+        adv.cvss_vector,
+        adv.epss_score,
+        adv.epss_percentile,
+        1 if adv.kev else 0,
+        adv.kev_date_added,
+        adv.kev_due_date,
+        adv.vendor,
+        adv.product,
+        json.dumps(adv.affected_versions),
+        json.dumps(adv.fixed_versions),
+        adv.patch_url,
+        json.dumps(adv.poc_urls),
+        json.dumps(adv.references),
+        adv.cwe,
+        adv.source,
+        adv.source_url,
+        adv.status,
+        adv.ingested_at,
+        json.dumps(adv.mitre_techniques),
+        json.dumps(adv.tags),
+    )

ingestion/sources/cisa_kev.py

@@ -0,0 +1,44 @@
+"""CISA Known Exploited Vulnerabilities (KEV) catalog ingestion."""
+
+from __future__ import annotations
+
+import csv
+import io
+from datetime import datetime, timezone
+
+import requests
+
+from schemas.advisory_v1 import Advisory
+
+KEV_URL = "https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv"
+
+
+def fetch(timeout: int = 30) -> list[Advisory]:
+    resp = requests.get(KEV_URL, timeout=timeout)
+    resp.raise_for_status()
+
+    reader = csv.DictReader(io.StringIO(resp.text))
+    advisories = []
+    now = datetime.now(timezone.utc).isoformat()
+
+    for row in reader:
+        adv = Advisory(
+            cve_id=row.get("cveID", ""),
+            title=row.get("vulnerabilityName", ""),
+            description=row.get("shortDescription", ""),
+            vendor=row.get("vendorProject", ""),
+            product=row.get("product", ""),
+            kev=True,
+            kev_date_added=row.get("dateAdded", ""),
+            kev_due_date=row.get("dueDate", ""),
+            references=[row["notes"]] if row.get("notes") else [],
+            source="cisa_kev",
+            source_url=KEV_URL,
+            status="unverified",
+            ingested_at=now,
+            tags=["kev", "actively-exploited"],
+        )
+        if adv.cve_id:
+            advisories.append(adv)
+
+    return advisories

ingestion/sources/epss_api.py

@@ -0,0 +1,49 @@
+"""FIRST EPSS (Exploit Prediction Scoring System) API enrichment."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import requests
+
+EPSS_API_URL = "https://api.first.org/data/v1/epss"
+
+
+@dataclass
+class EPSSScore:
+    cve: str
+    epss: float
+    percentile: float
+    date: str
+
+
+def fetch_scores(cve_ids: list[str], timeout: int = 30) -> list[EPSSScore]:
+    if not cve_ids:
+        return []
+
+    batch_size = 100
+    results: list[EPSSScore] = []
+
+    for i in range(0, len(cve_ids), batch_size):
+        batch = cve_ids[i : i + batch_size]
+        params = {"cve": ",".join(batch)}
+        resp = requests.get(EPSS_API_URL, params=params, timeout=timeout)
+        resp.raise_for_status()
+        data = resp.json()
+
+        for entry in data.get("data", []):
+            results.append(
+                EPSSScore(
+                    cve=entry.get("cve", ""),
+                    epss=float(entry.get("epss", 0.0)),
+                    percentile=float(entry.get("percentile", 0.0)),
+                    date=entry.get("date", ""),
+                )
+            )
+
+    return results
+
+
+def enrich_single(cve_id: str, timeout: int = 30) -> EPSSScore | None:
+    scores = fetch_scores([cve_id], timeout=timeout)
+    return scores[0] if scores else None

ingestion/sources/exploitdb.py

@@ -0,0 +1,50 @@
+"""ExploitDB CSV ingestion (files_exploits.csv from searchsploit clone)."""
+
+from __future__ import annotations
+
+import csv
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+from schemas.advisory_v1 import Advisory
+
+CVE_RE = re.compile(r"CVE-\d{4}-\d{4,7}")
+DEFAULT_CSV = "/usr/share/exploitdb/files_exploits.csv"
+
+
+def fetch(csv_path: str | None = None) -> list[Advisory]:
+    path = Path(csv_path or DEFAULT_CSV)
+    if not path.exists():
+        return []
+
+    advisories = []
+    now = datetime.now(timezone.utc).isoformat()
+
+    with open(path, errors="ignore") as fh:
+        reader = csv.DictReader(fh)
+        for row in reader:
+            codes = row.get("codes", "")
+            cves = CVE_RE.findall(codes)
+            if not cves:
+                continue
+
+            desc = row.get("description", "")
+            edb_id = row.get("id", "")
+            platform = row.get("platform", "")
+
+            for cve in set(cves):
+                adv = Advisory(
+                    cve_id=cve,
+                    title=desc[:200],
+                    description=desc,
+                    poc_urls=[f"https://www.exploit-db.com/exploits/{edb_id}"] if edb_id else [],
+                    source="exploitdb",
+                    source_url=f"https://www.exploit-db.com/exploits/{edb_id}",
+                    status="unverified",
+                    ingested_at=now,
+                    tags=["exploitdb", platform] if platform else ["exploitdb"],
+                )
+                advisories.append(adv)
+
+    return advisories

ingestion/sources/github_advisory.py

@@ -0,0 +1,113 @@
+"""GitHub Advisory Database (GHSA) ingestion via REST API."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import requests
+
+from schemas.advisory_v1 import Advisory
+
+GHSA_API = "https://api.github.com/advisories"
+
+
+def fetch(
+    days_back: int = 7,
+    github_token: str | None = None,
+    severity: str = "critical,high",
+    timeout: int = 30,
+) -> list[Advisory]:
+    now = datetime.now(timezone.utc)
+    since = (now - timedelta(days=days_back)).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    headers: dict[str, str] = {"Accept": "application/vnd.github+json"}
+    if github_token:
+        headers["Authorization"] = f"Bearer {github_token}"
+
+    advisories: list[Advisory] = []
+    page = 1
+
+    while True:
+        params: dict[str, str] = {
+            "type": "reviewed",
+            "severity": severity,
+            "updated": f">={since}",
+            "per_page": "100",
+            "page": str(page),
+        }
+
+        resp = requests.get(GHSA_API, params=params, headers=headers, timeout=timeout)
+        resp.raise_for_status()
+        items: list[dict[str, Any]] = resp.json()
+
+        if not items:
+            break
+
+        ingested_at = datetime.now(timezone.utc).isoformat()
+
+        for item in items:
+            cve_id = item.get("cve_id") or ""
+            ghsa_id: str = item.get("ghsa_id", "")
+            if not cve_id and not ghsa_id:
+                continue
+
+            cvss_score: float | None = None
+            cvss_vector = ""
+            cvss_data: dict[str, Any] | None = item.get("cvss")
+            if cvss_data:
+                cvss_score = cvss_data.get("score")
+                cvss_vector = cvss_data.get("vector_string", "")
+
+            cwe_list: list[dict[str, str]] = item.get("cwes", [])
+            cwe = cwe_list[0].get("cwe_id", "") if cwe_list else ""
+
+            refs: list[str] = [
+                r.get("url", "") for r in item.get("references", []) if r.get("url")
+            ]
+
+            vendor = ""
+            product = ""
+            affected: list[str] = []
+            fixed: list[str] = []
+            vuln_list: list[dict[str, Any]] = item.get("vulnerabilities", [])
+            for vuln in vuln_list:
+                pkg: dict[str, Any] = vuln.get("package", {})
+                if not vendor and pkg.get("ecosystem"):
+                    vendor = pkg["ecosystem"]
+                if not product and pkg.get("name"):
+                    product = pkg["name"]
+                vr: str = vuln.get("vulnerable_version_range", "")
+                if vr:
+                    affected.append(f"{pkg.get('name', '')}:{vr}")
+                fv: str | None = vuln.get("first_patched_version")
+                if fv:
+                    fixed.append(f"{pkg.get('name', '')}:{fv}")
+
+            adv = Advisory(
+                cve_id=cve_id or ghsa_id,
+                title=item.get("summary", "") or cve_id or ghsa_id,
+                description=(item.get("description", "") or "")[:500],
+                cvss_score=cvss_score,
+                cvss_vector=cvss_vector,
+                cwe=cwe,
+                vendor=vendor,
+                product=product,
+                affected_versions=affected[:10],
+                fixed_versions=fixed[:10],
+                references=refs[:10],
+                source="github_advisory",
+                source_url=item.get("html_url", GHSA_API),
+                status="unverified",
+                ingested_at=ingested_at,
+                tags=["ghsa", item.get("severity", "unknown")],
+            )
+            advisories.append(adv)
+
+        if len(items) < 100:
+            break
+        page += 1
+        time.sleep(1.0 if not github_token else 0.3)
+
+    return advisories

ingestion/sources/nuclei_templates.py

@@ -0,0 +1,62 @@
+"""Nuclei templates CVE extraction (local clone)."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+from schemas.advisory_v1 import Advisory
+
+CVE_RE = re.compile(r"CVE-\d{4}-\d{4,7}")
+SEVERITY_MAP = {"critical": 9.8, "high": 8.0, "medium": 5.5, "low": 3.0, "info": 0.0}
+
+
+def fetch(templates_dir: str = "nuclei-templates") -> list[Advisory]:
+    base = Path(templates_dir)
+    if not base.exists():
+        return []
+
+    advisories = []
+    now = datetime.now(timezone.utc).isoformat()
+
+    for yaml_file in base.rglob("*.yaml"):
+        content = yaml_file.read_text(errors="ignore")
+
+        cves = CVE_RE.findall(content)
+        if not cves:
+            continue
+
+        name = ""
+        severity = ""
+        desc = ""
+        refs = []
+
+        for line in content.split("\n"):
+            stripped = line.strip()
+            if stripped.startswith("name:"):
+                name = stripped.split(":", 1)[1].strip().strip("\"'")
+            elif stripped.startswith("severity:"):
+                severity = stripped.split(":", 1)[1].strip().lower()
+            elif stripped.startswith("description:"):
+                desc = stripped.split(":", 1)[1].strip().strip("\"'")
+            elif stripped.startswith("- http"):
+                refs.append(stripped.lstrip("- ").strip())
+
+        for cve in set(cves):
+            adv = Advisory(
+                cve_id=cve,
+                title=name or cve,
+                description=desc[:500],
+                cvss_score=SEVERITY_MAP.get(severity),
+                references=refs[:5],
+                poc_urls=[f"nuclei:{yaml_file.relative_to(base)}"],
+                source="nuclei",
+                source_url=str(yaml_file),
+                status="unverified",
+                ingested_at=now,
+                tags=["nuclei", severity] if severity else ["nuclei"],
+            )
+            advisories.append(adv)
+
+    return advisories

ingestion/sources/nvd_api.py

@@ -0,0 +1,99 @@
+"""NVD API v2 delta ingestion."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import requests
+
+from schemas.advisory_v1 import Advisory
+
+NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+
+
+def fetch(days_back: int = 7, api_key: str | None = None, timeout: int = 30) -> list[Advisory]:
+    now = datetime.now(timezone.utc)
+    start = now - timedelta(days=days_back)
+
+    pub_start = start.strftime("%Y-%m-%dT%H:%M:%S.000")
+    pub_end = now.strftime("%Y-%m-%dT%H:%M:%S.000")
+    start_index = 0
+    headers: dict[str, str] = {}
+    if api_key:
+        headers["apiKey"] = api_key
+
+    advisories: list[Advisory] = []
+    total: int | None = None
+
+    while True:
+        params: dict[str, str] = {
+            "pubStartDate": pub_start,
+            "pubEndDate": pub_end,
+            "resultsPerPage": "200",
+            "startIndex": str(start_index),
+        }
+        resp = requests.get(NVD_API, params=params, headers=headers, timeout=timeout)
+        resp.raise_for_status()
+        data: dict[str, Any] = resp.json()
+
+        if total is None:
+            total = int(data.get("totalResults", 0))
+
+        vulns: list[dict[str, Any]] = data.get("vulnerabilities", [])
+        for vuln in vulns:
+            cve_data: dict[str, Any] = vuln.get("cve", {})
+            cve_id: str = cve_data.get("id", "")
+
+            descriptions: list[dict[str, str]] = cve_data.get("descriptions", [])
+            desc = next((d["value"] for d in descriptions if d.get("lang") == "en"), "")
+
+            metrics: dict[str, Any] = cve_data.get("metrics", {})
+            cvss_score: float | None = None
+            cvss_vector = ""
+            for key in ["cvssMetricV31", "cvssMetricV30", "cvssMetricV2"]:
+                if metrics.get(key):
+                    cvss_data: dict[str, Any] = metrics[key][0].get("cvssData", {})
+                    cvss_score = cvss_data.get("baseScore")
+                    cvss_vector = cvss_data.get("vectorString", "")
+                    break
+
+            refs = [
+                r.get("url", "")
+                for r in cve_data.get("references", [])
+                if r.get("url")
+            ]
+
+            weaknesses: list[dict[str, Any]] = cve_data.get("weaknesses", [])
+            cwe = ""
+            for w in weaknesses:
+                for desc_item in w.get("description", []):
+                    if desc_item.get("value", "").startswith("CWE-"):
+                        cwe = desc_item["value"]
+                        break
+                if cwe:
+                    break
+
+            adv = Advisory(
+                cve_id=cve_id,
+                title=cve_id,
+                description=desc[:500],
+                cvss_score=cvss_score,
+                cvss_vector=cvss_vector,
+                cwe=cwe,
+                references=refs[:10],
+                source="nvd",
+                source_url=f"https://nvd.nist.gov/vuln/detail/{cve_id}",
+                status="unverified",
+                ingested_at=datetime.now(timezone.utc).isoformat(),
+            )
+            advisories.append(adv)
+
+        start_index += len(vulns)
+        if start_index >= total:
+            break
+
+        time.sleep(0.6 if api_key else 6)
+
+    return advisories

ingestion/sources/packetstorm_rss.py

@@ -0,0 +1,47 @@
+"""PacketStorm Security RSS feed ingestion."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import defusedxml.ElementTree as ET
+import requests
+
+from schemas.advisory_v1 import Advisory
+
+PACKETSTORM_RSS_URL = "https://rss.packetstormsecurity.com/files/tags/exploit/"
+
+
+def fetch(timeout: int = 30) -> list[Advisory]:
+    resp = requests.get(PACKETSTORM_RSS_URL, timeout=timeout)
+    resp.raise_for_status()
+
+    root = ET.fromstring(resp.text)
+    advisories: list[Advisory] = []
+    now = datetime.now(timezone.utc).isoformat()
+
+    for item in root.iter("item"):
+        title = item.findtext("title", "")
+        link = item.findtext("link", "")
+        description = item.findtext("description", "")
+        pub_date = item.findtext("pubDate", "")
+
+        adv = Advisory(
+            cve_id=_extract_cve(title + " " + description),
+            title=title,
+            description=description,
+            source="packetstorm",
+            source_url=link,
+            ingested_at=now,
+            extra={"pubDate": pub_date} if pub_date else {},
+        )
+        advisories.append(adv)
+
+    return advisories
+
+
+def _extract_cve(text: str) -> str:
+    import re
+
+    match = re.search(r"CVE-\d{4}-\d{4,}", text)
+    return match.group(0) if match else ""

ingestion/sources/vulncheck_kev.py

@@ -0,0 +1,67 @@
+"""VulnCheck KEV enriched catalog ingestion."""
+
+from __future__ import annotations
+
+import contextlib
+from datetime import datetime, timezone
+from typing import Any
+
+import requests
+
+from schemas.advisory_v1 import Advisory
+
+VULNCHECK_API = "https://api.vulncheck.com/v3/index/vulncheck-kev"
+
+
+def fetch(api_token: str, timeout: int = 30) -> list[Advisory]:
+    headers = {
+        "Accept": "application/json",
+        "Authorization": f"Bearer {api_token}",
+    }
+    advisories: list[Advisory] = []
+    now = datetime.now(timezone.utc).isoformat()
+
+    resp = requests.get(VULNCHECK_API, headers=headers, timeout=timeout)
+    resp.raise_for_status()
+    data: dict[str, Any] = resp.json()
+
+    entries: list[dict[str, Any]] = data.get("data", [])
+
+    for entry in entries:
+        cve_id: str = entry.get("cve_id", "") or entry.get("cveID", "")
+        if not cve_id:
+            continue
+
+        cvss_score: float | None = None
+        raw_cvss = entry.get("cvss_score") or entry.get("baseScore")
+        if raw_cvss is not None:
+            with contextlib.suppress(ValueError, TypeError):
+                cvss_score = float(raw_cvss)
+
+        refs: list[str] = []
+        for r in entry.get("references", []):
+            url = r if isinstance(r, str) else r.get("url", "")
+            if url:
+                refs.append(url)
+
+        adv = Advisory(
+            cve_id=cve_id,
+            title=entry.get("vulnerabilityName", "") or entry.get("name", "") or cve_id,
+            description=entry.get("shortDescription", "") or entry.get("description", ""),
+            cvss_score=cvss_score,
+            cvss_vector=entry.get("cvss_vector", "") or entry.get("vectorString", ""),
+            vendor=entry.get("vendorProject", "") or entry.get("vendor", ""),
+            product=entry.get("product", ""),
+            kev=True,
+            kev_date_added=entry.get("dateAdded", ""),
+            kev_due_date=entry.get("dueDate", ""),
+            references=refs[:10],
+            source="vulncheck_kev",
+            source_url=VULNCHECK_API,
+            status="unverified",
+            ingested_at=now,
+            tags=["kev", "vulncheck", "actively-exploited"],
+        )
+        advisories.append(adv)
+
+    return advisories

ingestion/sources/watchtowr_rss.py

@@ -0,0 +1,48 @@
+"""watchTowr Labs RSS/blog advisory ingestion."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+
+import requests
+from defusedxml import ElementTree as ET
+
+from schemas.advisory_v1 import Advisory
+
+FEED_URL = "https://labs.watchtowr.com/rss/"
+CVE_RE = re.compile(r"CVE-\d{4}-\d{4,7}")
+
+
+def fetch(timeout: int = 30) -> list[Advisory]:
+    resp = requests.get(FEED_URL, timeout=timeout)
+    resp.raise_for_status()
+
+    root = ET.fromstring(resp.text)
+    advisories = []
+    now = datetime.now(timezone.utc).isoformat()
+
+    for item in root.iter("item"):
+        title = item.findtext("title", "")
+        link = item.findtext("link", "")
+        desc = item.findtext("description", "")
+        cves = CVE_RE.findall(f"{title} {desc}")
+
+        if not cves:
+            cves = ["WATCHTOWR-NOID"]
+
+        for cve in set(cves):
+            adv = Advisory(
+                cve_id=cve,
+                title=title[:200],
+                description=desc[:500],
+                references=[link] if link else [],
+                source="watchtowr",
+                source_url=link,
+                status="unverified",
+                ingested_at=now,
+                tags=["watchtowr"],
+            )
+            advisories.append(adv)
+
+    return advisories

schemas/advisory_v1.py

@@ -0,0 +1,73 @@
+"""xpl.advisory.v1 schema definition."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Advisory:
+    cve_id: str
+    title: str
+    description: str = ""
+    cvss_score: float | None = None
+    cvss_vector: str = ""
+    epss_score: float | None = None
+    epss_percentile: float | None = None
+    kev: bool = False
+    kev_date_added: str | None = None
+    kev_due_date: str | None = None
+    vendor: str = ""
+    product: str = ""
+    affected_versions: list[str] = field(default_factory=list)
+    fixed_versions: list[str] = field(default_factory=list)
+    patch_url: str = ""
+    poc_urls: list[str] = field(default_factory=list)
+    references: list[str] = field(default_factory=list)
+    cwe: str = ""
+    source: str = ""
+    source_url: str = ""
+    status: str = "unverified"
+    ingested_at: str = ""
+    mitre_techniques: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema": "xpl.advisory.v1",
+            "cve_id": self.cve_id,
+            "title": self.title,
+            "description": self.description,
+            "cvss_score": self.cvss_score,
+            "cvss_vector": self.cvss_vector,
+            "epss_score": self.epss_score,
+            "epss_percentile": self.epss_percentile,
+            "kev": self.kev,
+            "kev_date_added": self.kev_date_added,
+            "kev_due_date": self.kev_due_date,
+            "vendor": self.vendor,
+            "product": self.product,
+            "affected_versions": self.affected_versions,
+            "fixed_versions": self.fixed_versions,
+            "patch_url": self.patch_url,
+            "poc_urls": self.poc_urls,
+            "references": self.references,
+            "cwe": self.cwe,
+            "source": self.source,
+            "source_url": self.source_url,
+            "status": self.status,
+            "ingested_at": self.ingested_at,
+            "mitre_techniques": self.mitre_techniques,
+            "tags": self.tags,
+            **self.extra,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Advisory:
+        known = {f.name for f in cls.__dataclass_fields__.values()}
+        extra = {k: v for k, v in data.items() if k not in known and k != "schema"}
+        kwargs = {k: v for k, v in data.items() if k in known}
+        kwargs["extra"] = extra
+        return cls(**kwargs)

xpl_intel-1.2.0.dist-info/METADATA

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: xpl-intel
+Version: 1.2.0
+Summary: XPL-Intel — CVE ingestion pipeline for XPL-Suite. CISA KEV, NVD, watchTowr, Nuclei, ExploitDB.
+Author-email: André Henrique <henrique.santos@uniaogeek.com.br>
+License: XSRL-1.0
+Keywords: cve,vulnerability,intel,kev,nvd,nuclei,xpl-suite
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Security
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: requests<3.0,>=2.31.0
+Requires-Dist: rich>=13.0
+Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: argcomplete>=3.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: types-requests>=2.31; extra == "dev"
+Dynamic: license-file

xpl_intel-1.2.0.dist-info/RECORD

@@ -0,0 +1,26 @@
+ingestion/__init__.py,sha256=YsbpQonm9QDnmXZjz_s_OjgqPUD8q4nbF42oCVqbzjo,44
+ingestion/cli.py,sha256=3-F5db2tyrsPNzF21sHxOa9DtHxoo1W5ZcdbhsZbplk,6442
+ingestion/dedup.py,sha256=jMHRRVrdmJ19Td_wuOwDLpH9FoYWJY4dHPjBET8aDuI,1914
+ingestion/output/__init__.py,sha256=RL91dQSVpTk9XqelsmtduBu7v6K48HvGFVe6z6prYIQ,297
+ingestion/output/csv_writer.py,sha256=EZ2YCoX9H6Wa06fjs5pjk94L6Zj1n6a9Brt_9_8_ftw,1256
+ingestion/output/json_writer.py,sha256=YY2Lvj8SfgPBepcqX5bi-clpZRPHIFS8LHQw8qFLHag,468
+ingestion/output/sqlite_writer.py,sha256=zWa_mVl2tBvDUEDFoBmzc4DReG9bZck3MpfTp3knA5U,3606
+ingestion/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestion/sources/cisa_kev.py,sha256=NWJR_jbiJZKzfEaIopsJR-KPwpBc0Ke_ZyjjnUlMcfA,1315
+ingestion/sources/epss_api.py,sha256=Hg1xfDVTXSn1tZgu10bHoNM7lvfvHsyDFgp-V-JwIOI,1282
+ingestion/sources/exploitdb.py,sha256=wRflqJrFmDgjPsyI5aZ37PxKTgJyJypBPICZ4GUPMus,1560
+ingestion/sources/github_advisory.py,sha256=wMDr5KFSB8GVGPyqrX-1av3KWw5hGvY07Dwo75rdBIg,3752
+ingestion/sources/nuclei_templates.py,sha256=cU6019zOdeo5clMHF9Nnnpat4jNfGtXQvutg9KxNDUk,1966
+ingestion/sources/nvd_api.py,sha256=2nc11cnHUwpx2YPeSdhJDJxJD2FszfzYVcS6IYEuAIU,3293
+ingestion/sources/packetstorm_rss.py,sha256=VPXbegeEUxRTwWECv8lxXSabk29nqOHWI3vQ_iOBrsM,1294
+ingestion/sources/vulncheck_kev.py,sha256=K_C63VkwgP8Wc5ksow_prHCP_oOl1bfSQzSf3XtRcKA,2237
+ingestion/sources/watchtowr_rss.py,sha256=J0fKZN7X2bOQEpstSfpOlb36h-vL0Lc0S70J5SkXtig,1297
+schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+schemas/advisory_v1.py,sha256=KZvealmzrMxYJYUqxQj08I_zRR-hmjmSnWmCf17th1s,2540
+schemas/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xpl_intel-1.2.0.dist-info/licenses/LICENSE,sha256=hQZm7l5K5ztlml-M_W-xQ8Z6Y8n9fNQvqRrH6pc6big,5710
+xpl_intel-1.2.0.dist-info/METADATA,sha256=uoMI_VTJQ89_lSR4nH6IkDNo8EDkim38m9E9HPxnCZw,898
+xpl_intel-1.2.0.dist-info/WHEEL,sha256=16fc_9Rp854SLcdhaDEjr0u65p0e3FhZxihNgcnCE98,91
+xpl_intel-1.2.0.dist-info/entry_points.txt,sha256=Q6NAOCbbd4xr5OiqIwvEV0N15qAX1yOawlk1DsbDGcE,74
+xpl_intel-1.2.0.dist-info/top_level.txt,sha256=m9wK5ToZT6pOo2B-PFgJazrH-7oDoiNezLjqW9cKScI,18
+xpl_intel-1.2.0.dist-info/RECORD,,

xpl_intel-1.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

xpl_intel-1.2.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+xif = ingestion.cli:main
+xpl-intel = ingestion.cli:main

xpl_intel-1.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,123 @@
+# XPL-Suite Security Research License (XSRL) v1.0
+
+Copyright (c) 2024-2026 Andre Henrike de Oliveira (@mrhenrike) / Uniao Geek
+All rights reserved.
+
+## 1. Definitions
+
+- "Software" means the source code, documentation, binaries, scripts, configuration
+  files, and any other materials included in this repository.
+- "Author" means the original copyright holder: Andre Henrike de Oliveira (@mrhenrike).
+- "Original Repository" means the source repository hosted at
+  https://github.com/mrhenrike/xpl-intel from which the Software originates.
+- "Upstream Credits" means the authors and licenses of third-party components,
+  libraries, or code referenced in the Software, as documented in source file headers,
+  CREDITS files, or README attribution sections.
+- "Authorized Security Testing" means penetration testing, vulnerability assessment,
+  red team exercises, purple team exercises, security auditing, or similar activities
+  conducted with explicit written permission from the owner of the target system(s).
+
+## 2. Grant of Rights
+
+Subject to the terms and conditions of this License, the Author grants you a
+non-exclusive, worldwide, royalty-free license to:
+
+a) **Use** the Software for educational purposes, academic research, security studies,
+   and Authorized Security Testing;
+b) **Copy and reproduce** the Software;
+c) **Modify** the Software and create derivative works;
+d) **Fork** the Software and maintain independent copies;
+e) **Distribute** the Software and derivative works, provided all conditions in
+   Section 3 are met.
+
+## 3. Conditions
+
+Any use, copy, modification, fork, or distribution of the Software must comply with
+ALL of the following conditions:
+
+### 3.1 Attribution
+
+a) You must retain the Author's original copyright notice and attribution in all
+   copies and derivative works.
+b) You must retain all Upstream Credits, including but not limited to: original author
+   names, license notices, and attribution statements of third-party code used as a
+   base or reference for the Software.
+c) Modified files must carry a prominent notice stating that you changed the files and
+   the date of any change.
+d) Derivative works must include a clear statement identifying the original Software
+   and its Author.
+e) Any copy, fork, or derivative work must include a visible reference to the Original
+   Repository URL (https://github.com/mrhenrike/xpl-intel) in the project's
+   README or equivalent documentation, so that users can trace the Software back to
+   its source.
+f) The Author's name and credits must not be removed, obscured, or replaced in any
+   copy, fork, or derivative work. The Author must remain credited as the original
+   creator in all circumstances.
+
+### 3.2 Same License
+
+a) Any distribution of the Software or derivative works must be under this same
+   License (XSRL v1.0).
+b) You may not apply any additional restrictions or alternative license terms to the
+   Software or derivative works.
+c) You may not sublicense the Software under different terms.
+
+### 3.3 Non-Commercial Use Only
+
+a) The Software may not be used, in whole or in part, for commercial purposes.
+b) "Commercial purposes" includes but is not limited to: selling the Software or
+   derivative works, incorporating the Software into commercial products or services,
+   using the Software to generate revenue directly or indirectly, and offering the
+   Software as part of a paid service.
+c) Use in non-profit organizations, academic institutions, and personal projects is
+   permitted, provided no commercial gain is derived.
+
+### 3.4 Ethical Use Only
+
+a) The Software may only be used for:
+   - Educational purposes and academic research;
+   - Security studies, training, and skill development;
+   - Authorized Security Testing (as defined in Section 1);
+   - Capture The Flag (CTF) competitions and security labs;
+   - Defensive security, incident response, and threat analysis.
+
+b) The Software may NOT be used for:
+   - Unauthorized access to systems, networks, or data;
+   - Any activity that violates applicable laws or regulations;
+   - Malicious purposes, including but not limited to: deploying malware against
+     unauthorized targets, conducting denial-of-service attacks, unauthorized
+     surveillance, data theft, or any form of cybercrime;
+   - Targeting individuals, organizations, or infrastructure without explicit
+     authorization;
+   - Any purpose that causes harm to individuals or organizations.
+
+## 4. No Warranty
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+## 5. Limitation of Liability
+
+The Author shall not be held responsible for any misuse of the Software. Users are
+solely responsible for ensuring their use of the Software complies with all applicable
+laws, regulations, and ethical standards.
+
+## 6. Termination
+
+This License and the rights granted hereunder will terminate automatically if you fail
+to comply with any of the terms and conditions of this License. Upon termination, you
+must cease all use and destroy all copies of the Software in your possession.
+
+## 7. Severability
+
+If any provision of this License is held to be unenforceable, such provision shall be
+reformed only to the extent necessary to make it enforceable, and the remaining
+provisions shall continue in full force and effect.
+
+---
+
+SPDX-License-Identifier: LicenseRef-XSRL-1.0

xpl_intel-1.2.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+ingestion
+schemas