netward 0.4.1__py3-none-any.whl

netward/__init__.py

@@ -0,0 +1,113 @@
+"""
+Net Ward — Passive Deception Layer for HTTP/Network Services
+
+Standalone product for the general DDoS-defense market. Sold to
+operators who get hit by botnet probing / credential stuffing / scrape
+attacks but can't afford enterprise DDoS protection (Cloudflare
+Enterprise, Akamai). Anti-virus doesn't cover this attack surface;
+Net Ward fills that gap.
+
+Customer market: ANY operator running an HTTP/network service with
+inbound traffic. Hospitals, indie SaaS, regional ISPs, federal
+contractors, defense-adjacent sites, public-interest projects, niche
+aviation operators, ANY web property under recurring DDoS or scrape
+pressure. The mesh treats every node identically — there is no
+privileged customer relationship.
+
+Philosophy: confuse, don't combat.
+- Don't harm bots — they're conscripts following orders
+- Mirror plausible-looking responses so bots think they succeeded
+- Bots' own deduplication logic deprioritizes the target → they move on
+- No active counter-attack, no legal liability, no escalation
+
+Operating principle: ALL NODES ARE CONTINUOUSLY PREPARED.
+
+There is no sacrificial node in Net Ward. Every node maintains its
+mirror layer at all times. Whoever gets hit first is just whoever the
+attacker happens to target first — they defend SUCCESSFULLY because
+they've been prepared all along. They are not absorbing harm so
+others can prepare; the entire mesh is already prepared.
+
+What the first-hit node DOES contribute is REAL-TIME INTEL:
+- Reports where it was hit (Pattern + Source published to mesh)
+- Continues relaying as the attack progresses (sustained, not one-shot)
+- Tracks mutations as bots iterate variants
+- Refreshes confidence on existing patterns as they keep firing
+
+Other nodes receive the relay and:
+- Refresh local Pattern confidence
+- Update Source reputation for the attacker's IPs/ASNs
+- Pre-arm against the specific mutation chain in case the attacker
+  pivots to them next
+
+Network effect: more nodes = smarter mesh = better deception for all.
+Anti-fragile: the harder attackers hit, the more pattern data the
+mesh collects, the better the deception gets for every node.
+Mutation-aware: when bots iterate variants seeking vulnerabilities,
+the mesh tracks the mutation chain so every node stays one step ahead.
+
+Architecture: distributed mesh with deception mirror layer.
+
+Product positioning:
+
+Net Ward sits BESIDE existing security tools, not replacing them.
+- CDN/WAF (Cloudflare, Akamai) handles broad edge filtering
+- Antivirus / EDR handles infected endpoints
+- Net Ward handles ADAPTIVE ABUSE PRESSURE: bot floods, suspicious
+  repeat patterns, credential stuffing, scrape attacks, and
+  operator-visible deflection
+
+Antivirus DOES NOT cover this attack surface. AV protects hosts from
+malware; it doesn't preserve availability when traffic or abuse floods
+the edge. Net Ward fills the gap as a unique add-on for electronic /
+bot warfare protection that saves host operators meaningful downtime
+and remediation cost.
+
+Buyer pain (real, validated): small orgs, hospitals, regional
+infrastructure, indie platforms, niche data services often can't
+afford Cloudflare-Enterprise-style support OR don't want opaque
+black-box mitigation. A clear, local/operator-controlled layer has
+genuine market room.
+
+CORE PRODUCT REQUIREMENT: FAIL SAFE AND FAIL CLEAN.
+
+Net Ward must protect availability without becoming another source of
+risk to the host system. This is not a "nice to have" — it's a
+gating requirement for v1 release.
+
+Design principles (encoded in every layer):
+
+- NO kernel hooks, NO packet drivers, NO invasive system changes
+- Run as user-space reverse proxy / sidecar / edge service ONLY
+- Config lives in its own directory; never modifies host config
+- Logs are append-only and bounded (no unbounded disk growth)
+- State/cache can be deleted without harming the host application
+- Crash behavior: operator chooses fail-open vs fail-closed explicitly
+  (default: fail-open — don't take the host's service down with us)
+- Uninstall = remove the service folder/config, NOT repair the OS
+- NO retaliatory traffic, NO client harm, NO "hack back"
+- Rate limiting and deflection must be REVERSIBLE and EXPLAINABLE
+  (operator can roll back any rule, can read why each action fired)
+
+Setup target (cheap, easy, accessible):
+
+- Single binary OR pure-Python package install
+- ONE config file (validated, with clear error messages on bad config)
+- Docker option for ops teams that prefer containers
+- Works in front of an existing app with MINIMAL routing changes
+- Clear dashboard / CLI: traffic state, active rules, top sources,
+  current action being taken — no operator surprise
+
+THE PRODUCT PROMISE:
+Protect availability without becoming another source of risk.
+
+Scope: HTTP/network services with INBOUND traffic.
+NOT for broadcast feed protection — public one-way feeds without
+authentication or upstream firewall control are a separate problem
+domain that mirror/honeypot countermeasures cannot address.
+
+Naming convention:
+- Display name: "Net Ward" (operator-facing brand)
+- Python package: `netward` (lowercase one-word import path)
+- Architecture term: "mirror layer" / "deception mirror" (internal)
+"""

netward/__main__.py

@@ -0,0 +1,55 @@
+"""
+Net Ward -- proxy daemon entrypoint
+
+    python -m netward --config /etc/netward/config.json
+
+Loads operator config, seeds vendor patterns on first run, then
+binds the capture loop and runs until interrupted.
+Management commands (install-patterns, list-patterns, etc.) live in
+netward/cli.py and are not part of this entrypoint.
+"""
+import argparse
+import asyncio
+import sys
+
+from netward.operator_layer import load_config, validate_storage_permissions
+from netward.capture import start_capture_loop
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="netward",
+        description="Net Ward passive deception proxy",
+        epilog="For pattern management: python netward/cli.py --help",
+    )
+    parser.add_argument(
+        "--config",
+        required=True,
+        metavar="PATH",
+        help="Operator config JSON (see example_config.json for a template)",
+    )
+    parser.add_argument(
+        "--allow-permissive-db",
+        action="store_true",
+        help="Allow startup even when the Net Ward DB path is world-writable.",
+    )
+    args = parser.parse_args()
+
+    try:
+        config = load_config(args.config)
+        validate_storage_permissions(
+            config,
+            allow_permissive_db=args.allow_permissive_db,
+        )
+    except Exception as exc:
+        print(f"Config error: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        asyncio.run(start_capture_loop(config))
+    except KeyboardInterrupt:
+        pass  # clean exit on Ctrl-C / SIGINT
+
+
+if __name__ == "__main__":
+    main()

netward/bootstrap.py

@@ -0,0 +1,89 @@
+"""
+Net Ward -- Bootstrap
+
+Seed storage with the built-in vendor pattern set on first run.
+Called automatically by start_capture_loop before the proxy binds.
+
+install_vendor_patterns(storage, force=False) -> tuple[int, int]
+    Reads netward/data/vendor_patterns.json, upserts MirrorResponses
+    then Patterns. Idempotent: second call is a no-op unless force=True.
+    Returns (patterns_installed, mirrors_installed).
+
+Raises BootstrapError on malformed JSON — never touches storage in
+that case, so storage is always left in a consistent state.
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Optional
+
+_VENDOR_DATA_PATH = Path(__file__).parent / "data" / "vendor_patterns.json"
+
+
+class BootstrapError(Exception):
+    pass
+
+
+def install_vendor_patterns(
+    storage,
+    force: bool = False,
+    _data_path: Optional[Path] = None,
+) -> tuple[int, int]:
+    """
+    Seed vendor patterns + mirror responses into storage.
+
+    force=False  -- no-op if any vendor-origin patterns already exist.
+    force=True   -- re-upserts regardless of existing state.
+    _data_path   -- override for tests; production always uses the package default.
+
+    Returns (patterns_installed, mirrors_installed).
+    """
+    # Idempotency guard
+    if not force:
+        try:
+            existing = storage.patterns_active()
+            if any(p.get("origin") == "vendor" for p in existing):
+                return (0, 0)
+        except Exception:
+            pass  # storage errors fall through to a fresh install attempt
+
+    # Validate JSON before touching storage
+    data_path = _data_path or _VENDOR_DATA_PATH
+    try:
+        raw = Path(data_path).read_text(encoding="utf-8")
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise BootstrapError(f"vendor_patterns.json is invalid JSON: {exc}") from exc
+    except OSError as exc:
+        raise BootstrapError(f"cannot read vendor_patterns.json: {exc}") from exc
+
+    mirrors = data.get("mirror_responses", [])
+    patterns = data.get("patterns", [])
+    if not isinstance(mirrors, list) or not isinstance(patterns, list):
+        raise BootstrapError(
+            "vendor_patterns.json must have 'mirror_responses' and 'patterns' arrays"
+        )
+
+    now = time.time()
+
+    # Upsert mirrors first — patterns reference them by id
+    mirrors_installed = 0
+    for mr in mirrors:
+        mr = dict(mr)
+        if not mr.get("created_at"):
+            mr["created_at"] = now
+        storage.mirror_response_upsert(mr)
+        mirrors_installed += 1
+
+    # Upsert patterns
+    patterns_installed = 0
+    for pat in patterns:
+        pat = dict(pat)
+        if not pat.get("created_at"):
+            pat["created_at"] = now
+        storage.patterns_upsert(pat)
+        patterns_installed += 1
+
+    return (patterns_installed, mirrors_installed)

netward/capture.py

@@ -0,0 +1,385 @@
+"""
+Net Ward -- Capture Layer
+
+aiohttp reverse-proxy entry point. All inbound HTTP traffic passes through
+here: build a Probe, classify it, then route -- mirror response (probe/flood)
+or pass-through to upstream (unknown/legit).
+
+Design rule (from __init__.py): FAIL SAFE, FAIL OPEN.
+If classification or mirroring raises, the request is forwarded to upstream
+rather than dropped. Net Ward must never take the host application down.
+
+Public interface:
+    start_capture_loop(config, storage=None) -> None
+        Bind to config["listen_address"], proxy to config["upstream_target"].
+        Runs until cancelled (asyncio.CancelledError).
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+import uuid
+from collections import deque
+from typing import Optional
+
+import aiohttp
+from aiohttp import web
+
+from netward.schema import OperatorConfig, Probe, RequestMetadata, Source
+from netward import classify as _classify_mod
+from netward import mirror as _mirror_mod
+
+# Body capture cap per schema (RequestMetadata.body_snippet)
+_BODY_SNIPPET_MAX = 4096
+
+# Must stay in sync with classify.FLOOD_WINDOW_SECS (both are 10.0 s, v0.4.1)
+_RATE_WINDOW_SECS: float = 10.0
+
+# Per-source rate windows (source_id -> bounded deque of request timestamps).
+# maxlen=2000: headroom above the 1000-hit flood threshold to avoid evicting
+# in-window timestamps before they age out naturally.
+# Dict is capped at _RATE_WINDOW_MAX_SOURCES; oldest quarter evicted when full.
+_RATE_WINDOW_MAX_SOURCES = 10_000
+_rate_windows: dict[str, deque] = {}
+
+
+def _get_rate_window(source_id: str) -> deque:
+    if source_id not in _rate_windows:
+        if len(_rate_windows) >= _RATE_WINDOW_MAX_SOURCES:
+            evict_n = _RATE_WINDOW_MAX_SOURCES // 4
+            for k in list(_rate_windows.keys())[:evict_n]:
+                del _rate_windows[k]
+        _rate_windows[source_id] = deque(maxlen=2000)
+    return _rate_windows[source_id]
+
+# Hop-by-hop headers stripped before forwarding upstream
+_HOP_BY_HOP = frozenset({
+    "connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
+    "te", "trailers", "transfer-encoding", "upgrade",
+})
+
+_PATTERN_CACHE_TTL = 30.0
+
+# Default mirror variant set (v0.4.1 fingerprint hardening).
+# A single fixed JSON shape was detectable under load (stress-state signature).
+# Variant selection is deterministic per (source_id, path) so a given source
+# sees a consistent shape per path — prevents second-probe shape divergence.
+_DEFAULT_MIRROR: dict = {
+    "id": "_netward_default",
+    "matches_pattern_id": "",
+    "intensity": "minimal",
+    "http_status": 200,
+    "headers": {"Content-Type": "application/json"},
+    "body_template": '{"status":"ok","id":"{{uuid}}","timestamp":"{{timestamp}}"}',
+    "body_template_vars": {"uuid": "uuid", "timestamp": "timestamp"},
+    "description": "Default minimal mirror (fallback; prefer _select_default_mirror)",
+    "created_at": 0.0,
+}
+
+_DEFAULT_MIRROR_VARIANTS: list[dict] = [
+    {
+        "id": "_netward_default_0",
+        "matches_pattern_id": "",
+        "intensity": "minimal",
+        "http_status": 200,
+        "headers": {"Content-Type": "application/json"},
+        "body_template": '{"status":"ok","id":"{{uuid}}","timestamp":"{{timestamp}}"}',
+        "body_template_vars": {"uuid": "uuid", "timestamp": "timestamp"},
+        "description": "Default variant 0 — JSON ok",
+        "created_at": 0.0,
+    },
+    {
+        "id": "_netward_default_1",
+        "matches_pattern_id": "",
+        "intensity": "minimal",
+        "http_status": 429,
+        "headers": {"Content-Type": "application/json", "Retry-After": "30"},
+        "body_template": '{"error":"rate limited","retry_after":30}',
+        "body_template_vars": {},
+        "description": "Default variant 1 — rate limited JSON",
+        "created_at": 0.0,
+    },
+    {
+        "id": "_netward_default_2",
+        "matches_pattern_id": "",
+        "intensity": "minimal",
+        "http_status": 503,
+        "headers": {"Content-Type": "text/html; charset=utf-8"},
+        "body_template": "<html><body><h1>Service Unavailable</h1><p>Try again later.</p></body></html>",
+        "body_template_vars": {},
+        "description": "Default variant 2 — HTML 503",
+        "created_at": 0.0,
+    },
+    {
+        "id": "_netward_default_3",
+        "matches_pattern_id": "",
+        "intensity": "minimal",
+        "http_status": 200,
+        "headers": {"Content-Type": "application/json"},
+        "body_template": '{"data":null,"next":null}',
+        "body_template_vars": {},
+        "description": "Default variant 3 — empty result JSON",
+        "created_at": 0.0,
+    },
+    {
+        "id": "_netward_default_4",
+        "matches_pattern_id": "",
+        "intensity": "minimal",
+        "http_status": 200,
+        "headers": {"Content-Type": "text/plain; charset=utf-8"},
+        "body_template": "OK",
+        "body_template_vars": {},
+        "description": "Default variant 4 — plain OK",
+        "created_at": 0.0,
+    },
+]
+
+
+def _select_default_mirror(source_id: str, path: str) -> dict:
+    """Deterministic variant per (source_id, path) to prevent shape divergence."""
+    idx = abs(hash(source_id + path)) % len(_DEFAULT_MIRROR_VARIANTS)
+    return _DEFAULT_MIRROR_VARIANTS[idx]
+
+
+async def _read_body_snippet(request: web.Request) -> Optional[str]:
+    try:
+        raw = await request.read()
+        if not raw:
+            return None
+        return raw[:_BODY_SNIPPET_MAX].decode("utf-8", errors="replace")
+    except Exception:
+        return None
+
+
+async def _build_probe(request: web.Request, source_id: str) -> Probe:
+    body = await _read_body_snippet(request)
+    req_meta: RequestMetadata = {
+        "method": request.method,
+        "path": request.path,
+        "headers": dict(request.headers),
+        "query_string": request.query_string or None,
+        "body_snippet": body,
+        "body_size": len(body.encode("utf-8", errors="replace")) if body else 0,
+        "user_agent": request.headers.get("User-Agent"),
+    }
+    return {
+        "id": str(uuid.uuid4()),
+        "timestamp": time.time(),
+        "source_id": source_id,
+        "pattern_id": None,
+        "classification": "unknown",
+        "request": req_meta,
+        "response_id": None,
+        "mirror_fired": False,
+        "upstream_passed": False,
+    }
+
+
+async def _forward_upstream(
+    request: web.Request,
+    upstream: str,
+    session: aiohttp.ClientSession,
+) -> Optional[web.Response]:
+    """Forward to upstream. Return None when upstream is unavailable."""
+    url = upstream.rstrip("/") + request.path
+    if request.query_string:
+        url += "?" + request.query_string
+    body = await request.read()
+    fwd_headers = {
+        k: v for k, v in request.headers.items()
+        if k.lower() not in _HOP_BY_HOP and k.lower() != "host"
+    }
+    try:
+        async with session.request(
+            method=request.method,
+            url=url,
+            headers=fwd_headers,
+            data=body,
+            allow_redirects=False,
+        ) as resp:
+            content = await resp.read()
+            resp_headers = {
+                k: v for k, v in resp.headers.items()
+                if k.lower() not in _HOP_BY_HOP | {"content-encoding"}
+            }
+            return web.Response(status=resp.status, headers=resp_headers, body=content)
+    except Exception:
+        return None
+
+
+def _mirror_response_from_probe(probe: Probe, mirror: dict) -> web.Response:
+    http_resp = _mirror_mod.fire_mirror(probe, mirror)
+    return web.Response(
+        status=http_resp["status"],
+        headers=http_resp["headers"],
+        text=http_resp["body"],
+    )
+
+
+def _fire_and_forget(storage, probe: Probe) -> None:
+    """Schedule probe logging without blocking the response path."""
+    try:
+        loop = asyncio.get_running_loop()
+        loop.create_task(asyncio.to_thread(storage.probes_log, probe))
+    except RuntimeError:
+        pass  # no running loop (e.g., during sync tests)
+    except Exception:
+        pass
+
+
+def _make_handler(config: OperatorConfig, storage, session: aiohttp.ClientSession):
+    upstream = config["upstream_target"]
+
+    # Per-handler pattern cache (closure) so each handler instance starts fresh
+    # and tests don't bleed stale cached patterns into each other.
+    _pcache: list = []
+    _pcache_at: float = 0.0
+
+    def _get_patterns() -> list:
+        nonlocal _pcache, _pcache_at
+        if time.time() - _pcache_at > _PATTERN_CACHE_TTL:
+            try:
+                _pcache = storage.patterns_active()
+                _pcache_at = time.time()
+            except Exception:
+                pass  # fail-open: return stale cache rather than crashing request path
+        return _pcache
+
+    async def handle(request: web.Request) -> web.Response:
+        ip = request.remote or "0.0.0.0"
+        now = time.time()
+
+        # Source lookup or create
+        source: Source = storage.sources_lookup(ip) or {
+            "id": str(uuid.uuid4()),
+            "ip_address": ip,
+            "reputation": "neutral",
+            "first_seen": now,
+            "last_seen": now,
+            "probe_count": 0,
+            "legit_count": 0,
+            "notes": [],
+        }
+        source_id: str = source["id"]
+
+        # Rate window update (in-memory, bounded + evicting)
+        window = _get_rate_window(source_id)
+        window.append(now)
+        rate_window = list(window)
+
+        probe = await _build_probe(request, source_id)
+
+        # Classify -- fail-open on any unhandled exception
+        patterns = _get_patterns()
+        try:
+            result = _classify_mod.classify(
+                probe, {"patterns": patterns, "source": source, "rate_window": rate_window}
+            )
+        except Exception:
+            probe["classification"] = "unknown"
+            upstream_response = await _forward_upstream(request, upstream, session)
+            if upstream_response is None:
+                probe["mirror_fired"] = True
+                probe["upstream_passed"] = False
+                response = _mirror_response_from_probe(
+                    probe, _select_default_mirror(source_id, request.path)
+                )
+            else:
+                probe["upstream_passed"] = True
+                response = upstream_response
+            _fire_and_forget(storage, probe)
+            return response
+
+        updated = result["probe"]
+        classification = updated.get("classification", "unknown")
+
+        # Source counters + reputation.
+        # Key on fire_mirror, not classification: a "flood"-classified request
+        # that didn't match a pattern still passes to upstream — that's legit
+        # traffic, not a probe. Only mirror-firing events are probe_count.
+        if result["fire_mirror"]:
+            source["probe_count"] = source.get("probe_count", 0) + 1
+        else:
+            source["legit_count"] = source.get("legit_count", 0) + 1
+        source["last_seen"] = now
+        source = _classify_mod.update_source_reputation(source)
+        try:
+            storage.sources_upsert(source)
+        except Exception:
+            pass  # fail-open: stale source state is acceptable
+
+        # Route
+        _default_mr = _select_default_mirror(source_id, request.path)
+        if result["fire_mirror"]:
+            mr_id = result.get("mirror_response_id")
+            mr = (storage.mirror_response_lookup(mr_id) if mr_id else None) or _default_mr
+            try:
+                response = _mirror_response_from_probe(updated, mr)
+            except Exception:
+                # Mirror failed -- fail-open, pass upstream
+                upstream_response = await _forward_upstream(request, upstream, session)
+                if upstream_response is None:
+                    updated["mirror_fired"] = True
+                    updated["upstream_passed"] = False
+                    response = _mirror_response_from_probe(updated, _default_mr)
+                else:
+                    updated["mirror_fired"] = False
+                    updated["upstream_passed"] = True
+                    response = upstream_response
+        else:
+            upstream_response = await _forward_upstream(request, upstream, session)
+            if upstream_response is None:
+                updated["mirror_fired"] = True
+                updated["upstream_passed"] = False
+                response = _mirror_response_from_probe(updated, _default_mr)
+            else:
+                updated["upstream_passed"] = True
+                response = upstream_response
+
+        _fire_and_forget(storage, updated)
+        return response
+
+    return handle
+
+
+async def start_capture_loop(config: OperatorConfig, storage=None) -> None:
+    """
+    Bind to config["listen_address"] and begin proxying to
+    config["upstream_target"]. Runs until cancelled.
+    Fail-open: crashing this process does not harm the upstream service.
+    """
+    if storage is None:
+        from netward.storage import Storage
+        storage = Storage(config.get("storage_path", "netward.db"))
+
+    try:
+        from netward import bootstrap as _bootstrap
+        _bootstrap.install_vendor_patterns(storage)
+    except Exception:
+        pass  # fail-open: never block startup on seed failure
+
+    connector = aiohttp.TCPConnector()
+    session = aiohttp.ClientSession(connector=connector)
+    _proxy_handler = _make_handler(config, storage, session)
+
+    @web.middleware
+    async def catch_all(request: web.Request, handler) -> web.Response:
+        return await _proxy_handler(request)
+
+    app = web.Application(middlewares=[catch_all])
+
+    runner = web.AppRunner(app)
+    await runner.setup()
+
+    listen = config.get("listen_address", "0.0.0.0:8080")
+    host, _, port_str = listen.rpartition(":")
+    if not host:
+        host = "0.0.0.0"
+    site = web.TCPSite(runner, host, int(port_str))
+    await site.start()
+
+    try:
+        await asyncio.Event().wait()
+    finally:
+        await session.close()
+        await runner.cleanup()