loghunter-cli 0.1.0.dev0__py3-none-any.whl

tests/test_allowlist_defaults_accessor.py

@@ -0,0 +1,90 @@
+"""Allowlist fallback paths read from the single config defaults accessor.
+
+After the rework, ``common/allowlist.py`` no longer carries its own copy of
+default paths. When config keys are absent (raw / notebook config), the
+fallback comes from ``cfg.default_allowlist_paths()`` — a deep copy of
+``_DEFAULTS["allowlist"]``. All three keys are covered: ``domain_patterns``,
+``connection_rules``, and ``allowlist_dir`` (Glenn's amendment).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from loghunter.common import allowlist, config as cfg
+
+
+def test_default_allowlist_paths_returns_deep_copy_of_defaults() -> None:
+    paths = cfg.default_allowlist_paths()
+    assert paths == cfg._DEFAULTS["allowlist"]
+    paths["domain_patterns"] = ["mutated"]
+    assert cfg._DEFAULTS["allowlist"]["domain_patterns"] != ["mutated"], (
+        "default_allowlist_paths must return a deep copy"
+    )
+
+
+def test_build_matcher_domain_patterns_fallback_uses_accessor(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    """When config has no domain_patterns, accessor supplies the path. Patch
+    the accessor and observe the result."""
+    fake = tmp_path / "fake_domains.txt"
+    fake.write_text("example.com\n", encoding="utf-8")
+    monkeypatch.setattr(
+        cfg, "default_allowlist_paths",
+        lambda: {"domain_patterns": [str(fake)], "connection_rules": [], "allowlist_dir": ""},
+    )
+    # Config with NO allowlist subkeys — forces the fallback.
+    matcher = allowlist.build_matcher({"allowlist": {}})
+    assert "example.com" in matcher._domain_patterns
+
+
+def test_build_matcher_connection_rules_fallback_uses_accessor(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    fake = tmp_path / "fake_conn.txt"
+    fake.write_text("192.0.2.1\n", encoding="utf-8")
+    monkeypatch.setattr(
+        cfg, "default_allowlist_paths",
+        lambda: {"domain_patterns": [], "connection_rules": [str(fake)], "allowlist_dir": ""},
+    )
+    matcher = allowlist.build_matcher({"allowlist": {}})
+    assert len(matcher._numeric_rules) == 1
+
+
+def test_build_matcher_allowlist_dir_fallback_uses_accessor(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    """Glenn's amendment: allowlist_dir gets the same single-source treatment."""
+    fake_dir = tmp_path / "fake_allowlist.d"
+    fake_dir.mkdir()
+    (fake_dir / "users.toml").write_text(
+        '[[allowlist.entry]]\nmatch = "example.com"\ncomment = "x"\n',
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(
+        cfg, "default_allowlist_paths",
+        lambda: {
+            "domain_patterns": [],
+            "connection_rules": [],
+            "allowlist_dir": str(fake_dir),
+        },
+    )
+    matcher = allowlist.build_matcher({"allowlist": {}})
+    assert len(matcher._entries) == 1
+    assert matcher._entries[0].match == "example.com"
+
+
+def test_shipped_domain_files_stay_package_local_not_routed_through_root(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
+) -> None:
+    """_SHIPPED_DOMAIN_FILES are package data, NOT routed through LH_ROOT.
+    Setting a bogus root must not displace them."""
+    monkeypatch.setenv("LOGHUNTER_ROOT", str(tmp_path / "nonexistent"))
+    # Build with no allowlist config — only shipped patterns load.
+    matcher = allowlist.build_matcher({"allowlist": {}})
+    # Shipped files include large universal lists; at least one entry must load.
+    # If the shipped path were routed through bogus root, they'd be absent.
+    assert len(matcher._domain_patterns) > 0

tests/test_architecture_spine.py

@@ -0,0 +1,302 @@
+"""Focused tests for detector discovery and output plumbing."""
+
+from __future__ import annotations
+
+import io
+import json
+import tempfile
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+from types import SimpleNamespace
+
+import pandas as pd
+
+from loghunter.common.allowlist import build_matcher
+from loghunter.common.finding import Finding, RunSummary, Severity
+from loghunter.common.output import Reporter, get_handler
+from loghunter.runner import build_run_plan, discover_detectors, resolve_detect
+
+
+def _summary() -> RunSummary:
+    now = datetime(2026, 5, 30, tzinfo=timezone.utc)
+    return RunSummary(
+        data_window=(now, now),
+        record_counts={"conn*.log*": 1},
+        data_size_bytes=0,
+        detectors_run=["beacon"],
+        detectors_skipped={},
+    )
+
+
+def _finding() -> Finding:
+    now = datetime(2026, 5, 30, tzinfo=timezone.utc)
+    return Finding(
+        detector="beacon",
+        severity=Severity.MEDIUM,
+        title="periodic flow",
+        description="A periodic flow was observed.",
+        evidence={"beacon_score": 0.61, "conn_count": 20},
+        next_steps=["Review the source host."],
+        ts_generated=now,
+        data_window=(now, now),
+    )
+
+
+class ArchitectureSpineTests(unittest.TestCase):
+    """Small checks for the app's detector and output boundaries."""
+
+    def test_discover_detectors_excludes_planned_stubs(self) -> None:
+        detectors = discover_detectors()
+
+        self.assertIn("beacon", detectors)
+        self.assertIn("dns", detectors)
+        self.assertIn("duration", detectors)
+        for planned in ("auth", "ssl", "protocol", "weird", "dnsblock"):
+            self.assertNotIn(planned, detectors)
+
+    def test_resolve_detect_all_uses_available_detectors_only(self) -> None:
+        available = sorted(discover_detectors())
+
+        self.assertEqual(resolve_detect("all", available), available)
+        self.assertIn("duration", resolve_detect("all", available))
+
+    def test_reporter_delivers_to_registered_json_handler(self) -> None:
+        stream = io.StringIO()
+        handler_cls = get_handler("json")
+        handler = handler_cls(stream=stream)
+
+        Reporter([handler]).run([_finding()], _summary())
+
+        payload = json.loads(stream.getvalue())
+        self.assertEqual(payload["run_summary"]["detectors_run"], ["beacon"])
+        self.assertEqual(payload["findings"][0]["detector"], "beacon")
+        self.assertEqual(payload["findings"][0]["evidence"]["beacon_score"], 0.61)
+
+    def test_csv_handler_collects_evidence_columns_before_writing(self) -> None:
+        stream = io.StringIO()
+        handler_cls = get_handler("csv")
+        handler = handler_cls(stream=stream)
+
+        Reporter([handler]).run([_finding()], _summary())
+
+        output = stream.getvalue()
+        self.assertIn("evidence.beacon_score", output.splitlines()[0])
+        self.assertIn("0.61", output)
+
+    def test_connection_rules_are_local_only_by_default(self) -> None:
+        matcher = build_matcher({"allowlist": {"domain_patterns": []}})
+
+        self.assertEqual(matcher._numeric_rules, [])
+
+    def test_configured_connection_rule_file_still_filters_rows(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            rules_path = Path(tmp) / "connections.txt"
+            rules_path.write_text("192.0.2.10  198.51.100.20  :443/tcp\n", encoding="utf-8")
+            matcher = build_matcher({
+                "allowlist": {
+                    "domain_patterns": [],
+                    "connection_rules": [str(rules_path)],
+                }
+            })
+
+        df = pd.DataFrame([
+            {"src": "192.0.2.10", "dst": "198.51.100.20", "port": 443, "proto": "tcp"},
+            {"src": "192.0.2.11", "dst": "203.0.113.20", "port": 443, "proto": "tcp"},
+        ])
+
+        filtered = matcher.filter_df(df, "beacon")
+
+        self.assertEqual(len(filtered), 1)
+        self.assertEqual(filtered.iloc[0]["src"], "192.0.2.11")
+
+    def test_configured_connection_rule_file_filters_duration_rows(self) -> None:
+        """Unscoped flat-file rule suppresses duration the same way it suppresses beacon.
+
+        Locks the filter-before-analyze pass-through for duration: omission of
+        scope is permission for every connection detector that groups on the
+        canonical (src, dst, port, proto) tuple, not just beacon.
+        """
+        with tempfile.TemporaryDirectory() as tmp:
+            rules_path = Path(tmp) / "connections.txt"
+            rules_path.write_text("192.0.2.10  198.51.100.20  :443/tcp\n", encoding="utf-8")
+            matcher = build_matcher({
+                "allowlist": {
+                    "domain_patterns": [],
+                    "connection_rules": [str(rules_path)],
+                }
+            })
+
+        df = pd.DataFrame([
+            {"src": "192.0.2.10", "dst": "198.51.100.20", "port": 443, "proto": "tcp"},
+            {"src": "192.0.2.11", "dst": "203.0.113.20", "port": 443, "proto": "tcp"},
+        ])
+
+        filtered = matcher.filter_df(df, "duration")
+
+        self.assertEqual(len(filtered), 1)
+        self.assertEqual(filtered.iloc[0]["src"], "192.0.2.11")
+
+    def test_scoped_stanza_filters_duration_and_gates_unlisted_detector(self) -> None:
+        """A stanza scoped to duration suppresses duration and only duration.
+
+        Locks the scoping half of the contract: a stanza with
+        ``detectors = ["duration", "beacon"]`` drops the matching row when
+        called for "duration", and the same matcher leaves the row in place
+        when called for a detector outside the scope.
+        """
+        matcher = build_matcher({
+            "allowlist": {
+                "domain_patterns": "",
+                "entry": [
+                    {
+                        "match": "ip_pair",
+                        "src": "192.0.2.10",
+                        "dst_port": 443,
+                        "comment": "Example scoped flow",
+                        "detectors": ["duration", "beacon"],
+                    }
+                ],
+            }
+        })
+
+        df = pd.DataFrame([
+            {"src": "192.0.2.10", "dst": "198.51.100.20", "port": 443, "proto": "tcp"},
+            {"src": "192.0.2.11", "dst": "203.0.113.20", "port": 443, "proto": "tcp"},
+        ])
+
+        filtered_duration = matcher.filter_df(df, "duration")
+        self.assertEqual(len(filtered_duration), 1)
+        self.assertEqual(filtered_duration.iloc[0]["src"], "192.0.2.11")
+
+        # Same matcher, same frame, a detector outside the scope — the rule
+        # must NOT fire. (filter_df routes by column shape, so a connection
+        # frame still flows through _filter_numeric_df for "dns"; what we are
+        # asserting is that the scope check gates suppression, not the shape.)
+        filtered_dns = matcher.filter_df(df, "dns")
+        self.assertEqual(len(filtered_dns), 2)
+
+    def test_connection_rule_path_can_be_operator_friendly_string(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            rules_path = Path(tmp) / "connections.txt"
+            rules_path.write_text("192.0.2.10  :443/tcp\n", encoding="utf-8")
+            matcher = build_matcher({
+                "allowlist": {
+                    "domain_patterns": "",
+                    "connection_rules": str(rules_path),
+                }
+            })
+
+        assert len(matcher._numeric_rules) == 1
+        assert matcher._numeric_rules[0].ip1 == "192.0.2.10"
+
+    def test_stanza_detector_scope_can_be_comma_string(self) -> None:
+        matcher = build_matcher({
+            "allowlist": {
+                "domain_patterns": "",
+                "entry": [
+                    {
+                        "match": "dst_port",
+                        "value": 443,
+                        "comment": "Example scoped service",
+                        "detectors": "beacon, scan",
+                    }
+                ],
+            }
+        })
+
+        assert matcher._numeric_rules[-1].detectors == ["beacon", "scan"]
+
+    def test_domain_patterns_filter_dns_rows_before_detection(self) -> None:
+        matcher = build_matcher({"allowlist": {"domain_patterns": []}})
+        matcher._domain_patterns = ["*.example.test", "re:\\.allowed\\.test$"]
+        df = pd.DataFrame([
+            {"src": "192.0.2.10", "query": "updates.example.test"},
+            {"src": "192.0.2.11", "query": "allowed.test"},
+            {"src": "192.0.2.12", "query": "suspicious.invalid"},
+        ])
+
+        filtered = matcher.filter_df(df, "dns")
+
+        self.assertEqual(filtered["query"].tolist(), ["suspicious.invalid"])
+
+    def test_shipped_universal_domains_filter_dns_infrastructure(self) -> None:
+        matcher = build_matcher({"allowlist": {"domain_patterns": []}})
+        df = pd.DataFrame([
+            {"src": "192.0.2.10", "query": "2.0.192.in-addr.arpa"},
+            {"src": "192.0.2.11", "query": "suspicious.invalid"},
+        ])
+
+        filtered = matcher.filter_df(df, "dns")
+
+        self.assertEqual(filtered["query"].tolist(), ["suspicious.invalid"])
+
+
+def test_build_run_plan_records_skips_and_needed_logs(tmp_path: Path) -> None:
+    zeek_dir = tmp_path / "zeek"
+    zeek_dir.mkdir()
+    (zeek_dir / "conn.log").write_text("", encoding="utf-8")
+    # Optional log must exist for the satisfiable-only filter to include it in needed_logs.
+    (zeek_dir / "conn_summary.log").write_text("", encoding="utf-8")
+    detectors = {
+        "beacon": SimpleNamespace(
+            REQUIRED_LOGS=[{"source": "zeek_dir", "pattern": "conn*.log*"}],
+            OPTIONAL_LOGS=[{"source": "zeek_dir", "pattern": "conn_summary*.log*"}],
+        ),
+        "dns": SimpleNamespace(
+            REQUIRED_LOGS=[{"source": "zeek_dir", "pattern": "dns*.log*"}],
+            OPTIONAL_LOGS=[],
+        ),
+    }
+
+    plan = build_run_plan("all", zeek_dir=zeek_dir, syslog_dir=None, detectors=detectors)
+
+    assert plan.selected == ["beacon", "dns"]
+    assert plan.will_run == ["beacon"]
+    assert plan.skipped == {"dns": f"dns*.log* not found in {zeek_dir}"}
+    assert plan.needed_logs == {
+        "conn*.log*": "zeek_dir",
+        "conn_summary*.log*": "zeek_dir",
+    }
+
+
+def test_build_run_plan_honors_exclusion_syntax(tmp_path: Path) -> None:
+    zeek_dir = tmp_path / "zeek"
+    zeek_dir.mkdir()
+    (zeek_dir / "conn.log").write_text("", encoding="utf-8")
+    detectors = {
+        "beacon": SimpleNamespace(
+            REQUIRED_LOGS=[{"source": "zeek_dir", "pattern": "conn*.log*"}],
+            OPTIONAL_LOGS=[],
+        ),
+        "dns": SimpleNamespace(
+            REQUIRED_LOGS=[{"source": "zeek_dir", "pattern": "dns*.log*"}],
+            OPTIONAL_LOGS=[],
+        ),
+    }
+
+    plan = build_run_plan("all,!dns", zeek_dir=zeek_dir, syslog_dir=None, detectors=detectors)
+
+    assert plan.selected == ["beacon"]
+    assert plan.will_run == ["beacon"]
+    assert plan.skipped == {}
+
+
+def test_allowlist_filter_tolerates_pihole_query_none_rows() -> None:
+    """Domain filter must not crash on query=None rows (unknown/validation events from pihole)."""
+    matcher = build_matcher({"allowlist": {"domain_patterns": []}})
+    matcher._domain_patterns = ["bad.example.test"]
+    df = pd.DataFrame([
+        {"src": "192.0.2.1", "query": "bad.example.test"},      # matches pattern — filtered out
+        {"src": "192.0.2.2", "query": "harmless.example.test"},  # no match — survives
+        {"src": "192.0.2.3", "query": None},                     # unknown/validation — survives
+    ])
+    filtered = matcher.filter_df(df, "dns")
+    assert len(filtered) == 2
+    surviving_queries = filtered["query"].tolist()
+    assert any(pd.isna(q) for q in surviving_queries)
+    assert "bad.example.test" not in surviving_queries
+
+
+if __name__ == "__main__":
+    unittest.main()