loghunter-cli 0.1.0.dev0__py3-none-any.whl

tests/test_loader_package_surface.py

@@ -0,0 +1,115 @@
+"""A-phase guard: the loader/ package presents the same import surface as the
+former single common/loader.py module, and the test-patchable I/O seams
+(``progress`` / ``_open_log``) remain SETTABLE at the package boundary AND
+patch-through to the load pipeline.
+
+This is the extraction's safety net (Glenn execution caution #1): a dropped
+re-export or a facade that snapshots a pre-patch object fails here loudly.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import loghunter.common.loader as loader
+
+
+# The full re-export inventory. Every name that resolved at
+# loghunter.common.loader.<name> before the package split MUST still resolve.
+# Pinned literally so a dropped re-export is a hard failure, not silent drift.
+_SURFACE = [
+    # display re-export (imported module-global, monkeypatched in 12 tests)
+    "progress",
+    # io
+    "_open_log", "_safe_resolve", "_union_dedupe",
+    # types
+    "LoadResult", "CoverageTracker", "SourceCoverage", "RotationSkipInfo",
+    "_data_window", "_PIHOLE_COLUMNS", "_CLOUDTRAIL_COLUMNS", "_SYSLOG_COLUMNS",
+    "_LOG_SUFFIXES",
+    # diagnostics
+    "_log_type", "_schema_warning", "_zeek_file_read_warning",
+    "_cloudtrail_parse_warning",
+    # sniff
+    "sniff_format", "sniff_format_detailed", "SniffResult", "_is_ndjson",
+    "_looks_like_syslog", "_SNIFF_MAX_PEEK", "_SNIFF_ORIGIN", "_SNIFF_RECOGNIZERS",
+    "_SYSLOG_SNIFF_BYTES",
+    # windowing
+    "_apply_ts_filter", "_missing_ts", "is_bounded", "is_zeek_bounded",
+    "_classify_rotation_name", "_rotation_base_and_index", "_peek_first_ts",
+    "_select_group", "_group_order_conflict", "_rotation_windowed_files",
+    "_COMPRESSION_EXTS", "_ROTATION_NUM_RE", "_DATE_RANK_BASE", "_EXPORT_WINDOW_RE",
+    # windowing — B+D named window model
+    "LoadWindow", "apply_default_window",
+    # discovery
+    "discover_files", "_DATE_DIR_RE", "_zeek_date_subdirs", "_file_matches_pattern",
+    "discover_zeek_files", "_syslog_files", "_discover_syslog_files",
+    "_dir_has_regular_files", "discover_cloudtrail_files", "_stem_hostname",
+    # discovery — B+D strategy resolvers (folded accessors)
+    "_zeek_dated_window", "_flat_default_floor", "_default_resolve_window",
+    "_zeek_resolve_window", "_flat_resolve_window",
+    # pipeline
+    "SourceLoader", "_SOURCE_LOADERS", "run_load", "load_required_logs",
+    "load_logs", "load_zeek_log", "load_syslog", "load_pihole", "load_cloudtrail",
+    "_zeek_records_from_lines", "_zeek_parse_from_lines", "_parse_ndjson_file",
+    "_parse_lines", "_zeek_strategy_parse", "_zeek_normalize",
+    "_syslog_strategy_parse", "_pihole_strategy_parse", "_cloudtrail_strategy_parse",
+    "_events_from_whole_document", "_syslog_should_skip", "_pihole_should_skip",
+    "_NORMALIZER_MAP", "resolve_load_windows",
+]
+
+
+def test_full_surface_resolves():
+    missing = [name for name in _SURFACE if not hasattr(loader, name)]
+    assert not missing, f"loader package dropped re-exports: {missing}"
+
+
+def test_source_loaders_registry_identity():
+    # The name imported by string-path must BE the registry the pipeline reads.
+    from loghunter.common.loader import _SOURCE_LOADERS as imported
+    assert imported is loader._SOURCE_LOADERS
+    # Every detector source key has a registered strategy (the completeness rail
+    # this extraction must not break).
+    for key in ("zeek_dir", "syslog_dir", "pihole_dir", "cloudtrail_dir"):
+        assert key in loader._SOURCE_LOADERS
+
+
+def _write_conn(tmp_path: Path) -> Path:
+    d = tmp_path / "zeek"
+    d.mkdir()
+    (d / "conn.log").write_text(
+        '{"ts": 1.0, "id.orig_h": "192.0.2.1", "id.resp_h": "192.0.2.2", '
+        '"id.resp_p": 53, "proto": "udp"}\n'
+    )
+    return d
+
+
+def test_progress_patch_through(tmp_path, monkeypatch):
+    """Patching loader.progress (the package attr) must reach pipeline.run_load —
+    proves the facade reads progress at call time, not via an import-time snapshot.
+    """
+    hits = {"n": 0}
+    real = loader.progress
+
+    def spy(it, **kwargs):
+        hits["n"] += 1
+        return real(it, **kwargs)
+
+    monkeypatch.setattr(loader, "progress", spy)
+    df = loader.load_logs(_write_conn(tmp_path), "conn*.log*")
+    assert len(df) == 1
+    assert hits["n"] >= 1, "progress patch did not reach the load pipeline"
+
+
+def test_open_log_patch_through(tmp_path, monkeypatch):
+    """Patching loader._open_log (the package attr) must reach pipeline.run_load."""
+    hits = {"n": 0}
+    real = loader._open_log
+
+    def spy(path):
+        hits["n"] += 1
+        return real(path)
+
+    monkeypatch.setattr(loader, "_open_log", spy)
+    df = loader.load_logs(_write_conn(tmp_path), "conn*.log*")
+    assert len(df) == 1
+    assert hits["n"] >= 1, "_open_log patch did not reach the load pipeline"

tests/test_loader_window_model.py

@@ -0,0 +1,215 @@
+"""B+D: the named window model — one LoadWindow type, one resolve_load_windows
+resolver shared by run() and run_digest(), and the contributor contract (a new
+source declares its temporal policy on ONE registry entry — zero runner edits,
+zero new accessor, zero digest twin).
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+import loghunter.common.loader as loader
+from loghunter import runner
+
+
+# ── resolve_load_windows — short-circuits ────────────────────────────────────
+
+
+def test_resolve_load_windows_short_circuits_on_explicit_window(tmp_path):
+    d = tmp_path / "zeek"
+    d.mkdir()
+    sources = {"conn*.log*": "zeek_dir"}
+    dirs = {"zeek_dir": [d]}
+    since = datetime(2026, 6, 1, tzinfo=timezone.utc)
+    assert loader.resolve_load_windows(
+        sources, dirs, "1d", since=since, until=None, load_all=False
+    ) == []
+    assert loader.resolve_load_windows(
+        sources, dirs, "1d", since=None, until=None, load_all=True
+    ) == []
+    # empty/"all"/invalid default spec → no windows
+    assert loader.resolve_load_windows(
+        sources, dirs, "all", since=None, until=None, load_all=False
+    ) == []
+
+
+def test_resolve_load_windows_skips_bounded_file_input(tmp_path):
+    f = tmp_path / "conn.log"
+    f.write_text("{}\n", encoding="utf-8")
+    assert loader.resolve_load_windows(
+        {"conn*.log*": "zeek_dir"}, {"zeek_dir": [f]}, "1d",
+        since=None, until=None, load_all=False,
+    ) == []
+
+
+# ── per-family resolution shapes (the strategy resolver bodies) ───────────────
+
+
+def test_resolve_load_windows_zeek_dated_precise_no_trim(tmp_path):
+    """Dated Zeek layout → precise (since, until) select_window, trim_span None."""
+    zd = tmp_path / "zeek"
+    zd.mkdir()
+    (zd / "2026-01-05").mkdir()
+    windows = loader.resolve_load_windows(
+        {"conn*.log*": "zeek_dir"}, {"zeek_dir": [zd]}, "1d",
+        since=None, until=None, load_all=False,
+    )
+    assert len(windows) == 1
+    w = windows[0]
+    assert w.source == "zeek_dir"
+    assert w.select_window == (
+        datetime(2026, 1, 5, 0, 0, 0, tzinfo=timezone.utc),
+        datetime(2026, 1, 5, 23, 59, 59, tzinfo=timezone.utc),
+    )
+    assert w.trim_span is None
+    assert w.keep_null is False  # zeek drops unparseable-ts rows
+
+
+def test_resolve_load_windows_zeek_flat_load_full_trim(tmp_path):
+    """Flat Zeek layout → load full (select_window None) + post-load trim_span."""
+    zd = tmp_path / "zeek"
+    zd.mkdir()
+    (zd / "conn.log").write_text("{}\n", encoding="utf-8")  # flat, no dated subdirs
+    windows = loader.resolve_load_windows(
+        {"conn*.log*": "zeek_dir"}, {"zeek_dir": [zd]}, "1d",
+        since=None, until=None, load_all=False,
+    )
+    assert len(windows) == 1
+    w = windows[0]
+    assert w.select_window is None
+    assert w.trim_span == timedelta(days=1)
+
+
+def test_resolve_load_windows_flat_family_conservative_floor(tmp_path):
+    """syslog → conservative (floor, None) select_window + precise trim_span;
+    keep_null True (syslog retains unparseable-ts rows through the implicit window)."""
+    sd = tmp_path / "syslog"
+    sd.mkdir()
+    (sd / "messages").write_text(
+        "Jun  5 12:00:00 host kernel: line\n", encoding="utf-8"
+    )
+    span = timedelta(days=1)
+    windows = loader.resolve_load_windows(
+        {"*.log*": "syslog_dir"}, {"syslog_dir": [sd]}, "1d",
+        since=None, until=None, load_all=False,
+    )
+    assert len(windows) == 1
+    w = windows[0]
+    assert w.select_window is not None and w.select_window[1] is None
+    assert w.select_window[0] == loader._peek_first_ts(sd / "messages") - span
+    assert w.trim_span == span
+    assert w.keep_null is True
+
+
+def test_resolve_load_windows_cloudtrail_opts_out(tmp_path):
+    """CloudTrail is baseline-relative → default_window_eligible False → no window."""
+    ct = tmp_path / "ct"
+    ct.mkdir()
+    (ct / "events.json").write_text("[]\n", encoding="utf-8")
+    assert loader.resolve_load_windows(
+        {"*.json*": "cloudtrail_dir"}, {"cloudtrail_dir": [ct]}, "1d",
+        since=None, until=None, load_all=False,
+    ) == []
+
+
+# ── the contributor contract (Doneness #2) ───────────────────────────────────
+
+
+def _fake_flat_source(**overrides) -> loader.SourceLoader:
+    """A hypothetical new flat source: ONE registry entry declaring only the
+    genuinely-variable bits — NO resolve_window, NO window_select, default
+    default_window_eligible=True. The fixture for the zero-runner-edits contract."""
+    base = loader.SourceLoader(
+        discover=lambda p, pattern, since, until: (
+            sorted(p.glob("*.log")) if p.is_dir() else [p]
+        ),
+        mode="stream",
+        parse=lambda line_iter, *, path, warnings: iter(()),
+        ts_policy="keep",
+        columns=["ts", "message"],
+        should_skip=None,
+        normalize=None,
+    )
+    return replace(base, **overrides) if overrides else base
+
+
+def test_contributor_contract_new_source_inherits_universal_default(
+    tmp_path, monkeypatch
+):
+    """A new flat source declaring NO resolve_window inherits the universal default
+    window (load full + post-load trim) with zero runner edits, zero new accessor,
+    and zero digest twin — resolved by the ONE resolve_load_windows entry point."""
+    monkeypatch.setitem(loader._SOURCE_LOADERS, "fake_dir", _fake_flat_source())
+    d = tmp_path / "fakesrc"
+    d.mkdir()
+    windows = loader.resolve_load_windows(
+        {"*.log": "fake_dir"}, {"fake_dir": [d]}, "1d",
+        since=None, until=None, load_all=False,
+    )
+    assert len(windows) == 1
+    w = windows[0]
+    assert w.source == "fake_dir"
+    assert w.select_window is None        # universal default = load full
+    assert w.trim_span == timedelta(days=1)
+    assert w.keep_null is True            # read straight off ts_policy="keep"
+
+
+def test_contributor_contract_new_source_can_opt_out(tmp_path, monkeypatch):
+    """A baseline-relative new source opts out via default_window_eligible=False on
+    its entry (the cloudtrail pattern) — still zero runner edits, no source-name
+    branch — and mints no LoadWindow."""
+    monkeypatch.setitem(
+        loader._SOURCE_LOADERS,
+        "fake_dir",
+        _fake_flat_source(default_window_eligible=False),
+    )
+    d = tmp_path / "fakesrc"
+    d.mkdir()
+    assert loader.resolve_load_windows(
+        {"*.log": "fake_dir"}, {"fake_dir": [d]}, "1d",
+        since=None, until=None, load_all=False,
+    ) == []
+
+
+# ── digest preservation: window resolution is Zeek-ONLY (caller-side gate) ─────
+
+
+def test_digest_window_resolution_is_zeek_only(tmp_path, monkeypatch, capsys):
+    """run_digest invokes the SHARED resolver for the Zeek source ONLY; non-Zeek
+    digest directories (syslog/cloudtrail) never resolve a default window → load
+    full, exactly as before the twin was deleted. Pinned via a spy on the one
+    resolver, exercised on the dry-run path (window resolution runs pre-load)."""
+    calls: list[Any] = []
+    real = loader.resolve_load_windows
+
+    def spy(needed_sources, *a, **k):
+        calls.append(needed_sources)
+        return real(needed_sources, *a, **k)
+
+    monkeypatch.setattr(loader, "resolve_load_windows", spy)
+
+    zd = tmp_path / "zeek"
+    zd.mkdir()
+    runner.run_digest(
+        config={"loghunter": {"zeek_dir": str(zd)}}, schema="conn", dry_run=True
+    )
+    assert len(calls) == 1, "zeek digest resolves the default window"
+
+    calls.clear()
+    sd = tmp_path / "syslog"
+    sd.mkdir()
+    runner.run_digest(
+        config={"loghunter": {"syslog_dir": str(sd)}}, schema="syslog", dry_run=True
+    )
+    ct = tmp_path / "ct"
+    ct.mkdir()
+    runner.run_digest(
+        config={"loghunter": {"cloudtrail_dir": str(ct)}},
+        schema="cloudtrail", dry_run=True,
+    )
+    assert calls == [], "non-Zeek digests never resolve a default window (load full)"