loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/common/loader/__init__.py

@@ -0,0 +1,136 @@
+"""Log discovery, decompression, parsing, and timeframe filtering.
+
+All file I/O for log data flows through this package. Detectors never open files
+directly. This ``__init__`` re-exports the FULL public + private symbol surface
+that previously lived in the single ``common/loader.py`` module, so every name
+importable or monkeypatchable at ``loghunter.common.loader.<name>`` continues to
+resolve (and stays settable) after the package split.
+
+Submodule layout (acyclic; ``io``/``types``/``diagnostics`` are leaves):
+- ``io``          — ``_open_log`` + path-normalization primitives.
+- ``types``       — ``LoadResult`` / coverage / rotation-skip dataclasses, the
+                    cross-frame window helper, column constants.
+- ``diagnostics`` — log-type + warning/wording helpers.
+- ``sniff``       — the digest recognizer cascade + the syslog content gate.
+- ``windowing``   — ts filter, boundedness, the rotation-peek subsystem.
+- ``discovery``   — per-family file discovery + dated-Zeek default window.
+- ``pipeline``    — ``run_load`` + the ``_SOURCE_LOADERS`` registry + the
+                    public ``load_*`` shims + registry-policy accessors.
+
+Canonical connection record schema
+───────────────────────────────────
+All conn log DataFrames returned by this package use these column names.
+Detectors, runner, and matcher never reference Zeek-specific column names.
+
+  src        — source IP (str)
+  dst        — destination IP (str)
+  port       — destination port (int)
+  proto      — protocol: tcp / udp / icmp (str)
+  ts         — unix epoch timestamp (float)
+  duration   — connection duration in seconds (float, nullable)
+  bytes      — originator bytes (int, nullable)
+  conn_state — connection state (str, nullable)
+  local_orig — bool (nullable)
+"""
+
+from __future__ import annotations
+
+# progress is re-exported so loghunter.common.loader.progress both RESOLVES and
+# is SETTABLE — the load pipeline reads it through this package attribute (the
+# call-time facade), so monkeypatching it here takes effect.
+from loghunter.common.display import progress
+
+from loghunter.common.loader.io import (
+    _open_log,
+    _safe_resolve,
+    _union_dedupe,
+)
+from loghunter.common.loader.types import (
+    _CLOUDTRAIL_COLUMNS,
+    _LOG_SUFFIXES,
+    _PIHOLE_COLUMNS,
+    _SYSLOG_COLUMNS,
+    CoverageTracker,
+    LoadResult,
+    RotationSkipInfo,
+    SourceCoverage,
+    _data_window,
+)
+from loghunter.common.loader.diagnostics import (
+    _cloudtrail_parse_warning,
+    _log_type,
+    _schema_warning,
+    _zeek_file_read_warning,
+)
+from loghunter.common.loader.sniff import (
+    _SNIFF_MAX_PEEK,
+    _SNIFF_ORIGIN,
+    _SNIFF_RECOGNIZERS,
+    _SYSLOG_SNIFF_BYTES,
+    SniffResult,
+    _is_ndjson,
+    _looks_like_syslog,
+    sniff_format,
+    sniff_format_detailed,
+)
+from loghunter.common.loader.windowing import (
+    _COMPRESSION_EXTS,
+    _DATE_RANK_BASE,
+    _EXPORT_WINDOW_RE,
+    _ROTATION_NUM_RE,
+    LoadWindow,
+    _apply_ts_filter,
+    _classify_rotation_name,
+    _group_order_conflict,
+    _missing_ts,
+    _peek_first_ts,
+    _rotation_base_and_index,
+    _rotation_windowed_files,
+    _select_group,
+    _strip_compression_ext,
+    apply_default_window,
+    is_bounded,
+    is_zeek_bounded,
+)
+from loghunter.common.loader.discovery import (
+    _DATE_DIR_RE,
+    _default_resolve_window,
+    _dir_has_regular_files,
+    _discover_syslog_files,
+    _file_matches_pattern,
+    _flat_default_floor,
+    _flat_resolve_window,
+    _stem_hostname,
+    _syslog_files,
+    _zeek_date_subdirs,
+    _zeek_dated_window,
+    _zeek_resolve_window,
+    discover_cloudtrail_files,
+    discover_files,
+    discover_zeek_files,
+)
+from loghunter.common.loader.pipeline import (
+    _NORMALIZER_MAP,
+    _SOURCE_LOADERS,
+    _cloudtrail_strategy_parse,
+    _events_from_whole_document,
+    _parse_lines,
+    _parse_ndjson_file,
+    _pihole_should_skip,
+    _pihole_strategy_parse,
+    _syslog_should_skip,
+    _syslog_strategy_parse,
+    _zeek_normalize,
+    _zeek_parse_from_lines,
+    _zeek_records_from_lines,
+    _zeek_strategy_parse,
+    SourceLoader,
+    load_cloudtrail,
+    load_logs,
+    load_pihole,
+    load_required_logs,
+    load_syslog,
+    load_zeek_log,
+    resolve_load_windows,
+    run_load,
+)

loghunter/common/loader/diagnostics.py

@@ -0,0 +1,94 @@
+"""Loader warning/wording + log-type helpers (leaf utility module).
+
+The non-dataclass glue James flagged out of ``types``: ``_log_type`` (glob →
+canonical log type), ``_schema_warning`` (actionable missing-field message),
+``_zeek_file_read_warning`` / ``_cloudtrail_parse_warning`` (privacy-safe
+per-file failure wording). Imports the parser schema constants only.
+"""
+
+from __future__ import annotations
+
+import gzip
+import lzma
+from pathlib import Path
+
+import pandas as pd
+
+from loghunter.parsers.zeek import (
+    _OPTIONAL_COLUMNS,
+    _REQUIRED_COLUMNS,
+)
+
+
+def _log_type(pattern: str) -> str | None:
+    """Return the canonical log type inferred from a loader glob pattern."""
+    if pattern.startswith("conn"):
+        return "conn"
+    if pattern.startswith("dns"):
+        return "dns"
+    if pattern.startswith("ssl"):
+        return "ssl"
+    if pattern.startswith("weird"):
+        return "weird"
+    if pattern.startswith("notice"):
+        return "notice"
+    if pattern.startswith("auth"):
+        return "auth"
+    if pattern.startswith("files"):
+        return "files"
+    if pattern.startswith("pihole"):
+        return "pihole"
+    if pattern.startswith("syslog"):
+        return "syslog"
+    return None
+
+
+def _schema_warning(pattern: str, df: pd.DataFrame) -> str | None:
+    """Return an actionable warning when a loaded DataFrame lacks canonical fields."""
+    if df.empty:
+        return None
+
+    log_type = _log_type(pattern)
+    if log_type is None or log_type not in _REQUIRED_COLUMNS:
+        return None
+
+    opt = _OPTIONAL_COLUMNS.get(log_type, set())
+    missing = sorted((_REQUIRED_COLUMNS[log_type] - opt) - set(df.columns))
+    if not missing:
+        return None
+
+    if log_type == "conn":
+        return (
+            f"conn.log fields not found: {', '.join(missing)} — "
+            "is this a Zeek conn.log?"
+        )
+    if log_type == "dns":
+        return (
+            f"dns.log fields not found: {', '.join(missing)} — "
+            "is this a Zeek dns.log?"
+        )
+    if log_type == "syslog":
+        return (
+            f"syslog.log fields not found: {', '.join(missing)} — "
+            "is this a Zeek syslog.log?"
+        )
+    return None
+
+
+def _zeek_file_read_warning(path: Path, exc: BaseException) -> str:
+    """Return a privacy-safe warning for a Zeek file that could not be read."""
+    if isinstance(exc, (EOFError, gzip.BadGzipFile, lzma.LZMAError)):
+        reason = "compressed file is incomplete or corrupt"
+    else:
+        reason = f"could not be read ({exc.__class__.__name__})"
+    return f"{path.name} could not be read — {reason}; skipping"
+
+
+def _cloudtrail_parse_warning(path: Path) -> str:
+    """Return a privacy-safe warning for a CloudTrail file with malformed JSON.
+
+    Parallels ``_zeek_file_read_warning``'s tone for I/O failures; this variant
+    covers the parse-failure case (file readable, but the contents are not JSON
+    we can use).
+    """
+    return f"{path.name} could not be read — not valid JSON; skipping"

loghunter/common/loader/discovery.py

@@ -0,0 +1,335 @@
+"""File discovery for each source family + the dated-Zeek default window.
+
+Per-family discovery (Zeek flat/dated, syslog content-gated, pihole glob,
+CloudTrail recursive), the hostname stem helper, and the per-strategy
+default-window resolvers (``_zeek_resolve_window`` / ``_flat_resolve_window`` /
+``_default_resolve_window`` — the ``SourceLoader.resolve_window`` bodies; the
+dated-layout selection ``_zeek_dated_window`` reads ``_zeek_date_subdirs``).
+Imports the rotation sort key + first-ts peek from ``windowing``, the union dedupe
+from ``io``, and the syslog content gate from ``sniff``; none import discovery, so
+the package stays acyclic.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import math
+import re
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path
+
+from loghunter.common.loader.io import _union_dedupe
+from loghunter.common.loader.sniff import _looks_like_syslog
+from loghunter.common.loader.types import _LOG_SUFFIXES
+from loghunter.common.loader.windowing import _peek_first_ts, _rotation_base_and_index
+
+
+def discover_files(directory: Path, pattern: str) -> list[Path]:
+    """Return all files in directory matching the glob pattern, sorted by name."""
+    return sorted(directory.glob(pattern))
+
+
+# Matches YYYY-MM-DD at the start of a Zeek log-rotation directory name.
+# Suffix (e.g. -TSVPRE) is ignored for date extraction; see discover_zeek_files.
+_DATE_DIR_RE = re.compile(r"^\d{4}-\d{2}-\d{2}")
+
+
+def _zeek_date_subdirs(directory: Path) -> list[Path]:
+    """Return immediate child directories whose names begin with YYYY-MM-DD, sorted."""
+    result = []
+    for child in directory.iterdir():
+        if child.is_dir() and _DATE_DIR_RE.match(child.name):
+            result.append(child)
+    return sorted(result, key=lambda p: p.name)
+
+
+def _file_matches_pattern(path: Path, pattern: str) -> bool:
+    """Return True if path's basename matches the glob pattern (single-file Zeek mode)."""
+    return fnmatch.fnmatch(path.name, pattern)
+
+
+def _zeek_dated_window(
+    paths: list[Path], span: timedelta
+) -> tuple[datetime, datetime] | None:
+    """Compute the default analysis window for a union of Zeek inputs.
+
+    PURELY-DATED predicate: every input is a directory AND every directory has
+    non-empty YYYY-MM-DD subdirs. When that predicate holds, generalizes the
+    single-input selection across the union — gather every discovered date
+    subdir across all inputs, dedupe by date prefix, sort by date, select the
+    newest ``N = ceil(span_days)`` (min 1), and return ``00:00:00`` UTC of the
+    earliest selected → ``23:59:59`` UTC of the newest selected.
+
+    Returns ``None`` when ANY input is a file (file + dated-dir mix) or ANY
+    directory is flat (mixed/flat present) — the caller falls through to the
+    flat-layout default path, which computes the window post-load from the
+    COMBINED loaded Zeek frame's max-ts.
+
+    Single-input behavior is BYTE-IDENTICAL with the prior scalar helper: a
+    one-element list of a single dated dir runs the same selection
+    (``_zeek_date_subdirs(input)``, newest N, earliest-midnight →
+    newest-23:59:59).
+
+    Sparse archives behave correctly: subdirs ``[2026-01-01, 2026-01-05]``
+    with span=2d → BOTH selected, window Jan 1 → Jan 5. Cross-input
+    duplicate dates count once toward N (dedup by date prefix).
+    """
+    if not paths:
+        return None
+    all_date_dirs: list[Path] = []
+    for p in paths:
+        if not p.is_dir():
+            return None
+        date_dirs = _zeek_date_subdirs(p)
+        if not date_dirs:
+            return None
+        all_date_dirs.extend(date_dirs)
+    # Dedup by date prefix so N counts DISTINCT dates across the union, not
+    # duplicates contributed by multiple inputs carrying the same day.
+    seen: set[str] = set()
+    unique_by_date: list[Path] = []
+    for d in sorted(all_date_dirs, key=lambda p: p.name[:10]):
+        prefix = d.name[:10]
+        if prefix in seen:
+            continue
+        seen.add(prefix)
+        unique_by_date.append(d)
+    n = max(1, math.ceil(span.total_seconds() / 86400))
+    selected = unique_by_date[-n:]
+    earliest_date = date.fromisoformat(selected[0].name[:10])
+    newest_date = date.fromisoformat(selected[-1].name[:10])
+    since = datetime(earliest_date.year, earliest_date.month, earliest_date.day,
+                     0, 0, 0, tzinfo=timezone.utc)
+    until = datetime(newest_date.year, newest_date.month, newest_date.day,
+                     23, 59, 59, tzinfo=timezone.utc)
+    return since, until
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Per-strategy default-window resolvers — the SourceLoader.resolve_window bodies.
+#
+# Signature is uniform: (strategy, dirs, pattern, span) -> (select_window,
+# trim_span). resolve_load_windows (pipeline) loops eligible/unbounded/in-play
+# families and calls each strategy's resolver (or _default_resolve_window when a
+# strategy declares none). The owning ``strategy`` is passed so the flat resolver
+# can reach ``strategy.discover`` for its candidate universe WITHOUT a source-name
+# ladder or a registry import — that is why this hook takes ``strategy`` and the
+# other strategy callables (parse/discover/should_skip) do not.
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def _default_resolve_window(
+    strategy, dirs: list[Path], pattern: str, span: timedelta
+) -> tuple[None, timedelta]:
+    """Universal default for a source that declares no ``resolve_window``: load the
+    family full and trim post-load to its own last-``span`` window. A new flat
+    source inherits this with zero runner edits — mirroring today's
+    ``default_window_eligible=True`` / ``window_select=None`` behavior.
+    """
+    return None, span
+
+
+def _zeek_resolve_window(
+    strategy, dirs: list[Path], pattern: str, span: timedelta
+) -> tuple[tuple[datetime, datetime] | None, timedelta | None]:
+    """Zeek strategy resolver. Dated layout → a precise ``(since, until)``
+    ``select_window`` and NO post-load trim (the load-time window already cut
+    exactly). Flat / mixed / file → ``(None, span)`` (load full, trim post-load).
+    """
+    dated = _zeek_dated_window(dirs, span)
+    if dated is not None:
+        return dated, None
+    return None, span
+
+
+def _flat_default_floor(
+    strategy, dir_paths: list[Path], pattern: str, span: timedelta
+) -> tuple[datetime, None] | None:
+    """Conservative default-window floor for a flat family (syslog / pihole).
+
+    Discovers the family's DIRECTORY candidates via the passed ``strategy.discover``
+    over the ``is_dir()`` inputs (``_union_dedupe``d — the same universe as
+    :func:`load_required_logs`, explicit files excluded) and peeks each candidate's
+    first-ts (``_peek_first_ts`` — clock-parity with the loader's own filter).
+    Returns ``(f_max − span, None)`` — the conservative select-window the
+    rotation-peek prunes against (``until=None``; the precise cut is the post-load
+    trim) — where ``f_max`` is the max parseable first-ts across candidates.
+    Returns ``None`` when nothing is peekable (load-full fallback).
+    """
+    candidates = _union_dedupe(
+        [strategy.discover(d, pattern, None, None) for d in dir_paths if d.is_dir()]
+    )
+    peeked = [ts for ts in (_peek_first_ts(p) for p in candidates) if ts is not None]
+    if not peeked:
+        return None
+    return (max(peeked) - span, None)
+
+
+def _flat_resolve_window(
+    strategy, dirs: list[Path], pattern: str, span: timedelta
+) -> tuple[tuple[datetime, None] | None, timedelta]:
+    """Flat strategy resolver (syslog / pihole). Peek the directory candidates →
+    conservative ``(floor, None)`` ``select_window`` + precise post-load
+    ``trim_span``; unpeekable → ``(None, span)`` (load full, trim post-load).
+    """
+    return _flat_default_floor(strategy, dirs, pattern, span), span
+
+
+def discover_zeek_files(
+    directory: Path,
+    pattern: str,
+    since: datetime | None = None,
+    until: datetime | None = None,
+) -> list[Path]:
+    """Return Zeek log files matching pattern from a flat or dated-layout directory.
+
+    Single-file mode (directory.is_file() True): returns [directory] when its basename
+    matches the pattern, else []. Single-file zeek_dir is a bounded target.
+
+    Flat layout (no YYYY-MM-DD immediate children): sorted(directory.glob(pattern)).
+    Dated layout: globs pattern within YYYY-MM-DD subdirs, date-prunes when window given.
+    This is structural inspection of a known zeek_dir — not generic format discovery.
+
+    Mixed-root policy: if any immediate child is a YYYY-MM-DD directory, dated layout
+    is used and root-level files are not included. Zeek does not produce root-level files
+    alongside date subdirs; the hybrid is ambiguous and therefore unsupported.
+    """
+    if directory.is_file():
+        return [directory] if _file_matches_pattern(directory, pattern) else []
+
+    date_dirs = _zeek_date_subdirs(directory)
+
+    if not date_dirs:
+        # Flat layout — identical to discover_files behavior.
+        return sorted(directory.glob(pattern))
+
+    # Dated layout.
+    if since is not None or until is not None:
+        # Prune by directory date; skip non-date children entirely.
+        since_date = since.date() if since else None
+        until_date = until.date() if until else None
+        included: list[Path] = []
+        for d in date_dirs:
+            dir_date = date.fromisoformat(d.name[:10])
+            if since_date is not None and dir_date < since_date:
+                continue
+            if until_date is not None and dir_date > until_date:
+                continue
+            included.append(d)
+    else:
+        # No window — include all date dirs and any non-date dirs, deduped by realpath.
+        # Non-date dirs (current, export, mixed) are often symlinks to date dirs;
+        # resolving and deduping prevents double-loading.
+        candidates: list[Path] = list(date_dirs)
+        for child in directory.iterdir():
+            if child.is_dir() and not _DATE_DIR_RE.match(child.name):
+                candidates.append(child)
+        seen: set[Path] = set()
+        included = []
+        # Dedup by realpath — non-date children are often symlinks pointing to date dirs;
+        # resolving prevents double-loading regardless of iteration order.
+        for d in sorted(candidates, key=lambda p: p.name):
+            rp = d.resolve()
+            if rp not in seen:
+                seen.add(rp)
+                included.append(d)
+
+    files: list[Path] = []
+    for d in included:
+        files.extend(sorted(d.glob(pattern)))
+    return files
+
+
+def _syslog_files(path: Path, pattern: str = "*.log*") -> list[Path]:
+    """Return flat-source files to process by FILENAME glob: ``[path]`` if a file,
+    else files matching ``pattern`` in the directory — ``._``-prefixed AppleDouble
+    sidecars dropped, numeric rotation order.
+
+    Pi-hole's discovery helper (and the Pi-hole pattern-mismatch presence check).
+    Syslog discovery is NOT this — it content-sniffs via ``_discover_syslog_files``
+    (RHEL/Fedora streams carry no ``.log`` suffix, and ``dnf.log`` etc. would be
+    mis-claimed by a filename glob). The ``pattern`` applies to DIRECTORY discovery
+    ONLY (pihole passes ``pihole*.log*`` to avoid grabbing non-pihole files in a
+    shared dir). An explicitly-named FILE always loads as named — the pattern is
+    NOT applied to it, so a content-routed Pi-hole file like ``events.log`` still
+    loads. The AppleDouble filter and numeric ordering apply to DIRECTORY discovery
+    only: the junk filter targets glob noise, not operator intent.
+    """
+    if path.is_file():
+        return [path]
+    files = [p for p in discover_files(path, pattern) if not p.name.startswith("._")]
+    return sorted(files, key=lambda p: _rotation_base_and_index(p.name))
+
+
+def _discover_syslog_files(path: Path) -> list[Path]:
+    """The SINGLE syslog discovery universe — content-gated.
+
+    FILE input → ``[path]`` UNGATED (the explicit-named-file rail: an explicitly
+    named file always loads regardless of name; ``_syslog_should_skip`` still
+    guards a named NDJSON/Zeek-TSV at load). DIRECTORY input → all regular,
+    non-AppleDouble files that pass ``_looks_like_syslog``, in rotation order.
+    Non-recursive ``iterdir`` correctly skips the binary subdirs (``journal/``,
+    ``audit/``, ``sa/``) — they are not regular files.
+    """
+    if path.is_file():
+        return [path]
+    files = [
+        p for p in path.iterdir()
+        if p.is_file() and not p.name.startswith("._") and _looks_like_syslog(p)
+    ]
+    return sorted(files, key=lambda p: _rotation_base_and_index(p.name))
+
+
+def _dir_has_regular_files(path: Path) -> bool:
+    """True iff ``path`` holds >=1 regular, non-AppleDouble file.
+
+    Cheap ``iterdir`` presence check for the syslog zero-accepted disclosure —
+    NO sniff, NO ``*.log*`` test, so an extensionless-only dir does not fall
+    through the disclosure silently.
+    """
+    try:
+        return any(p.is_file() and not p.name.startswith("._") for p in path.iterdir())
+    except OSError:
+        return False
+
+
+def discover_cloudtrail_files(path: Path) -> list[Path]:
+    """Discover CloudTrail event files for the loader and runner satisfiability check.
+
+    File path → [path]. Directory → recursive sorted ``*.json*`` matches, excluding
+    any file whose path contains a ``CloudTrail-Digest`` component (integrity
+    manifests, not events — the same exclusion the exporter applies on the S3 side).
+
+    The ``*.json*`` glob covers ``.json``, ``.json.gz``, the exporter's ``.json.log``
+    and its ``_partNN`` splits. Recursion is what makes a native
+    ``AWSLogs/<acct>/CloudTrail/<region>/YYYY/MM/DD/`` tree work — users who pull
+    logs their own way can point ``cloudtrail_dir`` at any level of that tree.
+    """
+    if path.is_file():
+        return [path]
+    if not path.is_dir():
+        return []
+    files: list[Path] = []
+    for candidate in sorted(path.rglob("*.json*")):
+        if not candidate.is_file():
+            continue
+        if "CloudTrail-Digest" in candidate.parts:
+            continue
+        files.append(candidate)
+    return files
+
+
+def _stem_hostname(name: str) -> str:
+    """Strip log-file suffixes from a filename to derive a hostname stem.
+
+    Strips .log, .gz, and numeric rotation suffixes (.1, .42).
+    Dotted hostnames are preserved: host1.example.com.log → host1.example.com.
+    """
+    stem = name
+    while True:
+        suffix = Path(stem).suffix
+        if suffix in _LOG_SUFFIXES or (suffix and suffix[1:].isdigit()):
+            stem = Path(stem).stem
+        else:
+            break
+    return stem

loghunter/common/loader/io.py

@@ -0,0 +1,76 @@
+"""Low-level filesystem primitives for the loader package (leaf module).
+
+Decompression-transparent file opening plus the path-normalization helpers
+(``_safe_resolve`` / ``_union_dedupe``). These are the lowest leaf: every other
+loader submodule may import from here, and this module imports nothing from the
+package. ``_open_log`` is the SINGLE chokepoint every source flows through.
+"""
+
+from __future__ import annotations
+
+import bz2
+import gzip
+import lzma
+from pathlib import Path
+
+
+def _open_log(path: Path):
+    """Open a plain, gzip-, bzip2-, or xz-compressed log file for reading.
+
+    Suffix-gated (NOT magic-authoritative — the blob profiler is the magic-sniff
+    context; the loader routes by suffix, keeping the two contexts distinct).
+    `_open_log` is the SINGLE chokepoint every source flows through, so adding a
+    new format here closes the gap across conn/dns/syslog/pihole/cloudtrail/sniff
+    in one place.
+    """
+    if path.suffix == ".gz":
+        return gzip.open(path, "rt", encoding="utf-8", errors="replace")
+    if path.suffix == ".bz2":
+        return bz2.open(path, "rt", encoding="utf-8", errors="replace")
+    if path.suffix == ".xz":
+        return lzma.open(path, "rt", encoding="utf-8", errors="replace")
+    return path.open("r", encoding="utf-8", errors="replace")
+
+
+def _safe_resolve(p: Path) -> Path:
+    """``p.resolve()``, falling back to ``p`` on ``OSError``.
+
+    The single realpath-normalization primitive the loader uses for dedupe,
+    the rotation-windowing explicit-file partition, and rotation grouping —
+    one consistent notion of "same path" across all three.
+    """
+    try:
+        return p.resolve()
+    except OSError:
+        return p
+
+
+def _union_dedupe(per_input_files: list[list[Path]]) -> list[Path]:
+    """Concat per-input discovery results; dedupe by ``.resolve()`` preserving
+    first-seen order.
+
+    Single-ownership union point — the loader is the only place file lists
+    from multiple source-dir inputs are concatenated under one family. Dedup
+    by realpath catches:
+
+    - the same file appearing in two inputs (positional pointing at a file
+      that's ALSO inside a positional directory);
+    - symlink farms (a non-date child of a Zeek dated dir that resolves to a
+      date dir already in the list).
+
+    First-seen order preservation keeps user-visible file ordering predictable
+    (positionals before flag-supplied dirs, mirrors CLI bucket order).
+    Returns the deduped list; downstream accounting (``data_size_bytes`` sums,
+    warnings, ``load_*`` iteration) runs over this list so duplicates never
+    double-count.
+    """
+    seen: set[Path] = set()
+    out: list[Path] = []
+    for files in per_input_files:
+        for p in files:
+            key = _safe_resolve(p)
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(p)
+    return out