microstructure-tpw 0.3.0__py3-none-any.whl

odas_tpw/perturb/logging_setup.py

@@ -0,0 +1,181 @@
+# May-2026, Claude and Pat Welch, pat@mousebrains.com
+"""Logging fan-out for perturb runs.
+
+Three layers, each writing to a separate ``FileHandler`` on the root
+logger so a single ``logger.info(...)`` call lands in every relevant log:
+
+* **Per-CLI-invocation** — ``<output_root>/logs/run_<timestamp>.log``,
+  always written.  ``--stdout`` adds a stderr ``StreamHandler`` so the
+  user can also watch progress on the terminal.
+* **Per-worker** —
+  ``<output_root>/logs/worker_<runtimestamp>_<pid>.log``, installed by
+  ``init_worker_logging`` from a ``ProcessPoolExecutor`` initializer so
+  each worker writes its own file (the spawn-context default on macOS
+  doesn't inherit handlers).
+* **Per-stage scope** — ``stage_log(stage_dir, basename)`` is a context
+  manager that adds and removes a ``FileHandler`` for the duration of a
+  block, so e.g. all log records emitted while processing ``a.p``'s
+  diss stage land in ``diss_NN/a.log`` *as well as* the worker and run
+  logs.
+
+The handlers stack: a single record fans out to whichever handlers are
+attached at emission time.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+import sys
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import UTC, datetime
+from pathlib import Path
+
+_FILE_FORMAT = "%(asctime)s %(levelname)-7s %(name)s [%(processName)s]: %(message)s"
+_STREAM_FORMAT = "%(message)s"
+_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
+
+_FILE_FORMATTER = logging.Formatter(_FILE_FORMAT, datefmt=_DATE_FORMAT)
+_STREAM_FORMATTER = logging.Formatter(_STREAM_FORMAT)
+
+# Marker attribute: handlers we install carry this so repeat calls to
+# setup_root_logging can replace their own handlers without touching
+# anything the user (e.g. a pytest plugin) attached.
+_OWNED_ATTR = "_perturb_logging_owned"
+
+
+def run_timestamp() -> str:
+    """Return a UTC timestamp suitable for log file names (no separators that
+    confuse shells)."""
+    return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+
+
+_CURRENT_RUN_STAMP: str | None = None
+
+
+def current_run_stamp() -> str:
+    """Return the run stamp for this process, generating one on first call.
+
+    Used by the CLI for the per-invocation log file name and by
+    :mod:`pipeline` to seed worker log file names so all logs from a
+    single ``perturb`` invocation share the same prefix.
+    """
+    global _CURRENT_RUN_STAMP
+    if _CURRENT_RUN_STAMP is None:
+        _CURRENT_RUN_STAMP = run_timestamp()
+    return _CURRENT_RUN_STAMP
+
+
+def reset_run_stamp() -> None:
+    """Test helper: clear the cached run stamp."""
+    global _CURRENT_RUN_STAMP
+    _CURRENT_RUN_STAMP = None
+
+
+def _mark_owned(handler: logging.Handler) -> logging.Handler:
+    setattr(handler, _OWNED_ATTR, True)
+    return handler
+
+
+def _drop_owned_handlers(logger: logging.Logger) -> None:
+    for h in list(logger.handlers):
+        if getattr(h, _OWNED_ATTR, False):
+            logger.removeHandler(h)
+            with contextlib.suppress(Exception):
+                h.close()
+
+
+def setup_root_logging(
+    log_file: Path,
+    *,
+    level: int = logging.INFO,
+    stdout: bool = False,
+) -> Path:
+    """Configure the root logger for a CLI invocation.
+
+    Always installs a ``FileHandler`` at *log_file* (parent dir is
+    created).  When *stdout* is True, also installs a stderr
+    ``StreamHandler``.
+
+    Calling this twice in the same process is safe — handlers installed
+    by previous calls are removed before new ones are added.  Handlers
+    attached by other code (e.g. a test fixture's ``caplog``) are left
+    alone.
+    """
+    log_file = Path(log_file)
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+
+    root = logging.getLogger()
+    root.setLevel(level)
+
+    _drop_owned_handlers(root)
+
+    fh = logging.FileHandler(log_file)
+    fh.setLevel(level)
+    fh.setFormatter(_FILE_FORMATTER)
+    root.addHandler(_mark_owned(fh))
+
+    if stdout:
+        sh = logging.StreamHandler(sys.stderr)
+        sh.setLevel(level)
+        sh.setFormatter(_STREAM_FORMATTER)
+        root.addHandler(_mark_owned(sh))
+
+    return log_file
+
+
+def init_worker_logging(log_dir: Path, run_stamp: str) -> Path:
+    """Install a per-worker ``FileHandler`` on the root logger.
+
+    Designed for ``ProcessPoolExecutor(initializer=init_worker_logging,
+    initargs=(log_dir, run_stamp))``.  Each spawn-context worker runs
+    this exactly once; ``run_stamp`` is the parent run's timestamp so
+    all workers from one invocation share a prefix and old runs aren't
+    overwritten.
+    """
+    log_dir = Path(log_dir)
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / f"worker_{run_stamp}_{os.getpid()}.log"
+
+    root = logging.getLogger()
+    root.setLevel(logging.INFO)
+    _drop_owned_handlers(root)
+
+    fh = logging.FileHandler(log_file)
+    fh.setLevel(logging.INFO)
+    fh.setFormatter(_FILE_FORMATTER)
+    root.addHandler(_mark_owned(fh))
+
+    return log_file
+
+
+@contextmanager
+def stage_log(stage_dir: Path | None, basename: str) -> Iterator[Path | None]:
+    """Add a ``FileHandler`` at ``<stage_dir>/<basename>.log`` for the
+    duration of the block; remove and close on exit.
+
+    A no-op when *stage_dir* is None — lets call sites use the same
+    ``with`` construct whether or not a stage is enabled.
+    """
+    if stage_dir is None:
+        yield None
+        return
+
+    stage_dir = Path(stage_dir)
+    stage_dir.mkdir(parents=True, exist_ok=True)
+    log_file = stage_dir / f"{basename}.log"
+
+    handler = logging.FileHandler(log_file)
+    handler.setLevel(logging.INFO)
+    handler.setFormatter(_FILE_FORMATTER)
+
+    root = logging.getLogger()
+    root.addHandler(handler)
+    try:
+        yield log_file
+    finally:
+        root.removeHandler(handler)
+        with contextlib.suppress(Exception):
+            handler.close()

odas_tpw/perturb/merge.py

@@ -0,0 +1,320 @@
+# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
+"""Merge split .p files that were rolled over due to file size limits.
+
+Files are mergeable when they share the same configuration (config string
+hash), endianness, and record size, and have sequential file numbers.
+
+Reference: Code/merge_p_files.m
+"""
+
+import contextlib
+import hashlib
+import os
+import shutil
+import struct
+import tempfile
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from odas_tpw.rsi.p_file import _H, HEADER_BYTES, HEADER_WORDS, _detect_endian
+
+# Max gap between a chained file's start and the previous file's computed end
+# for them to count as a genuine size-limit rollover (vs two independent casts).
+_MERGE_GAP_TOL_S = 5.0
+
+# I/O chunk size for the bounded record-aligned copy in merge_p_files.
+_COPY_CHUNK = 1 << 20  # 1 MiB
+
+
+def _copy_n(in_f, out_f, length: int) -> None:
+    """Copy exactly *length* bytes from *in_f* to *out_f* (record-aligned copy).
+
+    Unlike shutil.copyfileobj, this stops after *length* bytes so a fractional
+    trailing record in the source is never appended (audit #96).
+    """
+    remaining = length
+    while remaining > 0:
+        chunk = in_f.read(min(_COPY_CHUNK, remaining))
+        if not chunk:
+            break  # source shorter than expected; copy what exists
+        out_f.write(chunk)
+        remaining -= len(chunk)
+
+
+def _read_merge_info(path: Path) -> dict:
+    """Read header fields needed for merge decisions."""
+    with open(path, "rb") as f:
+        raw_hdr = f.read(HEADER_BYTES)
+        if len(raw_hdr) < HEADER_BYTES:
+            raise ValueError(f"{path.name}: file too small for header")
+
+        endian = _detect_endian(raw_hdr, path)
+        fmt = f"{endian}{HEADER_WORDS}H"
+        words = struct.unpack(fmt, raw_hdr)
+        header = {name: words[idx] for name, idx in _H.items()}
+
+        header_size = header["header_size"]
+        config_size = header["config_size"]
+        record_size = header["record_size"]
+        file_number = header["file_number"]
+
+        # Read config string to compute hash for matching
+        f.seek(header_size)
+        config_bytes = f.read(config_size)
+        config_hash = hashlib.md5(config_bytes).hexdigest()
+
+        f.seek(0, 2)
+        file_size = f.tell()
+
+    # Recording start time and duration, for the rollover-continuity check.
+    # Both are None when the header lacks a valid date / clock geometry (e.g.
+    # a synthetic or corrupt header) — in that case continuity is treated as
+    # unverifiable and does not block a merge.
+    try:
+        start_dt: datetime | None = datetime(
+            header["year"], header["month"], header["day"],
+            header["hour"], header["minute"], header["second"],
+            header["millisecond"] * 1000,
+        )
+    except (ValueError, OverflowError):
+        start_dt = None
+
+    duration: float | None = None
+    n_cols = header["fast_cols"] + header["slow_cols"]
+    clock = header["clock_hz"] + header["clock_frac"] / 1000.0
+    if start_dt is not None and n_cols > 0 and clock > 0 and record_size > header_size:
+        fs_fast = clock / n_cols
+        n_records = (file_size - (header_size + config_size)) // record_size
+        scans_per_record = ((record_size - header_size) // 2) // n_cols
+        if fs_fast > 0 and n_records > 0 and scans_per_record > 0:
+            duration = (n_records * scans_per_record) / fs_fast
+
+    return {
+        "path": path,
+        "file_number": file_number,
+        "endian": endian,
+        "header_size": header_size,
+        "config_size": config_size,
+        "record_size": record_size,
+        "config_hash": config_hash,
+        "file_size": file_size,
+        "start_dt": start_dt,
+        "duration": duration,
+    }
+
+
+def _is_continuous(prev: dict, nxt: dict, tol: float = _MERGE_GAP_TOL_S) -> bool:
+    """True if *nxt* starts within *tol* seconds of where *prev* ended.
+
+    A size-limit rollover continuation begins essentially where the previous
+    file ended; two independent casts are minutes/hours apart. When either
+    file's start time or the previous file's duration cannot be computed,
+    continuity is unverifiable and we return True (do not block the merge on
+    missing metadata — the config/geometry/sequence checks still apply).
+    """
+    if prev["start_dt"] is None or nxt["start_dt"] is None or prev["duration"] is None:
+        return True
+    prev_end = prev["start_dt"] + timedelta(seconds=prev["duration"])
+    return bool(abs((nxt["start_dt"] - prev_end).total_seconds()) <= tol)
+
+
+def _file_group_key(info: dict) -> tuple:
+    """Key for grouping files that could potentially be merged.
+
+    header_size and config_size are part of the key, not just record_size:
+    the merged file is reparsed with the FIRST file's geometry (PFile._read
+    slices every record as header_size//2 header words + data words), but the
+    splice skips each continuation file's own header_size + config_size. If a
+    chained file's header_size differed, its records would be silently
+    mis-sliced. Keying on the full geometry keeps geometry-mismatched files in
+    separate groups so they never chain.
+    """
+    return (
+        info["config_hash"],
+        info["endian"],
+        info["record_size"],
+        info["header_size"],
+        info["config_size"],
+    )
+
+
+def find_mergeable_files(p_files: list[Path]) -> list[list[Path]]:
+    """Identify chains of split files that should be merged.
+
+    Files are chained when they share the same config hash, endianness,
+    and record size, and have sequential file numbers.
+
+    Parameters
+    ----------
+    p_files : list of Path
+        .p file paths (typically from :func:`perturb.discover.find_p_files`).
+
+    Returns
+    -------
+    list of list of Path
+        Each inner list is a chain of files to merge (length >= 2).
+        Single files (no merge partner) are not included.
+    """
+    if len(p_files) < 2:
+        return []
+
+    # Read headers
+    infos = []
+    for p in p_files:
+        try:
+            infos.append(_read_merge_info(p))
+        except (ValueError, OSError):
+            continue
+
+    # Group by matching properties
+    groups: dict[tuple, list[dict]] = {}
+    for info in infos:
+        key = _file_group_key(info)
+        groups.setdefault(key, []).append(info)
+
+    chains = []
+    for group in groups.values():
+        if len(group) < 2:
+            continue
+
+        # Sort by file number
+        group.sort(key=lambda x: x["file_number"])
+
+        # Build chains of sequential file numbers
+        chain = [group[0]]
+        for i in range(1, len(group)):
+            if group[i]["file_number"] == chain[-1]["file_number"] + 1 and _is_continuous(
+                chain[-1], group[i]
+            ):
+                chain.append(group[i])
+            else:
+                if len(chain) >= 2:
+                    chains.append([info["path"] for info in chain])
+                chain = [group[i]]
+        if len(chain) >= 2:
+            chains.append([info["path"] for info in chain])
+
+    return chains
+
+
+def merge_destination(
+    chain: list[Path],
+    output_dir: Path,
+    root: Path | str | None = None,
+) -> Path:
+    """Return the merged output path for *chain*.
+
+    The first file in the chain names the merged file.  When *root* is
+    supplied and that first file lives underneath it, the relative directory
+    structure is preserved under *output_dir*.
+    """
+    if not chain:
+        raise ValueError("Empty chain")
+    first = Path(chain[0])
+    output_dir = Path(output_dir)
+    if root is not None:
+        try:
+            rel = first.resolve().relative_to(Path(root).resolve())
+        except ValueError:
+            rel = Path(first.name)
+        return output_dir / rel
+    return output_dir / first.name
+
+
+def plan_merge_outputs(
+    p_files: list[Path],
+    output_dir: Path,
+    root: Path | str | None = None,
+) -> list[tuple[Path, list[Path]]]:
+    """Plan merge outputs for all inputs.
+
+    Returns ``(output_path, source_chain)`` pairs.  Mergeable chains map to a
+    new output path; non-mergeable singleton files map to themselves so the
+    caller can keep processing them unchanged.
+    """
+    chains = find_mergeable_files(p_files)
+    chain_members = {p.resolve() for chain in chains for p in chain}
+
+    plan: list[tuple[Path, list[Path]]] = []
+    for chain in chains:
+        plan.append((merge_destination(chain, output_dir, root=root), chain))
+    for p in p_files:
+        if p.resolve() not in chain_members:
+            plan.append((p, [p]))
+    return plan
+
+
+def merge_p_files(
+    chain: list[Path],
+    output_dir: Path,
+    root: Path | str | None = None,
+) -> Path:
+    """Merge a chain of split .p files into a single file.
+
+    The first file is copied in its entirety.  Subsequent files contribute
+    only their data records (header + config are skipped).
+
+    Parameters
+    ----------
+    chain : list of Path
+        Ordered list of .p files to merge (first file is the base).
+    output_dir : Path
+        Directory for the merged output file.
+    root : Path, optional
+        Source root used to preserve relative paths under *output_dir*.
+
+    Returns
+    -------
+    Path
+        Path to the merged output file (named after the first file in the chain).
+    """
+    if not chain:
+        raise ValueError("Empty chain")
+
+    dest = merge_destination(chain, output_dir, root=root)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+
+    if len(chain) == 1:
+        if dest.resolve() != chain[0].resolve():
+            shutil.copy2(chain[0], dest)
+        return dest
+
+    # Build the merged file in a temp file then atomically replace. If dest
+    # resolves to chain[0] (output_dir == the chain's own directory), opening
+    # dest "wb" directly would truncate the base file to 0 bytes BEFORE it is
+    # read — yielding a headerless, unparseable file and destroying the source.
+    # Writing via a separate temp fd keeps every source intact until os.replace.
+    fd, tmp = tempfile.mkstemp(dir=str(dest.parent), prefix=".merge-", suffix=".tmp")
+    try:
+        with os.fdopen(fd, "wb") as out_f:
+            # Defect (audit #96): the splice previously copied each member to
+            # EOF, so a fractional trailing record (e.g. when files.trim is off
+            # or a non-final file rolled over mid-record) was appended verbatim
+            # and shifted every subsequent file's records by a sub-record
+            # offset, silently mis-slicing the merged file's interior records.
+            # Copy only the integer-record prefix of every member so the
+            # merged geometry stays record-aligned regardless of trim setting.
+            for i, p in enumerate(chain):
+                info = _read_merge_info(p)
+                data_start = info["header_size"] + info["config_size"]
+                record_size = info["record_size"]
+                if record_size <= 0:
+                    raise ValueError(f"{p.name}: invalid record_size {record_size}")
+                data_bytes = info["file_size"] - data_start
+                if data_bytes < 0:
+                    raise ValueError(f"{p.name}: file shorter than header+config")
+                n_complete = data_bytes // record_size
+                # Base file (i == 0) keeps its header+config + complete records;
+                # continuations contribute only their complete data records.
+                copy_start = 0 if i == 0 else data_start
+                copy_len = (data_start - copy_start) + n_complete * record_size
+                with open(p, "rb") as in_f:
+                    in_f.seek(copy_start)
+                    _copy_n(in_f, out_f, copy_len)
+        os.replace(tmp, dest)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp)
+        raise
+
+    return dest