dsvmonkey 0.1.0__py3-none-any.whl

dsvmonkey/__init__.py

@@ -0,0 +1,46 @@
+"""dsvmonkey — detect, profile, normalize and repair DSV files."""
+
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+from dsvmonkey.columns import ColumnProfile, profile_columns
+from dsvmonkey.convert import to_jsonl
+from dsvmonkey.orchestrate import profile_bytes, profile_file
+from dsvmonkey.profile import (
+    DetectionAlternative,
+    DetectionResult,
+    DictOverflowError,
+    DSVProfile,
+    ReviewRecommendedError,
+    assert_clean,
+)
+from dsvmonkey.reader import read
+from dsvmonkey.repair import RepairReport, repair
+
+# pyproject.toml is the single source of truth for the version number.
+# Reading via importlib.metadata avoids hand-syncing a duplicate
+# string here — a previous foot-gun where __init__.py's hardcoded
+# "0.1.0" could drift from the packaged version on a release. The
+# fallback covers running directly from a source tree where the
+# package isn't installed (rare but legal during dev).
+try:
+    __version__ = _pkg_version("dsvmonkey")
+except PackageNotFoundError:
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "ColumnProfile",
+    "DetectionAlternative",
+    "DetectionResult",
+    "DictOverflowError",
+    "DSVProfile",
+    "RepairReport",
+    "ReviewRecommendedError",
+    "assert_clean",
+    "profile_bytes",
+    "profile_columns",
+    "profile_file",
+    "read",
+    "repair",
+    "to_jsonl",
+    "__version__",
+]

dsvmonkey/_internal.py

@@ -0,0 +1,222 @@
+"""Shared internal helpers for output paths.
+
+Lives here rather than under one feature module (`repair.py`) so the
+cross-module dependency is explicit. Both `repair.py` and `convert.py`
+need atomic writes, same-path safety checks, and formula-injection
+neutralisation; importing private helpers across modules works but
+makes future renames risky and obscures the dependency. Pulling
+them into a single private module makes the shared surface
+discoverable and refactorable in one place.
+
+Still underscore-prefixed (the module name and the helpers) — these
+are not part of the public API. External callers should not import
+from `dsvmonkey._internal`.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+import unicodedata
+from contextlib import contextmanager
+from pathlib import Path
+from typing import IO, Iterator
+
+
+# ---------- Atomic + durable text writer ----------
+
+
+@contextmanager
+def _atomic_text_writer(path: Path, *, encoding: str) -> Iterator[IO[str]]:
+    """Yield a text-mode file handle whose writes land atomically
+    and durably at `path` on successful close.
+
+    Writes go to a sibling temp file (`.<path.name>.<pid>.partial`),
+    the file's pages are flushed to disk (`fsync`), then
+    `os.replace()` swaps it into place — a single filesystem
+    operation that's atomic on POSIX and Windows. The parent
+    directory is then also `fsync`ed so the rename itself survives
+    power loss. If the caller raises, the temp file is removed
+    rather than left at the final path, so a killed-mid-write run
+    doesn't corrupt existing output.
+
+    Previously we used `os.replace` alone, which is atomic against
+    process crashes but not power loss — a sudden reboot between
+    the write and the write hitting the platter could still lose
+    the new contents despite "atomic" semantics. The fsync before
+    the rename, plus the dir fsync after, closes that window for
+    durability-critical pipelines (regulated ETL, audit trails).
+
+    Uses the same directory as the target so the replace doesn't
+    cross filesystems (cross-fs rename falls back to copy+delete
+    and isn't atomic).
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_fd, tmp_name = tempfile.mkstemp(
+        prefix=f".{path.name}.",
+        suffix=".partial",
+        dir=str(path.parent),
+    )
+    tmp_path = Path(tmp_name)
+    try:
+        fh = os.fdopen(tmp_fd, "w", encoding=encoding, newline="")
+        try:
+            yield fh
+            fh.flush()
+            try:
+                os.fsync(fh.fileno())
+            except OSError:
+                # Some filesystems (/dev/null, special devices,
+                # some network mounts) don't support fsync. The
+                # atomic rename still works; durability degrades
+                # gracefully to "regular POSIX" rather than raising
+                # on every /tmp write.
+                pass
+        finally:
+            fh.close()
+        os.replace(tmp_path, path)
+        # Also fsync the containing directory so the rename itself
+        # survives power loss. Without this, the new inode exists
+        # on disk but the directory entry pointing at it might not
+        # be flushed yet — crash recovery could "lose" the rename.
+        try:
+            dir_fd = os.open(str(path.parent), os.O_RDONLY)
+            try:
+                os.fsync(dir_fd)
+            finally:
+                os.close(dir_fd)
+        except OSError:
+            # Windows doesn't support directory fsync; on other
+            # filesystems where it fails, gracefully skip.
+            pass
+    except BaseException:
+        # Any failure: remove the temp so we don't leave
+        # `.something.123.partial` files scattered in user dirs.
+        # Suppress every OSError class (FileNotFoundError,
+        # PermissionError, race conditions on Windows, network
+        # mount weirdness) so a cleanup hiccup doesn't replace
+        # the real exception's traceback context. Diagnosis of
+        # the original failure matters more than perfect
+        # housekeeping when something already went wrong.
+        try:
+            tmp_path.unlink()
+        except OSError:
+            pass
+        raise
+
+
+# ---------- Same-file protection ----------
+
+
+def _reject_same_path(
+    input_path: Path, output_path: Path | None, op: str
+) -> None:
+    """Raise if src and dst point at the same underlying file.
+
+    Streamed read + truncate-on-open would race the filesystem:
+    `open(p, "w")` truncates the inode on disk, and while POSIX
+    semantics keep the read fd alive (the writer gets an independent
+    inode on most systems but NOT all filesystems), Windows raises
+    immediately and network filesystems can produce silently
+    truncated output. Failing fast is cheaper than debugging a lost
+    CSV at 2am.
+
+    Identity is checked via (st_dev, st_ino) when both files exist,
+    which catches hardlinks and bind-mounts that resolve to the
+    same inode via different paths. Falls back to resolved-path
+    equality when either file doesn't exist yet (a fresh output
+    path has no inode to compare) or when the stat call fails
+    (permission errors etc.) — same-path protection degrades
+    gracefully rather than blocking legitimate writes.
+    """
+    if output_path is None:
+        return
+    # Primary: inode identity. Handles hardlinks, bind mounts, and
+    # other paths-to-same-file cases that resolved-path equality
+    # alone misses.
+    try:
+        src_stat = input_path.stat()
+        dst_stat = output_path.stat()
+    except (OSError, ValueError):
+        # Output file doesn't exist yet (the common case) or can't
+        # be stat'ed — fall through to path-equality.
+        pass
+    else:
+        if (src_stat.st_dev, src_stat.st_ino) == (
+            dst_stat.st_dev,
+            dst_stat.st_ino,
+        ):
+            raise ValueError(
+                f"{op}(): input and output resolve to the same file "
+                f"(inode {src_stat.st_ino} on device "
+                f"{src_stat.st_dev}). Writing would truncate the "
+                f"source before the read completes. Pick a different "
+                f"output path, or write to a temp file and "
+                f"os.replace() it."
+            )
+
+    # Fallback: resolved-path equality, for the "dst doesn't exist
+    # yet" case and for filesystems where stat is unavailable.
+    try:
+        src_resolved = input_path.resolve(strict=False)
+        dst_resolved = output_path.resolve(strict=False)
+    except OSError:
+        return
+    if src_resolved == dst_resolved:
+        raise ValueError(
+            f"{op}(): input and output resolve to the same path "
+            f"({src_resolved}). Writing would truncate the source "
+            f"before the read completes. Pick a different output "
+            f"path, or write to a temp file and os.replace() it."
+        )
+
+
+# ---------- Formula-injection neutralisation ----------
+
+
+# Characters that Excel / Google Sheets / LibreOffice Calc treat as
+# the start of a formula. An attacker who can influence cell values
+# crafts something like `=SUM(A:A)*cmd|' /C calc.exe'!A1` — when the
+# CSV is opened in a spreadsheet, the cell evaluates as a formula
+# with side effects. Prepending `'` neutralises the formula marker
+# (the apostrophe is treated as a formatting hint by spreadsheets,
+# indicating "display this cell as literal text").
+_FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@")
+
+
+def _neutralize_formula(cell: str) -> str:
+    """Prepend `'` when `cell`'s first non-whitespace, non-invisible
+    character is a spreadsheet-formula trigger; return `cell`
+    unchanged otherwise.
+
+    Called only when the caller opts into formula sanitization
+    (`repair(sanitize_formulas=True)` /
+    `to_jsonl(sanitize_formulas=True)`). Off by default because the
+    prefix mutates visible content and is unwanted when the
+    consumer isn't a spreadsheet; turn on at the pipeline boundary
+    where output crosses into untrusted spreadsheet territory.
+
+    The skip set covers BOTH whitespace AND Unicode "Format" /
+    "Mark, Nonspacing" categories (BOM U+FEFF, ZWSP U+200B,
+    ZWNJ/ZWJ, combining marks, etc.). Excel and Sheets ignore
+    those when evaluating the cell, so a naive lstrip() — which
+    only removes whitespace — leaves `"\\ufeff=SUM(A:A)"`
+    un-neutralised. Under `clean_cells=False` the BOM survives
+    to output and the formula evaluates in the spreadsheet,
+    bypassing the defence the caller asked for.
+    """
+    if not cell:
+        return cell
+    for ch in cell:
+        if ch.isspace():
+            continue
+        # Cf = Format (BOM, ZWSP, ZWNJ, ZWJ, RLM, LRM…).
+        # Mn = Mark, Nonspacing (combining diacritics).
+        # Both are visually invisible / non-rendered as a leading
+        # cell character in spreadsheets.
+        if unicodedata.category(ch) in ("Cf", "Mn"):
+            continue
+        if ch in _FORMULA_INJECTION_PREFIXES:
+            return "'" + cell
+        break
+    return cell