datakrit 2.1.1__py3-none-any.whl

datakrit/__init__.py

@@ -0,0 +1,61 @@
+"""Datakrit: validate and explicitly clean CSV files with declared number and date formats.
+
+The public API is the set of names in ``__all__`` (see docs/STABILITY.md), mainly::
+
+    from datakrit import check_file, clean_file, export_clean, load_profile
+    from datakrit import profile_to_toml, run_batch, select_inputs
+
+See docs/ARCHITECTURE.md for the result model and exceptions.
+"""
+
+__version__ = "2.1.1"
+
+from .api import CheckResult, CleanResult, ScanInfo, check_file, clean_file, export_clean
+from .batch import BatchInput, BatchResult, FileOutcome, run_batch, select_inputs
+from .errors import DatakritError, InputError, OutputError, ProfileError
+from .normalization import FieldChange, RemovedRecord
+from .output import ArtifactOutcome
+from .reader import FatalError
+from .schema import (
+    ColumnSpec,
+    OutputSettings,
+    Overrides,
+    Profile,
+    Settings,
+    SourceSettings,
+    load_profile,
+    profile_to_toml,
+)
+from .validation import Issue
+
+__all__ = [
+    "ArtifactOutcome",
+    "BatchInput",
+    "BatchResult",
+    "CheckResult",
+    "CleanResult",
+    "ColumnSpec",
+    "DatakritError",
+    "FatalError",
+    "FieldChange",
+    "FileOutcome",
+    "InputError",
+    "Issue",
+    "OutputError",
+    "OutputSettings",
+    "Overrides",
+    "Profile",
+    "ProfileError",
+    "RemovedRecord",
+    "ScanInfo",
+    "Settings",
+    "SourceSettings",
+    "__version__",
+    "check_file",
+    "clean_file",
+    "export_clean",
+    "load_profile",
+    "profile_to_toml",
+    "run_batch",
+    "select_inputs",
+]

datakrit/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

datakrit/api.py

@@ -0,0 +1,367 @@
+"""Public entry points shared by the CLI and library users.
+
+Core functions never print or exit. Configuration and I/O problems raise
+``ProfileError``/``InputError``/``OutputError``; data problems are returned as issues.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from . import __version__
+from .errors import OutputError
+from .normalization import FieldChange, RemovedRecord, transform
+from .output import (
+    OUTPUT_CSV,
+    OUTPUT_PROFILE,
+    ArtifactOutcome,
+    encode_text,
+    render_csv,
+    validate_artifact_paths,
+    write_artifact,
+)
+from .reader import FatalError, Table, read_csv
+from .schema import (
+    Overrides,
+    Profile,
+    Settings,
+    load_profile,
+    output_profile,
+    profile_to_toml,
+    resolve_settings,
+)
+from .validation import ERROR, FATAL, WARNING, Issue, validate_table
+
+
+@dataclass(frozen=True)
+class ScanInfo:
+    """``complete`` is false when a fatal error stopped reading; counts cover what was read."""
+
+    complete: bool
+    data_records: int
+    physical_lines: int
+    blank_lines: int
+    bom: bool
+    fatal: FatalError | None
+
+
+@dataclass(frozen=True)
+class CheckResult:
+    """Source validation result. ``exit_code``: 0 valid, 1 source errors, 2 incomplete scan."""
+
+    source_path: str
+    profile_path: str | None
+    settings: Settings
+    scan: ScanInfo
+    issues: tuple[Issue, ...]
+    # Absolute paths captured when the files were read; used to protect them on export
+    # even if the working directory changes later.
+    source_file: Path | None = None
+    profile_file: Path | None = None
+    header: tuple[str, ...] | None = None
+
+    @property
+    def errors(self) -> int:
+        return sum(i.severity == ERROR for i in self.issues)
+
+    @property
+    def warnings(self) -> int:
+        return sum(i.severity == WARNING for i in self.issues)
+
+    @property
+    def exit_code(self) -> int:
+        if not self.scan.complete:
+            return 2
+        return 1 if self.errors else 0
+
+    @property
+    def ok(self) -> bool:
+        return self.exit_code == 0
+
+
+@dataclass(frozen=True)
+class CleanResult:
+    """Clean/dry-run result. ``exit_code`` reflects remaining *output* errors (0/1), or 2 when
+    the scan is incomplete. ``rows`` are the cleaned records in source order."""
+
+    source_path: str
+    profile_path: str | None
+    settings: Settings
+    scan: ScanInfo
+    header: tuple[str, ...] | None
+    rows: tuple[tuple[str, ...], ...]
+    source_issues: tuple[Issue, ...]
+    output_issues: tuple[Issue, ...]
+    changes: tuple[FieldChange, ...]
+    removed: tuple[RemovedRecord, ...]
+    output_profile: Profile
+    source_file: Path | None = None
+    profile_file: Path | None = None
+
+    @property
+    def source_errors(self) -> int:
+        """Errors of the source file, before the selected transformations."""
+        return sum(i.severity == ERROR for i in self.source_issues)
+
+    @property
+    def source_warnings(self) -> int:
+        return sum(i.severity == WARNING for i in self.source_issues)
+
+    @property
+    def output_errors(self) -> int:
+        """Errors that remain in the cleaned result; they block the export."""
+        return sum(i.severity == ERROR for i in self.output_issues)
+
+    @property
+    def output_warnings(self) -> int:
+        return sum(i.severity == WARNING for i in self.output_issues)
+
+    @property
+    def export_ready(self) -> bool:
+        return self.scan.complete and self.header is not None and self.output_errors == 0
+
+    @property
+    def export_blocked_reason(self) -> str | None:
+        """Why the result cannot be exported, with the next step; ``None`` when it can."""
+        if self.export_ready:
+            return None
+        fatal = self.scan.fatal
+        if fatal is not None:
+            where = f" at line {fatal.line}" if fatal.line is not None else ""
+            return (
+                f"the scan is incomplete ({fatal.code}{where}), so the file was not fully "
+                "read and nothing can be exported. Fix the file structure or encoding (or "
+                "the delimiter/encoding settings), then run the command again"
+            )
+        if self.header is None:
+            return (
+                "the file has no header row, so there is nothing to export. Provide a CSV "
+                "file whose first row contains the column names"
+            )
+        return (
+            f"{self.output_errors} output error(s) remain. Fix the reported values in the "
+            "source file or select a transformation that repairs them, then run clean again"
+        )
+
+    @property
+    def exit_code(self) -> int:
+        if not self.scan.complete:
+            return 2
+        return 0 if self.export_ready else 1
+
+    @property
+    def ok(self) -> bool:
+        """True when the cleaned result can be exported (exit code 0)."""
+        return self.exit_code == 0
+
+    def output_profile_toml(self) -> str:
+        """The effective output profile as TOML text."""
+        name = Path(self.source_path).name
+        header = (
+            f"Generated by datakrit {__version__} from the cleaned output of {name}.\n"
+            "It describes the cleaned file; use it with `datakrit check` or `datakrit clean`."
+        )
+        return profile_to_toml(self.output_profile, header)
+
+    def csv_text(self) -> str:
+        """The cleaned CSV serialized with the output delimiter and line ending."""
+        if self.header is None:
+            raise OutputError("there is no header to export")
+        out = self.settings.output
+        return render_csv(self.header, self.rows, out.delimiter, out.line_ending)
+
+
+def _profile(profile: Profile | str | Path | None) -> Profile | None:
+    if profile is None or isinstance(profile, Profile):
+        return profile
+    return load_profile(profile)
+
+
+def _profile_file(profile: Profile | None) -> Path | None:
+    if profile is None or profile.resolved_path is None:
+        return None
+    return Path(profile.resolved_path)
+
+
+def _read(path: str | Path, settings: Settings) -> Table:
+    src = settings.source
+    return read_csv(path, src.delimiter, src.encoding, src.max_field_chars)
+
+
+def _scan(table: Table) -> ScanInfo:
+    return ScanInfo(
+        table.complete,
+        len(table.records),
+        table.physical_lines,
+        len(table.blank_lines),
+        table.bom,
+        table.fatal,
+    )
+
+
+FATAL_REMEDIES = {
+    "decode_error": "encoding",
+    "malformed_csv": "quoting",
+    "field_too_large": "max_field_chars",
+}
+
+
+def _fatal_issue(fatal: FatalError, stage: str) -> Issue:
+    return Issue(
+        fatal.code,
+        FATAL,
+        fatal.message[:1].upper() + fatal.message[1:] + ".",
+        "Fix the file structure or encoding (or adjust the profile), then run the command again. "
+        "Records after this point were not scanned.",
+        stage,
+        line_start=fatal.line,
+        line_end=fatal.line,
+        remedy=FATAL_REMEDIES.get(fatal.code),
+    )
+
+
+def check_file(
+    path: str | Path,
+    profile: Profile | str | Path | None = None,
+    *,
+    overrides: Overrides | None = None,
+) -> CheckResult:
+    """Validate a CSV file against source rules without modifying it."""
+    loaded = _profile(profile)
+    settings = resolve_settings(loaded, overrides)
+    table = _read(path, settings)
+    issues = validate_table(table, settings, "source")
+    if table.fatal:
+        issues.append(_fatal_issue(table.fatal, "source"))
+    return CheckResult(
+        str(path),
+        loaded.path if loaded else None,
+        settings,
+        _scan(table),
+        tuple(issues),
+        source_file=Path(path).resolve(),
+        profile_file=_profile_file(loaded),
+        header=table.header,
+    )
+
+
+def clean_file(
+    path: str | Path,
+    profile: Profile | str | Path | None = None,
+    *,
+    overrides: Overrides | None = None,
+) -> CleanResult:
+    """Validate, apply selected transformations in memory, and validate the result.
+
+    Nothing is written; use ``export_clean`` to write the cleaned CSV.
+    """
+    loaded = _profile(profile)
+    settings = resolve_settings(loaded, overrides)
+    table = _read(path, settings)
+    source_issues = validate_table(table, settings, "source")
+    changes: list[FieldChange] = []
+    removed: list[RemovedRecord] = []
+    rows: tuple[tuple[str, ...], ...] = ()
+    if table.fatal is not None:  # the scan is incomplete
+        source_issues.append(_fatal_issue(table.fatal, "source"))
+        output_issues: list[Issue] = []
+    elif table.header is None:
+        output_issues = validate_table(table, settings, "output", include_blank_lines=False)
+    else:
+        result = transform(table.header, table.records, settings)
+        changes, removed = result.changes, result.removed
+        rows = tuple(r.fields for r in result.records)
+        cleaned = Table(
+            header=table.header,
+            header_line=table.header_line,
+            records=result.records,
+            physical_lines=table.physical_lines,
+            bom=table.bom,
+        )
+        output_issues = validate_table(
+            cleaned,
+            settings,
+            "output",
+            include_blank_lines=False,
+            unconverted=result.unconverted,
+        )
+
+    removed_ids = {r.record for r in removed}
+    output_keys = {(i.code, i.record, i.column) for i in output_issues}
+    source_keys = {(i.code, i.record, i.column) for i in source_issues}
+    statused = []
+    for issue in source_issues:
+        if not table.complete:
+            status = "unresolved"
+        elif issue.record is not None and issue.record in removed_ids:
+            status = "removed"
+        elif (issue.code, issue.record, issue.column) in output_keys:
+            status = "unresolved"
+        else:
+            status = "resolved"
+        statused.append(replace(issue, status=status))
+    introduced = [
+        replace(i, introduced=(i.code, i.record, i.column) not in source_keys)
+        for i in output_issues
+    ]
+    return CleanResult(
+        source_path=str(path),
+        profile_path=loaded.path if loaded else None,
+        settings=settings,
+        scan=_scan(table),
+        header=table.header,
+        rows=rows,
+        source_issues=tuple(statused),
+        output_issues=tuple(introduced),
+        changes=tuple(changes),
+        removed=tuple(removed),
+        output_profile=output_profile(settings),
+        source_file=Path(path).resolve(),
+        profile_file=_profile_file(loaded),
+    )
+
+
+def protected_inputs(result: CheckResult | CleanResult) -> dict[str, Path]:
+    protected = {"input CSV": result.source_file or Path(result.source_path).resolve()}
+    if result.profile_file is not None:
+        protected["input profile"] = result.profile_file
+    elif result.profile_path:
+        protected["input profile"] = Path(result.profile_path).resolve()
+    return protected
+
+
+def export_clean(
+    result: CleanResult,
+    output: str | Path,
+    *,
+    output_profile: str | Path | None = None,
+    overwrite: bool = False,
+) -> tuple[ArtifactOutcome, ...]:
+    """Write the cleaned CSV (and optionally its output profile) after safety checks.
+
+    Raises ``OutputError`` if output errors remain, a path aliases an input or another
+    artifact, a file exists without ``overwrite``, or a write fails. Each file is replaced
+    atomically; if the profile write fails after the CSV was written, the error's
+    ``outcomes`` record that the CSV was written.
+    """
+    if not result.export_ready:
+        raise OutputError(f"export blocked: {result.export_blocked_reason}")
+    output = Path(output)
+    profile = Path(output_profile) if output_profile is not None else None
+    validate_artifact_paths(
+        protected_inputs(result), {OUTPUT_CSV: output, OUTPUT_PROFILE: profile}, overwrite
+    )
+    out = result.settings.output
+    csv_bytes = encode_text(result.csv_text(), out.encoding, "output CSV")
+    profile_bytes = (
+        encode_text(result.output_profile_toml(), "utf-8", "output profile") if profile else None
+    )
+    outcomes = [write_artifact(output, csv_bytes, overwrite, OUTPUT_CSV)]
+    if profile is not None and profile_bytes is not None:
+        try:
+            outcomes.append(write_artifact(profile, profile_bytes, overwrite, OUTPUT_PROFILE))
+        except OutputError as exc:
+            outcomes.append(ArtifactOutcome(OUTPUT_PROFILE, str(profile), "failed", str(exc)))
+            raise OutputError(str(exc), tuple(outcomes)) from None
+    return tuple(outcomes)