solverpilot 0.1__py3-none-any.whl

solverpilot/io/csv.py

@@ -0,0 +1,230 @@
+"""Safe deterministic local CSV and explicitly declared CSV-bundle ingestion."""
+
+from __future__ import annotations
+
+import csv
+import io
+from pathlib import Path
+from typing import Any
+
+from .errors import (
+    DataIOConfigurationError,
+    DataIOMappingError,
+    DataIOParseError,
+    DataIOSourceError,
+)
+from .hashing import canonicalize_json_value, deep_thaw_json, sha256_bytes, sha256_json
+from .mapping import apply_rules, set_path
+from .model import CsvMapping, CsvTable, DataIssue, LoadPolicy
+from .provenance import SourceFileProvenance, SourceProvenance
+from .result import LoadedData
+from .source import read_limited_bytes, safe_bundle_child, validate_local_path
+
+_CSV_LOADER_VERSION = "1.0"
+
+
+def _parse_table(
+    raw_bytes: bytes,
+    *,
+    table: CsvTable,
+    encoding: str,
+    delimiter: str,
+    policy: LoadPolicy,
+) -> tuple[list[tuple[int, dict[str, str]]], tuple[str, ...]]:
+    try:
+        text = raw_bytes.decode(encoding)
+    except LookupError as exc:
+        raise DataIOSourceError("Declared CSV encoding is not supported.") from exc
+    except UnicodeDecodeError as exc:
+        raise DataIOParseError("CSV source cannot be decoded with the declared encoding.") from exc
+
+    try:
+        reader = csv.reader(io.StringIO(text, newline=""), delimiter=delimiter, strict=True)
+        try:
+            raw_headers = next(reader)
+        except StopIteration as exc:
+            raise DataIOParseError("CSV source is missing a header row.") from exc
+        headers = tuple(str(header).strip() for header in raw_headers)
+        if any(not header for header in headers):
+            raise DataIOParseError("CSV header names must be non-empty.")
+        if len(headers) != len(set(headers)):
+            raise DataIOParseError("CSV header names must be unique.")
+
+        required_headers = set(table.required_headers)
+        required_headers.update(
+            rule.source_path
+            for rule in table.fields
+            if rule.source_path is not None and rule.required and not rule.has_constant
+        )
+        missing = sorted(str(item) for item in required_headers.difference(headers))
+        if missing:
+            raise DataIOParseError(f"CSV source is missing required headers: {', '.join(missing)}.")
+
+        rows: list[tuple[int, dict[str, str]]] = []
+        for raw_row in reader:
+            row_number = reader.line_num
+            if not raw_row:
+                raise DataIOParseError(f"CSV row {row_number} is blank; blank data rows are not allowed.")
+            if len(raw_row) != len(headers):
+                raise DataIOParseError(
+                    f"CSV row {row_number} has a different field count than the header."
+                )
+            rows.append((row_number, dict(zip(headers, map(str, raw_row)))))
+        return rows, headers
+    except DataIOParseError:
+        raise
+    except (csv.Error, ValueError) as exc:
+        raise DataIOParseError("CSV source or delimiter configuration is invalid.") from exc
+
+
+def _resolve_table_path(root: Path, table: CsvTable, policy: LoadPolicy) -> Path:
+    if root.is_file():
+        if table.file_name not in {None, root.name}:
+            raise DataIOConfigurationError(
+                "Single-file CSV mapping file_name must be omitted or match the source name."
+            )
+        return root
+    if table.file_name is None:
+        raise DataIOConfigurationError("Every CSV table in a directory bundle must declare file_name.")
+    return safe_bundle_child(root, table.file_name, reject_symlinks=policy.reject_symlinks)
+
+
+def load_csv(
+    source: str | Path,
+    *,
+    mapping: CsvMapping,
+    policy: LoadPolicy | None = None,
+) -> LoadedData:
+    """Load one CSV file or an explicitly declared local CSV directory bundle."""
+    policy = LoadPolicy() if policy is None else policy
+    if not isinstance(mapping, CsvMapping):
+        raise DataIOConfigurationError("mapping must be CsvMapping.")
+    if not isinstance(policy, LoadPolicy):
+        raise DataIOConfigurationError("policy must be LoadPolicy.")
+
+    root = validate_local_path(
+        source,
+        expect_directory=None,
+        reject_symlinks=policy.reject_symlinks,
+    )
+    if root.is_dir() and any(table.file_name is None for table in mapping.tables):
+        raise DataIOConfigurationError("Every CSV table in a directory bundle must declare file_name.")
+    if root.is_file() and len(mapping.tables) != 1:
+        raise DataIOConfigurationError("A single CSV file source requires exactly one table mapping.")
+
+    normalized: Any = canonicalize_json_value(deep_thaw_json(mapping.constants))
+    if not isinstance(normalized, dict):
+        raise DataIOConfigurationError("CSV constants must form an object.")
+
+    source_files: list[SourceFileProvenance] = []
+    issues: list[DataIssue] = []
+    total_size = 0
+    total_records = 0
+    table_metadata: list[dict[str, Any]] = []
+
+    for table in mapping.tables:
+        table_path = _resolve_table_path(root, table, policy)
+        source_name = table_path.name if root.is_file() else str(table.file_name)
+        remaining = policy.max_source_bytes - total_size
+        if remaining <= 0:
+            raise DataIOSourceError("Combined CSV bundle size exceeds the configured maximum.")
+        raw_bytes = read_limited_bytes(table_path, max_bytes=remaining)
+        total_size += len(raw_bytes)
+        encoding = table.encoding or policy.encoding
+        delimiter = table.delimiter or policy.delimiter
+        rows, headers = _parse_table(
+            raw_bytes,
+            table=table,
+            encoding=encoding,
+            delimiter=delimiter,
+            policy=policy,
+        )
+        if len(rows) < table.min_rows:
+            raise DataIOParseError(
+                f"CSV table {source_name!r} has {len(rows)} data rows; minimum required is {table.min_rows}."
+            )
+        used_headers = {rule.source_path for rule in table.fields if rule.source_path is not None}
+        unused = tuple(header for header in headers if header not in used_headers)
+        if unused:
+            issues.append(
+                DataIssue(
+                    severity="info",
+                    code="unused_csv_headers",
+                    message=f"CSV table {source_name!r} contains headers not used by the explicit mapping.",
+                    path=source_name,
+                )
+            )
+
+        records: list[dict[str, Any]] = []
+        for row_number, row in rows:
+            try:
+                records.append(
+                    apply_rules(
+                        row,
+                        table.fields,
+                        strict_numbers=policy.strict_numbers,
+                        max_json_depth=policy.max_json_depth,
+                        row_number=row_number,
+                    )
+                )
+            except DataIOMappingError as exc:
+                raise DataIOMappingError(f"CSV mapping failed in {source_name!r}: {exc}") from exc
+
+        if table.target_path:
+            set_path(normalized, table.target_path, records)
+        else:
+            if len(mapping.tables) != 1 or mapping.constants:
+                raise DataIOConfigurationError(
+                    "An empty CSV target_path is only valid for one table with no top-level constants."
+                )
+            normalized = records
+
+        file_hash = sha256_bytes(raw_bytes)
+        source_files.append(
+            SourceFileProvenance(name=source_name, sha256=file_hash, size_bytes=len(raw_bytes))
+        )
+        total_records += len(records)
+        table_metadata.append(
+            {
+                "source_name": source_name,
+                "target_path": table.target_path,
+                "row_count": len(records),
+                "encoding": encoding,
+                "delimiter": delimiter,
+            }
+        )
+
+    evidence = tuple(sorted(source_files, key=lambda item: item.name.casefold()))
+    combined_source_hash = (
+        evidence[0].sha256
+        if root.is_file() and len(evidence) == 1
+        else sha256_json([item.to_dict() for item in evidence])
+    )
+    provenance = SourceProvenance(
+        loader="solverpilot.io.csv",
+        loader_version=_CSV_LOADER_VERSION,
+        source_type="csv_bundle" if root.is_dir() else "csv_file",
+        source_name=root.name,
+        source_sha256=combined_source_hash,
+        source_size_bytes=total_size,
+        mapping_sha256=sha256_json(mapping.to_dict()),
+        normalized_payload_sha256=sha256_json(normalized),
+        policy_sha256=sha256_json(policy.to_dict()),
+        record_count=total_records,
+        encoding=policy.encoding,
+        delimiter=policy.delimiter,
+        source_files=evidence,
+        metadata={
+            "mapping_version": mapping.version,
+            "reject_symlinks": policy.reject_symlinks,
+            "strict_numbers": policy.strict_numbers,
+            "bundle_name_policy": "normalized_posix_casefold_unique_v1",
+            "tables": table_metadata,
+        },
+    )
+    return LoadedData(
+        payload=normalized,
+        provenance=provenance,
+        issues=tuple(issues),
+        metadata={"mapping_kind": "csv", "table_count": len(mapping.tables)},
+    )

solverpilot/io/errors.py

@@ -0,0 +1,25 @@
+"""Fail-closed exceptions for local data ingestion."""
+
+from __future__ import annotations
+
+from solverpilot.exceptions import SolverPilotError
+
+
+class DataIOError(SolverPilotError):
+    """Base class for deterministic SolverPilot data-I/O failures."""
+
+
+class DataIOConfigurationError(DataIOError, ValueError):
+    """Raised when an I/O or mapping configuration is invalid."""
+
+
+class DataIOSourceError(DataIOError):
+    """Raised when a local source cannot be inspected or read safely."""
+
+
+class DataIOParseError(DataIOError, ValueError):
+    """Raised when source bytes cannot be parsed under the declared policy."""
+
+
+class DataIOMappingError(DataIOError, ValueError):
+    """Raised when explicit normalization or field mapping fails."""

solverpilot/io/hashing.py

@@ -0,0 +1,111 @@
+"""Canonical JSON snapshots and SHA-256 helpers for data provenance."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from types import MappingProxyType
+from typing import Any
+
+from .errors import DataIOConfigurationError
+
+
+def _validate_text(value: str) -> str:
+    try:
+        value.encode("utf-8")
+    except UnicodeEncodeError as exc:
+        raise DataIOConfigurationError("JSON-compatible text contains an invalid Unicode surrogate.") from exc
+    return value
+
+
+def canonicalize_json_value(value: Any) -> Any:
+    """Return a detached JSON-compatible value with deterministic mapping order."""
+    if value is None or isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return _validate_text(value)
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise DataIOConfigurationError("JSON-compatible numbers must be finite.")
+        return value
+    if isinstance(value, Mapping):
+        keys = tuple(value.keys())
+        if any(not isinstance(key, str) for key in keys):
+            raise DataIOConfigurationError("Canonical JSON mapping keys must be strings.")
+        return {
+            _validate_text(key): canonicalize_json_value(value[key])
+            for key in sorted(keys)
+        }
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [canonicalize_json_value(item) for item in value]
+    raise DataIOConfigurationError(
+        f"Value of type {type(value).__name__} is not JSON-compatible for provenance."
+    )
+
+
+def _freeze(value: Any) -> Any:
+    if isinstance(value, dict):
+        return MappingProxyType({key: _freeze(item) for key, item in value.items()})
+    if isinstance(value, list):
+        return tuple(_freeze(item) for item in value)
+    return value
+
+
+def deep_freeze_json(value: Any) -> Any:
+    """Create a detached, recursively immutable JSON-compatible snapshot."""
+    try:
+        return _freeze(canonicalize_json_value(value))
+    except DataIOConfigurationError:
+        raise
+    except RecursionError as exc:
+        raise DataIOConfigurationError(
+            "JSON-compatible configuration exceeds supported nesting or contains a cycle."
+        ) from exc
+
+
+def deep_thaw_json(value: Any) -> Any:
+    """Return a detached mutable JSON-compatible copy of a frozen snapshot."""
+    if isinstance(value, Mapping):
+        return {str(key): deep_thaw_json(item) for key, item in value.items()}
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [deep_thaw_json(item) for item in value]
+    return canonicalize_json_value(value)
+
+
+def canonical_json_bytes(value: Any) -> bytes:
+    try:
+        normalized = canonicalize_json_value(value)
+        return json.dumps(
+            normalized,
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+            allow_nan=False,
+        ).encode("utf-8")
+    except DataIOConfigurationError:
+        raise
+    except (TypeError, ValueError, UnicodeEncodeError, RecursionError) as exc:
+        raise DataIOConfigurationError("Value cannot be serialized as canonical JSON.") from exc
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def sha256_json(value: Any) -> str:
+    return sha256_bytes(canonical_json_bytes(value))
+
+
+def sha256_file(path: str | Path, *, chunk_size: int = 1024 * 1024) -> str:
+    if type(chunk_size) is not int or chunk_size <= 0:
+        raise ValueError("chunk_size must be a positive integer")
+    digest = hashlib.sha256()
+    with Path(path).open("rb") as handle:
+        for chunk in iter(lambda: handle.read(chunk_size), b""):
+            digest.update(chunk)
+    return digest.hexdigest()

solverpilot/io/json.py

@@ -0,0 +1,90 @@
+"""Safe deterministic local JSON ingestion."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .errors import DataIOParseError, DataIOSourceError
+from .hashing import sha256_bytes, sha256_json
+from .mapping import apply_object_mapping
+from .model import LoadPolicy, ObjectMapping
+from .provenance import SourceFileProvenance, SourceProvenance
+from .result import LoadedData
+from .source import read_limited_bytes, validate_local_path
+from .strict_json import loads_strict_json
+
+_JSON_LOADER_VERSION = "1.0"
+
+
+def load_json(
+    source: str | Path,
+    *,
+    mapping: ObjectMapping | None = None,
+    policy: LoadPolicy | None = None,
+) -> LoadedData:
+    """Load one local JSON file with strict parsing, mapping, and provenance.
+
+    No problem type is inferred and no solver-specific object is materialized. The
+    returned normalized payload is immutable; use ``mutable_payload()`` when a
+    mutable detached copy is needed.
+    """
+    mapping = ObjectMapping.identity_mapping() if mapping is None else mapping
+    policy = LoadPolicy() if policy is None else policy
+    if not isinstance(mapping, ObjectMapping):
+        from .errors import DataIOConfigurationError
+
+        raise DataIOConfigurationError("mapping must be ObjectMapping.")
+    if not isinstance(policy, LoadPolicy):
+        from .errors import DataIOConfigurationError
+
+        raise DataIOConfigurationError("policy must be LoadPolicy.")
+
+    path = validate_local_path(
+        source,
+        expect_directory=False,
+        reject_symlinks=policy.reject_symlinks,
+    )
+    raw_bytes = read_limited_bytes(path, max_bytes=policy.max_source_bytes)
+    try:
+        text = raw_bytes.decode(policy.encoding)
+    except LookupError as exc:
+        raise DataIOSourceError("Declared JSON encoding is not supported.") from exc
+    except UnicodeDecodeError as exc:
+        raise DataIOParseError("JSON source cannot be decoded with the declared encoding.") from exc
+
+    parsed = loads_strict_json(
+        text,
+        context="JSON source",
+        max_depth=policy.max_json_depth,
+        strict_numbers=policy.strict_numbers,
+    )
+    normalized = apply_object_mapping(
+        parsed,
+        mapping,
+        strict_numbers=policy.strict_numbers,
+        max_json_depth=policy.max_json_depth,
+    )
+    source_hash = sha256_bytes(raw_bytes)
+    provenance = SourceProvenance(
+        loader="solverpilot.io.json",
+        loader_version=_JSON_LOADER_VERSION,
+        source_type="json_file",
+        source_name=path.name,
+        source_sha256=source_hash,
+        source_size_bytes=len(raw_bytes),
+        mapping_sha256=sha256_json(mapping.to_dict()),
+        normalized_payload_sha256=sha256_json(normalized),
+        policy_sha256=sha256_json(policy.to_dict()),
+        record_count=len(parsed) if isinstance(parsed, list) else 1,
+        encoding=policy.encoding,
+        source_files=(
+            SourceFileProvenance(name=path.name, sha256=source_hash, size_bytes=len(raw_bytes)),
+        ),
+        metadata={
+            "mapping_version": mapping.version,
+            "reject_symlinks": policy.reject_symlinks,
+            "strict_numbers": policy.strict_numbers,
+            "strict_json_policy": "solverpilot_strict_json_v1",
+        },
+    )
+    return LoadedData(payload=normalized, provenance=provenance, metadata={"mapping_kind": "object"})

solverpilot/io/mapping.py

@@ -0,0 +1,195 @@
+"""Collision-safe explicit field mapping and scalar normalization."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from .errors import DataIOConfigurationError, DataIOMappingError
+from .hashing import canonicalize_json_value, deep_thaw_json
+from .model import MappingRule, ObjectMapping
+from .numeric import parse_boolean, parse_integer, parse_number
+from .strict_json import loads_strict_json
+
+_MISSING = object()
+_PATH_TOKEN_RE = re.compile(r"([^.[\]]+)|\[([0-9]+)\]")
+
+
+def _path_tokens(path: str) -> tuple[str | int, ...]:
+    if path == "":
+        return ()
+    tokens: list[str | int] = []
+    position = 0
+    while position < len(path):
+        match = _PATH_TOKEN_RE.match(path, position)
+        if match is None:
+            raise DataIOMappingError(f"Invalid source path syntax at position {position}.")
+        token = match.group(1)
+        tokens.append(token if token is not None else int(match.group(2)))
+        position = match.end()
+        if position == len(path):
+            break
+        if path[position] == "[":
+            continue
+        if path[position] != ".":
+            raise DataIOMappingError(f"Invalid source path syntax at position {position}.")
+        position += 1
+        if position == len(path) or path[position] in ".[":
+            raise DataIOMappingError(f"Invalid source path syntax at position {position}.")
+    return tuple(tokens)
+
+
+def get_path(source: Any, path: str) -> Any:
+    current = source
+    for token in _path_tokens(path):
+        if isinstance(token, int):
+            if not isinstance(current, Sequence) or isinstance(current, (str, bytes, bytearray)):
+                return _MISSING
+            if token >= len(current):
+                return _MISSING
+            current = current[token]
+        else:
+            if not isinstance(current, Mapping) or token not in current:
+                return _MISSING
+            current = current[token]
+    return current
+
+
+def set_path(target: dict[str, Any], path: str, value: Any) -> None:
+    if not path:
+        raise DataIOMappingError("Target path cannot be empty for object field mapping.")
+    parts = path.split(".")
+    if any(not part or "[" in part or "]" in part for part in parts):
+        raise DataIOMappingError(f"Invalid target path syntax: {path!r}.")
+    current = target
+    for part in parts[:-1]:
+        if part not in current:
+            nested: dict[str, Any] = {}
+            current[part] = nested
+            current = nested
+            continue
+        existing = current[part]
+        if not isinstance(existing, dict):
+            raise DataIOMappingError(
+                f"Target path {path!r} collides with a non-object value at {part!r}."
+            )
+        current = existing
+    leaf = parts[-1]
+    if leaf in current:
+        raise DataIOMappingError(f"Target path {path!r} would overwrite an existing value.")
+    current[leaf] = value
+
+
+def _canonical_mapped(value: Any, *, field_name: str) -> Any:
+    try:
+        return canonicalize_json_value(value)
+    except DataIOConfigurationError as exc:
+        raise DataIOMappingError(f"{field_name} is not a supported JSON-compatible value.") from exc
+
+
+def normalize_value(
+    value: Any,
+    rule: MappingRule,
+    *,
+    field_name: str,
+    strict_numbers: bool,
+    max_json_depth: int,
+) -> Any:
+    is_null = value is None or (
+        isinstance(value, str) and value.strip().lower() in rule.null_values
+    )
+    if is_null:
+        if rule.has_default:
+            value = deep_thaw_json(rule.default)
+        elif rule.required:
+            raise DataIOMappingError(f"{field_name} is required and cannot be null.")
+        else:
+            return None
+    if rule.value_type == "any":
+        if isinstance(value, str) and rule.trim:
+            return value.strip()
+        return _canonical_mapped(value, field_name=field_name)
+    if rule.value_type == "string":
+        if not isinstance(value, str):
+            raise DataIOMappingError(f"{field_name} must be a string.")
+        return value.strip() if rule.trim else value
+    if rule.value_type == "integer":
+        return parse_integer(value, field_name=field_name)
+    if rule.value_type == "number":
+        return parse_number(value, field_name=field_name, strict_binary64=strict_numbers)
+    if rule.value_type == "boolean":
+        return parse_boolean(value, field_name=field_name)
+    if rule.value_type == "json":
+        if isinstance(value, str):
+            value = loads_strict_json(
+                value,
+                context=f"{field_name} JSON text",
+                max_depth=max_json_depth,
+                strict_numbers=strict_numbers,
+            )
+        return _canonical_mapped(value, field_name=field_name)
+    raise DataIOMappingError(f"Unsupported normalization type {rule.value_type!r}.")
+
+
+def apply_rules(
+    source: Any,
+    rules: Sequence[MappingRule],
+    *,
+    strict_numbers: bool,
+    max_json_depth: int,
+    row_number: int | None = None,
+) -> dict[str, Any]:
+    target: dict[str, Any] = {}
+    for rule in rules:
+        if rule.has_constant:
+            raw_value = deep_thaw_json(rule.constant)
+        else:
+            if rule.source_path is None:
+                raise DataIOMappingError("Mapping rule without a constant requires source_path.")
+            raw_value = get_path(source, rule.source_path)
+            if raw_value is _MISSING:
+                if rule.has_default:
+                    raw_value = deep_thaw_json(rule.default)
+                elif rule.required:
+                    suffix = "" if row_number is None else f" at row {row_number}"
+                    raise DataIOMappingError(
+                        f"Required source field {rule.source_path!r} is missing{suffix}."
+                    )
+                else:
+                    raw_value = None
+        suffix = "" if row_number is None else f" at row {row_number}"
+        normalized = normalize_value(
+            raw_value,
+            rule,
+            field_name=f"Field {rule.source_path or rule.target_path!r}{suffix}",
+            strict_numbers=strict_numbers,
+            max_json_depth=max_json_depth,
+        )
+        set_path(target, rule.target_path, normalized)
+    return target
+
+
+def apply_object_mapping(
+    source: Any,
+    config: ObjectMapping,
+    *,
+    strict_numbers: bool,
+    max_json_depth: int,
+) -> Any:
+    if config.identity:
+        return _canonical_mapped(source, field_name="Identity-mapped source")
+    target = _canonical_mapped(deep_thaw_json(config.constants), field_name="Object mapping constants")
+    if not isinstance(target, dict):
+        raise DataIOMappingError("Object mapping constants must form an object.")
+    mapped = apply_rules(
+        source,
+        config.rules,
+        strict_numbers=strict_numbers,
+        max_json_depth=max_json_depth,
+    )
+    for key, value in mapped.items():
+        if key in target:
+            raise DataIOMappingError(f"Object mapping would overwrite constant key {key!r}.")
+        target[key] = value
+    return target