tieout 0.1.0__py3-none-any.whl

tieout/__init__.py

@@ -0,0 +1,39 @@
+"""tieout: an open-source reconciliation engine.
+
+from tieout import load_spec, reconcile
+from tieout.connectors import read_source
+
+spec = load_spec("recon.yaml")
+bank = read_source("bank", "statement.xml", spec.sources["bank"])
+ledger = read_source("ledger", "ledger.csv", spec.sources["ledger"])
+result = reconcile([bank, ledger], spec)
+"""
+
+from tieout.engine import ENGINE_VERSION as __version__
+from tieout.engine import reconcile
+from tieout.errors import ConnectorError, InputError, MoneyError, SpecError, TieoutError
+from tieout.records import Record, Source
+from tieout.result import Break, BreakReason, Match, Result, RunInfo
+from tieout.spec import Rule, RuleKind, Spec, load_spec, parse_spec
+
+__all__ = [
+    "Break",
+    "BreakReason",
+    "ConnectorError",
+    "InputError",
+    "Match",
+    "MoneyError",
+    "Record",
+    "Result",
+    "Rule",
+    "RuleKind",
+    "RunInfo",
+    "Source",
+    "Spec",
+    "SpecError",
+    "TieoutError",
+    "__version__",
+    "load_spec",
+    "parse_spec",
+    "reconcile",
+]

tieout/cli.py

@@ -0,0 +1,121 @@
+"""Command-line interface for tieout."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+
+from tieout.connectors import read_source, validate_spec
+from tieout.engine import ENGINE_VERSION, reconcile
+from tieout.errors import TieoutError
+from tieout.report import format_summary, to_json, write_breaks_csv, write_matches_csv
+from tieout.spec import load_spec
+
+
+class _CliError(click.ClickException):
+    """Configuration or input error; exits with code 2 to stay distinct from breaks (1)."""
+
+    exit_code = 2
+
+
+@click.group()
+@click.version_option(ENGINE_VERSION, prog_name="tieout")
+def main() -> None:
+    """Reconcile money records across banks, payment providers, ledgers, and invoices."""
+
+
+@main.command()
+@click.argument("spec_path", type=click.Path(dir_okay=False, path_type=Path))
+def validate(spec_path: Path) -> None:
+    """Validate a YAML reconciliation spec and its source options."""
+    try:
+        spec = load_spec(spec_path)
+        validate_spec(spec)
+    except TieoutError as exc:
+        raise _CliError(str(exc)) from exc
+
+    click.echo(
+        f"Valid spec: {len(spec.sources)} source(s), {len(spec.rules)} rule(s) ({spec.digest()})"
+    )
+
+
+@main.command()
+@click.argument("spec_path", type=click.Path(dir_okay=False, path_type=Path))
+@click.option(
+    "--input",
+    "-i",
+    "inputs",
+    multiple=True,
+    required=True,
+    metavar="NAME=PATH",
+    help="Bind a declared source name to an input file (repeat for each source).",
+)
+@click.option(
+    "--output-dir",
+    "-o",
+    type=click.Path(file_okay=False, path_type=Path),
+    default=None,
+    help="Write result.json, matches.csv, and breaks.csv into this directory.",
+)
+@click.option("--json", "as_json", is_flag=True, help="Print full result JSON to stdout.")
+@click.option(
+    "--fail-on-breaks",
+    is_flag=True,
+    help="Exit with status 1 if any breaks remain after running all rules.",
+)
+def run(
+    spec_path: Path,
+    inputs: tuple[str, ...],
+    output_dir: Path | None,
+    as_json: bool,
+    fail_on_breaks: bool,
+) -> None:
+    """Run a reconciliation spec against input files."""
+    bindings = _parse_inputs(inputs)
+    try:
+        spec = load_spec(spec_path)
+        validate_spec(spec)
+        for name in bindings:
+            if name not in spec.sources:
+                declared = ", ".join(sorted(spec.sources)) or "(none)"
+                raise _CliError(
+                    f"input {name!r} is not declared in spec.sources (declared: {declared})"
+                )
+        sources = [read_source(name, path, spec.sources[name]) for name, path in bindings.items()]
+        result = reconcile(sources, spec)
+    except TieoutError as exc:
+        raise _CliError(str(exc)) from exc
+
+    if output_dir is not None:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        (output_dir / "result.json").write_text(to_json(result) + "\n", encoding="utf-8")
+        write_matches_csv(result, output_dir / "matches.csv")
+        write_breaks_csv(result, output_dir / "breaks.csv")
+
+    if as_json:
+        click.echo(to_json(result))
+        if output_dir is not None:
+            click.echo(f"Wrote reports to {output_dir}", err=True)
+    else:
+        click.echo(format_summary(result))
+        if output_dir is not None:
+            click.echo(f"\nWrote reports to {output_dir}")
+
+    if fail_on_breaks and result.breaks:
+        sys.exit(1)
+
+
+def _parse_inputs(raw_inputs: tuple[str, ...]) -> dict[str, Path]:
+    bindings: dict[str, Path] = {}
+    for item in raw_inputs:
+        name, sep, path_str = item.partition("=")
+        name = name.strip()
+        path_str = path_str.strip()
+        if not sep or not name or not path_str:
+            raise _CliError(f"invalid --input {item!r}: expected NAME=PATH")
+        if name in bindings:
+            raise _CliError(f"duplicate --input for source {name!r}")
+        bindings[name] = Path(path_str)
+    return bindings

tieout/conditions.py

@@ -0,0 +1,226 @@
+"""How a rule judges a candidate pairing, and how it explains the verdict.
+
+A pairing is a group of left records against a group of right records. For a
+one-to-one rule both groups hold one record; for grouping rules one side holds
+several. Every check is defined over groups so all rule kinds share one meaning:
+
+- amount: right total must equal the expected total (left total, or its negation
+  for ``sign: opposite``) within the tolerance;
+- date: *every* right date minus *every* left date must fall in the window;
+- references: the condition must hold for *every* left/right record pair.
+
+Containment checks (``left_in_right``/``right_in_left``) match whole tokens only:
+the needle must start and end where the haystack has a token boundary (a
+separator, or a switch between letters and digits). ``"INV-0042"`` is found in
+``"Payment for inv 0042"``, but ``"7712"`` is not found in ``"Payment #77712"``.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from functools import lru_cache
+
+from tieout.money import format_amount
+from tieout.records import Record
+from tieout.spec import ReferenceCondition, Rule
+
+_NON_ALPHANUMERIC = re.compile(r"[^0-9A-Z]+")
+_TOKEN = re.compile(r"[A-Z]+|[0-9]+")
+
+
+@dataclass(frozen=True, slots=True)
+class DateSpan:
+    """Earliest and latest date of a group of records, as proleptic ordinals."""
+
+    first: int
+    last: int
+
+
+def date_span(records: Sequence[Record], rule: Rule) -> DateSpan | None:
+    """Date span used by the rule's date check, or None if any record lacks the date.
+
+    With a date condition, only the field it names counts. Without one, dates only
+    rank otherwise equal candidates, so a record's value date is used if it has
+    one and its booking date if not.
+    """
+    ordinals = []
+    for record in records:
+        if rule.date is not None:
+            value = record.date_of(rule.date.field)
+        else:
+            value = record.value_date or record.booking_date
+        if value is None:
+            return None
+        ordinals.append(value.toordinal())
+    return DateSpan(min(ordinals), max(ordinals))
+
+
+def dates_fit(rule: Rule, left: DateSpan, right: DateSpan) -> bool:
+    if rule.date is None:
+        return True
+    return (
+        right.first - left.last >= rule.date.min_days
+        and right.last - left.first <= rule.date.max_days
+    )
+
+
+def date_distance(left: DateSpan, right: DateSpan) -> int:
+    """Largest absolute gap between any left and any right date; used to rank candidates."""
+    return max(abs(right.last - left.first), abs(left.last - right.first))
+
+
+def expected_right_total(rule: Rule, left_total: int) -> int:
+    return left_total if rule.amount.sign == "same" else -left_total
+
+
+def amount_difference(rule: Rule, left_total: int, right_total: int) -> int:
+    return right_total - expected_right_total(rule, left_total)
+
+
+@lru_cache(maxsize=524288)
+def normalize_reference(text: str) -> str:
+    return _NON_ALPHANUMERIC.sub("", text.upper())
+
+
+@lru_cache(maxsize=262144)
+def _token_boundaries(text: str) -> frozenset[int]:
+    """Offsets in the normalised text where a token starts or ends."""
+    boundaries = {0}
+    offset = 0
+    for token in _TOKEN.findall(text.upper()):
+        offset += len(token)
+        boundaries.add(offset)
+    return frozenset(boundaries)
+
+
+def contains_token_sequence(haystack: str, needle: str) -> bool:
+    """Whether normalised ``needle`` occurs in ``haystack`` aligned to token boundaries."""
+    normalized_haystack = normalize_reference(haystack)
+    normalized_needle = normalize_reference(needle)
+    if not normalized_needle:
+        return False
+    boundaries = _token_boundaries(haystack)
+    start = normalized_haystack.find(normalized_needle)
+    while start != -1:
+        if start in boundaries and start + len(normalized_needle) in boundaries:
+            return True
+        start = normalized_haystack.find(normalized_needle, start + 1)
+    return False
+
+
+def reference_holds(condition: ReferenceCondition, left: Record, right: Record) -> bool:
+    raw_left, raw_right = left.value_of(condition.left), right.value_of(condition.right)
+    left_value, right_value = normalize_reference(raw_left), normalize_reference(raw_right)
+    if not left_value or not right_value:
+        return False  # an absent reference never counts as agreement
+    if condition.op == "equals":
+        return left_value == right_value
+    needle, haystack = (
+        (raw_left, raw_right)
+        if condition.op == "left_in_right"
+        else (
+            raw_right,
+            raw_left,
+        )
+    )
+    if len(normalize_reference(needle)) < condition.min_length:
+        return False
+    return contains_token_sequence(haystack, needle)
+
+
+def references_hold(rule: Rule, lefts: Sequence[Record], rights: Sequence[Record]) -> bool:
+    return all(
+        reference_holds(condition, left, right)
+        for condition in rule.references
+        for left in lefts
+        for right in rights
+    )
+
+
+def explain(
+    rule: Rule,
+    lefts: Sequence[Record],
+    rights: Sequence[Record],
+    *,
+    grouping_note: str | None = None,
+) -> tuple[str, ...]:
+    """Plain-language account of why this pairing satisfies the rule."""
+    currency = lefts[0].currency
+    left_total = sum(record.amount for record in lefts)
+    right_total = sum(record.amount for record in rights)
+    difference = amount_difference(rule, left_total, right_total)
+
+    lines = [
+        f"rule {rule.id} ({rule.kind.value}): "
+        f"{_count(len(lefts), 'left record')} against {_count(len(rights), 'right record')}",
+        _explain_amount(rule, left_total, right_total, difference, currency),
+    ]
+    if rule.date is not None:
+        lines.append(_explain_dates(rule, lefts, rights))
+    lines.extend(_explain_reference(condition, lefts, rights) for condition in rule.references)
+    if grouping_note:
+        lines.append(grouping_note)
+    return tuple(lines)
+
+
+_OP_WORDS = {
+    "equals": "equals",
+    "left_in_right": "is contained in",
+    "right_in_left": "contains",
+}
+
+
+def _explain_amount(
+    rule: Rule, left_total: int, right_total: int, difference: int, currency: str
+) -> str:
+    sign_note = " (opposite signs)" if rule.amount.sign == "opposite" else ""
+    totals = (
+        f"amount: left {format_amount(left_total, currency)} {currency}, "
+        f"right {format_amount(right_total, currency)} {currency}{sign_note}"
+    )
+    if difference == 0:
+        return f"{totals}, exact"
+    return (
+        f"{totals}, difference {format_amount(difference, currency)} {currency} "
+        f"within tolerance {format_amount(rule.amount.tolerance, currency)}"
+    )
+
+
+def _explain_reference(
+    condition: ReferenceCondition, lefts: Sequence[Record], rights: Sequence[Record]
+) -> str:
+    left_values = _distinct(record.value_of(condition.left) for record in lefts)
+    right_values = _distinct(record.value_of(condition.right) for record in rights)
+    return (
+        f"reference: left.{condition.left} {_quote(left_values)} {_OP_WORDS[condition.op]} "
+        f"right.{condition.right} {_quote(right_values)}"
+    )
+
+
+def _distinct(values: Iterable[str]) -> list[str]:
+    return sorted(set(values))
+
+
+def _quote(values: list[str]) -> str:
+    shown = ", ".join(repr(value) for value in values[:3])
+    more = f" and {len(values) - 3} more" if len(values) > 3 else ""
+    return shown + more
+
+
+def _explain_dates(rule: Rule, lefts: Sequence[Record], rights: Sequence[Record]) -> str:
+    assert rule.date is not None
+    left, right = date_span(lefts, rule), date_span(rights, rule)
+    assert left is not None
+    assert right is not None
+    low, high = right.first - left.last, right.last - left.first
+    gap = f"{low} days" if low == high else f"{low} to {high} days"
+    return (
+        f"date: right {rule.date.field} minus left is {gap} "
+        f"(allowed {rule.date.min_days} to {rule.date.max_days})"
+    )
+
+
+def _count(n: int, noun: str) -> str:
+    return f"{n} {noun}" if n == 1 else f"{n} {noun}s"

tieout/connectors/__init__.py

@@ -0,0 +1,122 @@
+"""Source connectors: parse external files into canonical `Source` objects."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from pydantic import ValidationError
+
+from tieout.connectors.camt import CamtOptions, parse_camt
+from tieout.connectors.csv import CsvOptions, parse_csv
+from tieout.connectors.stripe import StripeOptions, parse_stripe
+from tieout.errors import ConnectorError, SpecError
+from tieout.records import Exclusion, Record, Source
+from tieout.spec import RuleKind, SourceSpec, Spec
+
+_Parser = Callable[[str, Path, bytes, dict[str, Any]], tuple[list[Record], list[Exclusion]]]
+
+_CONNECTORS: dict[str, tuple[_Parser, type[CamtOptions | CsvOptions | StripeOptions]]] = {
+    "camt": (parse_camt, CamtOptions),
+    "csv": (parse_csv, CsvOptions),
+    "stripe": (parse_stripe, StripeOptions),
+}
+
+
+def validate_source_spec(spec: SourceSpec) -> frozenset[str]:
+    """Verify a source's connector format and options without reading a file.
+
+    Returns the field names records from that source can carry.
+    """
+    entry = _CONNECTORS.get(spec.format)
+    if entry is None:
+        supported = ", ".join(sorted(_CONNECTORS))
+        raise SpecError(f"unknown source format {spec.format!r} (expected one of: {supported})")
+    _, option_model = entry
+    try:
+        options = option_model.model_validate(spec.options)
+    except ValidationError as exc:
+        lines = [
+            f"{'.'.join(str(p) for p in err['loc']) or 'options'}: {err['msg']}"
+            for err in exc.errors()
+        ]
+        raise SpecError(f"invalid {spec.format} options:\n  " + "\n  ".join(lines)) from exc
+    return options.fields()
+
+
+def validate_spec(spec: Spec) -> None:
+    """Check every source's options, and that every field a rule names can exist.
+
+    A misspelt field (``payout_idd``) would otherwise match nothing, silently: an
+    absent reference never counts as agreement and an absent group key forms no
+    group. Only declared sources are checked.
+    """
+    available = {name: validate_source_spec(source) for name, source in spec.sources.items()}
+    problems: list[str] = []
+    for rule in spec.rules:
+        wanted: list[tuple[str, str, str]] = []  # (side, source, field)
+        for condition in rule.references:
+            wanted.append(("left", rule.left, condition.left))
+            wanted.append(("right", rule.right, condition.right))
+        if rule.date is not None:
+            wanted.append(("left", rule.left, rule.date.field))
+            wanted.append(("right", rule.right, rule.date.field))
+        if rule.group is not None and rule.group.by is not None:
+            side, source = (
+                ("right", rule.right) if rule.kind is RuleKind.ONE_TO_MANY else ("left", rule.left)
+            )
+            wanted.append((side, source, rule.group.by))
+        for side, source, field in wanted:
+            fields = available.get(source)
+            if fields is not None and field not in fields:
+                problems.append(
+                    f"rule {rule.id!r}: {side} source {source!r} has no field {field!r} "
+                    f"(available: {', '.join(sorted(fields))})"
+                )
+    if problems:
+        raise SpecError("invalid spec:\n  " + "\n  ".join(dict.fromkeys(problems)))
+
+
+def read_source(
+    name: str,
+    path: str | Path,
+    spec: SourceSpec | str,
+    **options: Any,
+) -> Source:
+    """Read a file into a `Source` using the named connector and record its byte digest."""
+    if isinstance(spec, str):
+        fmt = spec
+        merged_options: dict[str, Any] = dict(options)
+    else:
+        fmt = spec.format
+        merged_options = {**spec.options, **options}
+
+    entry = _CONNECTORS.get(fmt)
+    if entry is None:
+        supported = ", ".join(sorted(_CONNECTORS))
+        raise SpecError(f"unknown source format {fmt!r} (expected one of: {supported})")
+    parser, _ = entry
+
+    file_path = Path(path)
+    try:
+        raw_bytes = file_path.read_bytes()
+    except OSError as exc:
+        detail = exc.strerror or str(exc)
+        raise ConnectorError(f"cannot read file: {detail}", path=str(file_path)) from exc
+
+    digest = f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}"
+    try:
+        records, excluded = parser(name, file_path, raw_bytes, merged_options)
+    except ValidationError as exc:
+        lines = [
+            f"{'.'.join(str(p) for p in err['loc']) or 'options'}: {err['msg']}"
+            for err in exc.errors()
+        ]
+        raise SpecError(f"invalid {fmt} options:\n  " + "\n  ".join(lines)) from exc
+
+    return Source(name=name, records=tuple(records), digest=digest, excluded=tuple(excluded))
+
+
+__all__ = ["read_source", "validate_source_spec", "validate_spec"]