xml-auditfile 0.0.1__py3-none-any.whl

xml_auditfile/__init__.py

@@ -0,0 +1,88 @@
+"""Streaming reader for Dutch tax audit files (XAF and related), following auditfile-spec."""
+
+from ._json import JsonFormatError
+from .auditfile import Auditfile, AuditfileReader, SinkAbortError
+from .custom import DialectRegistry, ViewRegistry
+from .definitions import (
+    AuditfileManifest,
+    ColumnType,
+    Dialect,
+    DialectAction,
+    DialectCorrection,
+    ElementDefinition,
+    LocalizedText,
+    ManifestColumn,
+    ManifestKey,
+    ManifestReference,
+    ManifestTable,
+    TextLanguage,
+    ViewAggregate,
+    ViewColumn,
+    ViewDefinition,
+    ViewJoin,
+    ViewJoinCondition,
+    ViewOperator,
+    ViewParameter,
+    ViewValue,
+)
+from .diagnostics import AuditfileReport, Diagnostic, DiagnosticSeverity
+from .memory import MemorySink
+from .options import ReadCancelledError, ReadMode, ReadOptions, ReadProgress
+from .rows import AuditfileRow
+from .schema import AuditfileSchema, SchemaColumn, SchemaTable, VersionSource
+from .sinks import ImportContext, Sink, SinkCapabilities
+from .source import AuditfileSource, PartInfo, SourcePart
+from .versions import AuditfileVersion
+from .views import ViewResult
+from .xsd import XsdError
+
+__all__ = [
+    "Auditfile",
+    "AuditfileManifest",
+    "AuditfileReader",
+    "AuditfileReport",
+    "AuditfileRow",
+    "AuditfileSchema",
+    "AuditfileSource",
+    "AuditfileVersion",
+    "ColumnType",
+    "Diagnostic",
+    "DiagnosticSeverity",
+    "Dialect",
+    "DialectAction",
+    "DialectCorrection",
+    "DialectRegistry",
+    "ElementDefinition",
+    "ImportContext",
+    "JsonFormatError",
+    "LocalizedText",
+    "ManifestColumn",
+    "ManifestKey",
+    "ManifestReference",
+    "ManifestTable",
+    "MemorySink",
+    "PartInfo",
+    "ReadCancelledError",
+    "ReadMode",
+    "ReadOptions",
+    "ReadProgress",
+    "SchemaColumn",
+    "SchemaTable",
+    "Sink",
+    "SinkAbortError",
+    "SinkCapabilities",
+    "SourcePart",
+    "TextLanguage",
+    "VersionSource",
+    "ViewAggregate",
+    "ViewColumn",
+    "ViewDefinition",
+    "ViewJoin",
+    "ViewJoinCondition",
+    "ViewOperator",
+    "ViewParameter",
+    "ViewRegistry",
+    "ViewResult",
+    "ViewValue",
+    "XsdError",
+]

xml_auditfile/_checks.py

@@ -0,0 +1,279 @@
+"""Checks that see every row as it is emitted: keys and references, and the XAF business rules."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+
+from ._decimal import add, format_decimal
+from ._keys import SEPARATOR as _SEPARATOR
+from .definitions import ColumnType
+from .diagnostics import DiagnosticCollector, Location
+from .rows import AuditfileRow
+from .schema import AuditfileSchema, SchemaColumn, SchemaTable
+from .values import to_text
+
+_MAX_PENDING_PER_REFERENCE = 100_000
+
+
+def _composite(row: AuditfileRow, columns: tuple[int, ...]) -> str | None:
+    parts = []
+    for column in columns:
+        value = row.values[column]
+        if value is None:
+            return None
+
+        parts.append(to_text(value))
+
+    return _SEPARATOR.join(parts)
+
+
+def _display(value: str) -> str:
+    return value.replace(_SEPARATOR, "/")
+
+
+def _indexes(table: SchemaTable, columns: tuple[str, ...]) -> tuple[int, ...]:
+    result = []
+    for name in columns:
+        column = table.find_column(name)
+        if column is None:
+            raise LookupError(f"Key column {name} not in {table.name}.")
+
+        result.append(column.index)
+
+    return tuple(result)
+
+
+class _KeyState:
+    __slots__ = ("table", "columns", "values")
+
+    def __init__(self, table: SchemaTable, columns: tuple[int, ...]) -> None:
+        self.table = table
+        self.columns = columns
+        self.values: set[str] = set()
+
+
+class _ReferenceState:
+    __slots__ = ("table", "column_names", "columns", "key", "pending", "overflowed")
+
+    def __init__(self, table: SchemaTable, column_names: str, columns: tuple[int, ...], key: _KeyState) -> None:
+        self.table = table
+        self.column_names = column_names
+        self.columns = columns
+        self.key = key
+        self.pending: set[str] = set()
+        self.overflowed = False
+        """True once the pending limit was reached and XAF-R017 was reported."""
+
+
+class ReferenceChecker:
+    """
+    Checks keys (unique) and references (existing) from the schema.
+    Only key values and unresolved reference values are kept in memory; references are resolved at the end,
+    so a reference to a key that appears later (e.g. in a later part) is still found.
+    """
+
+    def __init__(self, schema: AuditfileSchema, diagnostics: DiagnosticCollector) -> None:
+        self._diagnostics = diagnostics
+        keys_by_name: dict[str, _KeyState] = {}
+        self._keys_by_table: dict[int, list[_KeyState]] = {}
+        self._references_by_table: dict[int, list[_ReferenceState]] = {}
+
+        for table in schema.tables:
+            for key in table.keys:
+                state = _KeyState(table, _indexes(table, key.columns))
+                keys_by_name[key.name] = state
+                self._keys_by_table.setdefault(id(table), []).append(state)
+
+        for table in schema.tables:
+            for reference in table.references:
+                target = keys_by_name.get(reference.key)
+                if target is not None:
+                    self._references_by_table.setdefault(id(table), []).append(
+                        _ReferenceState(table, ", ".join(reference.columns), _indexes(table, reference.columns), target)
+                    )
+
+    def on_row(self, row: AuditfileRow) -> None:
+        for key in self._keys_by_table.get(id(row.table), ()):
+            value = _composite(row, key.columns)
+            if value is not None:
+                if value in key.values:
+                    self._diagnostics.add("XAF-R004", Location(row.part, None, None, row.table.path), value=_display(value), table=row.table.name)
+                else:
+                    key.values.add(value)
+
+        for reference in self._references_by_table.get(id(row.table), ()):
+            value = _composite(row, reference.columns)
+            if value is None or value in reference.key.values or value in reference.pending:
+                continue
+
+            if len(reference.pending) < _MAX_PENDING_PER_REFERENCE:
+                reference.pending.add(value)
+            elif not reference.overflowed:
+                # From here on new values are not checked; say so once, so a clean result is not mistaken for a complete one.
+                reference.overflowed = True
+                self._diagnostics.add(
+                    "XAF-R017", Location(row.part, None, None, row.table.path),
+                    table=row.table.name, column=reference.column_names, target=reference.key.table.name, max=_MAX_PENDING_PER_REFERENCE,
+                )
+
+    def on_end(self) -> None:
+        for references in self._references_by_table.values():
+            for reference in references:
+                for value in sorted(v for v in reference.pending if v not in reference.key.values):
+                    self._diagnostics.add(
+                        "XAF-R003", Location(None, None, None, reference.table.path),
+                        table=reference.table.name, column=reference.column_names, value=_display(value), target=reference.key.table.name,
+                    )
+
+
+XAF = "xaf"
+"""Audit file type the XAF rules apply to."""
+
+_AMOUNT = "amnt"
+_AMOUNT_TYPE = "amnt_tp"
+_DEBIT = "D"
+_CREDIT = "C"
+
+
+class _Lines:
+    """A table of lines with an amount and a debit/credit indicator, with its columns looked up once."""
+
+    __slots__ = ("table", "amount", "amount_type")
+
+    def __init__(self, table: SchemaTable, amount: SchemaColumn, amount_type: SchemaColumn) -> None:
+        self.table = table
+        self.amount = amount
+        self.amount_type = amount_type
+
+    @staticmethod
+    def find(table: SchemaTable | None) -> _Lines | None:
+        """None when the table or a column does not exist, or has another type, in this version."""
+        amount = table.find_column(_AMOUNT) if table is not None else None
+        amount_type = table.find_column(_AMOUNT_TYPE) if table is not None else None
+        if table is None or amount is None or amount_type is None:
+            return None
+
+        if amount.type not in (ColumnType.DECIMAL, ColumnType.INTEGER) or amount_type.type is not ColumnType.STRING:
+            return None
+
+        return _Lines(table, amount, amount_type)
+
+
+class _Totals:
+    __slots__ = ("count", "debit", "credit")
+
+    def __init__(self) -> None:
+        self.count = 0
+        self.debit = Decimal(0)
+        self.credit = Decimal(0)
+
+    def add(self, row: AuditfileRow, lines: _Lines) -> None:
+        self.count += 1
+        amount = row.get_decimal(lines.amount)
+        if amount is None:
+            return
+
+        type_ = row.get_string(lines.amount_type)
+        if type_ == _DEBIT:
+            self.debit = add(self.debit, amount)
+        elif type_ == _CREDIT:
+            self.credit = add(self.credit, amount)
+
+
+def _text(value: object) -> str:
+    return format_decimal(value) if isinstance(value, Decimal) else str(value)
+
+
+def _compare(row: AuditfileRow, field: str, expected: object, actual: object, diagnostics: DiagnosticCollector) -> None:
+    if expected is None or expected == actual:
+        return
+
+    diagnostics.add(
+        "XAF-R002", Location(row.part, None, None, row.table.path),
+        field=field, table=row.table.name, expected=_text(expected), actual=_text(actual),
+    )
+
+
+class _TotalsRule:
+    def __init__(self, lines: _Lines, owner: SchemaTable, per_parent: bool, count: SchemaColumn, debit: SchemaColumn, credit: SchemaColumn) -> None:
+        self._lines = lines
+        self._owner = owner
+        self._per_parent = per_parent
+        self._count = count
+        self._debit = debit
+        self._credit = credit
+        self._by_parent: dict[int, _Totals] = {}
+        self._all = _Totals()
+
+    def on_row(self, row: AuditfileRow, diagnostics: DiagnosticCollector) -> None:
+        if row.table is self._lines.table:
+            totals = self._by_parent.setdefault(row.parent_id, _Totals()) if self._per_parent and row.parent_id is not None else self._all
+            totals.add(row, self._lines)
+            return
+
+        if row.table is not self._owner:
+            return
+
+        actual = self._by_parent.pop(row.id, _Totals()) if self._per_parent else self._all
+        _compare(row, self._count.name, row.get_integer(self._count), actual.count, diagnostics)
+        _compare(row, self._debit.name, row.get_decimal(self._debit), actual.debit, diagnostics)
+        _compare(row, self._credit.name, row.get_decimal(self._credit), actual.credit, diagnostics)
+
+
+class XafRules:
+    """
+    Business rules for Auditfile Financieel (XAF):
+    every transaction balances, and the counts and totals in the headers match the lines.
+    Rules whose tables or columns do not exist in a version are skipped. Columns are looked up once, not per row.
+    """
+
+    def __init__(self, schema: AuditfileSchema, diagnostics: DiagnosticCollector) -> None:
+        self._diagnostics = diagnostics
+        self._transactions: dict[int, _Totals] = {}
+        self._rules: list[_TotalsRule] = []
+        self._lines = _Lines.find(schema.find_table("tr_line"))
+        self._transaction = schema.find_table("transaction")
+        self._transaction_number = self._transaction.find_column("nr") if self._transaction is not None else None
+
+        self._add_rule(schema, "tr_line", "company", False, "transactions_lines_count", "transactions_total_debit", "transactions_total_credit")
+        self._add_rule(schema, "ob_line", "company", False, "opening_balance_lines_count", "opening_balance_total_debit", "opening_balance_total_credit")
+        self._add_rule(schema, "sb_line", "subledger", True, "lines_count", "total_debit", "total_credit")
+        self._add_rule(schema, "ob_sb_line", "ob_subledger", True, "lines_count", "total_debit", "total_credit")
+
+    def on_row(self, row: AuditfileRow) -> None:
+        if self._lines is not None and row.table is self._lines.table and row.parent_id is not None:
+            self._transactions.setdefault(row.parent_id, _Totals()).add(row, self._lines)
+        elif row.table is self._transaction and row.id in self._transactions:
+            lines = self._transactions.pop(row.id)
+            if lines.debit != lines.credit:
+                number = row[self._transaction_number] if self._transaction_number is not None else None
+                self._diagnostics.add(
+                    "XAF-R001", Location(row.part, None, None, row.table.path), transaction=number, debit=lines.debit, credit=lines.credit
+                )
+
+        for rule in self._rules:
+            rule.on_row(row, self._diagnostics)
+
+    def on_end(self) -> None:
+        pass
+
+    def _add_rule(self, schema: AuditfileSchema, child: str, owner: str, per_parent: bool, count: str, debit: str, credit: str) -> None:
+        lines = _Lines.find(schema.find_table(child))
+        owner_table = schema.find_table(owner)
+        if lines is None or owner_table is None:
+            return
+
+        count_column = owner_table.find_column(count)
+        debit_column = owner_table.find_column(debit)
+        credit_column = owner_table.find_column(credit)
+        if (
+            count_column is None
+            or count_column.type is not ColumnType.INTEGER
+            or debit_column is None
+            or debit_column.type is not ColumnType.DECIMAL
+            or credit_column is None
+            or credit_column.type is not ColumnType.DECIMAL
+        ):
+            return
+
+        self._rules.append(_TotalsRule(lines, owner_table, per_parent, count_column, debit_column, credit_column))

xml_auditfile/_decimal.py

@@ -0,0 +1,44 @@
+"""Exact decimals in XSD notation, within the range of .NET decimal so every implementation accepts the same values."""
+
+from __future__ import annotations
+
+import decimal
+import re
+from decimal import Decimal
+
+_PATTERN = re.compile(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)")
+_MAX_SCALE = 28
+_MAX = Decimal("79228162514264337593543950335")
+
+# Additions never round: an inexact result raises instead of silently losing a cent.
+EXACT = decimal.Context(prec=200, traps=[decimal.Inexact, decimal.InvalidOperation, decimal.Overflow])
+
+
+def parse_decimal(text: str) -> Decimal | None:
+    """Parses XSD notation (sign, digits, optional point; no exponent). Returns None when invalid or out of range."""
+    if _PATTERN.fullmatch(text) is None:
+        return None
+
+    fraction = text.partition(".")[2]
+    if len(fraction) > _MAX_SCALE:
+        return None
+
+    digits = text.lstrip("+-").replace(".", "").lstrip("0")
+    if len(digits) > 29:
+        return None
+
+    value = Decimal(text)
+    if abs(value) > _MAX:
+        return None
+
+    return value.copy_abs() if value.is_zero() else value
+
+
+def add(a: Decimal, b: Decimal) -> Decimal:
+    return EXACT.add(a, b)
+
+
+def format_decimal(value: Decimal) -> str:
+    """Invariant notation with the scale as written, never scientific: 100.00, -0.0000001."""
+    text = format(value, "f")
+    return text[1:] if text.startswith("-") and value.is_zero() else text

xml_auditfile/_frames.py

@@ -0,0 +1,147 @@
+"""pandas and polars DataFrames from tables and views. Both are optional:
+``pip install xml-auditfile[pandas]`` or ``pip install xml-auditfile[polars]``."""
+
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import Iterable, Sequence
+from decimal import Decimal
+from typing import TYPE_CHECKING, Any, Literal
+
+from .definitions import ColumnType
+
+if TYPE_CHECKING:
+    import pandas
+    import polars
+
+Decimals = Literal["decimal", "float"]
+"""How decimal columns end up in a DataFrame: exact decimals or float64."""
+
+Columns = Sequence[tuple[str, "ColumnType | None"]]
+"""(name, type) per column; a type of None is inferred from the values."""
+
+
+def _module(name: str, method: str) -> Any:
+    try:
+        return __import__(name)
+    except ImportError as error:
+        raise ImportError(f"{method} needs {name}: pip install xml-auditfile[{name}]") from error
+
+
+def _infer(values: Sequence[object]) -> ColumnType | None:
+    """The column type of view values, from the first value that is not None; None when every value is None."""
+    for value in values:
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            return ColumnType.BOOLEAN
+        if isinstance(value, Decimal):
+            return ColumnType.DECIMAL
+        if isinstance(value, int):
+            return ColumnType.INTEGER
+        if isinstance(value, dt.datetime):
+            return ColumnType.DATETIME
+        if isinstance(value, dt.date):
+            return ColumnType.DATE
+        if isinstance(value, dt.time):
+            return ColumnType.TIME
+        return ColumnType.STRING
+
+    return None
+
+
+def _cells(columns: Columns, rows: Iterable[Sequence[object]], decimals: Decimals) -> list[tuple[str, ColumnType | None, list[object]]]:
+    """(name, type, values) per column, with the types of None inferred."""
+    if decimals not in ("decimal", "float"):
+        raise ValueError(f"decimals must be 'decimal' or 'float', not {decimals!r}.")
+
+    cells: list[list[object]] = [[] for _ in columns]
+    for row in rows:
+        for i, value in enumerate(row):
+            cells[i].append(value)
+
+    return [(name, type_ if type_ is not None else _infer(values), values) for (name, type_), values in zip(columns, cells)]
+
+
+def _is_aware(values: list[object]) -> bool:
+    # One time zone per column: with any aware value, everything is converted to UTC (naive values count as UTC).
+    return any(isinstance(v, dt.datetime) and v.tzinfo is not None for v in values)
+
+
+def _pandas_series(pd: Any, name: str, values: list[object], type_: ColumnType | None, decimals: Decimals) -> pandas.Series:
+    if type_ is ColumnType.STRING:
+        return pd.Series(values, name=name, dtype="string")
+
+    if type_ is ColumnType.INTEGER:
+        return pd.Series(values, name=name, dtype="Int64")
+
+    if type_ is ColumnType.BOOLEAN:
+        return pd.Series(values, name=name, dtype="boolean")
+
+    if type_ is ColumnType.DECIMAL:
+        if decimals == "float":
+            return pd.Series([None if v is None else float(v) for v in values], name=name, dtype="float64")  # type: ignore[arg-type]
+        return pd.Series(values, name=name, dtype="object")
+
+    if type_ is ColumnType.DATE:
+        return pd.Series(pd.to_datetime(values), name=name)
+
+    if type_ is ColumnType.DATETIME:
+        return pd.Series(pd.to_datetime(values, utc=_is_aware(values)), name=name)
+
+    return pd.Series(values, name=name, dtype="object")
+
+
+def pandas_frame(columns: Columns, rows: Iterable[Sequence[object]], decimals: Decimals, index: tuple[str, list[int]] | None = None) -> pandas.DataFrame:
+    """A pandas DataFrame; decimals stay Decimal objects (object dtype) unless decimals is "float"."""
+    cells = _cells(columns, rows, decimals)
+    pd = _module("pandas", "to_pandas")
+    result = pd.DataFrame({name: _pandas_series(pd, name, values, type_, decimals) for name, type_, values in cells},
+                          columns=[name for name, _ in columns])
+    if index is not None:
+        result.index = pd.Index(index[1], name=index[0], dtype="int64")
+
+    return result
+
+
+def _scale(values: list[object]) -> int:
+    """The largest number of decimals in the column, so the Decimal dtype holds every value exactly."""
+    scales = [-v.as_tuple().exponent for v in values if isinstance(v, Decimal) and isinstance(v.as_tuple().exponent, int)]
+    return max([0, *scales])
+
+
+def _polars_series(pl: Any, name: str, values: list[object], type_: ColumnType | None, decimals: Decimals) -> polars.Series:
+    if type_ is ColumnType.STRING:
+        return pl.Series(name, values, dtype=pl.String)
+
+    if type_ is ColumnType.INTEGER:
+        return pl.Series(name, values, dtype=pl.Int64)
+
+    if type_ is ColumnType.BOOLEAN:
+        return pl.Series(name, values, dtype=pl.Boolean)
+
+    if type_ is ColumnType.DECIMAL:
+        if decimals == "float":
+            return pl.Series(name, [None if v is None else float(v) for v in values], dtype=pl.Float64)  # type: ignore[arg-type]
+        return pl.Series(name, values, dtype=pl.Decimal(38, _scale(values)))
+
+    if type_ is ColumnType.DATE:
+        return pl.Series(name, values, dtype=pl.Date)
+
+    if type_ is ColumnType.DATETIME:
+        if not _is_aware(values):
+            return pl.Series(name, values, dtype=pl.Datetime("us"))
+        utc = [v.astimezone(dt.timezone.utc).replace(tzinfo=None) if isinstance(v, dt.datetime) and v.tzinfo else v for v in values]
+        return pl.Series(name, utc, dtype=pl.Datetime("us")).dt.replace_time_zone("UTC")
+
+    if type_ is ColumnType.TIME:
+        return pl.Series(name, values, dtype=pl.Time)
+
+    return pl.Series(name, values, dtype=pl.Null)
+
+
+def polars_frame(columns: Columns, rows: Iterable[Sequence[object]], decimals: Decimals) -> polars.DataFrame:
+    """A polars DataFrame; decimals get an exact Decimal dtype unless decimals is "float"."""
+    cells = _cells(columns, rows, decimals)
+    pl = _module("polars", "to_polars")
+    return pl.DataFrame([_polars_series(pl, name, values, type_, decimals) for name, type_, values in cells])

xml_auditfile/_json.py

@@ -0,0 +1,92 @@
+"""Typed access to parsed JSON. Every accessor checks the type, so malformed input fails with a clear message."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+class JsonFormatError(ValueError):
+    """JSON that does not have the expected structure."""
+
+
+def parse_object(text: str, name: str) -> dict[str, Any]:
+    try:
+        value = json.loads(text)
+    except json.JSONDecodeError as error:
+        raise JsonFormatError(f"{name} is not valid JSON: {error}") from error
+
+    if not isinstance(value, dict):
+        raise JsonFormatError(f"{name} is not a JSON object.")
+
+    return value
+
+
+def _get(obj: dict[str, Any], key: str) -> Any:
+    return obj.get(key)
+
+
+def string(obj: dict[str, Any], key: str) -> str:
+    value = _get(obj, key)
+    if not isinstance(value, str):
+        raise JsonFormatError(f"JSON: '{key}' must be a string.")
+
+    return value
+
+
+def optional_string(obj: dict[str, Any], key: str) -> str | None:
+    return None if _get(obj, key) is None else string(obj, key)
+
+
+def integer(obj: dict[str, Any], key: str) -> int:
+    value = _get(obj, key)
+    if not isinstance(value, int) or isinstance(value, bool):
+        raise JsonFormatError(f"JSON: '{key}' must be an integer.")
+
+    return value
+
+
+def optional_integer(obj: dict[str, Any], key: str) -> int | None:
+    return None if _get(obj, key) is None else integer(obj, key)
+
+
+def boolean(obj: dict[str, Any], key: str) -> bool:
+    return _get(obj, key) is True
+
+
+def obj(parent: dict[str, Any], key: str) -> dict[str, Any] | None:
+    value = _get(parent, key)
+    if value is None:
+        return None
+
+    if not isinstance(value, dict):
+        raise JsonFormatError(f"JSON: '{key}' must be an object.")
+
+    return value
+
+
+def array(parent: dict[str, Any], key: str) -> list[Any]:
+    value = _get(parent, key)
+    if value is None:
+        return []
+
+    if not isinstance(value, list):
+        raise JsonFormatError(f"JSON: '{key}' must be an array.")
+
+    return value
+
+
+def objects(parent: dict[str, Any], key: str) -> list[dict[str, Any]]:
+    items = array(parent, key)
+    if not all(isinstance(item, dict) for item in items):
+        raise JsonFormatError(f"JSON: '{key}' must contain objects.")
+
+    return items
+
+
+def strings(parent: dict[str, Any], key: str) -> list[str]:
+    items = array(parent, key)
+    if not all(isinstance(item, str) for item in items):
+        raise JsonFormatError(f"JSON: '{key}' must contain strings.")
+
+    return items

xml_auditfile/_keys.py

@@ -0,0 +1,4 @@
+"""Joins the values of composite keys, view groups and joins."""
+
+SEPARATOR = "\u001f"
+"""A character that cannot occur in XML text."""