paradoxdb 0.1.0__py3-none-any.whl

paradoxdb/__init__.py

@@ -0,0 +1,94 @@
+"""paradoxdb: pure-Python reader and writer for Paradox tables (.DB / .MB / .PX).
+
+Quick start::
+
+    from paradoxdb import open_table
+
+    table = open_table("ORDERS.DB")
+    for record in table.records():
+        cell = record["Amount"]
+        if cell.is_ok:
+            print(cell.value)
+        else:
+            print(cell.state, cell.reason, cell.raw_bytes.hex())
+"""
+
+from __future__ import annotations
+
+from .blocks import Anomaly
+from .errors import (
+    DuplicateKeyError,
+    EncodingError,
+    EncryptedTableError,
+    FormatError,
+    ParadoxError,
+    WriteError,
+)
+from .fields import FieldDef, FieldType
+from .header import TableHeader, parse_header
+from .index import DataBlock, build_index, read_data_blocks, verify_index
+from .memo import MemoFile, MemoPointer
+from .profile import TableProfile, export_csv, export_json, profile_table
+from .table import Decryptor, ParadoxTable, Record, open_table
+from .values import (
+    CellValue,
+    ValueState,
+    decode_value,
+    encode_alpha,
+    encode_bcd,
+    encode_bool,
+    encode_date,
+    encode_double,
+    encode_int,
+    encode_null,
+    encode_time,
+    encode_timestamp,
+)
+from .writer import WriteRecord, WriteResult, legacy_sort_key, write_table
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Anomaly",
+    "CellValue",
+    "DataBlock",
+    "Decryptor",
+    "DuplicateKeyError",
+    "EncodingError",
+    "EncryptedTableError",
+    "FieldDef",
+    "FieldType",
+    "FormatError",
+    "MemoFile",
+    "MemoPointer",
+    "ParadoxError",
+    "ParadoxTable",
+    "Record",
+    "TableHeader",
+    "TableProfile",
+    "ValueState",
+    "WriteError",
+    "WriteRecord",
+    "WriteResult",
+    "__version__",
+    "build_index",
+    "decode_value",
+    "encode_alpha",
+    "encode_bcd",
+    "encode_bool",
+    "encode_date",
+    "encode_double",
+    "encode_int",
+    "encode_null",
+    "encode_time",
+    "encode_timestamp",
+    "export_csv",
+    "export_json",
+    "legacy_sort_key",
+    "open_table",
+    "parse_header",
+    "profile_table",
+    "read_data_blocks",
+    "verify_index",
+    "write_table",
+]

paradoxdb/__main__.py

@@ -0,0 +1,10 @@
+"""Allow ``python -m paradoxdb ...`` as an alias of the ``paradoxdb`` console script."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

paradoxdb/blocks.py

@@ -0,0 +1,138 @@
+"""Data blocks of ``.DB`` and ``.PX`` files: walking the chain and packing records.
+
+Blocks are numbered from 1 and start at ``header_size + (number - 1) * block_size``.
+Each block begins with a 6-byte header: ``next`` (u16), ``previous`` (u16) and
+the offset of the last record relative to the start of the block data (i16).
+An empty block stores ``-record_size`` there.
+"""
+
+from __future__ import annotations
+
+import struct
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+
+from .header import TableHeader
+
+__all__ = [
+    "BLOCK_HEADER_SIZE",
+    "Anomaly",
+    "RawBlock",
+    "iter_blocks",
+    "pack_block_groups",
+    "pack_blocks",
+]
+
+BLOCK_HEADER_SIZE = 6
+_BLOCK_HEADER = struct.Struct("<HHh")
+MAX_BLOCKS = 0xFFFF
+
+
+@dataclass(frozen=True, slots=True)
+class Anomaly:
+    """Something odd found while walking the block chain. Never fatal."""
+
+    block_no: int
+    message: str
+
+
+@dataclass(frozen=True, slots=True)
+class RawBlock:
+    number: int
+    next_block: int
+    prev_block: int
+    last_offset: int
+    records: tuple[bytes, ...]
+
+    @property
+    def record_count(self) -> int:
+        return len(self.records)
+
+
+def block_start(header: TableHeader, number: int) -> int:
+    return header.header_size + (number - 1) * header.block_size
+
+
+def iter_blocks(data: bytes, header: TableHeader, anomalies: list[Anomaly]) -> Iterator[RawBlock]:
+    """Follow the block chain from the header's first block.
+
+    Every record present in a block is yielded, whatever the declared record
+    count says. Cycles, blocks outside the file and blocks declaring more
+    records than they can hold are appended to ``anomalies`` (the latter count
+    as empty); none of them aborts the walk.
+    """
+    number = header.first_block
+    seen: set[int] = set()
+    record_size = header.record_size
+    capacity = header.block_capacity
+    while number:
+        if number in seen:
+            anomalies.append(Anomaly(number, "block chain loops back to an earlier block"))
+            return
+        seen.add(number)
+        start = block_start(header, number)
+        end = start + header.block_size
+        if start < header.header_size or end > len(data):
+            anomalies.append(Anomaly(number, "block lies outside the file"))
+            return
+        next_block, prev_block, last_offset = _BLOCK_HEADER.unpack_from(data, start)
+        if last_offset < 0:
+            count = 0
+        else:
+            count = last_offset // record_size + 1
+            if count > capacity:
+                anomalies.append(
+                    Anomaly(
+                        number,
+                        f"block declares {count} records but can hold {capacity}; counted as 0",
+                    )
+                )
+                count = 0
+        base = start + BLOCK_HEADER_SIZE
+        records = tuple(
+            bytes(data[base + i * record_size : base + (i + 1) * record_size]) for i in range(count)
+        )
+        yield RawBlock(number, next_block, prev_block, last_offset, records)
+        number = next_block
+
+
+def pack_blocks(records: Sequence[bytes], record_size: int, block_size: int) -> tuple[bytes, int]:
+    """Lay ``records`` out in a chain of full-size blocks.
+
+    Returns the block bytes and the number of blocks. An empty record set still
+    produces one empty block, as Paradox does.
+    """
+    capacity = (block_size - BLOCK_HEADER_SIZE) // record_size
+    if capacity < 1:
+        raise ValueError("record does not fit in a block")
+    groups = [records[i : i + capacity] for i in range(0, len(records), capacity)] or [()]
+    return pack_block_groups(groups, record_size, block_size)
+
+
+def pack_block_groups(
+    groups: Sequence[Sequence[bytes]], record_size: int, block_size: int
+) -> tuple[bytes, int]:
+    """Write one block per group, chained in order; groups must respect the block capacity."""
+    capacity = (block_size - BLOCK_HEADER_SIZE) // record_size
+    if capacity < 1:
+        raise ValueError("record does not fit in a block")
+    if not groups:
+        groups = [()]
+    if len(groups) > MAX_BLOCKS:
+        raise ValueError("too many blocks for a 16-bit block number")
+    out = bytearray()
+    total = len(groups)
+    for i, chunk in enumerate(groups, start=1):
+        if len(chunk) > capacity:
+            raise ValueError(f"{len(chunk)} records exceed the block capacity of {capacity}")
+        block = bytearray(block_size)
+        last_offset = (len(chunk) - 1) * record_size if chunk else -record_size
+        _BLOCK_HEADER.pack_into(block, 0, i + 1 if i < total else 0, i - 1, last_offset)
+        pos = BLOCK_HEADER_SIZE
+        for rec in chunk:
+            if len(rec) != record_size:
+                raise ValueError(f"record of {len(rec)} bytes, expected {record_size}")
+            block[pos : pos + record_size] = rec
+            pos += record_size
+        out += block
+    return bytes(out), total

paradoxdb/cli.py

@@ -0,0 +1,140 @@
+"""Command line interface: ``paradoxdb profile | export | verify-index``."""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+from collections.abc import Sequence
+from importlib.metadata import entry_points
+from pathlib import Path
+
+from . import __version__
+from .errors import ParadoxError
+from .index import read_data_blocks, verify_index
+from .profile import export_csv, export_json, profile_directory
+from .table import Decryptor, find_sibling
+
+DECRYPTOR_ENTRY_POINT_GROUP = "paradoxdb.decryptors"
+
+
+def load_decryptor(spec: str | None) -> Decryptor:
+    """Instantiate a decryptor from ``module:attribute`` or from the first registered entry point.
+
+    The attribute may be a ready object or a zero-argument factory (class or
+    function) returning one.
+    """
+    if spec:
+        module_name, _, attr = spec.partition(":")
+        if not attr:
+            raise SystemExit("--decrypt expects MODULE:ATTRIBUTE")
+        target = getattr(importlib.import_module(module_name), attr)
+    else:
+        candidates = list(entry_points(group=DECRYPTOR_ENTRY_POINT_GROUP))
+        if not candidates:
+            raise SystemExit(
+                f"no decryptor installed: register one under the {DECRYPTOR_ENTRY_POINT_GROUP!r} "
+                "entry-point group or pass --decrypt MODULE:ATTRIBUTE"
+            )
+        target = candidates[0].load()
+    instance = target() if callable(target) and not hasattr(target, "decrypt") else target
+    if not hasattr(instance, "decrypt"):
+        raise SystemExit(f"{spec or 'entry point'} does not provide a decrypt() method")
+    return instance
+
+
+def _cmd_profile(args: argparse.Namespace) -> int:
+    decryptor = load_decryptor(args.decrypt) if args.decrypt is not None else None
+    profiles, errors = profile_directory(
+        args.directory, decryptor=decryptor, default_codec=args.codec
+    )
+    payload = {
+        "tool": f"paradoxdb {__version__}",
+        "directory": str(args.directory),
+        "tables": [p.to_dict() for p in profiles],
+        "errors": errors,
+    }
+    text = json.dumps(payload, indent=2, ensure_ascii=False)
+    if args.output:
+        Path(args.output).write_text(text, encoding="utf-8")
+        print(f"{len(profiles)} table(s) profiled, {len(errors)} error(s) -> {args.output}")
+    else:
+        print(text)
+    return 1 if errors and not profiles else 0
+
+
+def _cmd_export(args: argparse.Namespace) -> int:
+    decryptor = load_decryptor(args.decrypt) if args.decrypt is not None else None
+    table = Path(args.table)
+    output = Path(args.output) if args.output else table.with_suffix(f".{args.format}")
+    if args.format == "csv":
+        count = export_csv(table, output, decryptor=decryptor)
+    else:
+        count = export_json(table, output, decryptor=decryptor)
+    print(f"{count} record(s) -> {output}")
+    return 0
+
+
+def _cmd_verify_index(args: argparse.Namespace) -> int:
+    table = Path(args.table)
+    px = find_sibling(table, ".PX")
+    if px is None:
+        print(f"{table.name}: no .PX file found next to the table", file=sys.stderr)
+        return 2
+    problems = verify_index(px, read_data_blocks(table))
+    if problems:
+        print(f"{px.name}: {len(problems)} problem(s)")
+        for problem in problems:
+            print(f"  - {problem}")
+        return 1
+    print(f"{px.name}: index is consistent with {table.name}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="paradoxdb", description="Read, profile and check Paradox tables."
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("profile", help="aggregate every .DB in a directory into a JSON profile")
+    p.add_argument("directory", type=Path)
+    p.add_argument("--output", "-o", help="write the JSON here instead of stdout")
+    p.add_argument(
+        "--decrypt",
+        nargs="?",
+        const="",
+        default=None,
+        metavar="MODULE:ATTR",
+        help="use an external decryptor (entry point if no value is given)",
+    )
+    p.add_argument("--codec", default="cp437", help="fallback code page (default cp437)")
+    p.set_defaults(func=_cmd_profile)
+
+    e = sub.add_parser("export", help="dump a table with value, state and reason per cell")
+    e.add_argument("table", help="path to the .DB file")
+    e.add_argument("--format", "-f", choices=("csv", "json"), default="csv")
+    e.add_argument("--output", "-o", help="destination file (default: table name + format)")
+    e.add_argument("--decrypt", nargs="?", const="", default=None, metavar="MODULE:ATTR")
+    e.set_defaults(func=_cmd_export)
+
+    v = sub.add_parser("verify-index", help="check the .PX of a table against its data blocks")
+    v.add_argument("table", help="path to the .DB file")
+    v.set_defaults(func=_cmd_verify_index)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return args.func(args)
+    except ParadoxError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

paradoxdb/errors.py

@@ -0,0 +1,27 @@
+"""Exception hierarchy of :mod:`paradoxdb`."""
+
+from __future__ import annotations
+
+
+class ParadoxError(Exception):
+    """Base class for every error raised by the library."""
+
+
+class FormatError(ParadoxError):
+    """The file does not look like a coherent Paradox table or index."""
+
+
+class EncryptedTableError(ParadoxError):
+    """The table is encrypted and no decryptor was supplied (or writing was requested)."""
+
+
+class WriteError(ParadoxError):
+    """A record set cannot be written on top of the given template."""
+
+
+class DuplicateKeyError(WriteError):
+    """Two records share the same primary key."""
+
+
+class EncodingError(ParadoxError):
+    """A Python value cannot be represented in the requested Paradox field."""

paradoxdb/fields.py

@@ -0,0 +1,132 @@
+"""Field types, field definitions and code pages of Paradox tables."""
+
+from __future__ import annotations
+
+import codecs
+from dataclasses import dataclass
+from enum import IntEnum
+
+__all__ = ["CODE_PAGES", "FieldDef", "FieldType", "codec_for_code_page", "physical_size"]
+
+
+class FieldType(IntEnum):
+    """Field type codes as stored in the 2-byte field descriptors."""
+
+    ALPHA = 0x01
+    DATE = 0x02
+    SHORT = 0x03
+    LONG = 0x04
+    CURRENCY = 0x05
+    NUMBER = 0x06
+    LOGICAL = 0x09
+    MEMO = 0x0C
+    BLOB = 0x0D
+    FORMATTED_MEMO = 0x0E
+    OLE = 0x0F
+    GRAPHIC = 0x10
+    TIME = 0x14
+    TIMESTAMP = 0x15
+    AUTOINCREMENT = 0x16
+    BCD = 0x17
+    BYTES = 0x18
+
+    @property
+    def is_blob(self) -> bool:
+        """True for every type whose content lives (at least partly) in the ``.MB`` file."""
+        return self in _BLOB_TYPES
+
+    @property
+    def is_text_blob(self) -> bool:
+        """True for memo types, whose content is text decoded with the table code page."""
+        return self in (FieldType.MEMO, FieldType.FORMATTED_MEMO)
+
+    @property
+    def is_numeric(self) -> bool:
+        return self in _NUMERIC_TYPES
+
+    @property
+    def is_fixed_size(self) -> bool:
+        return self in _FIXED_SIZES
+
+
+_BLOB_TYPES = frozenset(
+    {FieldType.MEMO, FieldType.BLOB, FieldType.FORMATTED_MEMO, FieldType.OLE, FieldType.GRAPHIC}
+)
+_NUMERIC_TYPES = frozenset(
+    {
+        FieldType.SHORT,
+        FieldType.LONG,
+        FieldType.CURRENCY,
+        FieldType.NUMBER,
+        FieldType.AUTOINCREMENT,
+        FieldType.BCD,
+    }
+)
+_FIXED_SIZES: dict[FieldType, int] = {
+    FieldType.DATE: 4,
+    FieldType.SHORT: 2,
+    FieldType.LONG: 4,
+    FieldType.CURRENCY: 8,
+    FieldType.NUMBER: 8,
+    FieldType.LOGICAL: 1,
+    FieldType.TIME: 4,
+    FieldType.TIMESTAMP: 8,
+    FieldType.AUTOINCREMENT: 4,
+    FieldType.BCD: 17,
+}
+
+#: Length of the pointer that closes every BLOB-like field inside the record.
+BLOB_POINTER_SIZE = 10
+
+#: DOS / Windows code pages found in the header extension and their Python codecs.
+CODE_PAGES: dict[int, str] = {437: "cp437", 850: "cp850", 1252: "cp1252"}
+
+
+def codec_for_code_page(code_page: int | None, default: str) -> str:
+    """Return the Python codec for a Paradox code page, or ``default`` if unknown."""
+    if code_page is None:
+        return default
+    return CODE_PAGES.get(code_page, default)
+
+
+def physical_size(field_type: FieldType, size_byte: int) -> int:
+    """Number of bytes the field occupies in the record.
+
+    ``size_byte`` is the second byte of the descriptor: a length for alpha,
+    bytes and BLOB fields, the number of decimals for BCD, ignored otherwise.
+    """
+    fixed = _FIXED_SIZES.get(field_type)
+    if fixed is not None:
+        return fixed
+    return size_byte
+
+
+@dataclass(frozen=True, slots=True)
+class FieldDef:
+    """One column of the table, with its position inside the fixed-size record."""
+
+    name: str
+    type: FieldType
+    size_byte: int
+    offset: int
+    byte_length: int
+
+    @property
+    def decimals(self) -> int:
+        """Number of decimals of a BCD field (``size_byte`` for that type)."""
+        return self.size_byte if self.type is FieldType.BCD else 0
+
+    @property
+    def leader_length(self) -> int:
+        """Bytes of BLOB content kept inside the record, before the 10-byte pointer."""
+        if not self.type.is_blob:
+            return 0
+        return max(self.byte_length - BLOB_POINTER_SIZE, 0)
+
+    def slice(self, record: bytes) -> bytes:
+        return record[self.offset : self.offset + self.byte_length]
+
+
+def validate_codec(name: str) -> str:
+    """Normalise a codec name, raising ``LookupError`` if Python does not know it."""
+    return codecs.lookup(name).name