tallyd 0.0.0-stage → 0.1.0

package/dist/statement/columns.js

@@ -0,0 +1,221 @@
+/**
+ * Working out which column is which.
+ *
+ * Header names are a hint, not an answer: banks ship `Transaction Date`,
+ * `Datum`, `Post Date`, `Value Date` and sometimes nothing at all. So each
+ * column is scored twice — once on its name, once on what its cells actually
+ * look like — and the content score is what saves an unlabelled export.
+ */
+import { looksLikeAmount } from "./amount.js";
+import { looksLikeDate } from "./dates.js";
+/** Header aliases, lowercased. Longer, more specific names come first. */
+const NAME_HINTS = [
+    {
+        role: "valueDate",
+        patterns: [/^value\s*date$/, /^val\s*date$/, /^settle(ment)?\s*date$/, /^valuta/],
+        weight: 10,
+    },
+    {
+        role: "date",
+        patterns: [
+            /^(transaction|posting|post|booking|entry|trans)\s*date$/,
+            /^date$/,
+            /^datum$/,
+            /^fecha$/,
+            /^date\s*of\s*transaction$/,
+        ],
+        weight: 10,
+    },
+    {
+        role: "debit",
+        patterns: [/^debit/, /^paid\s*out$/, /^money\s*out$/, /^withdrawal/, /^afschrijving/, /^soll$/],
+        weight: 10,
+    },
+    {
+        role: "credit",
+        patterns: [/^credit/, /^paid\s*in$/, /^money\s*in$/, /^deposit/, /^bijschrijving/, /^haben$/],
+        weight: 10,
+    },
+    {
+        role: "balance",
+        patterns: [/balance/, /^saldo$/, /^running\s*total$/],
+        weight: 10,
+    },
+    {
+        role: "amount",
+        patterns: [/^amount$/, /^value$/, /^bedrag$/, /^betrag$/, /^importe$/, /amount/],
+        weight: 8,
+    },
+    {
+        role: "description",
+        patterns: [
+            /^description$/,
+            /^narrative$/,
+            /^details?$/,
+            /^payee$/,
+            /^memo$/,
+            /^particulars$/,
+            /^omschrijving$/,
+            /^merchant$/,
+            /description|narrative|details/,
+        ],
+        weight: 8,
+    },
+    {
+        role: "reference",
+        patterns: [/^reference$/, /^ref$/, /^transaction\s*id$/, /^cheque\s*(no|number)$/, /ref/],
+        weight: 7,
+    },
+    {
+        role: "currency",
+        patterns: [/^currency$/, /^ccy$/, /^valuta\s*code$/],
+        weight: 9,
+    },
+    {
+        role: "type",
+        patterns: [/^type$/, /^transaction\s*type$/, /^tx\s*type$/, /^code$/],
+        weight: 7,
+    },
+];
+function nameScore(header, role) {
+    const name = header.trim().toLowerCase().replace(/[_.]/g, " ").replace(/\s+/g, " ");
+    if (name === "")
+        return 0;
+    for (const hint of NAME_HINTS) {
+        if (hint.role !== role)
+            continue;
+        for (let i = 0; i < hint.patterns.length; i++) {
+            const pattern = hint.patterns[i];
+            if (pattern.test(name)) {
+                // Earlier patterns are the more exact ones.
+                return hint.weight - i * 0.1;
+            }
+        }
+    }
+    return 0;
+}
+function fraction(values, predicate) {
+    const nonEmpty = values.filter((v) => v.trim() !== "");
+    if (nonEmpty.length === 0)
+        return 0;
+    return nonEmpty.filter(predicate).length / nonEmpty.length;
+}
+function contentScore(values, role) {
+    switch (role) {
+        case "date":
+        case "valueDate":
+            return fraction(values, looksLikeDate) * 8;
+        case "amount":
+        case "debit":
+        case "credit":
+        case "balance": {
+            const numeric = fraction(values, looksLikeAmount);
+            const dateish = fraction(values, looksLikeDate);
+            return Math.max(0, numeric - dateish) * 6;
+        }
+        case "description": {
+            const wordy = fraction(values, (v) => !looksLikeAmount(v) && !looksLikeDate(v) && v.trim().length >= 3);
+            return wordy * 5;
+        }
+        case "currency":
+            return fraction(values, (v) => /^[A-Za-z]{3}$/.test(v.trim())) * 6;
+        default:
+            return 0;
+    }
+}
+/** Roles at most one column may hold. */
+const EXCLUSIVE = [
+    "date",
+    "valueDate",
+    "description",
+    "amount",
+    "debit",
+    "credit",
+    "balance",
+    "reference",
+    "currency",
+    "type",
+];
+/**
+ * Assign roles to columns.
+ *
+ * Greedy over the highest-scoring (column, role) pairs: each column takes one
+ * role and each role one column. Greedy is enough here because the scores are
+ * well separated in practice, and it is explainable — every assignment carries
+ * the score that won it, which matters when a user asks why their file was
+ * read the way it was.
+ */
+export function inferColumns(header, rows, overrides = {}) {
+    const columnCount = Math.max(header.length, ...rows.map((r) => r.length), 0);
+    const columns = [];
+    for (let i = 0; i < columnCount; i++) {
+        columns.push(rows.map((r) => r[i] ?? ""));
+    }
+    const scored = [];
+    for (let index = 0; index < columnCount; index++) {
+        const name = header[index] ?? "";
+        const values = columns[index];
+        for (const role of EXCLUSIVE) {
+            const score = nameScore(name, role) + contentScore(values, role);
+            if (score > 0)
+                scored.push({ index, role, score });
+        }
+    }
+    scored.sort((a, b) => (b.score === a.score ? a.index - b.index : b.score - a.score));
+    const roleToColumn = new Map();
+    const columnToRole = new Map();
+    const scoreByColumn = new Map();
+    for (const [role, index] of Object.entries(overrides)) {
+        roleToColumn.set(role, index);
+        columnToRole.set(index, role);
+        scoreByColumn.set(index, Number.POSITIVE_INFINITY);
+    }
+    for (const candidate of scored) {
+        if (roleToColumn.has(candidate.role))
+            continue;
+        if (columnToRole.has(candidate.index))
+            continue;
+        roleToColumn.set(candidate.role, candidate.index);
+        columnToRole.set(candidate.index, candidate.role);
+        scoreByColumn.set(candidate.index, candidate.score);
+    }
+    const assignments = [];
+    for (let index = 0; index < columnCount; index++) {
+        assignments.push(Object.freeze({
+            index,
+            header: header[index] ?? "",
+            role: columnToRole.get(index) ?? "unknown",
+            score: scoreByColumn.get(index) ?? 0,
+        }));
+    }
+    const at = (role) => roleToColumn.get(role) ?? null;
+    const warnings = [];
+    const amount = at("amount");
+    const debit = at("debit");
+    const credit = at("credit");
+    if (amount === null && (debit === null || credit === null)) {
+        warnings.push("No amount column found, and no debit/credit pair either — rows cannot be valued");
+    }
+    if (amount !== null && debit !== null && credit !== null) {
+        warnings.push("Found both an amount column and a debit/credit pair; the amount column wins");
+    }
+    if (at("date") === null)
+        warnings.push("No date column found");
+    if (at("description") === null) {
+        warnings.push("No description column found; matching will rely on amount and date alone");
+    }
+    return Object.freeze({
+        assignments: Object.freeze(assignments),
+        date: at("date"),
+        valueDate: at("valueDate"),
+        description: at("description"),
+        amount,
+        debit,
+        credit,
+        balance: at("balance"),
+        reference: at("reference"),
+        currency: at("currency"),
+        type: at("type"),
+        warnings: Object.freeze(warnings),
+    });
+}

package/dist/statement/csv.d.ts

@@ -0,0 +1,50 @@
+/**
+ * A CSV reader for files exported by banks, which is a narrower and stranger
+ * problem than "parse CSV".
+ *
+ * Real exports arrive with a UTF-8 BOM, CRLF endings, a preamble of two or
+ * three junk lines before the header, semicolon delimiters from European
+ * banks, and quoted fields containing the delimiter. None of that is exotic;
+ * all of it breaks a `line.split(",")`.
+ *
+ * The tokenizer follows RFC 4180 with one deliberate relaxation: a bare quote
+ * inside an unquoted field is treated as data rather than an error, because
+ * descriptions like `JOHN'S 24" TV` appear in the wild and rejecting the whole
+ * import over one is not a service to anybody.
+ */
+export declare const DELIMITERS: readonly [",", ";", "\t", "|"];
+export type Delimiter = (typeof DELIMITERS)[number];
+export declare class CsvParseError extends Error {
+    readonly line: number;
+    constructor(message: string, line: number);
+}
+/** Strip a UTF-8 byte-order mark, which Excel adds and nothing else wants. */
+export declare function stripBom(text: string): string;
+/**
+ * Split into records. Quoted fields may span physical lines, so this cannot be
+ * done by splitting on newlines first.
+ */
+export declare function parseCsv(input: string, delimiter?: Delimiter): string[][];
+/**
+ * Guess the delimiter by looking for the one that gives a consistent column
+ * count across the most lines. Counting occurrences on line one alone gets
+ * this wrong whenever a description contains a comma.
+ */
+export declare function sniffDelimiter(input: string): Delimiter;
+export interface CsvTable {
+    readonly header: readonly string[];
+    readonly rows: readonly (readonly string[])[];
+    readonly delimiter: Delimiter;
+    /** How many leading junk lines were skipped before the header. */
+    readonly preambleLines: number;
+}
+/**
+ * Read a table, locating the header row rather than assuming it is first.
+ *
+ * Banks routinely prefix an export with the account name, the statement period
+ * and a blank line. The header is taken to be the first row whose width equals
+ * the modal width of the file — a preamble line is almost always narrower.
+ */
+export declare function readTable(input: string, options?: {
+    delimiter?: Delimiter;
+}): CsvTable;

package/dist/statement/csv.js

@@ -0,0 +1,185 @@
+/**
+ * A CSV reader for files exported by banks, which is a narrower and stranger
+ * problem than "parse CSV".
+ *
+ * Real exports arrive with a UTF-8 BOM, CRLF endings, a preamble of two or
+ * three junk lines before the header, semicolon delimiters from European
+ * banks, and quoted fields containing the delimiter. None of that is exotic;
+ * all of it breaks a `line.split(",")`.
+ *
+ * The tokenizer follows RFC 4180 with one deliberate relaxation: a bare quote
+ * inside an unquoted field is treated as data rather than an error, because
+ * descriptions like `JOHN'S 24" TV` appear in the wild and rejecting the whole
+ * import over one is not a service to anybody.
+ */
+export const DELIMITERS = [",", ";", "\t", "|"];
+export class CsvParseError extends Error {
+    line;
+    constructor(message, line) {
+        super(`${message} (line ${line})`);
+        this.line = line;
+        this.name = "CsvParseError";
+    }
+}
+/** Strip a UTF-8 byte-order mark, which Excel adds and nothing else wants. */
+export function stripBom(text) {
+    return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+}
+/**
+ * Split into records. Quoted fields may span physical lines, so this cannot be
+ * done by splitting on newlines first.
+ */
+export function parseCsv(input, delimiter = ",") {
+    const text = stripBom(input);
+    const rows = [];
+    let row = [];
+    let field = "";
+    let inQuotes = false;
+    let fieldWasQuoted = false;
+    let line = 1;
+    let sawAnyContent = false;
+    const endField = () => {
+        row.push(fieldWasQuoted ? field : field.trim());
+        field = "";
+        fieldWasQuoted = false;
+    };
+    const endRow = () => {
+        endField();
+        // Drop rows that are entirely empty — trailing newlines, blank separators.
+        if (row.some((f) => f !== ""))
+            rows.push(row);
+        row = [];
+    };
+    for (let i = 0; i < text.length; i++) {
+        const char = text[i];
+        if (inQuotes) {
+            if (char === '"') {
+                if (text[i + 1] === '"') {
+                    field += '"';
+                    i++;
+                }
+                else {
+                    inQuotes = false;
+                }
+            }
+            else {
+                if (char === "\n")
+                    line++;
+                field += char;
+            }
+            continue;
+        }
+        if (char === '"' && field.trim() === "") {
+            // A quote only opens a quoted field at the start of one; elsewhere it is
+            // literal data.
+            inQuotes = true;
+            fieldWasQuoted = true;
+            field = "";
+            sawAnyContent = true;
+            continue;
+        }
+        if (char === delimiter) {
+            endField();
+            sawAnyContent = true;
+            continue;
+        }
+        if (char === "\r") {
+            if (text[i + 1] === "\n")
+                i++;
+            line++;
+            endRow();
+            continue;
+        }
+        if (char === "\n") {
+            line++;
+            endRow();
+            continue;
+        }
+        field += char;
+        if (char.trim() !== "")
+            sawAnyContent = true;
+    }
+    if (inQuotes)
+        throw new CsvParseError("Unterminated quoted field", line);
+    if (field !== "" || row.length > 0)
+        endRow();
+    if (!sawAnyContent)
+        return [];
+    return rows;
+}
+/**
+ * Guess the delimiter by looking for the one that gives a consistent column
+ * count across the most lines. Counting occurrences on line one alone gets
+ * this wrong whenever a description contains a comma.
+ */
+export function sniffDelimiter(input) {
+    const sample = stripBom(input).split(/\r?\n/).filter((l) => l.trim() !== "").slice(0, 20);
+    if (sample.length === 0)
+        return ",";
+    let best = ",";
+    let bestScore = -1;
+    for (const delimiter of DELIMITERS) {
+        let rows;
+        try {
+            rows = parseCsv(sample.join("\n"), delimiter);
+        }
+        catch {
+            continue;
+        }
+        if (rows.length === 0)
+            continue;
+        const counts = rows.map((r) => r.length);
+        const maxColumns = Math.max(...counts);
+        if (maxColumns < 2)
+            continue;
+        // Reward consistency and column count; a delimiter that never appears
+        // yields one column everywhere and scores zero.
+        const modal = mode(counts);
+        const consistent = counts.filter((c) => c === modal).length / counts.length;
+        const score = consistent * 10 + modal;
+        if (score > bestScore) {
+            bestScore = score;
+            best = delimiter;
+        }
+    }
+    return best;
+}
+function mode(values) {
+    const tally = new Map();
+    for (const v of values)
+        tally.set(v, (tally.get(v) ?? 0) + 1);
+    let best = values[0] ?? 0;
+    let bestCount = 0;
+    for (const [value, count] of tally) {
+        if (count > bestCount || (count === bestCount && value > best)) {
+            best = value;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+/**
+ * Read a table, locating the header row rather than assuming it is first.
+ *
+ * Banks routinely prefix an export with the account name, the statement period
+ * and a blank line. The header is taken to be the first row whose width equals
+ * the modal width of the file — a preamble line is almost always narrower.
+ */
+export function readTable(input, options = {}) {
+    const delimiter = options.delimiter ?? sniffDelimiter(input);
+    const rows = parseCsv(input, delimiter);
+    if (rows.length === 0) {
+        return { header: [], rows: [], delimiter, preambleLines: 0 };
+    }
+    const widths = rows.map((r) => r.length);
+    const modal = mode(widths);
+    const headerIndex = rows.findIndex((r) => r.length === modal);
+    const header = rows[headerIndex].map((h) => h.trim());
+    const body = rows.slice(headerIndex + 1).filter((r) => r.some((f) => f.trim() !== ""));
+    return {
+        header,
+        rows: body,
+        delimiter,
+        preambleLines: headerIndex,
+    };
+}

package/dist/statement/dates.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Reading the date column.
+ *
+ * `03/04/2026` is the third of April in London and the fourth of March in New
+ * York, and a bank export rarely says which it means. Guessing per row is the
+ * worst option: it produces a statement where some rows are right and some are
+ * silently six weeks out.
+ *
+ * So the format is decided once for the whole column. A single value with a
+ * component above 12 settles it for every other row. Where the column is
+ * genuinely ambiguous — every day-of-month is 12 or under — the caller is told
+ * so and can insist on a format rather than accept a coin flip.
+ */
+import { type CalendarDate } from "../ledger/date.js";
+export type DateFormat = "YYYY-MM-DD" | "DD/MM/YYYY" | "MM/DD/YYYY" | "DD-MM-YYYY" | "MM-DD-YYYY" | "DD.MM.YYYY" | "YYYY/MM/DD" | "DD-MMM-YYYY" | "MMM DD, YYYY";
+export declare const DATE_FORMATS: readonly DateFormat[];
+export declare class DateParseError extends Error {
+    readonly input: string;
+    readonly format: DateFormat | null;
+    constructor(input: string, format: DateFormat | null);
+}
+/** Parse one value in a known format. */
+export declare function parseDateIn(input: string, format: DateFormat): CalendarDate;
+export declare function canParseIn(input: string, format: DateFormat): boolean;
+export interface DateFormatDetection {
+    readonly format: DateFormat;
+    /** Every format that parses all the samples. More than one means ambiguity. */
+    readonly candidates: readonly DateFormat[];
+    /** False when day/month order could not be settled from the data. */
+    readonly confident: boolean;
+    /** Samples no candidate format could read. */
+    readonly unparsed: readonly string[];
+}
+/**
+ * Work out which format a column of dates is in.
+ *
+ * A format survives only if it reads *every* sample. Where both DD/MM and
+ * MM/DD survive, the column contains no value with a component above 12 and
+ * the order is genuinely undecidable from the data alone.
+ */
+export declare function detectDateFormat(samples: readonly string[], options?: {
+    prefer?: DateFormat;
+}): DateFormatDetection;
+/** Parse a column, detecting the format from the column itself. */
+export declare function parseDateColumn(values: readonly string[], options?: {
+    prefer?: DateFormat;
+}): {
+    dates: readonly (CalendarDate | null)[];
+    detection: DateFormatDetection;
+};
+/** True when a string parses as a date in any supported format. */
+export declare function looksLikeDate(input: string): boolean;