tallyd 0.0.0-stage → 0.1.0

package/dist/reconcile/memory.js

@@ -0,0 +1,281 @@
+/**
+ * What the reviewer already told us.
+ *
+ * Every month the same supplier arrives with a new reference. The bank writes
+ * `FPO ASHGROVE SUPPLIES 4471 BILL-3104` in March and `SO ASHGROVE SUPPLIES
+ * 8822 BILL-3391` in April; the books say `Payment — Ashgrove Supplies` both
+ * times. The description rule scores the two months identically, which means a
+ * pair a human confirmed in March lands in the review queue again in April, and
+ * again in May, for as long as anyone uses the tool.
+ *
+ * A confirmed match is a person stating a fact the matcher is trying and
+ * failing to infer: *these two counterparties are the same counterparty*. This
+ * remembers that fact.
+ *
+ * The key idea is what gets remembered. Not the transaction — that never
+ * repeats — but the **counterparty**, which is what is left of a description
+ * once the parts that change every month are removed. Anything carrying a digit
+ * is a reference, a terminal id or a date, so it goes; what remains is the
+ * name, and the name is stable on both sides.
+ *
+ * Three things are worth remembering, not one:
+ *
+ * - **A confirmation** is evidence for the pairing.
+ * - **A rejection** is evidence against, and matters more. Proposing a pair a
+ *   reviewer has already refused twice is worse than proposing nothing: it
+ *   teaches them the queue is not worth reading.
+ * - **A contradiction** — this bank name seen before, but always against a
+ *   different ledger counterparty — is evidence against too, and it is the
+ *   case that absence-of-memory would silently score as neutral.
+ *
+ * The store is immutable in the same way the ledger is: `learn` returns a new
+ * memory. A matcher run cannot mutate the memory it is reading from, which is
+ * what makes "reconcile with the memory as it stood before this run" a thing
+ * that can be said precisely.
+ */
+import { normaliseDescription } from "../statement/line.js";
+/**
+ * The stable part of a description: the counterparty name.
+ *
+ * Tokens carrying a digit are dropped — they are references, terminal ids and
+ * dates, and they are precisely the parts that differ between two months of the
+ * same relationship. Very short tokens go too, since `TO`, `A` and `RE` carry
+ * no identity. What is left is sorted, so word order in the two systems does
+ * not matter: `ASHGROVE SUPPLIES PAYMENT` and `PAYMENT ASHGROVE SUPPLIES` are
+ * one counterparty written two ways.
+ */
+export function counterpartyKey(description) {
+    const normalised = normaliseDescription(description);
+    const tokens = normalised
+        .split(" ")
+        .filter((token) => token.length >= 3 && !/\d/.test(token));
+    return [...new Set(tokens)].sort().join(" ");
+}
+const UNKNOWN = Object.freeze({
+    kind: "unknown",
+    score: 0,
+    confirmed: 0,
+    rejected: 0,
+    detail: "nothing remembered about this counterparty",
+});
+const pairKey = (statementKey, bookKey) => `${statementKey}\u0000${bookKey}`;
+const times = (count) => (count === 1 ? "once" : `${count} times`);
+export class MemoryDocumentError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "MemoryDocumentError";
+    }
+}
+export class MatchMemory {
+    byPair;
+    /** Statement keys to the book keys they have been confirmed against. */
+    confirmedFor;
+    constructor(byPair, confirmedFor) {
+        this.byPair = byPair;
+        this.confirmedFor = confirmedFor;
+    }
+    static empty() {
+        return new MatchMemory(new Map(), new Map());
+    }
+    static from(decisions) {
+        let memory = MatchMemory.empty();
+        for (const decision of decisions)
+            memory = memory.learn(decision);
+        return memory;
+    }
+    get size() {
+        return this.byPair.size;
+    }
+    /** Every entry, in a stable order: most confirmed first, then by key. */
+    get entries() {
+        return Object.freeze([...this.byPair.values()].sort((a, b) => b.confirmed - a.confirmed ||
+            b.rejected - a.rejected ||
+            (a.statementKey < b.statementKey ? -1 : a.statementKey > b.statementKey ? 1 : 0) ||
+            (a.bookKey < b.bookKey ? -1 : a.bookKey > b.bookKey ? 1 : 0)));
+    }
+    /**
+     * Record a decision, returning the memory that results.
+     *
+     * A decision about a description with no stable part — a bank line reading
+     * only `4471 8822` — is dropped rather than stored under an empty key, which
+     * would collide with every other nameless line.
+     */
+    learn(decision) {
+        const statementKey = counterpartyKey(decision.statementDescription);
+        const bookKey = counterpartyKey(decision.bookDescription);
+        if (statementKey === "" || bookKey === "")
+            return this;
+        const key = pairKey(statementKey, bookKey);
+        const existing = this.byPair.get(key);
+        const entry = Object.freeze({
+            statementKey,
+            bookKey,
+            confirmed: (existing?.confirmed ?? 0) + (decision.accepted ? 1 : 0),
+            rejected: (existing?.rejected ?? 0) + (decision.accepted ? 0 : 1),
+            lastSeen: existing === undefined || decision.on > existing.lastSeen ? decision.on : existing.lastSeen,
+        });
+        const byPair = new Map(this.byPair);
+        byPair.set(key, entry);
+        const confirmedFor = new Map([...this.confirmedFor].map(([k, v]) => [k, new Set(v)]));
+        if (decision.accepted) {
+            const set = confirmedFor.get(statementKey) ?? new Set();
+            set.add(bookKey);
+            confirmedFor.set(statementKey, set);
+        }
+        return new MatchMemory(byPair, confirmedFor);
+    }
+    learnAll(decisions) {
+        let memory = this;
+        for (const decision of decisions)
+            memory = memory.learn(decision);
+        return memory;
+    }
+    /**
+     * What is remembered about pairing these two descriptions?
+     *
+     * The order of the checks is the order of how much they should be believed.
+     * A rejection outweighs a confirmation of the same pair: if a reviewer has
+     * both accepted and refused this pairing, the refusal is the more recent
+     * correction of the two more often than not, and being wrong in the cautious
+     * direction costs a minute rather than an afternoon.
+     */
+    recall(statementDescription, bookDescription) {
+        const statementKey = counterpartyKey(statementDescription);
+        const bookKey = counterpartyKey(bookDescription);
+        if (statementKey === "" || bookKey === "")
+            return UNKNOWN;
+        const entry = this.byPair.get(pairKey(statementKey, bookKey));
+        if (entry !== undefined && entry.rejected > 0) {
+            return Object.freeze({
+                kind: "rejected",
+                score: 0,
+                confirmed: entry.confirmed,
+                rejected: entry.rejected,
+                detail: entry.confirmed > 0
+                    ? `rejected ${times(entry.rejected)} before, accepted ${times(entry.confirmed)}`
+                    : `this pairing was rejected ${times(entry.rejected)} before`,
+            });
+        }
+        if (entry !== undefined && entry.confirmed > 0) {
+            return Object.freeze({
+                kind: "confirmed",
+                score: 1,
+                confirmed: entry.confirmed,
+                rejected: 0,
+                detail: `confirmed ${times(entry.confirmed)} before, most recently ${entry.lastSeen}`,
+            });
+        }
+        // Seen this bank name, but always against somebody else. Absence of a
+        // matching entry would score as neutral; this is not neutral.
+        const known = this.confirmedFor.get(statementKey);
+        if (known !== undefined && known.size > 0 && !known.has(bookKey)) {
+            return Object.freeze({
+                kind: "contradicted",
+                score: 0,
+                confirmed: 0,
+                rejected: 0,
+                detail: `"${statementKey}" has only ever been confirmed against something else`,
+            });
+        }
+        return UNKNOWN;
+    }
+    /** Forget a pairing entirely — the undo for a decision made in error. */
+    forget(statementDescription, bookDescription) {
+        const statementKey = counterpartyKey(statementDescription);
+        const bookKey = counterpartyKey(bookDescription);
+        const key = pairKey(statementKey, bookKey);
+        if (!this.byPair.has(key))
+            return this;
+        const byPair = new Map(this.byPair);
+        byPair.delete(key);
+        const confirmedFor = new Map();
+        for (const entry of byPair.values()) {
+            if (entry.confirmed === 0)
+                continue;
+            const set = confirmedFor.get(entry.statementKey) ?? new Set();
+            set.add(entry.bookKey);
+            confirmedFor.set(entry.statementKey, set);
+        }
+        return new MatchMemory(byPair, confirmedFor);
+    }
+    toDocument() {
+        return Object.freeze({ version: 1, entries: this.entries });
+    }
+    static fromDocument(document) {
+        if (typeof document !== "object" || document === null) {
+            throw new MemoryDocumentError("A memory document must be an object");
+        }
+        const raw = document;
+        if (raw.version !== 1) {
+            throw new MemoryDocumentError(`Unsupported memory version: ${String(raw.version)}`);
+        }
+        if (!Array.isArray(raw.entries)) {
+            throw new MemoryDocumentError("A memory document needs an entries array");
+        }
+        const byPair = new Map();
+        const confirmedFor = new Map();
+        raw.entries.forEach((value, index) => {
+            const entry = value;
+            if (typeof entry.statementKey !== "string" || typeof entry.bookKey !== "string") {
+                throw new MemoryDocumentError(`Entry ${index} needs a statementKey and a bookKey`);
+            }
+            const confirmed = entry.confirmed ?? 0;
+            const rejected = entry.rejected ?? 0;
+            if (!Number.isInteger(confirmed) || confirmed < 0 || !Number.isInteger(rejected) || rejected < 0) {
+                throw new MemoryDocumentError(`Entry ${index} has a nonsensical count`);
+            }
+            if (confirmed === 0 && rejected === 0) {
+                throw new MemoryDocumentError(`Entry ${index} records neither a confirmation nor a rejection`);
+            }
+            if (typeof entry.lastSeen !== "string") {
+                throw new MemoryDocumentError(`Entry ${index} needs a lastSeen date`);
+            }
+            byPair.set(pairKey(entry.statementKey, entry.bookKey), Object.freeze({
+                statementKey: entry.statementKey,
+                bookKey: entry.bookKey,
+                confirmed,
+                rejected,
+                lastSeen: entry.lastSeen,
+            }));
+            if (confirmed > 0) {
+                const set = confirmedFor.get(entry.statementKey) ?? new Set();
+                set.add(entry.bookKey);
+                confirmedFor.set(entry.statementKey, set);
+            }
+        });
+        return new MatchMemory(byPair, confirmedFor);
+    }
+    toJson(indent = 2) {
+        return `${JSON.stringify(this.toDocument(), null, indent)}\n`;
+    }
+    static fromJson(text) {
+        let parsed;
+        try {
+            parsed = JSON.parse(text);
+        }
+        catch (error) {
+            throw new MemoryDocumentError(`Not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return MatchMemory.fromDocument(parsed);
+    }
+}
+/** What the memory holds, as something a person can read down. */
+export function renderMemory(memory) {
+    if (memory.size === 0)
+        return "Nothing remembered yet.";
+    const rows = memory.entries.map((entry) => [
+        entry.statementKey,
+        entry.bookKey,
+        `${entry.confirmed}/${entry.rejected}`,
+        entry.lastSeen,
+    ]);
+    const header = ["bank", "ledger", "yes/no", "last seen"];
+    const widths = header.map((cell, column) => Math.max(cell.length, ...rows.map((row) => row[column].length)));
+    const format = (cells) => cells.map((cell, column) => cell.padEnd(widths[column])).join("  ").trimEnd();
+    return [
+        `${memory.size} remembered ${memory.size === 1 ? "pairing" : "pairings"}`,
+        "",
+        format(header),
+        ...rows.map(format),
+    ].join("\n");
+}

package/dist/reconcile/posting.d.ts

@@ -0,0 +1,185 @@
+/**
+ * What a reconciliation implies the books are missing.
+ *
+ * A reconciliation ends with statement lines the ledger has never heard of:
+ * the bank charge nobody entered, the interest, the direct debit that was set
+ * up two years ago, the card processor's fee. They are not matching failures —
+ * there is genuinely nothing to match them against — and the reconciliation
+ * will not balance until somebody books them.
+ *
+ * "Somebody books them" is where the cycle currently breaks. The matcher knows
+ * exactly which lines are missing and says nothing about what to do with them,
+ * so a person retypes each one into the ledger, guesses the account, and the
+ * guess is recorded nowhere. This module makes the guess explicit, attributable
+ * to a rule, and repeatable.
+ *
+ * ## Three deliberate refusals
+ *
+ * **It proposes; it does not post.** Every proposal carries the statement line
+ * it came from, the rule that classified it, and a balanced entry. Applying
+ * them is a separate step a caller has to ask for.
+ *
+ * **It never invents an account.** A line no rule matches is reported as
+ * unclassified. Sweeping the remainder into a suspense account the chart does
+ * not have is how a chart of accounts rots: the account appears, everything
+ * awkward goes in it, and nobody ever empties it. A caller who wants that
+ * behaviour names an account that already exists and takes responsibility for
+ * it.
+ *
+ * **Posting the same statement twice is a no-op.** Entry ids are derived from
+ * the statement line's fingerprint — its date, amount and normalised
+ * description — so the same line always implies the same entry id, and an id
+ * already in the ledger is skipped with a reason rather than posted again.
+ * Overlapping statement exports are the normal case, not an edge case.
+ */
+import { Money } from "../money/money.js";
+import type { ChartOfAccounts } from "../accounts/chart.js";
+import type { Ledger } from "../ledger/ledger.js";
+import { JournalEntry } from "../ledger/entry.js";
+import type { StatementLine } from "../statement/line.js";
+import type { ReconciliationResult } from "./matcher.js";
+import type { MatchMemory } from "./memory.js";
+/** Which way the money went, from the account holder's point of view. */
+export type Direction = "in" | "out" | "either";
+/**
+ * A test against a statement line's description.
+ *
+ * Matching is done on the *normalised* description — uppercased, with card
+ * scheme noise, references and dates stripped — because that is the form in
+ * which two months of the same standing order look alike.
+ */
+export interface RuleMatch {
+    /** Every one of these must appear. */
+    readonly all?: readonly string[];
+    /** At least one of these must appear. */
+    readonly any?: readonly string[];
+    /** None of these may appear. Checked last, and it wins. */
+    readonly none?: readonly string[];
+    /** Applied to the normalised description, case-insensitively. */
+    readonly regex?: string;
+}
+/**
+ * What to do with a line that matches.
+ *
+ * `book` is the usual case. `skip` exists because some statement lines are not
+ * transactions at all: an opening balance line is the bank telling you where it
+ * thinks you started, and booking it would double-count the whole period.
+ */
+export type RuleAction = "book" | "skip";
+export interface PostingRule {
+    readonly id: string;
+    /** What a person reading the output should understand this rule to mean. */
+    readonly describe: string;
+    readonly match: RuleMatch;
+    /** Restrict to money in or money out. Interest earned and interest paid are
+     *  the same word and different accounts. */
+    readonly direction?: Direction;
+    readonly action?: RuleAction;
+    /** The other side of the entry. Required when the action is `book`. */
+    readonly account?: string;
+    /** Narration for the entry. Defaults to the statement description. */
+    readonly narration?: string;
+}
+export declare class PostingRuleError extends Error {
+    constructor(message: string);
+}
+/**
+ * The rules that come in the box, against the standard chart.
+ *
+ * Ordered, and the order is load-bearing: `INTEREST` outbound is a cost of
+ * borrowing and `INTEREST` inbound is income, so the two interest rules must
+ * be narrower than the general charges rule that follows them. Opening balance
+ * lines are skipped first, before anything can classify them as a receipt.
+ *
+ * This is a starting point and not a claim about anybody's business. A chart
+ * that is not this chart needs its own rules; that is what `--rules` is for.
+ */
+export declare function standardRules(): readonly PostingRule[];
+/** Why a proposal came out the way it did. */
+export type ProposalOutcome = "book" | "skip" | "unclassified" | "already-booked";
+export interface Proposal {
+    readonly line: StatementLine;
+    readonly outcome: ProposalOutcome;
+    /** The rule that fired, when one did. */
+    readonly rule: PostingRule | null;
+    /** The entry to post. Present only when the outcome is `book`. */
+    readonly entry: JournalEntry | null;
+    /** The account the other side lands in, when there is one. */
+    readonly account: string | null;
+    /** One line a person can read. */
+    readonly reason: string;
+}
+export interface ProposalOptions {
+    /** The account the statement belongs to. */
+    readonly account: string;
+    readonly rules?: readonly PostingRule[];
+    /**
+     * Where lines no rule matched should go. Must already exist in the chart;
+     * nothing here creates an account. Omitted means they stay unclassified.
+     */
+    readonly suspenseAccount?: string;
+    /** Prefix for generated entry ids. */
+    readonly idPrefix?: string;
+    /**
+     * `1` when a debit to the account is money in — an asset, the normal case —
+     * and `-1` for a liability such as a credit card. Matches `bankView`.
+     */
+    readonly inflowSign?: 1 | -1;
+    /** Skip lines whose implied entry the ledger already holds. */
+    readonly ledger?: Ledger;
+    readonly chart?: ChartOfAccounts;
+    /** Tags to put on every entry, so an import can be found again. */
+    readonly tags?: readonly string[];
+}
+/** The entry id a statement line implies, whoever asks and however often. */
+export declare function impliedEntryId(line: StatementLine, prefix?: string): string;
+/**
+ * What the ledger is missing, one proposal per statement line handed in.
+ *
+ * Callers pass the lines that reconciliation could not explain. Every line
+ * gets a proposal, including the ones that come to nothing: a queue of
+ * eleven lines that produced four entries should say so, and say why the
+ * other seven did not.
+ */
+export declare function proposeEntries(lines: readonly StatementLine[], options: ProposalOptions): readonly Proposal[];
+/** Post everything a proposal set says to post, in the order it was proposed. */
+export declare function applyProposals(ledger: Ledger, proposals: readonly Proposal[]): Ledger;
+export interface ProposalSummary {
+    readonly total: number;
+    readonly booked: number;
+    readonly skipped: number;
+    readonly unclassified: number;
+    readonly alreadyBooked: number;
+    /** Net effect on the bank account, in the statement's own direction. */
+    readonly net: Money;
+    /** Booked totals per contra account, most material first. */
+    readonly byAccount: readonly {
+        readonly account: string;
+        readonly amount: Money;
+        readonly count: number;
+    }[];
+}
+export declare function summariseProposals(proposals: readonly Proposal[], currency: Money["currency"] | string): ProposalSummary;
+/** Read a rules file. */
+export declare function parseRules(text: string): readonly PostingRule[];
+/** The proposals, as something a person can read down. */
+export declare function renderProposals(proposals: readonly Proposal[], summary: ProposalSummary): string;
+/**
+ * Which statement lines still need booking.
+ *
+ * The unmatched ones, plainly — nothing in the books corresponds to them. And
+ * then the ones sitting under a suggestion the reviewer has already refused:
+ * a rejected pairing is a person saying this line does *not* belong to that
+ * entry, which leaves the line with no counterpart at all.
+ *
+ * A suggestion nobody has decided is left alone. The matcher believes there is
+ * a ledger entry behind it, and booking a second one would double-count the
+ * transaction. Undecided means unproposed, which is the conservative reading
+ * and the right one: a missed entry is a reconciliation that does not balance,
+ * and a duplicated entry is a reconciliation that balances and is wrong.
+ *
+ * "Refused" means every description pair in the suggestion recalls as rejected.
+ * A batch where the reviewer refused one of four suppliers is still, in part,
+ * something the books know about.
+ */
+export declare function linesNeedingEntries(result: ReconciliationResult, memory?: MatchMemory): readonly StatementLine[];