shutro-erp 0.2.1 → 0.3.1

package/dist/import/csv.d.ts

@@ -0,0 +1,28 @@
+/**
+ * A CSV reader without dependencies (RFC 0014 §5.1): the encoding is read from the byte-order mark, else UTF-8 when
+ * the bytes are valid UTF-8, else Windows-1252 (spreadsheets saved by older Excel); the separator is guessed from the
+ * first lines (`,` `;` tab `|`); quoted cells may hold separators, quotes (`""`) and newlines (RFC 4180).
+ */
+export type CsvEncoding = 'utf-8' | 'utf-16le' | 'utf-16be' | 'windows-1252';
+export interface CsvOptions {
+    /** Default: guessed. */
+    readonly separator?: string;
+    /** Default: from the byte-order mark or the bytes. */
+    readonly encoding?: CsvEncoding;
+}
+export interface CsvTable {
+    readonly rows: string[][];
+    readonly separator: string;
+    readonly encoding: CsvEncoding;
+}
+/** The text of `bytes` and the encoding it was read in. */
+export declare function decodeText(bytes: Uint8Array, given?: CsvEncoding): {
+    text: string;
+    encoding: CsvEncoding;
+};
+/** The separator found the same number of times on the most of the first lines (ties: the earlier in the list). */
+export declare function guessSeparator(text: string): string;
+/** Splits `text` into rows of cells. Trailing empty rows are dropped. */
+export declare function parseCsvText(text: string, separator: string): string[][];
+/** Reads a CSV file. */
+export declare function readCsv(bytes: Uint8Array, options?: CsvOptions): CsvTable;

package/dist/import/csv.js

@@ -0,0 +1,115 @@
+/**
+ * A CSV reader without dependencies (RFC 0014 §5.1): the encoding is read from the byte-order mark, else UTF-8 when
+ * the bytes are valid UTF-8, else Windows-1252 (spreadsheets saved by older Excel); the separator is guessed from the
+ * first lines (`,` `;` tab `|`); quoted cells may hold separators, quotes (`""`) and newlines (RFC 4180).
+ */
+const SEPARATORS = [',', ';', '\t', '|'];
+/** The text of `bytes` and the encoding it was read in. */
+export function decodeText(bytes, given) {
+    const [a, b, c] = bytes;
+    let encoding = given;
+    if (encoding === undefined) {
+        if (a === 0xef && b === 0xbb && c === 0xbf)
+            encoding = 'utf-8';
+        else if (a === 0xff && b === 0xfe)
+            encoding = 'utf-16le';
+        else if (a === 0xfe && b === 0xff)
+            encoding = 'utf-16be';
+    }
+    if (encoding !== undefined)
+        // TextDecoder drops the byte-order mark of its own encoding.
+        return { text: new TextDecoder(encoding).decode(bytes), encoding };
+    try {
+        return { text: new TextDecoder('utf-8', { fatal: true }).decode(bytes), encoding: 'utf-8' };
+    }
+    catch {
+        return { text: new TextDecoder('windows-1252').decode(bytes), encoding: 'windows-1252' };
+    }
+}
+/** Counts of `separator` outside quotes in each of the first lines. */
+function counts(text, separator, lines) {
+    const out = [];
+    let quoted = false;
+    let n = 0;
+    for (let i = 0; i < text.length && out.length < lines; i++) {
+        const ch = text[i];
+        if (ch === '"')
+            quoted = !quoted;
+        else if (!quoted && ch === separator)
+            n += 1;
+        else if (!quoted && ch === '\n') {
+            out.push(n);
+            n = 0;
+        }
+    }
+    if (out.length < lines && n > 0)
+        out.push(n);
+    return out;
+}
+/** The separator found the same number of times on the most of the first lines (ties: the earlier in the list). */
+export function guessSeparator(text) {
+    let best = ',';
+    let bestScore = 0;
+    for (const separator of SEPARATORS) {
+        const seen = counts(text, separator, 10).filter((n) => n > 0);
+        const first = seen[0];
+        if (first === undefined)
+            continue;
+        const score = seen.filter((n) => n === first).length * 1000 + first;
+        if (score > bestScore) {
+            best = separator;
+            bestScore = score;
+        }
+    }
+    return best;
+}
+/** Splits `text` into rows of cells. Trailing empty rows are dropped. */
+export function parseCsvText(text, separator) {
+    const rows = [];
+    let row = [];
+    let cell = '';
+    let quoted = false;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        if (quoted) {
+            if (ch !== '"')
+                cell += ch;
+            else if (text[i + 1] === '"') {
+                cell += '"';
+                i += 1;
+            }
+            else
+                quoted = false;
+        }
+        else if (ch === '"' && cell === '')
+            quoted = true;
+        else if (ch === separator) {
+            row.push(cell);
+            cell = '';
+        }
+        else if (ch === '\n' || ch === '\r') {
+            if (ch === '\r' && text[i + 1] === '\n')
+                i += 1;
+            row.push(cell);
+            rows.push(row);
+            row = [];
+            cell = '';
+        }
+        else
+            cell += ch;
+    }
+    if (cell !== '' || row.length > 0) {
+        row.push(cell);
+        rows.push(row);
+    }
+    while (rows.length > 0 && rows.at(-1).every((c) => c === ''))
+        rows.pop();
+    return rows;
+}
+/** Reads a CSV file. */
+export function readCsv(bytes, options = {}) {
+    const { text, encoding } = decodeText(bytes, options.encoding);
+    const body = text.startsWith('') ? text.slice(1) : text;
+    const separator = options.separator ?? guessSeparator(body);
+    return { rows: parseCsvText(body, separator), separator, encoding };
+}

package/dist/import/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * @packageDocumentation
+ * `shutro-erp/import` (RFC 0014): CSV and XLSX files become records of any served model — columns mapped to fields
+ * (guessed from keys and labels), cells converted (decimals exact, dates in the file's order, Bangla digits),
+ * relations matched by external id, unique key or name as the user, a dry run with every error on its row and cell,
+ * and an all-or-nothing import that is audited. Nothing is bypassed: record rules, field security, constraints and
+ * sequences apply as for a record typed in a form.
+ */
+import { type KoshModule } from '../modules-sdk/index.js';
+import { type ImportRoutesOptions } from './routes.js';
+export { asciiDigits, type Cell, type CellError, type Converted, type ConvertField, type ConvertOptions, convertCell, importable, parseDate, parseDateTime, plainNumber, } from './convert.js';
+export { type CsvEncoding, type CsvTable, decodeText, guessSeparator, parseCsvText, readCsv, } from './csv.js';
+export { type ColumnMapping, guessMapping, type MappableField, normalize } from './mapping.js';
+export { type Match, type MatchTarget, matchAll, matchOne } from './match.js';
+export { importShutro } from './models.js';
+export { type ImportRoutesOptions, registerImportRoutes } from './routes.js';
+export { EXTERNAL_ID_KEY, externalIdOf, ID_KEY, IMPORT_MODULE, type ImportField, type ImportModels, type ImportOptions, type ImportResult, importFields, type RowError, type RowErrorCode, runImport, type UndoData, type UndoResult, undoImport, WRITE_BATCH, } from './run.js';
+export { createImportService, FINISHED_TOPIC, IMPORT_MAX_BYTES, IMPORT_ROWS, type ImportContext, type ImportService, INLINE_ROWS, parseOptions, RUN_JOB, type SavedMapping, type Session, UNDO_DAYS, } from './service.js';
+export { importWorker } from './worker.js';
+export { fromSerial, isDateFormat, readXlsx, unzip, type XlsxCell, XlsxError, type XlsxSheet, xlsxSheetNames, } from './xlsx.js';
+/** Package identity. */
+export declare const PACKAGE: {
+    readonly name: string;
+    readonly rfc: string;
+};
+export interface ImportModuleOptions extends Omit<ImportRoutesOptions, 'service'> {
+    /** Run background imports in this process (default: unless `KOSH_SCHEDULES=off`). */
+    readonly worker?: boolean;
+}
+/** The import module with explicit limits (tests). */
+export declare function createImport(options?: ImportModuleOptions): KoshModule;
+/** Data import, limits from the environment (`KOSH_IMPORT_MAX_BYTES`, `KOSH_IMPORT_MAX_ROWS`). */
+export declare const dataImport: KoshModule;

package/dist/import/index.js

@@ -0,0 +1,60 @@
+/**
+ * @packageDocumentation
+ * `shutro-erp/import` (RFC 0014): CSV and XLSX files become records of any served model — columns mapped to fields
+ * (guessed from keys and labels), cells converted (decimals exact, dates in the file's order, Bangla digits),
+ * relations matched by external id, unique key or name as the user, a dry run with every error on its row and cell,
+ * and an all-or-nothing import that is audited. Nothing is bypassed: record rules, field security, constraints and
+ * sequences apply as for a record typed in a form.
+ */
+import { defineModule } from '../modules-sdk/index.js';
+import { base } from '../server/index.js';
+import { importShutro } from './models.js';
+import { registerImportRoutes } from './routes.js';
+import { createImportService } from './service.js';
+import { importWorker } from './worker.js';
+export { asciiDigits, convertCell, importable, parseDate, parseDateTime, plainNumber, } from './convert.js';
+export { decodeText, guessSeparator, parseCsvText, readCsv, } from './csv.js';
+export { guessMapping, normalize } from './mapping.js';
+export { matchAll, matchOne } from './match.js';
+export { importShutro } from './models.js';
+export { registerImportRoutes } from './routes.js';
+export { EXTERNAL_ID_KEY, externalIdOf, ID_KEY, IMPORT_MODULE, importFields, runImport, undoImport, WRITE_BATCH, } from './run.js';
+export { createImportService, FINISHED_TOPIC, IMPORT_MAX_BYTES, IMPORT_ROWS, INLINE_ROWS, parseOptions, RUN_JOB, UNDO_DAYS, } from './service.js';
+export { importWorker } from './worker.js';
+export { fromSerial, isDateFormat, readXlsx, unzip, XlsxError, xlsxSheetNames, } from './xlsx.js';
+/** Package identity. */
+export const PACKAGE = Object.freeze({
+    name: 'shutro-erp/import',
+    rfc: 'RFC 0014',
+});
+/** The import module with explicit limits (tests). */
+export function createImport(options = {}) {
+    const env = process.env;
+    const maxBytes = Number(env['KOSH_IMPORT_MAX_BYTES'] ?? Number.NaN);
+    const maxRows = Number(env['KOSH_IMPORT_MAX_ROWS'] ?? Number.NaN);
+    return defineModule({
+        shutro: importShutro,
+        kosh: '^0.1.0',
+        depends: [base],
+        accessLevels: { user: { label: 'Import data', groups: ['import.user'] } },
+        privateModels: ['import.session', 'import.progress', 'import.mapping'],
+        routes: (context) => {
+            const { worker, ...limits } = options;
+            const service = createImportService(context, limits.inlineRows === undefined ? {} : { inlineRows: limits.inlineRows });
+            registerImportRoutes(context, {
+                ...(Number.isFinite(maxBytes) && maxBytes > 0 ? { maxBytes } : {}),
+                ...(Number.isFinite(maxRows) && maxRows > 0 ? { maxRows } : {}),
+                ...limits,
+                service,
+            });
+            // Background work is off in the CLI (kosh init, migrate), like the schedules.
+            if (!(worker ?? env['KOSH_SCHEDULES'] !== 'off'))
+                return;
+            const running = importWorker(context, service);
+            running.start();
+            context.app.addHook('onClose', () => running.stop());
+        },
+    });
+}
+/** Data import, limits from the environment (`KOSH_IMPORT_MAX_BYTES`, `KOSH_IMPORT_MAX_ROWS`). */
+export const dataImport = createImport();

package/dist/import/mapping.d.ts

@@ -0,0 +1,15 @@
+export interface MappableField {
+    readonly key: string;
+    readonly kind: string;
+    readonly label: string;
+}
+/** A column of the file and the field it fills, or null to skip it. */
+export interface ColumnMapping {
+    readonly index: number;
+    readonly header: string;
+    readonly field: string | null;
+}
+/** Lower case, letters and digits only (Bangla letters kept). */
+export declare function normalize(text: string): string;
+/** The field each header most likely fills: an exact key, then an exact label; each field once. */
+export declare function guessMapping(headers: readonly string[], fields: readonly MappableField[]): ColumnMapping[];

package/dist/import/mapping.js

@@ -0,0 +1,33 @@
+/**
+ * Columns to fields (RFC 0014 §5.1 step 2): each header is matched to a field by its key or its label, ignoring case,
+ * spaces and punctuation, so an export of Kosh (headers are labels) re-imports without edits. Unmatched columns
+ * are skipped until the user maps them.
+ */
+import { importable } from './convert.js';
+/** Lower case, letters and digits only (Bangla letters kept). */
+export function normalize(text) {
+    return text
+        .normalize('NFKC')
+        .toLowerCase()
+        .replace(/[^\p{L}\p{M}\p{N}]+/gu, '');
+}
+/** The field each header most likely fills: an exact key, then an exact label; each field once. */
+export function guessMapping(headers, fields) {
+    const candidates = fields.filter((f) => importable(f.kind));
+    const taken = new Set();
+    const byKey = new Map(candidates.map((f) => [normalize(f.key), f.key]));
+    const byLabel = new Map();
+    for (const f of candidates) {
+        const label = normalize(f.label);
+        if (!byLabel.has(label))
+            byLabel.set(label, f.key);
+    }
+    return headers.map((header, index) => {
+        const n = normalize(header);
+        const found = byKey.get(n) ?? byLabel.get(n) ?? null;
+        if (found === null || taken.has(found) || n === '')
+            return { index, header, field: null };
+        taken.add(found);
+        return { index, header, field: found };
+    });
+}

package/dist/import/match.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Relation cells to record ids (RFC 0014 §5.1 step 3), read as the user: an external id (`contacts.partner_acme`),
+ * then a field the target keeps unique (a code, an email), then its display name — exactly, ignoring case, then as
+ * the one record whose name contains the text. Several matches are an error, never a guess.
+ */
+import { type Env } from '../server/orm.js';
+/** How the records of a target model are found from text. */
+export interface MatchTarget {
+    readonly model: string;
+    /** The field the display name comes from. */
+    readonly recName: string;
+    /** Stored char fields with a unique constraint of their own (besides the company). */
+    readonly keys: readonly string[];
+}
+export type Match = {
+    readonly id: number;
+    readonly by: 'id' | 'external-id' | 'key' | 'name';
+} | {
+    readonly error: 'not-found' | 'ambiguous';
+};
+/** The record `text` names in `target`, as the user sees it. */
+export declare function matchOne(env: Env, target: MatchTarget, text: string): Promise<Match>;
+/** Each distinct text once (a file names the same customer on many rows). */
+export declare function matchAll(env: Env, target: MatchTarget, texts: Iterable<string>): Promise<Map<string, Match>>;

package/dist/import/match.js

@@ -0,0 +1,60 @@
+/**
+ * Relation cells to record ids (RFC 0014 §5.1 step 3), read as the user: an external id (`contacts.partner_acme`),
+ * then a field the target keeps unique (a code, an email), then its display name — exactly, ignoring case, then as
+ * the one record whose name contains the text. Several matches are an error, never a guess.
+ */
+import { orm } from '../server/orm.js';
+const ref = (env, name) => env.model(name);
+const EXTERNAL_ID = /^[a-z0-9_]+\.[a-z0-9_.-]+$/i;
+const escapeLike = (text) => text.replace(/[%_\\]/g, (m) => `\\${m}`);
+/** The record `text` names in `target`, as the user sees it. */
+export async function matchOne(env, target, text) {
+    const model = ref(env, target.model);
+    if (/^#\d+$/.test(text)) {
+        // `#42`: a record id, as an export with ids writes it.
+        const found = await model.search({ where: { id: Number(text.slice(1)) }, limit: 1 });
+        return found.length === 1 ? { id: found.ids[0], by: 'id' } : { error: 'not-found' };
+    }
+    if (EXTERNAL_ID.test(text)) {
+        try {
+            const id = Number(await env.refId(target.model, text));
+            const found = await model.search({ where: { id }, limit: 1 });
+            if (found.length === 1)
+                return { id, by: 'external-id' };
+        }
+        catch (error) {
+            if (!(error instanceof orm.ShutroError))
+                throw error; // an unknown id: try the other ways
+        }
+    }
+    for (const key of target.keys) {
+        const found = await model.search({ where: { [key]: text }, limit: 2 });
+        if (found.length === 1)
+            return { id: found.ids[0], by: 'key' };
+        if (found.length > 1)
+            return { error: 'ambiguous' };
+    }
+    const exact = await model.search({
+        where: { [target.recName]: { ilike: escapeLike(text) } },
+        limit: 2,
+    });
+    if (exact.length === 1)
+        return { id: exact.ids[0], by: 'name' };
+    if (exact.length > 1)
+        return { error: 'ambiguous' };
+    const partial = await model.search({
+        where: { [target.recName]: { ilike: `%${escapeLike(text)}%` } },
+        limit: 2,
+    });
+    if (partial.length === 1)
+        return { id: partial.ids[0], by: 'name' };
+    return { error: partial.length > 1 ? 'ambiguous' : 'not-found' };
+}
+/** Each distinct text once (a file names the same customer on many rows). */
+export async function matchAll(env, target, texts) {
+    const out = new Map();
+    for (const text of texts)
+        if (!out.has(text))
+            out.set(text, await matchOne(env, target, text));
+    return out;
+}

package/dist/import/models.d.ts

@@ -0,0 +1,2 @@
+import { orm } from '../server/orm.js';
+export declare const importShutro: ReturnType<typeof orm.defineModule>;

package/dist/import/models.js

@@ -0,0 +1,102 @@
+/**
+ * The import's models (RFC 0014):
+ *
+ * - `import.session` (§5.1): the uploaded file, its mapping and options, and what became of it. A session belongs to
+ *   the user who uploaded it; the routes read it as that user (record rule) and nobody else sees the file.
+ * - `import.progress` (§5.3): how far a background run is, written by the job in short transactions of its own (the
+ *   job's transaction commits only at the end); read through the routes.
+ * - `import.mapping` (§5.4): a saved mapping (columns by header, options) for a model, the user's own or shared with
+ *   every import user of the company.
+ */
+import { baseShutro } from '../server/index.js';
+import { orm } from '../server/orm.js';
+const { defineGroup, defineModel, defineModule, f } = orm;
+const Session = defineModel('import.session', {
+    label: 'Import',
+    order: ['id desc'],
+    companyScoped: { required: true },
+    access: { 'import.user': ['read', 'write', 'create', 'unlink'] },
+    rules: {
+        own: {
+            groups: ['import.user'],
+            perms: ['read', 'write', 'create', 'unlink'],
+            domain: { user: { $env: 'uid' } },
+        },
+    },
+    fields: {
+        user: f.many2one('res.users', { required: true, ondelete: 'cascade' }),
+        /** The model the rows become (`res.partner`). */
+        resModel: f.char({ required: true, label: 'Model' }),
+        fileName: f.char({ required: true, size: 255 }),
+        fileType: f.selection({ csv: 'CSV', xlsx: 'Excel' }, { required: true }),
+        file: f.binary({ required: true }),
+        /** The XLSX sheet read (the first by default). */
+        sheet: f.char({ size: 100 }),
+        /** `[{ index, header, field }]`. */
+        columns: f.json({ required: true, default: [] }),
+        /** Conversion options and the mode (`ImportOptions`). */
+        options: f.json({ required: true, default: {} }),
+        state: f.selection({ draft: 'Draft', running: 'Running', done: 'Imported', undone: 'Undone' }, { required: true, default: 'draft' }),
+        rowCount: f.integer({ required: true, default: 0, label: 'Rows' }),
+        /** The ids the commit created. */
+        createdIds: f.json({ label: 'Created records' }),
+        /** What undo needs (`UndoData`, RFC 0014 §5.5). */
+        undo: f.json(),
+        /** The errors and file rows of the records left out (`onError: 'skip'`), for the rejected-rows file. */
+        rejected: f.json(),
+        importedAt: f.datetime({ label: 'Imported on' }),
+        /** The last dry run or commit as the API shows it (background runs leave it here). */
+        result: f.json(),
+        /** Whether `result` is a dry run's. */
+        resultDry: f.boolean({ required: true, default: false }),
+        /** Why a background run could not finish (a fault, not the data). */
+        failure: f.char({ size: 500 }),
+    },
+});
+const Progress = defineModel('import.progress', {
+    label: 'Import progress',
+    access: {},
+    fields: {
+        session: f.many2one('import.session', { required: true, ondelete: 'cascade' }),
+        done: f.integer({ required: true, default: 0 }),
+        total: f.integer({ required: true, default: 0 }),
+    },
+    sqlConstraints: {
+        onePerSession: { unique: ['session'], message: 'One progress row per import.' },
+    },
+});
+const Mapping = defineModel('import.mapping', {
+    label: 'Import mapping',
+    order: ['name'],
+    companyScoped: { required: true },
+    access: { 'import.user': ['read', 'write', 'create', 'unlink'] },
+    rules: {
+        visible: {
+            groups: ['import.user'],
+            perms: ['read'],
+            domain: { or: [{ user: { $env: 'uid' } }, { shared: true }] },
+        },
+        own: {
+            groups: ['import.user'],
+            perms: ['write', 'create', 'unlink'],
+            domain: { user: { $env: 'uid' } },
+        },
+    },
+    fields: {
+        name: f.char({ required: true, size: 100 }),
+        resModel: f.char({ required: true, label: 'Model' }),
+        user: f.many2one('res.users', { required: true, ondelete: 'cascade', label: 'Owner' }),
+        /** Every import user of the company sees it (only its owner changes it). */
+        shared: f.boolean({ required: true, default: false }),
+        /** `[{ header, field }]`: by header, so a file with the columns in another order still matches. */
+        columns: f.json({ required: true, default: [] }),
+        options: f.json({ required: true, default: {} }),
+    },
+});
+export const importShutro = defineModule({
+    name: 'import',
+    version: '0.1.0',
+    depends: [baseShutro],
+    groups: [defineGroup({ code: 'import.user', label: 'Import data', implies: ['base.user'] })],
+    models: [Session, Progress, Mapping],
+});

package/dist/import/routes.d.ts

@@ -0,0 +1,13 @@
+import { type ModuleContext } from '../server/index.js';
+import { type ImportService } from './service.js';
+export interface ImportRoutesOptions {
+    /** Bytes per file. */
+    readonly maxBytes?: number;
+    /** Rows per file. */
+    readonly maxRows?: number;
+    /** Rows run in the request; above, a background job. */
+    readonly inlineRows?: number;
+    /** The service the worker shares (made from the context by default). */
+    readonly service?: ImportService;
+}
+export declare function registerImportRoutes(context: ModuleContext, options?: ImportRoutesOptions): void;