sfora-cli 0.9.0 → 0.11.0

package/dist/format/index.d.ts

@@ -1,13 +1,28 @@
 /**
  * The sfora markdown format — the single source of truth for how posts, tasks
- * (cards), and docs (notes) serialize to/from markdown files.
+ * (cards), and docs (notes) serialize to/from markdown files, plus the shared
+ * grammar every surface used to reimplement (wiki links, checklists,
+ * plaintext flattening, structured blocks).
  *
  * This package is canonical: the Convex backend re-exports these modules
- * (convex/lib/* are thin shims), so cloud files and local `.sfora/` files are
- * byte-identical by construction. Round-trip tests live in convex/lib/__tests__
- * and exercise this code through the shims.
+ * (convex/lib/* are thin shims), the published CLI syncs this source into its
+ * own tree at build time (packages/sfora/scripts/sync-format.mjs), and the web
+ * app reaches it through the `@sfora/markdown` alias — so cloud files, local
+ * `.sfora/` files, and everything in between are byte-identical by
+ * construction. Round-trip and parse-health tests live in `src/__tests__/`.
  */
 export * from "./markdown/index.js";
 export * from "./cardMarkdown.js";
 export * from "./postMarkdown.js";
 export * from "./noteMarkdown.js";
+export * from "./wikiLinks.js";
+export * from "./plaintext.js";
+export * from "./checklist.js";
+export * from "./parseWithFallback.js";
+export * from "./callout.js";
+export * from "./lint/index.js";
+export * from "./blocks/structured-block-schema.js";
+export * from "./blocks/markdown-block-catalog.js";
+export * from "./blocks/dropClosure.js";
+export * from "./formatAxes.js";
+export * from "./wayfinder.js";

package/dist/format/index.js

@@ -1,13 +1,37 @@
+// GENERATED by scripts/sync-format.mjs from packages/markdown/src — DO NOT EDIT.
+// Edit packages/markdown/src and re-run the sync (any sfora-cli build does it).
 /**
  * The sfora markdown format — the single source of truth for how posts, tasks
- * (cards), and docs (notes) serialize to/from markdown files.
+ * (cards), and docs (notes) serialize to/from markdown files, plus the shared
+ * grammar every surface used to reimplement (wiki links, checklists,
+ * plaintext flattening, structured blocks).
  *
  * This package is canonical: the Convex backend re-exports these modules
- * (convex/lib/* are thin shims), so cloud files and local `.sfora/` files are
- * byte-identical by construction. Round-trip tests live in convex/lib/__tests__
- * and exercise this code through the shims.
+ * (convex/lib/* are thin shims), the published CLI syncs this source into its
+ * own tree at build time (packages/sfora/scripts/sync-format.mjs), and the web
+ * app reaches it through the `@sfora/markdown` alias — so cloud files, local
+ * `.sfora/` files, and everything in between are byte-identical by
+ * construction. Round-trip and parse-health tests live in `src/__tests__/`.
  */
 export * from "./markdown/index.js";
 export * from "./cardMarkdown.js";
 export * from "./postMarkdown.js";
 export * from "./noteMarkdown.js";
+export * from "./wikiLinks.js";
+export * from "./plaintext.js";
+export * from "./checklist.js";
+export * from "./parseWithFallback.js";
+export * from "./callout.js";
+export * from "./lint/index.js";
+export * from "./blocks/structured-block-schema.js";
+export * from "./blocks/markdown-block-catalog.js";
+export * from "./blocks/dropClosure.js";
+// The format-DOF vocabulary the ledger above spends (card #288). It is data
+// and types only — no parser, no walker — so it rides the CLI sync alongside
+// the ledger that would not compile without it.
+export * from "./formatAxes.js";
+export * from "./wayfinder.js";
+// htmlToMarkdown and parseMarkdownAst are deliberately NOT in the barrel:
+// they carry the package's only heavy dependencies (unified/rehype/remark)
+// and are excluded from the CLI sync — consumers import the subpaths
+// `@sfora/markdown/htmlToMarkdown` and `@sfora/markdown/parseMarkdownAst`.

package/dist/format/lineGeometry.d.ts

@@ -0,0 +1,100 @@
+/** A fenced code block, delimiters included. */
+export interface FenceRegion {
+    /** Line index of the opening delimiter. */
+    open: number;
+    /** Line index of the closing delimiter, or of the last line when unclosed. */
+    close: number;
+    /** True when a closing delimiter was actually found. */
+    closed: boolean;
+    /** First word of the info string, lowercased. Empty for a bare fence. */
+    lang: string;
+    /** Leading spaces on the opening delimiter — content is dedented by these. */
+    indent: number;
+    /** First content line index (exclusive of the delimiter). */
+    contentStart: number;
+    /** One past the last content line. */
+    contentEnd: number;
+}
+/** The line without its trailing `\r`, so `$`-anchored rules work on CRLF. */
+export declare function lineText(lines: readonly string[], i: number): string;
+/**
+ * Frontmatter extent, or null. Only a document that OPENS with `---` has
+ * frontmatter — a `---` further down is a thematic break, and treating it as a
+ * fence would make every horizontal rule the start of a lint dead zone.
+ */
+export declare function frontmatterExtent(lines: readonly string[]): {
+    open: number;
+    close: number;
+} | null;
+/**
+ * The first line of the document's markdown: 0, or the line after a leading
+ * frontmatter block. A frontmatter block that never closes takes the whole
+ * document with it, which is what a reader sees too.
+ *
+ * Lint and the mask both need this number and must not each compute it: one of
+ * them being wrong about where the YAML ends is exactly how a `<!--` in a
+ * summary field opens a comment over a whole file.
+ */
+export declare function markdownStart(lines: readonly string[]): number;
+/**
+ * Every fenced code block in the document, in order. `from` skips the
+ * frontmatter block, whose contents are not markdown either.
+ */
+export declare function fencedRegions(lines: readonly string[], from?: number): FenceRegion[];
+/** True for every line covered by a fenced block, delimiters included. */
+export declare function fenceMask(lines: readonly string[], from?: number): boolean[];
+/**
+ * True where a line is inside an inline code span, delimiters included.
+ *
+ * Rules and parsers mask their matches against this so a token quoted as
+ * `` `[[c:1]]` `` — the way documentation talks ABOUT the grammar — is never
+ * read as using it, and so a pipe inside `` `a|b` `` is content rather than a
+ * cell boundary. Line-local by design: a code span cannot cross a blank line,
+ * and the callers hold one line at a time.
+ */
+export declare function codeSpanMask(line: string): boolean[];
+/**
+ * The non-rendering mask, line by line. Each returned line has exactly the
+ * length of the line it masks, so `masked[i][k]` and `lines[i][k]` are the
+ * same byte position.
+ *
+ * `from` is where the document's markdown begins — everything above it is a
+ * frontmatter block, which is not markdown at all and is returned verbatim for
+ * whoever does read it (lint, with `inFrontmatter`). Scanning it would be
+ * worse than useless: a `<!--` in a YAML value would open a comment over the
+ * whole document. Callers holding a BODY pass 0 — a leading `---` there is a
+ * thematic break, not frontmatter — and `maskNonRenderingContexts` works the
+ * boundary out with `markdownStart`.
+ *
+ * A fence decides the whole line, but only where nothing is open already: a
+ * ``` line inside an unterminated comment is comment text, and the document
+ * below the `-->` renders. That ordering is CommonMark's — an HTML block ends
+ * at its closing condition, not at the next thing that looks like a fence —
+ * and it is the one place fence geometry does not go first. Where the line is
+ * clear, `fenceRegionAt` answers, so there is still exactly one idea in the
+ * package of where a fence starts and ends.
+ *
+ * Lines are whatever the caller split; a bare `\r` inside one is not a line
+ * ending here. `maskNonRenderingContexts` splits the way CommonMark does.
+ */
+export declare function maskNonRenderingLines(lines: readonly string[], from?: number): string[];
+/**
+ * The whole source, masked. `masked.length === source.length` always: the two
+ * strings are the same document, one of them with the non-markdown blanked
+ * out, and every offset holds across both.
+ *
+ * This is the DOCUMENT entry point — cm-lint and `demoteNonRenderingLinks`
+ * hand it whole files — so the frontmatter boundary is worked out here rather
+ * than assumed to be 0. Pass `from` explicitly when the caller knows better:
+ * 0 says "these bytes are all markdown", which is what a body is.
+ */
+export declare function maskNonRenderingContexts(source: string, from?: number): string;
+/**
+ * True when the source span `[start, end)` holds bytes but no rendering ones —
+ * the predicate a consumer asks before trusting anything it found by scanning.
+ *
+ * A span that was blank to begin with is not "non-rendering", it is empty, and
+ * answering true for it would demote every piece of whitespace in the
+ * document.
+ */
+export declare function isNonRenderingRange(source: string, masked: string, start: number, end: number): boolean;

package/dist/format/lineGeometry.js

@@ -0,0 +1,424 @@
+// GENERATED by scripts/sync-format.mjs from packages/markdown/src — DO NOT EDIT.
+// Edit packages/markdown/src and re-run the sync (any sfora-cli build does it).
+// Line geometry: where the fences and the frontmatter are, and which bytes
+// never reach a reader at all.
+//
+// Lint built this first and is still its heaviest user, but it is not lint's
+// to own — a checkbox-looking line inside a fence is code to the reader, to
+// `checklistProgress`, and to every other tool that walks lines instead of the
+// tree. Those callers must not carry a second, subtly different idea of where
+// a fence starts, so the scan lives here and `lint/lintSource` re-exports it.
+//
+// The second half of the file is the NON-RENDERING MASK: one same-length
+// string in which every byte that does not render — fenced code, inline code
+// spans, HTML comments, raw `<pre>`/`<code>` — has been replaced by a space.
+// Substitution, never deletion, so offset N in the mask is offset N in the
+// source by construction and a line scanner and the parsed tree can never
+// disagree about which bytes are markdown. Every scanner in the package reads
+// it; there is no second copy.
+//
+// Pure line arithmetic: no parser, no dependencies. It ships in the CLI.
+// The optional BOM only ever occurs on the first line, and allowing it here is
+// not cosmetic: without it a file that opens with a fence has no fence at all,
+// and every rule that skips fenced code would walk straight into one.
+const FENCE_OPEN = /^?( {0,3})(`{3,}|~{3,})(.*)$/;
+/** The line without its trailing `\r`, so `$`-anchored rules work on CRLF. */
+export function lineText(lines, i) {
+    const raw = lines[i] ?? "";
+    return raw.endsWith("\r") ? raw.slice(0, -1) : raw;
+}
+/**
+ * Frontmatter extent, or null. Only a document that OPENS with `---` has
+ * frontmatter — a `---` further down is a thematic break, and treating it as a
+ * fence would make every horizontal rule the start of a lint dead zone.
+ */
+export function frontmatterExtent(lines) {
+    if (lines.length === 0)
+        return null;
+    const first = lineText(lines, 0).replace(/^/, "");
+    if (first !== "---")
+        return null;
+    for (let i = 1; i < lines.length; i++) {
+        const text = lineText(lines, i).trimEnd();
+        if (text === "---" || text === "...")
+            return { open: 0, close: i };
+    }
+    return { open: 0, close: -1 }; // opened, never closed
+}
+/**
+ * The first line of the document's markdown: 0, or the line after a leading
+ * frontmatter block. A frontmatter block that never closes takes the whole
+ * document with it, which is what a reader sees too.
+ *
+ * Lint and the mask both need this number and must not each compute it: one of
+ * them being wrong about where the YAML ends is exactly how a `<!--` in a
+ * summary field opens a comment over a whole file.
+ */
+export function markdownStart(lines) {
+    const frontmatter = frontmatterExtent(lines);
+    if (frontmatter === null)
+        return 0;
+    return frontmatter.close === -1 ? lines.length : frontmatter.close + 1;
+}
+/** The fenced block opening at line `i`, or null if none opens there. */
+function fenceRegionAt(lines, i) {
+    const match = FENCE_OPEN.exec(lineText(lines, i));
+    if (!match)
+        return null;
+    const delimiter = match[2];
+    const info = match[3];
+    // A backtick fence's info string may not contain a backtick — that rules
+    // out inline code like ```` ```a``` ```` opening a block.
+    if (delimiter.startsWith("`") && info.includes("`"))
+        return null;
+    const char = delimiter[0];
+    const closer = new RegExp(`^ {0,3}[${char}]{${delimiter.length},}[ \\t]*$`);
+    let close = lines.length - 1;
+    let closed = false;
+    for (let j = i + 1; j < lines.length; j++) {
+        if (closer.test(lineText(lines, j))) {
+            close = j;
+            closed = true;
+            break;
+        }
+    }
+    return {
+        open: i,
+        close,
+        closed,
+        lang: info.trim().split(/\s+/)[0]?.toLowerCase() ?? "",
+        indent: match[1].length,
+        contentStart: i + 1,
+        contentEnd: closed ? close : lines.length,
+    };
+}
+/**
+ * Every fenced code block in the document, in order. `from` skips the
+ * frontmatter block, whose contents are not markdown either.
+ */
+export function fencedRegions(lines, from = 0) {
+    const regions = [];
+    for (let i = from; i < lines.length; i++) {
+        const region = fenceRegionAt(lines, i);
+        if (region === null)
+            continue;
+        regions.push(region);
+        i = region.close;
+    }
+    return regions;
+}
+/** True for every line covered by a fenced block, delimiters included. */
+export function fenceMask(lines, from = 0) {
+    const mask = new Array(lines.length).fill(false);
+    for (const region of fencedRegions(lines, from)) {
+        for (let i = region.open; i <= region.close; i++)
+            mask[i] = true;
+    }
+    return mask;
+}
+// ── The non-rendering mask ──────────────────────────────────────────────────
+/**
+ * One past the closing backtick run of the code span opening at `start`, or
+ * null when the run never closes.
+ *
+ * CommonMark wants the closing run to be EXACTLY as long as the opening one,
+ * which is why this is a scan and not an `indexOf`: in `` `a`` `` the double
+ * run does not close the single one and nothing else on the line does either,
+ * so there is no code span at all — where a search for the next backtick would
+ * end one after `a` and mask two characters of prose as code. An unterminated
+ * run opens nothing: a stray backtick must not swallow the rest of the line.
+ */
+function codeSpanEnd(text, start) {
+    let open = 0;
+    while (text[start + open] === "`")
+        open++;
+    if (open === 0)
+        return null;
+    let i = start + open;
+    while (i < text.length) {
+        if (text[i] !== "`") {
+            i++;
+            continue;
+        }
+        let run = 0;
+        while (text[i + run] === "`")
+            run++;
+        if (run === open)
+            return i + run;
+        i += run;
+    }
+    return null;
+}
+/**
+ * True where a line is inside an inline code span, delimiters included.
+ *
+ * Rules and parsers mask their matches against this so a token quoted as
+ * `` `[[c:1]]` `` — the way documentation talks ABOUT the grammar — is never
+ * read as using it, and so a pipe inside `` `a|b` `` is content rather than a
+ * cell boundary. Line-local by design: a code span cannot cross a blank line,
+ * and the callers hold one line at a time.
+ */
+export function codeSpanMask(line) {
+    const mask = new Array(line.length).fill(false);
+    let i = 0;
+    while (i < line.length) {
+        if (line[i] !== "`") {
+            i++;
+            continue;
+        }
+        const end = codeSpanEnd(line, i);
+        if (end === null) {
+            while (line[i] === "`")
+                i++;
+            continue;
+        }
+        for (let k = i; k < end; k++)
+            mask[k] = true;
+        i = end;
+    }
+    return mask;
+}
+/** The raw-text tag opening at `start`, or null. */
+function rawTextTagAt(line, start) {
+    if (line[start] !== "<")
+        return null;
+    const lower = line.slice(start, start + 6).toLowerCase();
+    for (const tag of ["pre", "code"]) {
+        if (!lower.startsWith(`<${tag}`))
+            continue;
+        // `<pre>`, `<pre/>`, `<pre class=…>` — but not `<precise`.
+        const boundary = line[start + tag.length + 1];
+        if (boundary === ">" || boundary === "/" || boundary === " " || boundary === "\t") {
+            return tag;
+        }
+    }
+    return null;
+}
+/** One past the `>` closing the tag that opens at `start`, or null. */
+function htmlTagEnd(line, start) {
+    let quote = null;
+    for (let i = start; i < line.length; i++) {
+        const char = line[i];
+        if (quote !== null) {
+            if (char === quote)
+                quote = null;
+        }
+        else if (char === '"' || char === "'") {
+            quote = char;
+        }
+        else if (char === ">") {
+            return i + 1;
+        }
+    }
+    return null;
+}
+/** One past `</pre>` / `</code>` at or after `start`, or null. */
+function rawTextCloseEnd(line, start, tag) {
+    const match = new RegExp(`</${tag}\\s*>`, "i").exec(line.slice(start));
+    return match === null ? null : start + match.index + match[0].length;
+}
+/**
+ * Blank one line's non-rendering spans, carrying `state` forward.
+ *
+ * Order matters: a backtick inside a comment is not a code span, and a `<!--`
+ * inside a code span is not a comment, so whichever construct opens first owns
+ * everything up to its own close. Left open at the end of the line, a comment
+ * or a raw tag keeps consuming into the lines below — which is what an
+ * unterminated `<!--` does to a reader too.
+ */
+function maskLine(raw, state) {
+    const chars = raw.split("");
+    const blank = (from, to) => {
+        for (let i = from; i < to; i++)
+            chars[i] = " ";
+    };
+    let i = 0;
+    while (i < raw.length) {
+        if (state.comment) {
+            const close = raw.indexOf("-->", i);
+            blank(i, close < 0 ? raw.length : close + 3);
+            if (close < 0)
+                break;
+            state.comment = false;
+            i = close + 3;
+            continue;
+        }
+        if (state.rawTag !== null) {
+            const end = rawTextCloseEnd(raw, i, state.rawTag);
+            blank(i, end ?? raw.length);
+            if (end === null)
+                break;
+            state.rawTag = null;
+            i = end;
+            continue;
+        }
+        const char = raw[i];
+        if (char === "`") {
+            const end = codeSpanEnd(raw, i);
+            if (end === null) {
+                while (raw[i] === "`")
+                    i++;
+                continue;
+            }
+            blank(i, end);
+            i = end;
+            continue;
+        }
+        if (char === "<") {
+            if (raw.startsWith("<!--", i)) {
+                // CommonMark 0.31 admits the two degenerate spellings `<!-->` and
+                // `<!--->` as complete comments. Without them the `-->` search runs
+                // off the end, the comment is read as unterminated, and everything
+                // after it in the document is masked on the strength of five bytes.
+                const short = raw.startsWith("<!--->", i)
+                    ? 6
+                    : raw.startsWith("<!-->", i)
+                        ? 5
+                        : 0;
+                if (short > 0) {
+                    blank(i, i + short);
+                    i += short;
+                    continue;
+                }
+                const close = raw.indexOf("-->", i + 4);
+                blank(i, close < 0 ? raw.length : close + 3);
+                state.comment = close < 0;
+                if (state.comment)
+                    break;
+                i = close + 3;
+                continue;
+            }
+            const tag = rawTextTagAt(raw, i);
+            if (tag !== null) {
+                const openEnd = htmlTagEnd(raw, i);
+                // An opening tag that runs past the end of the line: nothing after it
+                // on this line can be read with any confidence, so stop here.
+                if (openEnd === null)
+                    break;
+                const closeEnd = rawTextCloseEnd(raw, openEnd, tag);
+                blank(i, closeEnd ?? raw.length);
+                state.rawTag = closeEnd === null ? tag : null;
+                if (closeEnd === null)
+                    break;
+                i = closeEnd;
+                continue;
+            }
+        }
+        i++;
+    }
+    return chars.join("");
+}
+/**
+ * The non-rendering mask, line by line. Each returned line has exactly the
+ * length of the line it masks, so `masked[i][k]` and `lines[i][k]` are the
+ * same byte position.
+ *
+ * `from` is where the document's markdown begins — everything above it is a
+ * frontmatter block, which is not markdown at all and is returned verbatim for
+ * whoever does read it (lint, with `inFrontmatter`). Scanning it would be
+ * worse than useless: a `<!--` in a YAML value would open a comment over the
+ * whole document. Callers holding a BODY pass 0 — a leading `---` there is a
+ * thematic break, not frontmatter — and `maskNonRenderingContexts` works the
+ * boundary out with `markdownStart`.
+ *
+ * A fence decides the whole line, but only where nothing is open already: a
+ * ``` line inside an unterminated comment is comment text, and the document
+ * below the `-->` renders. That ordering is CommonMark's — an HTML block ends
+ * at its closing condition, not at the next thing that looks like a fence —
+ * and it is the one place fence geometry does not go first. Where the line is
+ * clear, `fenceRegionAt` answers, so there is still exactly one idea in the
+ * package of where a fence starts and ends.
+ *
+ * Lines are whatever the caller split; a bare `\r` inside one is not a line
+ * ending here. `maskNonRenderingContexts` splits the way CommonMark does.
+ */
+export function maskNonRenderingLines(lines, from = 0) {
+    const state = { comment: false, rawTag: null };
+    const out = [];
+    let fenceEnd = -1;
+    for (let i = 0; i < lines.length; i++) {
+        const raw = lines[i];
+        if (i < from) {
+            out.push(raw);
+            continue;
+        }
+        if (i <= fenceEnd) {
+            out.push(" ".repeat(raw.length));
+            continue;
+        }
+        if (!state.comment && state.rawTag === null) {
+            const region = fenceRegionAt(lines, i);
+            if (region !== null) {
+                fenceEnd = region.close;
+                out.push(" ".repeat(raw.length));
+                continue;
+            }
+        }
+        out.push(maskLine(raw, state));
+    }
+    return out;
+}
+/**
+ * Split a document the way CommonMark does: `\n`, `\r\n` and a bare `\r` all
+ * end a line. The terminators are not part of the returned text and are never
+ * rewritten, which is how the mask stays byte-aligned with the source.
+ */
+function documentLines(source) {
+    const lines = [];
+    let start = 0;
+    for (let i = 0; i < source.length; i++) {
+        const char = source[i];
+        if (char === "\n") {
+            lines.push({ text: source.slice(start, i), start });
+            start = i + 1;
+        }
+        else if (char === "\r") {
+            lines.push({ text: source.slice(start, i), start });
+            if (source[i + 1] === "\n")
+                i++;
+            start = i + 1;
+        }
+    }
+    lines.push({ text: source.slice(start), start });
+    return lines;
+}
+/**
+ * The whole source, masked. `masked.length === source.length` always: the two
+ * strings are the same document, one of them with the non-markdown blanked
+ * out, and every offset holds across both.
+ *
+ * This is the DOCUMENT entry point — cm-lint and `demoteNonRenderingLinks`
+ * hand it whole files — so the frontmatter boundary is worked out here rather
+ * than assumed to be 0. Pass `from` explicitly when the caller knows better:
+ * 0 says "these bytes are all markdown", which is what a body is.
+ */
+export function maskNonRenderingContexts(source, from) {
+    const lines = documentLines(source);
+    const texts = lines.map((line) => line.text);
+    const masked = maskNonRenderingLines(texts, from ?? markdownStart(texts));
+    // Written back in place, never re-joined: the source keeps its own line
+    // terminators, whatever mix of them it has.
+    const out = source.split("");
+    for (let i = 0; i < lines.length; i++) {
+        const { start } = lines[i];
+        const text = masked[i];
+        for (let k = 0; k < text.length; k++)
+            out[start + k] = text[k];
+    }
+    return out.join("");
+}
+/**
+ * True when the source span `[start, end)` holds bytes but no rendering ones —
+ * the predicate a consumer asks before trusting anything it found by scanning.
+ *
+ * A span that was blank to begin with is not "non-rendering", it is empty, and
+ * answering true for it would demote every piece of whitespace in the
+ * document.
+ */
+export function isNonRenderingRange(source, masked, start, end) {
+    if (end <= start)
+        return false;
+    if (source.slice(start, end).trim() === "")
+        return false;
+    return masked.slice(start, end).trim() === "";
+}