tina4-nodejs 3.13.133 → 3.13.134

package/packages/core/src/docs.ts

@@ -19,6 +19,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { fileURLToPath } from "node:url";
+import { parseTypeScript, type ParsedFile } from "./docsParser.js";
 
 // ── Types ────────────────────────────────────────────────────────────
 
@@ -223,505 +224,6 @@ function docblockBody(doc: string): string {
   return lines.join(" ");
 }
 
-// ── TS regex parser ──────────────────────────────────────────────────
-
-interface ParsedClass {
-  name: string;
-  line: number;
-  doc: string;
-  exported: boolean;
-  methods: ParsedMethod[];
-}
-
-interface ParsedMethod {
-  name: string;
-  line: number;
-  doc: string;
-  signature: string;
-  visibility: "public" | "protected" | "private";
-  static: boolean;
-}
-
-interface ParsedFile {
-  classes: ParsedClass[];
-  functions: ParsedMethod[];
-}
-
-const CLASS_RE = /(?:^|\n)([ \t]*)((?:export\s+(?:default\s+)?(?:abstract\s+)?)?class\s+([A-Za-z_$][\w$]*))[\s\S]*?(?=\n[ \t]*(?:export\s+(?:default\s+)?(?:abstract\s+)?class|export\s+function|function|$))/g;
-
-/**
- * Parse a TS source string. Lightweight — finds top-level classes and their
- * public methods, plus top-level exported functions. Captures preceding JSDoc.
- *
- * Strategy: scan token-by-token. We don't need a full AST — we only care
- * about identifying class declarations, brace depth (to find class members),
- * method/function declarations, and JSDoc comments immediately above.
- */
-function parseTypeScript(source: string, _debugTag = ""): ParsedFile {
-  const classes: ParsedClass[] = [];
-  const functions: ParsedMethod[] = [];
-
-  // Strip line comments and string contents (preserve length for line numbers).
-  const stripped = stripStrings(source);
-  const lines = source.split(/\r?\n/);
-
-  let i = 0;
-  let line = 1;
-  let pendingDoc = "";
-  let pendingDocLine = 0;
-  const len = stripped.length;
-  let braceDepth = 0;
-  let classStack: { name: string; bodyStartDepth: number; entry: ParsedClass; isExport: boolean }[] = [];
-
-  function lineOf(offset: number): number {
-    // Count newlines up to offset.
-    let l = 1;
-    for (let k = 0; k < offset && k < source.length; k++) {
-      if (source.charCodeAt(k) === 10) l++;
-    }
-    return l;
-  }
-
-  while (i < len) {
-    const ch = stripped[i];
-
-    // JSDoc detection
-    if (ch === "/" && stripped[i + 1] === "*" && stripped[i + 2] === "*") {
-      const end = stripped.indexOf("*/", i + 3);
-      if (end === -1) break;
-      pendingDoc = source.slice(i, end + 2);
-      pendingDocLine = lineOf(i);
-      i = end + 2;
-      continue;
-    }
-
-    // Skip line comments (already stripped → '/' followed by '/' won't appear in stripped, but be safe)
-    if (ch === "/" && stripped[i + 1] === "/") {
-      while (i < len && stripped[i] !== "\n") i++;
-      continue;
-    }
-
-    // Brace tracking (only outside strings; strings are zeroed in stripped)
-    if (ch === "{") {
-      braceDepth++;
-      i++;
-      continue;
-    }
-    if (ch === "}") {
-      braceDepth--;
-      // Pop classes whose body just closed
-      while (classStack.length > 0 && braceDepth <= classStack[classStack.length - 1].bodyStartDepth) {
-        const cls = classStack.pop()!;
-        classes.push(cls.entry);
-      }
-      i++;
-      continue;
-    }
-
-    // Look for "class <Name>"
-    if (isWordBoundary(stripped, i) && matchKeyword(stripped, i, "class")) {
-      // Read modifiers backwards on this line — already covered by pendingDoc capture.
-      const after = i + "class".length;
-      const nameMatch = /^\s+([A-Za-z_$][\w$]*)/.exec(stripped.slice(after));
-      if (nameMatch) {
-        const name = nameMatch[1];
-        const classLine = lineOf(i);
-        // Look for opening '{' starting from after the name
-        let j = after + nameMatch[0].length;
-        while (j < len && stripped[j] !== "{") j++;
-        if (j < len) {
-          const isExport = isExportedAt(stripped, lines, classLine, name);
-          const entry: ParsedClass = {
-            name,
-            line: classLine,
-            doc: pendingDoc,
-            exported: isExport,
-            methods: [],
-          };
-          // Push class context with its bodyStartDepth = current braceDepth
-          // (the upcoming '{' will increment braceDepth one above this).
-          classStack.push({ name, bodyStartDepth: braceDepth, entry, isExport });
-          pendingDoc = "";
-          // Move past '{' and increment depth
-          braceDepth++;
-          i = j + 1;
-          continue;
-        }
-      }
-    }
-
-    // Method or top-level function detection — we only care about either:
-    //   * methods inside a class body (classStack non-empty AND directly inside class body)
-    //   * top-level "export function" or "function" declarations
-    if (classStack.length > 0
-        && braceDepth === classStack[classStack.length - 1].bodyStartDepth + 1) {
-      // We're directly inside a class body.
-      const m = matchMethodSignature(stripped, source, i);
-      if (m) {
-        const methodLine = lineOf(m.nameStart ?? i);
-        const cls = classStack[classStack.length - 1];
-        // Skip private/protected based on TS modifier OR name prefix '_'
-        const visibility = m.visibility;
-        cls.entry.methods.push({
-          name: m.name,
-          line: methodLine,
-          doc: pendingDoc,
-          signature: m.signature,
-          visibility,
-          static: m.static,
-        });
-        pendingDoc = "";
-        i = m.endIndex;
-        continue;
-      }
-    } else if (braceDepth === 0) {
-      // Top-level — look for "export function" or "function"
-      const f = matchTopLevelFunction(stripped, source, i);
-      if (f) {
-        const fLine = lineOf(f.nameStart ?? i);
-        functions.push({
-          name: f.name,
-          line: fLine,
-          doc: pendingDoc,
-          signature: f.signature,
-          visibility: "public",
-          static: false,
-        });
-        pendingDoc = "";
-        i = f.endIndex;
-        continue;
-      }
-    }
-
-    // Whitespace doesn't reset pendingDoc — but most other tokens do.
-    if (!/\s/.test(ch)) {
-      // Non-whitespace, non-doc-comment — only reset doc if it was a long way back.
-      // Be conservative: only reset on punctuation that clearly terminates.
-      if (ch === ";") {
-        pendingDoc = "";
-      }
-    }
-
-    i++;
-  }
-
-  // Any unclosed class (shouldn't happen in valid TS) → flush.
-  while (classStack.length > 0) classes.push(classStack.pop()!.entry);
-
-  // Suppress unused-warning
-  void pendingDocLine;
-
-  return { classes, functions };
-}
-
-function isWordBoundary(text: string, i: number): boolean {
-  if (i === 0) return true;
-  const prev = text.charCodeAt(i - 1);
-  // Word chars: A-Z a-z 0-9 _ $
-  if ((prev >= 65 && prev <= 90) || (prev >= 97 && prev <= 122) || (prev >= 48 && prev <= 57) || prev === 95 || prev === 36) {
-    return false;
-  }
-  return true;
-}
-
-function matchKeyword(text: string, i: number, kw: string): boolean {
-  if (text.substr(i, kw.length) !== kw) return false;
-  const after = i + kw.length;
-  if (after >= text.length) return true;
-  const nextCode = text.charCodeAt(after);
-  if ((nextCode >= 65 && nextCode <= 90) || (nextCode >= 97 && nextCode <= 122) || (nextCode >= 48 && nextCode <= 57) || nextCode === 95 || nextCode === 36) {
-    return false;
-  }
-  return true;
-}
-
-function isExportedAt(_stripped: string, lines: string[], lineNo: number, name: string): boolean {
-  // Walk back up to 8 lines and look for "export class <name>" / "export default class <name>"
-  const start = Math.max(0, lineNo - 1);
-  const exportClass = "export class " + name;
-  const exportDefaultClass = "export default class " + name;
-  const exportAbstractClass = "export abstract class " + name;
-  const justClass = "class " + name;
-  for (let l = start; l >= Math.max(0, start - 8); l--) {
-    const ln = lines[l] || "";
-    if (ln.includes(exportClass) || ln.includes(exportDefaultClass) || ln.includes(exportAbstractClass)) {
-      return true;
-    }
-    if (ln.includes(justClass)) {
-      // declared but not exported
-      return /\bexport\b/.test(ln);
-    }
-  }
-  return false;
-}
-
-interface MethodMatch {
-  name: string;
-  signature: string;
-  endIndex: number;
-  nameStart: number;
-  visibility: "public" | "protected" | "private";
-  static: boolean;
-}
-
-const METHOD_HEAD_RE =
-  /^([ \t]*)((?:public|protected|private|readonly|static|async|abstract|override|\s)*)([A-Za-z_$][\w$]*)\s*[<(]/;
-
-function matchMethodSignature(stripped: string, source: string, i: number): MethodMatch | null {
-  // Method must be at start-of-line-ish position.
-  if (i > 0) {
-    const prev = stripped.charCodeAt(i - 1);
-    if (prev !== 10 && prev !== 32 && prev !== 9 && prev !== 123) return null;
-  }
-  // Take the rest of the current line + a little ahead.
-  let lineEnd = stripped.indexOf("\n", i);
-  if (lineEnd === -1) lineEnd = stripped.length;
-  // Read up to 4 lines for multi-line signatures.
-  let chunkEnd = lineEnd;
-  for (let extra = 0; extra < 4 && chunkEnd < stripped.length; extra++) {
-    const next = stripped.indexOf("\n", chunkEnd + 1);
-    if (next === -1) break;
-    chunkEnd = next;
-  }
-  const chunk = stripped.slice(i, chunkEnd + 1);
-  const match = METHOD_HEAD_RE.exec(chunk);
-  if (!match) return null;
-  const modifiers = match[2] || "";
-  const name = match[3];
-  // Skip reserved words / control-flow that masquerade as method names.
-  const reserved = new Set([
-    "if", "for", "while", "switch", "return", "do", "try", "catch", "throw",
-    "const", "let", "var", "import", "export", "function", "class", "interface",
-    "type", "new", "yield", "await", "case", "break", "continue", "else",
-  ]);
-  if (reserved.has(name)) return null;
-
-  // Determine visibility from modifiers
-  let visibility: "public" | "protected" | "private" = "public";
-  if (/\bprivate\b/.test(modifiers)) visibility = "private";
-  else if (/\bprotected\b/.test(modifiers)) visibility = "protected";
-  const isStatic = /\bstatic\b/.test(modifiers);
-
-  // Capture signature — read from start of "name" up through matching ')' and optional return type.
-  const nameStart = i + match[1].length + match[2].length;
-  const result = captureSignature(stripped, source, nameStart, name);
-  if (!result) return null;
-
-  return {
-    name,
-    signature: result.signature,
-    endIndex: result.endIndex,
-    nameStart,
-    visibility,
-    static: isStatic,
-  };
-}
-
-const FN_HEAD_RE =
-  /^((?:export\s+(?:default\s+)?)?(?:async\s+)?function\s+)([A-Za-z_$][\w$]*)\s*[<(]/;
-
-function matchTopLevelFunction(stripped: string, source: string, i: number): MethodMatch | null {
-  if (i > 0) {
-    const prev = stripped.charCodeAt(i - 1);
-    if (prev !== 10 && prev !== 32 && prev !== 9) return null;
-  }
-  let lineEnd = stripped.indexOf("\n", i);
-  if (lineEnd === -1) lineEnd = stripped.length;
-  let chunkEnd = lineEnd;
-  for (let extra = 0; extra < 4 && chunkEnd < stripped.length; extra++) {
-    const next = stripped.indexOf("\n", chunkEnd + 1);
-    if (next === -1) break;
-    chunkEnd = next;
-  }
-  const chunk = stripped.slice(i, chunkEnd + 1);
-  const match = FN_HEAD_RE.exec(chunk);
-  if (!match) return null;
-  const name = match[2];
-  const nameStart = i + match[1].length;
-  const result = captureSignature(stripped, source, nameStart, name);
-  if (!result) return null;
-  return {
-    name,
-    signature: result.signature,
-    endIndex: result.endIndex,
-    nameStart,
-    visibility: "public",
-    static: false,
-  };
-}
-
-interface CapturedSig {
-  signature: string;
-  endIndex: number;
-}
-
-function captureSignature(stripped: string, source: string, nameStart: number, name: string): CapturedSig | null {
-  // Skip the name
-  let j = nameStart + name.length;
-  // Optional generic <...>
-  while (j < stripped.length && /\s/.test(stripped[j])) j++;
-  if (stripped[j] === "<") {
-    let depth = 0;
-    while (j < stripped.length) {
-      const c = stripped[j];
-      if (c === "<") depth++;
-      else if (c === ">") {
-        depth--;
-        if (depth === 0) { j++; break; }
-      }
-      j++;
-    }
-  }
-  // Whitespace
-  while (j < stripped.length && /\s/.test(stripped[j])) j++;
-  if (stripped[j] !== "(") return null;
-  // Capture (...) balanced on parens (ignore strings — already stripped)
-  const parenStart = j;
-  let depth = 0;
-  while (j < stripped.length) {
-    const c = stripped[j];
-    if (c === "(") depth++;
-    else if (c === ")") {
-      depth--;
-      if (depth === 0) { j++; break; }
-    }
-    j++;
-  }
-  const parenSegment = source.slice(parenStart, j); // pull from original source for human-readable
-  // Optional return type: ": Type" up to '{' or ';' or '=>' or end-of-line for arrow.
-  let retStart = j;
-  while (retStart < stripped.length && /[ \t]/.test(stripped[retStart])) retStart++;
-  let retEnd = retStart;
-  let returnType = "";
-  if (stripped[retStart] === ":") {
-    retEnd = retStart + 1;
-    let depthBracket = 0;
-    while (retEnd < stripped.length) {
-      const c = stripped[retEnd];
-      // Stop on a body-opening '{' or terminating ';' at the same depth.
-      if (depthBracket === 0 && (c === "{" || c === ";")) break;
-      if (depthBracket === 0 && c === "\n") {
-        // Arrow-return on next line — stop only if the next non-space is '{'.
-        let k2 = retEnd + 1;
-        while (k2 < stripped.length && (stripped[k2] === " " || stripped[k2] === "\t")) k2++;
-        if (stripped[k2] === "{") break;
-      }
-      if (c === "<" || c === "(" || c === "[") depthBracket++;
-      else if (c === ">" || c === ")" || c === "]") depthBracket--;
-      retEnd++;
-    }
-    returnType = source.slice(retStart, retEnd).trim();
-  }
-  const cleanedParens = parenSegment.replace(/\s+/g, " ");
-  const sig = name + cleanedParens + (returnType ? " " + returnType : "");
-  return { signature: sig, endIndex: retEnd };
-}
-
-/**
- * Replace string literals and template literal contents with spaces of equal
- * length so brace/paren scanning isn't fooled by characters inside strings.
- * Also strips line + block comments. Newlines are preserved so line numbers
- * line up with the original source.
- */
-function stripStrings(source: string): string {
-  const out: string[] = [];
-  const len = source.length;
-  let i = 0;
-  const BACKTICK = String.fromCharCode(96);
-  const DOLLAR = "$";
-  const OPEN_BRACE = "{";
-  const CLOSE_BRACE = "}";
-
-  while (i < len) {
-    const c = source[i];
-
-    if (c === "/" && source[i + 1] === "*") {
-      const end = source.indexOf("*/", i + 2);
-      if (end === -1) {
-        for (let k = i; k < len; k++) out.push(source[k] === "\n" ? "\n" : " ");
-        return out.join("");
-      }
-      for (let k = i; k < end + 2; k++) out.push(source[k] === "\n" ? "\n" : " ");
-      i = end + 2;
-      continue;
-    }
-
-    if (c === "/" && source[i + 1] === "/") {
-      while (i < len && source[i] !== "\n") {
-        out.push(" ");
-        i++;
-      }
-      continue;
-    }
-
-    if (c === '"' || c === "'") {
-      out.push(c);
-      i++;
-      while (i < len) {
-        const sc = source[i];
-        if (sc === "\\" && i + 1 < len) {
-          out.push("  ");
-          i += 2;
-          continue;
-        }
-        if (sc === c) {
-          out.push(c);
-          i++;
-          break;
-        }
-        out.push(sc === "\n" ? "\n" : " ");
-        i++;
-      }
-      continue;
-    }
-
-    if (c === BACKTICK) {
-      // Treat the entire template literal — including any ${...} expressions —
-      // as opaque content. Replace every char inside with spaces so brace/paren
-      // accounting in the outer parser isn't fooled. Newlines preserved.
-      out.push(c);
-      i++;
-      while (i < len) {
-        const tc = source[i];
-        if (tc === "\\" && i + 1 < len) {
-          out.push("  ");
-          i += 2;
-          continue;
-        }
-        if (tc === BACKTICK) {
-          out.push(BACKTICK);
-          i++;
-          break;
-        }
-        if (tc === DOLLAR && source[i + 1] === OPEN_BRACE) {
-          // Skip over the entire ${ ... } expression, replacing all chars with
-          // spaces (or newlines). Track real brace depth (ignoring nested
-          // template literals' own braces, but those are zeroed too).
-          out.push(" ");
-          out.push(" ");
-          i += 2;
-          let depth = 1;
-          while (i < len && depth > 0) {
-            const ic = source[i];
-            if (ic === OPEN_BRACE) depth++;
-            else if (ic === CLOSE_BRACE) depth--;
-            out.push(ic === "\n" ? "\n" : " ");
-            i++;
-          }
-          continue;
-        }
-        out.push(tc === "\n" ? "\n" : " ");
-        i++;
-      }
-      continue;
-    }
-
-    out.push(c);
-    i++;
-  }
-  return out.join("");
-}
-
 // ── Indexer ──────────────────────────────────────────────────────────
 
 function walkTsFiles(root: string): string[] {
@@ -870,6 +372,52 @@ function buildLineIndex(text: string): (offset: number) => number {
   };
 }
 
+function scoreNameToken(name: string, stripped: string, nameTokens: string[], token: string): number {
+  if (!token) return 0;
+  if (name.startsWith(token) || stripped.startsWith(token)) return 3;
+  for (const nameToken of nameTokens) {
+    if (nameToken === token) return 3;
+    if (nameToken.startsWith(token)) return 2;
+  }
+  return name.includes(token) ? 0.5 : 0;
+}
+
+function scoreName(name: string, stripped: string, nameTokens: string[], tokens: string[], joined: string): number {
+  let score = name === joined || stripped === joined ? 5 : 0;
+  for (const token of tokens) score += scoreNameToken(name, stripped, nameTokens, token);
+  return score;
+}
+
+function scoreText(text: string, tokens: string[], weight: number): number {
+  let score = 0;
+  for (const token of tokens) {
+    if (token && text.includes(token)) score += weight;
+  }
+  return score;
+}
+
+function scoreClassQualifier(entry: InternalEntry, name: string, tokens: string[], joined: string): number {
+  const parent = (entry.class ?? "").toLowerCase();
+  if (!parent) return 0;
+  let score = 0;
+  const normalized = joined.replace(/[:.]+/g, ".");
+  if (normalized === `${parent}.${name}` || normalized === `${parent}.${name.replace(/^_+/, "")}`) score += 6;
+  for (const token of tokens) {
+    if (token === parent) score += 2.5;
+    else if (token && parent.startsWith(token)) score += 1;
+  }
+  return score;
+}
+
+function scoreFqn(entry: InternalEntry, tokens: string[]): number {
+  const segments = new Set(entry.fqn.toLowerCase().split(/[.\s:]+/).filter(Boolean));
+  let score = 0;
+  for (const token of tokens) {
+    if (token && segments.has(token)) score += 1;
+  }
+  return score;
+}
+
 // ── Public Docs class ───────────────────────────────────────────────
 
 /**
@@ -1233,52 +781,12 @@ export class Docs {
   private scoreEntry(entry: InternalEntry, tokens: string[], joined: string): number {
     const name = entry.name.toLowerCase();
     const stripped = name.replace(/^_+/, "");
-    const summary = entry.summary.toLowerCase();
-    const doc = entry.docblock.toLowerCase();
-    let score = 0;
-
-    if (name === joined || stripped === joined) score += 5;
-
     const nameTokens = tokenise(entry.name);
-    for (const tk of tokens) {
-      if (!tk) continue;
-      if (name.startsWith(tk) || stripped.startsWith(tk)) {
-        score += 3;
-        continue;
-      }
-      let hit = false;
-      for (const nt of nameTokens) {
-        if (nt === tk) { score += 3; hit = true; break; }
-        if (nt.startsWith(tk)) { score += 2; hit = true; break; }
-      }
-      if (!hit && name.includes(tk)) score += 0.5;
-    }
-    for (const tk of tokens) {
-      if (tk && summary.includes(tk)) score += 2;
-    }
-    for (const tk of tokens) {
-      if (tk && doc.includes(tk)) score += 1;
-    }
-    // Class-qualified queries ("Frond.addTest" / "Frond addTest"): score the
-    // owning class so the qualifier steers ranking instead of being dead weight.
-    const parent = (entry.class ?? "").toLowerCase();
-    if (parent) {
-      // Normalise `.`/`:`/whitespace in the joined query to a single `.` so
-      // "frond.addtest", "frond:addtest" and "frondaddtest" all compare alike.
-      const qNorm = joined.replace(/[:.]+/g, ".");
-      if (qNorm === `${parent}.${name}` || qNorm === `${parent}.${stripped}`) {
-        score += 6; // exact "Class.method" intent — the strongest signal
-      }
-      for (const tk of tokens) {
-        if (tk === parent) score += 2.5;
-        else if (tk && parent.startsWith(tk)) score += 1;
-      }
-    }
-    // Any token that is a whole segment of the fqn (module / class / name).
-    const fqnSegs = new Set(entry.fqn.toLowerCase().split(/[.\s:]+/).filter(Boolean));
-    for (const tk of tokens) {
-      if (tk && fqnSegs.has(tk)) score += 1;
-    }
+    let score = scoreName(name, stripped, nameTokens, tokens, joined);
+    score += scoreText(entry.summary.toLowerCase(), tokens, 2);
+    score += scoreText(entry.docblock.toLowerCase(), tokens, 1);
+    score += scoreClassQualifier(entry, name, tokens, joined);
+    score += scoreFqn(entry, tokens);
     if (joined && score === 0 && name.includes(joined)) score += 2;
     return score;
   }