scribe-cms 0.0.10 → 0.0.12

package/dist/index.js

@@ -3,6 +3,9 @@ import path3 from 'path';
 import fs2 from 'fs';
 import matter from 'gray-matter';
 import Database from 'better-sqlite3';
+import remarkMdx from 'remark-mdx';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
 import { createJiti } from 'jiti';
 import { createHash } from 'crypto';
 import { GoogleGenAI } from '@google/genai';
@@ -687,6 +690,135 @@ function bulkLoadTranslations(db) {
   return db.prepare(`SELECT * FROM translations ORDER BY content_type, en_slug, locale`).all();
 }
 
+// src/translate/normalize-mdx-body.ts
+function needsMdxEscapeNormalization(body) {
+  return /\\n[ \t/>]|<[\w.-][^>]*\\n/.test(body);
+}
+function normalizeTranslatedMdxBody(body) {
+  if (!needsMdxEscapeNormalization(body)) {
+    return { body, adjusted: false };
+  }
+  const normalized = body.replace(/\\r\\n/g, "\n").replace(/\\n/g, "\n").replace(/\\t/g, "	").replace(/\\"/g, '"').replace(/\\\\/g, "\\");
+  return { body: normalized, adjusted: normalized !== body };
+}
+
+// src/translate/sanitize-mdx-jsx.ts
+function sanitizeMdxJsxAttributeQuotes(body) {
+  let adjusted = false;
+  let out = "";
+  let i = 0;
+  while (i < body.length) {
+    if (body[i] !== "<" || !/[\w.]/.test(body[i + 1] ?? "")) {
+      out += body[i];
+      i += 1;
+      continue;
+    }
+    const tagStart = i;
+    i += 1;
+    while (i < body.length && /[\w.-]/.test(body[i] ?? "")) i += 1;
+    const tagName = body.slice(tagStart + 1, i);
+    let tagOut = `<${tagName}`;
+    let tagAdjusted = false;
+    while (i < body.length && body[i] !== ">" && body[i] !== "/") {
+      while (i < body.length && /\s/.test(body[i] ?? "")) {
+        tagOut += body[i];
+        i += 1;
+      }
+      if (i >= body.length || body[i] === ">" || body[i] === "/") break;
+      const attrStart = i;
+      while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
+      body.slice(attrStart, i);
+      while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
+      if (body[i] !== "=") {
+        tagOut += body.slice(attrStart, i);
+        continue;
+      }
+      tagOut += body.slice(attrStart, i);
+      tagOut += "=";
+      i += 1;
+      while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
+      const quote = body[i];
+      if (quote !== '"' && quote !== "'") {
+        continue;
+      }
+      if (quote === "'") {
+        const valStart2 = i + 1;
+        i += 1;
+        while (i < body.length) {
+          if (body[i] === "\\") {
+            i += 2;
+            continue;
+          }
+          if (body[i] === "'") break;
+          i += 1;
+        }
+        tagOut += body.slice(valStart2 - 1, i + 1);
+        i += 1;
+        continue;
+      }
+      const valStart = i + 1;
+      i += 1;
+      let closeIdx = -1;
+      for (let scan = valStart; scan < body.length; scan += 1) {
+        if (body[scan] !== '"') continue;
+        let j = scan + 1;
+        while (j < body.length && /\s/.test(body[j] ?? "")) j += 1;
+        if (j < body.length && (body[j] === ">" || body[j] === "/" || /[a-zA-Z]/.test(body[j] ?? ""))) {
+          closeIdx = scan;
+          break;
+        }
+      }
+      if (closeIdx === -1) {
+        tagOut += body.slice(valStart - 1, i);
+        break;
+      }
+      const value = body.slice(valStart, closeIdx);
+      const hasInternalQuote = value.includes('"');
+      if (hasInternalQuote) {
+        const escaped = value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+        tagOut += `'${escaped}'`;
+        tagAdjusted = true;
+      } else {
+        tagOut += `"${value}"`;
+      }
+      i = closeIdx + 1;
+    }
+    if (tagAdjusted) adjusted = true;
+    tagOut += body[i] ?? "";
+    out += tagOut;
+    i += 1;
+  }
+  return { body: out, adjusted };
+}
+
+// src/translate/validate-mdx-body.ts
+var parser = unified().use(remarkParse).use(remarkMdx);
+function prepareTranslatedMdxBody(body) {
+  const normalized = normalizeTranslatedMdxBody(body);
+  const sanitized = sanitizeMdxJsxAttributeQuotes(normalized.body);
+  return {
+    body: sanitized.body,
+    adjusted: normalized.adjusted || sanitized.adjusted
+  };
+}
+function validateMdxBody(body) {
+  try {
+    parser.parse(body);
+    return { ok: true };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { ok: false, error: message };
+  }
+}
+function assertValidTranslatedMdxBody(body) {
+  const prepared = prepareTranslatedMdxBody(body);
+  const validated = validateMdxBody(prepared.body);
+  if (!validated.ok) {
+    throw new Error(`MDX validation failed: ${validated.error}`);
+  }
+  return prepared;
+}
+
 // src/loader/normalize-en.ts
 function normalizeEnFrontmatter(data) {
   const out = { ...data };
@@ -807,7 +939,7 @@ function buildDocumentFromTranslation(row, enDoc, type, config) {
     noindex: seo.noindex,
     canonicalPathOverride: seo.canonicalPathOverride,
     frontmatter,
-    content: row.body
+    content: prepareTranslatedMdxBody(row.body).body
   };
 }
 var DEV_REVALIDATE_MS = 1500;
@@ -2075,6 +2207,20 @@ function listEnSlugs3(rootDir, contentDir) {
   if (!fs2.existsSync(dir)) return [];
   return fs2.readdirSync(dir).filter(isPublishableContentFile).map((f) => f.replace(/\.(md|mdx)$/, ""));
 }
+function validateDocumentMdxBody(issues, input) {
+  const preparedBody = prepareTranslatedMdxBody(input.body).body;
+  const mdxValidation = validateMdxBody(preparedBody);
+  if (!mdxValidation.ok) {
+    issues.push({
+      level: "error",
+      contentType: input.contentType,
+      enSlug: input.enSlug,
+      locale: input.locale,
+      field: "body",
+      message: `Invalid MDX: ${mdxValidation.error}`
+    });
+  }
+}
 function validateProject(config) {
   const issues = [];
   const project = createProject(config);
@@ -2133,6 +2279,12 @@ function validateProject(config) {
       })) {
         issues.push(issue);
       }
+      validateDocumentMdxBody(issues, {
+        contentType: type.id,
+        enSlug,
+        locale: config.defaultLocale,
+        body: enDoc.content
+      });
       for (const locale of config.locales) {
         if (locale === config.defaultLocale) continue;
         const row = getTranslation(db, type.id, enSlug, locale);
@@ -2148,15 +2300,22 @@ function validateProject(config) {
             message: issue.message
           });
         }
+        const preparedBody = prepareTranslatedMdxBody(row.body).body;
         for (const issue of validateDocumentAssets(config, {
           contentType: type.id,
           enSlug,
           locale,
           frontmatter: localeFm,
-          body: row.body
+          body: preparedBody
         })) {
           issues.push(issue);
         }
+        validateDocumentMdxBody(issues, {
+          contentType: type.id,
+          enSlug,
+          locale,
+          body: row.body
+        });
       }
     }
     try {
@@ -2305,12 +2464,20 @@ function normalizeGeminiDisplayName(model) {
 // src/translate/gemini-client.ts
 var DEFAULT_MODEL = DEFAULT_GEMINI_MODEL;
 function extractJson(text) {
-  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+  const trimmed = text.trim();
+  const fenced = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
   if (fenced?.[1]) return fenced[1].trim();
-  const start = text.indexOf("{");
-  const end = text.lastIndexOf("}");
-  if (start >= 0 && end > start) return text.slice(start, end + 1);
-  return text.trim();
+  const start = trimmed.indexOf("{");
+  const end = trimmed.lastIndexOf("}");
+  if (start >= 0 && end > start) return trimmed.slice(start, end + 1);
+  return trimmed;
+}
+function parseGeminiResponse(text) {
+  try {
+    return JSON.parse(text.trim());
+  } catch {
+    return JSON.parse(extractJson(text));
+  }
 }
 async function translatePageWithGemini(input) {
   const apiKey = input.apiKey ?? process.env.GEMINI_API_KEY;
@@ -2331,7 +2498,7 @@ async function translatePageWithGemini(input) {
     }
   });
   const raw = response.text ?? "";
-  const parsed = JSON.parse(extractJson(raw));
+  const parsed = parseGeminiResponse(raw);
   if (!parsed.frontmatter || typeof parsed.body !== "string") {
     throw new Error("Gemini response missing frontmatter/body");
   }
@@ -2488,24 +2655,21 @@ function buildGeminiResponseSchema(schema, slugStrategy) {
 }
 
 // src/translate/resolve-translate-config.ts
-function slugStrategyRules(slugStrategy, preserveTerms) {
+var BUILTIN_TRANSLATE_RULES = [
+  "Do not translate brand or product names unless the brand has a well-known localized name in the target market (rare). Keep the original spelling and capitalization.",
+  "Return the MDX body with real line breaks; do not use JSON escape sequences like \\n or \\t in the body string.",
+  'In JSX attributes (e.g. FaqItem question="..."), use single quotes when the value contains double-quote characters (e.g. Hebrew \u05D3\u05D5\u05D0"\u05DC), or escape them as \\".'
+];
+function slugStrategyRules(slugStrategy) {
   if (slugStrategy === "localized") {
-    const rules = [
+    return [
       "Provide a URL slug in JSON field `slug` that is Localized into the target language.",
       "Base the slug on the meaning of the translated title \u2014 do NOT reuse the English slug words.",
       "Slug MUST be ASCII only: a-z, 0-9, hyphens. No uppercase, accents, underscores, or spaces.",
       "For non-Latin languages, write the words in the target language and transliterate them into Latin script (e.g. Russian -> romanized Russian, not English).",
       "Do NOT append locale codes to the slug (e.g. -fr, -he, -zh-cn). Locale routing is handled by the URL prefix, not the slug.",
-      'In JSX attributes (e.g. FaqItem question="..."), use single quotes when the value contains double-quote characters (e.g. Hebrew \u05D3\u05D5\u05D0"\u05DC), or escape them as \\".'
+      "Preserve 4-digit years and proper nouns in slugs."
     ];
-    if (preserveTerms?.length) {
-      rules.push(
-        `Preserve these terms verbatim in slugs (lowercase): ${preserveTerms.join(", ")}.`
-      );
-    } else {
-      rules.push("Preserve 4-digit years and proper nouns in slugs.");
-    }
-    return rules;
   }
   return ["Do NOT output a slug \u2014 slugStrategy is fixed."];
 }
@@ -2518,9 +2682,10 @@ function mergeTranslateConfig(project, type, slugStrategy) {
     promptOverride: type?.prompt ?? project?.prompt,
     context: mergeContext(project, type),
     rules: [
+      ...BUILTIN_TRANSLATE_RULES,
       ...project?.rules ?? [],
       ...type?.rules ?? [],
-      ...slugStrategyRules(slugStrategy, project?.slugPreserveTerms)
+      ...slugStrategyRules(slugStrategy)
     ],
     model: type?.model ?? project?.defaultModel
   };
@@ -2529,95 +2694,6 @@ function resolveTranslateConfig(config, type) {
   return mergeTranslateConfig(config.translate, type.translate, type.slugStrategy);
 }
 
-// src/translate/sanitize-mdx-jsx.ts
-function sanitizeMdxJsxAttributeQuotes(body) {
-  let adjusted = false;
-  let out = "";
-  let i = 0;
-  while (i < body.length) {
-    if (body[i] !== "<" || !/[\w.]/.test(body[i + 1] ?? "")) {
-      out += body[i];
-      i += 1;
-      continue;
-    }
-    const tagStart = i;
-    i += 1;
-    while (i < body.length && /[\w.-]/.test(body[i] ?? "")) i += 1;
-    const tagName = body.slice(tagStart + 1, i);
-    let tagOut = `<${tagName}`;
-    let tagAdjusted = false;
-    while (i < body.length && body[i] !== ">" && body[i] !== "/") {
-      while (i < body.length && /\s/.test(body[i] ?? "")) {
-        tagOut += body[i];
-        i += 1;
-      }
-      if (i >= body.length || body[i] === ">" || body[i] === "/") break;
-      const attrStart = i;
-      while (i < body.length && /[\w:.-]/.test(body[i] ?? "")) i += 1;
-      body.slice(attrStart, i);
-      while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
-      if (body[i] !== "=") {
-        tagOut += body.slice(attrStart, i);
-        continue;
-      }
-      tagOut += body.slice(attrStart, i);
-      tagOut += "=";
-      i += 1;
-      while (i < body.length && /\s/.test(body[i] ?? "")) i += 1;
-      const quote = body[i];
-      if (quote !== '"' && quote !== "'") {
-        continue;
-      }
-      if (quote === "'") {
-        const valStart2 = i + 1;
-        i += 1;
-        while (i < body.length) {
-          if (body[i] === "\\") {
-            i += 2;
-            continue;
-          }
-          if (body[i] === "'") break;
-          i += 1;
-        }
-        tagOut += body.slice(valStart2 - 1, i + 1);
-        i += 1;
-        continue;
-      }
-      const valStart = i + 1;
-      i += 1;
-      let closeIdx = -1;
-      for (let scan = valStart; scan < body.length; scan += 1) {
-        if (body[scan] !== '"') continue;
-        let j = scan + 1;
-        while (j < body.length && /\s/.test(body[j] ?? "")) j += 1;
-        if (j < body.length && (body[j] === ">" || body[j] === "/" || /[a-zA-Z]/.test(body[j] ?? ""))) {
-          closeIdx = scan;
-          break;
-        }
-      }
-      if (closeIdx === -1) {
-        tagOut += body.slice(valStart - 1, i);
-        break;
-      }
-      const value = body.slice(valStart, closeIdx);
-      const hasInternalQuote = value.includes('"');
-      if (hasInternalQuote) {
-        const escaped = value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
-        tagOut += `'${escaped}'`;
-        tagAdjusted = true;
-      } else {
-        tagOut += `"${value}"`;
-      }
-      i = closeIdx + 1;
-    }
-    if (tagAdjusted) adjusted = true;
-    tagOut += body[i] ?? "";
-    out += tagOut;
-    i += 1;
-  }
-  return { body: out, adjusted };
-}
-
 // src/translate/validate-translation.ts
 function formatZodIssues(error) {
   return error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`).join("; ");
@@ -2751,7 +2827,7 @@ async function translatePage(config, item, options = {}) {
     if (!validated.ok) {
       throw new Error(`Translation validation failed: ${validated.error}`);
     }
-    const { body: translatedBody, adjusted: mdxAdjusted } = sanitizeMdxJsxAttributeQuotes(
+    const { body: translatedBody, adjusted: mdxAdjusted } = assertValidTranslatedMdxBody(
       result.parsed.body
     );
     const writeDb = openStore(config, "readwrite");