regen.mde 0.2.2

package/bin/validate-katex.js

@@ -0,0 +1,93 @@
+const fs = require("fs");
+const path = require("path");
+const katex = require("katex");
+
+function extractMath(markdown) {
+  const spans = [];
+  const blockRe = /\$\$([\s\S]*?)\$\$/g;
+  let masked = markdown;
+  let match;
+  while ((match = blockRe.exec(markdown)) !== null) {
+    spans.push({ displayMode: true, tex: match[1].trim(), index: match.index });
+  }
+  masked = masked.replace(blockRe, "");
+
+  for (let i = 0; i < masked.length; i++) {
+    if (masked[i] !== "$" || isEscaped(masked, i)) continue;
+    let end = -1;
+    for (let j = i + 1; j < masked.length; j++) {
+      if (masked[j] === "\n") break;
+      if (masked[j] === "$" && !isEscaped(masked, j)) {
+        end = j;
+        break;
+      }
+    }
+    if (end > i + 1) {
+      spans.push({ displayMode: false, tex: masked.slice(i + 1, end).trim(), index: i });
+      i = end;
+    }
+  }
+  return spans;
+}
+
+function isEscaped(text, index) {
+  let slashCount = 0;
+  for (let i = index - 1; i >= 0 && text[i] === "\\"; i--) {
+    slashCount++;
+  }
+  return slashCount % 2 === 1;
+}
+
+function validateFile(filePath) {
+  const markdown = fs.readFileSync(filePath, "utf8");
+  const spans = extractMath(markdown);
+  const failures = [];
+  for (const span of spans) {
+    try {
+      katex.renderToString(span.tex, {
+        displayMode: span.displayMode,
+        throwOnError: true,
+        strict: "ignore",
+        trust: false,
+      });
+    } catch (error) {
+      failures.push({
+        index: span.index,
+        displayMode: span.displayMode,
+        tex: span.tex.slice(0, 500),
+        error: error.message,
+      });
+    }
+  }
+  return {
+    file: filePath,
+    math_spans: spans.length,
+    failures: failures.length,
+    failure_samples: failures.slice(0, 50),
+  };
+}
+
+function main() {
+  const target = process.argv[2];
+  if (!target) {
+    console.error("Usage: node validate_katex.js <markdown-file-or-directory>");
+    process.exit(2);
+  }
+  const stat = fs.statSync(target);
+  const files = [];
+  if (stat.isDirectory()) {
+    for (const entry of fs.readdirSync(target, { recursive: true })) {
+      const full = path.join(target, entry);
+      if (fs.existsSync(full) && fs.statSync(full).isFile() && full.endsWith(".md")) {
+        files.push(full);
+      }
+    }
+  } else {
+    files.push(target);
+  }
+  const results = files.map(validateFile);
+  console.log(JSON.stringify({ files: results.length, results }, null, 2));
+  process.exit(results.some((result) => result.failures > 0) ? 1 : 0);
+}
+
+main();

package/desktop/BuildCorpusEditor/BuildCorpusBridge.cs

@@ -0,0 +1,270 @@
+using System.Diagnostics;
+using System.Text.Json;
+
+namespace BuildCorpusEditor;
+
+internal sealed class BuildCorpusBridge
+{
+    private readonly string? initialPath;
+    private readonly string root;
+    private readonly string sessionRoot;
+
+    public BuildCorpusBridge(string? initialPath)
+    {
+        this.initialPath = string.IsNullOrWhiteSpace(initialPath) ? null : Path.GetFullPath(initialPath);
+        root = LocateRoot();
+        sessionRoot = Path.Combine(Path.GetTempPath(), "build-corpus-editor");
+        Directory.CreateDirectory(sessionRoot);
+    }
+
+    public StartupResult Startup() => new(initialPath);
+
+    public object? ChooseOpen()
+    {
+        using var dialog = new OpenFileDialog
+        {
+            Title = "Open Markdown, Word, or PowerPoint",
+            Filter = "Supported documents|*.md;*.markdown;*.docx;*.pptx;*.ppt|Markdown|*.md;*.markdown|Word|*.docx|PowerPoint|*.pptx;*.ppt|All files|*.*",
+            CheckFileExists = true,
+        };
+        return dialog.ShowDialog() == DialogResult.OK ? new { path = dialog.FileName } : new { path = "" };
+    }
+
+    public async Task<object?> ChooseImageAsync()
+    {
+        using var dialog = new OpenFileDialog
+        {
+            Title = "Insert Image",
+            Filter = "Images|*.png;*.jpg;*.jpeg;*.gif;*.webp;*.bmp|All files|*.*",
+            CheckFileExists = true,
+        };
+        if (dialog.ShowDialog() != DialogResult.OK) return new { src = "", alt = "" };
+
+        var bytes = await File.ReadAllBytesAsync(dialog.FileName);
+        var extension = Path.GetExtension(dialog.FileName).ToLowerInvariant();
+        var mime = extension switch
+        {
+            ".jpg" or ".jpeg" => "image/jpeg",
+            ".gif" => "image/gif",
+            ".webp" => "image/webp",
+            ".bmp" => "image/bmp",
+            _ => "image/png",
+        };
+        return new
+        {
+            src = $"data:{mime};base64,{Convert.ToBase64String(bytes)}",
+            alt = Path.GetFileNameWithoutExtension(dialog.FileName),
+        };
+    }
+
+    public async Task<OpenResult> OpenAsync(string? path)
+    {
+        var source = RequirePath(path);
+        if (!File.Exists(source)) throw new FileNotFoundException("File not found.", source);
+        var ext = Path.GetExtension(source).ToLowerInvariant();
+        if (ext is ".md" or ".markdown")
+        {
+            return new OpenResult(source, source, "markdown", await File.ReadAllTextAsync(source));
+        }
+
+        if (ext is not ".docx" and not ".pptx" and not ".ppt")
+        {
+            throw new InvalidOperationException($"Unsupported file type: {ext}");
+        }
+
+        var outDir = Path.Combine(sessionRoot, Path.GetFileNameWithoutExtension(source) + "-" + Guid.NewGuid().ToString("N")[..8]);
+        Directory.CreateDirectory(outDir);
+        var report = RunBuildCorpus(source, "--out", outDir);
+        var output = report.RootElement.GetProperty("outputs")[0].GetString();
+        if (string.IsNullOrWhiteSpace(output) || !File.Exists(output))
+        {
+            throw new InvalidOperationException("Build Corpus did not produce a Markdown file.");
+        }
+        return new OpenResult(source, output, ext.TrimStart('.'), await File.ReadAllTextAsync(output));
+    }
+
+    public SaveResult Save(string? path, string content)
+    {
+        var target = RequirePath(path);
+        if (Path.GetExtension(target).ToLowerInvariant() is not ".md" and not ".markdown")
+        {
+            target = Path.ChangeExtension(target, ".md");
+        }
+        Directory.CreateDirectory(Path.GetDirectoryName(target)!);
+        File.WriteAllText(target, content);
+        return new SaveResult(target);
+    }
+
+    public async Task<DocumentSelfTestResult> RunDocumentSelfTestAsync(string? inputPath, string? outputDir)
+    {
+        var source = RequirePath(inputPath);
+        var destination = string.IsNullOrWhiteSpace(outputDir)
+            ? Path.Combine(sessionRoot, "document-self-test-" + Guid.NewGuid().ToString("N")[..8])
+            : Path.GetFullPath(outputDir);
+        Directory.CreateDirectory(destination);
+
+        var opened = await OpenAsync(source);
+        if (string.IsNullOrWhiteSpace(opened.Content))
+        {
+            throw new InvalidOperationException("Opened document was empty.");
+        }
+
+        var marker = $"regen.mde functional edit {DateTimeOffset.UtcNow:yyyyMMddHHmmss}";
+        var edited = opened.Content.TrimEnd() + Environment.NewLine + Environment.NewLine + marker + Environment.NewLine;
+        var savedMarkdown = Path.Combine(destination, Path.GetFileNameWithoutExtension(source) + ".edited.md");
+        var save = Save(savedMarkdown, edited);
+        if (!File.ReadAllText(save.Output).Contains(marker, StringComparison.Ordinal))
+        {
+            throw new InvalidOperationException("Saved Markdown did not contain the edit marker.");
+        }
+
+        var exportedWord = Path.Combine(destination, Path.GetFileNameWithoutExtension(source) + ".edited.docx");
+        await ExportWordAsync(exportedWord, edited);
+        if (!File.Exists(exportedWord) || new FileInfo(exportedWord).Length == 0)
+        {
+            throw new InvalidOperationException("Word export was missing or empty.");
+        }
+
+        var reconvertDir = Path.Combine(destination, "reconverted");
+        Directory.CreateDirectory(reconvertDir);
+        var report = RunBuildCorpus(exportedWord, "--out", reconvertDir);
+        var reconvertedMarkdown = report.RootElement.GetProperty("outputs")[0].GetString();
+        if (string.IsNullOrWhiteSpace(reconvertedMarkdown) || !File.Exists(reconvertedMarkdown))
+        {
+            throw new InvalidOperationException("Exported Word document did not reconvert to Markdown.");
+        }
+
+        var reconvertedText = await File.ReadAllTextAsync(reconvertedMarkdown);
+        if (!reconvertedText.Contains("regen.mde functional edit", StringComparison.Ordinal))
+        {
+            throw new InvalidOperationException("Reconverted Markdown did not contain the edit marker.");
+        }
+
+        return new DocumentSelfTestResult(
+            source,
+            opened.WorkingPath,
+            save.Output,
+            exportedWord,
+            reconvertedMarkdown,
+            marker);
+    }
+
+    public async Task<SaveResult?> SaveAsAsync(string? suggestedPath, string format, string content)
+    {
+        using var dialog = new SaveFileDialog
+        {
+            Title = format == "word" ? "Export Word Document" : "Save Markdown",
+            Filter = format == "word" ? "Word document|*.docx" : "Markdown|*.md",
+            FileName = SuggestedFileName(suggestedPath, format == "word" ? ".docx" : ".md"),
+            AddExtension = true,
+            OverwritePrompt = true,
+        };
+        if (dialog.ShowDialog() != DialogResult.OK) return null;
+
+        if (format == "word")
+        {
+            return new SaveResult(await ExportWordAsync(dialog.FileName, content));
+        }
+        File.WriteAllText(dialog.FileName, content);
+        return new SaveResult(dialog.FileName);
+    }
+
+    public async Task<SaveResult> SaveAsDirectAsync(string? targetPath, string format, string content)
+    {
+        var target = RequirePath(targetPath);
+        Directory.CreateDirectory(Path.GetDirectoryName(target)!);
+        if (format == "word")
+        {
+            return new SaveResult(await ExportWordAsync(target, content));
+        }
+        if (Path.GetExtension(target).ToLowerInvariant() is not ".md" and not ".markdown")
+        {
+            target = Path.ChangeExtension(target, ".md");
+        }
+        await File.WriteAllTextAsync(target, content);
+        return new SaveResult(target);
+    }
+
+    private async Task<string> ExportWordAsync(string targetPath, string content)
+    {
+        var tmp = Path.Combine(sessionRoot, "export-" + Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(tmp);
+        var md = Path.Combine(tmp, Path.GetFileNameWithoutExtension(targetPath) + ".md");
+        await File.WriteAllTextAsync(md, content);
+        RunBuildCorpus(md, "--to", "word", "--out-same-dir");
+        var generated = Path.ChangeExtension(md, ".docx");
+        File.Copy(generated, targetPath, overwrite: true);
+        return targetPath;
+    }
+
+    private JsonDocument RunBuildCorpus(params string[] args)
+    {
+        var psi = new ProcessStartInfo
+        {
+            FileName = "node",
+            WorkingDirectory = root,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+        };
+        psi.ArgumentList.Add(Path.Combine(root, "bin", "build-corpus.js"));
+        foreach (var arg in args) psi.ArgumentList.Add(arg);
+        psi.Environment["PYTHONPATH"] = Path.Combine(root, "src");
+
+        using var process = Process.Start(psi) ?? throw new InvalidOperationException("Could not start build-corpus.");
+        var stdout = process.StandardOutput.ReadToEnd();
+        var stderr = process.StandardError.ReadToEnd();
+        process.WaitForExit();
+        if (process.ExitCode != 0)
+        {
+            throw new InvalidOperationException(stderr.Length > 0 ? stderr : stdout);
+        }
+
+        var start = stdout.IndexOf('{');
+        var end = stdout.LastIndexOf('}');
+        if (start < 0 || end < start) throw new InvalidOperationException("Build Corpus did not return JSON.");
+        return JsonDocument.Parse(stdout[start..(end + 1)]);
+    }
+
+    private static string RequirePath(string? path)
+    {
+        if (string.IsNullOrWhiteSpace(path)) throw new InvalidOperationException("Missing path.");
+        return Path.GetFullPath(path);
+    }
+
+    private static string SuggestedFileName(string? path, string extension)
+    {
+        if (string.IsNullOrWhiteSpace(path)) return "document" + extension;
+        return Path.ChangeExtension(Path.GetFileName(path), extension);
+    }
+
+    private static string LocateRoot()
+    {
+        var env = Environment.GetEnvironmentVariable("BUILD_CORPUS_ROOT");
+        if (!string.IsNullOrWhiteSpace(env) && File.Exists(Path.Combine(env, "bin", "build-corpus.js")))
+        {
+            return Path.GetFullPath(env);
+        }
+
+        var current = AppContext.BaseDirectory;
+        while (!string.IsNullOrWhiteSpace(current))
+        {
+            if (File.Exists(Path.Combine(current, "bin", "build-corpus.js"))) return current;
+            var parent = Directory.GetParent(current)?.FullName;
+            if (parent == current) break;
+            current = parent ?? "";
+        }
+        return Directory.GetCurrentDirectory();
+    }
+}
+
+internal sealed record StartupResult(string? InitialPath);
+internal sealed record OpenResult(string SourcePath, string WorkingPath, string OriginalFormat, string Content);
+internal sealed record SaveResult(string Output);
+internal sealed record DocumentSelfTestResult(
+    string SourcePath,
+    string WorkingPath,
+    string SavedMarkdown,
+    string ExportedWord,
+    string ReconvertedMarkdown,
+    string Marker);

package/desktop/BuildCorpusEditor/BuildCorpusEditor.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>WinExe</OutputType>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <UseWindowsForms>true</UseWindowsForms>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <ApplicationManifest>app.manifest</ApplicationManifest>
+    <AssemblyName>BuildCorpusEditor</AssemblyName>
+    <RootNamespace>BuildCorpusEditor</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Web.WebView2" Version="1.0.3537.50" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="wwwroot\**\*">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+</Project>