pyDiffTools 0.1.8__py3-none-any.whl

pydifftools/notebook/tex_to_qmd.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+"""this script has been entirely vibe-coded based on the tex example included
+in the repo!"""
+
+import re
+import sys
+import subprocess
+import tempfile
+import shutil
+from pathlib import Path
+
+from pydifftools.command_registry import register_command
+
+
+def find_matching(text: str, start: int, open_ch: str, close_ch: str) -> int:
+    """Return index of matching close_ch for open_ch at *start* or -1."""
+    depth = 1
+    i = start + 1
+    while i < len(text):
+        c = text[i]
+        if c == open_ch:
+            depth += 1
+        elif c == close_ch:
+            depth -= 1
+            if depth == 0:
+                return i
+        i += 1
+    return -1
+
+
+def preprocess_latex(src: str) -> str:
+    """Convert custom environments and observation macros before pandoc."""
+
+    def repl_python(m: re.Match) -> str:
+        """Preserve python blocks exactly using markers."""
+        code = m.group(1)
+        return (
+            "\\begin{verbatim}\n%%PYTHON_START%%\n"
+            + code
+            + "%%PYTHON_END%%\n\\end{verbatim}"
+        )
+
+    def repl_verbatim(m: re.Match) -> str:
+        """Mark generic verbatim blocks for fenced conversion."""
+        newline = m.group(1)
+        body = m.group(2)
+        if "%%PYTHON_START%%" in body:
+            return m.group(0)
+        return (
+            f"\\begin{{verbatim}}{newline}%%VERBATIM_START%%\n"
+            + body
+            + "%%VERBATIM_END%%\n\\end{verbatim}"
+        )
+
+    # replace python environment with verbatim + markers without touching
+    # the whitespace contained in the block
+    src = re.sub(
+        r"\\begin{python}(?:\[[^\]]*\])?\n(.*?)\\end{python}",
+        repl_python,
+        src,
+        flags=re.S,
+    )
+
+    # mark standard verbatim blocks so they convert to fenced code later
+    src = re.sub(
+        r"\\begin{verbatim}(\n?)(.*?)\\end{verbatim}",
+        repl_verbatim,
+        src,
+        flags=re.S,
+    )
+
+    # convert err environment so pandoc will parse inside while preserving
+    # the whitespace exactly
+    src = re.sub(
+        r"\\begin{err}\n?(.*?)\\end{err}",
+        lambda m: f"<err>{m.group(1)}</err>",
+        src,
+        flags=re.S,
+    )
+
+    # handle \o[...]{} and \o{} observations
+    out = []
+    i = 0
+    while True:
+        idx_bracket = src.find("\\o[", i)
+        idx_brace = src.find("\\o{", i)
+        idxs = [x for x in (idx_bracket, idx_brace) if x != -1]
+        idx = min(idxs) if idxs else -1
+        if idx == -1:
+            out.append(src[i:])
+            break
+        out.append(src[i:idx])
+        j = idx + 2
+        attrs = ""
+        if j < len(src) and src[j] == "[":
+            end_attrs = find_matching(src, j, "[", "]")
+            if end_attrs == -1:
+                out.append(src[idx:])
+                break
+            attrs = src[j + 1 : end_attrs]
+            j = end_attrs + 1
+        if j >= len(src) or src[j] != "{":
+            out.append(src[idx : idx + 2])
+            i = idx + 2
+            continue
+        end_body = find_matching(src, j, "{", "}")
+        if end_body == -1:
+            out.append(src[idx:])
+            break
+        body = src[j + 1 : end_body]
+        j = end_body + 1
+        if attrs:
+            m = re.match(r"(.*?)\s*(\(([^)]+)\))?$", attrs.strip())
+            time = m.group(1).strip() if m else attrs.strip()
+            author = m.group(3) if m else None
+            tag = (
+                f'<obs time="{time}"'
+                + (f' author="{author}"' if author else "")
+                + f">{body}</obs>"
+            )
+        else:
+            tag = f"<obs>{body}</obs>"
+        out.append(tag)
+        i = j
+    return "".join(out)
+
+
+def clean_html_escapes(text: str) -> str:
+    return text.replace("\\<", "<").replace("\\>", ">").replace('\\"', '"')
+
+
+def finalize_markers(text: str) -> str:
+    lines = []
+    in_py = False
+    need_reset = False
+    in_verb = False
+    for line in text.splitlines(keepends=True):
+        if re.match(r"^\s*%%PYTHON_START%%", line):
+            lines.append("```{python}\n")
+            in_py = True
+            need_reset = True
+            continue
+        if re.match(r"^\s*%%PYTHON_END%%", line):
+            lines.append("```\n")
+            in_py = False
+            continue
+        if re.match(r"^\s*%%VERBATIM_START%%", line):
+            lines.append("```\n")
+            in_verb = True
+            continue
+        if re.match(r"^\s*%%VERBATIM_END%%", line):
+            lines.append("```\n")
+            in_verb = False
+            continue
+        if in_py:
+            stripped = line[4:] if line.startswith("    ") else line
+            if need_reset:
+                if stripped.lstrip().startswith("%reset"):
+                    lines.append(stripped)
+                else:
+                    lines.append("%reset -f\n")
+                    lines.append(stripped)
+                need_reset = False
+            else:
+                lines.append(stripped)
+        elif in_verb and line.startswith("    "):
+            lines.append(line[4:])
+        else:
+            lines.append(line)
+    return "".join(lines)
+
+
+def format_observations(text: str) -> str:
+    """Ensure observation tags sit on a single line without altering
+    content."""
+
+    obs_re = re.compile(r"(<obs[^>]*>)(.*?)(</obs>)", flags=re.S)
+
+    def repl(match: re.Match) -> str:
+        open_tag = match.group(1).strip()
+        body = match.group(2)
+        close_tag = match.group(3).strip()
+        # trim newlines that may surround the body but keep internal whitespace
+        body = body.lstrip("\n").rstrip("\n")
+        return f"{open_tag}{body}{close_tag}"
+
+    return obs_re.sub(repl, text)
+
+
+def format_tags(text: str, indent_str: str = "  ") -> str:
+    """Format <err> blocks with indentation and tidy <obs> tags."""
+    text = format_observations(text)
+    # normalize whitespace around err tags
+    text = re.sub(r"<err>[ \t]*\n+", "<err>\n", text)
+    text = re.sub(r"<err>[ \t]+", "<err>\n", text)
+    text = re.sub(r"</err>[ \t]+", "</err>", text)
+    # ensure opening obs tags start on a new line without collapsing blank
+    # lines
+    text = re.sub(r"(\n+)[ \t]*(<obs)", r"\1\2", text)
+    text = re.sub(r"(?<!^)(?<!\n)(<obs)", r"\n\1", text)
+    # ensure a newline after closing obs tags but keep extra blank lines
+    text = re.sub(r"</obs>[ \t]+", "</obs>", text)
+    text = re.sub(r"</obs>(?!\n)", "</obs>\n", text)
+    pattern = re.compile(r"(<err>|</err>)")
+    parts = pattern.split(text)
+    out = []
+    indent = 0
+    prev_tag = None
+    for part in parts:
+        if not part:
+            continue
+        if part == "<err>":
+            if out and not out[-1].endswith("\n"):
+                out[-1] = out[-1].rstrip() + "\n"
+            out.append(indent_str * indent + "<err>\n")
+            indent += 1
+            prev_tag = "<err>"
+        elif part == "</err>":
+            if out and not out[-1].endswith("\n"):
+                out[-1] = out[-1].rstrip() + "\n"
+            indent -= 1
+            out.append(indent_str * indent + "</err>\n")
+            prev_tag = "</err>"
+        else:
+            # Keep err contents tight while
+            # forcing a blank line after
+            # closing tags so pandoc treats
+            # the debug block as a standalone
+            # HTML block. The newline handling
+            # below ensures content after a
+            # closing </err> tag always starts
+            # on its own paragraph line.
+            if prev_tag == "<err>" and part.startswith("\n"):
+                part = part[1:]
+            if prev_tag == "</err>" and not part.startswith("\n"):
+                part = "\n" + part
+            lines = part.splitlines(True)
+            for line in lines:
+                if line.strip():
+                    out.append(indent_str * indent + line)
+                else:
+                    out.append(line)
+            prev_tag = None
+    formatted = "".join(out)
+    return re.sub(r"[ \t]+(?=\n)", "", formatted)
+
+
+@register_command(
+    "Convert LaTeX sources to Quarto Markdown (.qmd) files",
+    help={"tex": "Input .tex file to convert"},
+)
+def tex2qmd(tex):
+    """Convert ``tex`` to a .qmd file and return the output path."""
+
+    inp = Path(tex)
+    if not inp.exists():
+        print(f"File not found: {inp}", file=sys.stderr)
+        sys.exit(1)
+
+    base = inp.with_suffix("")
+    src = inp.read_text()
+    pre_content = preprocess_latex(src)
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".tex") as pre:
+        pre.write(pre_content.encode())
+        pre_path = pre.name
+
+    mid_fd, mid_path = tempfile.mkstemp()
+    Path(mid_path).unlink()  # we just want the name; pandoc will create it
+
+    try:
+        # Prefer Quarto's bundled pandoc when available so the conversion
+        # matches Quarto defaults, but fall back to a standalone pandoc
+        # installation when Quarto is not on PATH.
+        quarto = shutil.which("quarto")
+        pandoc = shutil.which("pandoc")
+        if quarto:
+            cmd = [quarto, "pandoc"]
+        else:
+            cmd = [pandoc]
+        cmd += [
+            pre_path,
+            "-f",
+            "latex",
+            "-t",
+            "markdown",
+            "--wrap=none",
+            "-o",
+            mid_path,
+        ]
+        subprocess.run(cmd, check=True)
+    finally:
+        Path(pre_path).unlink(missing_ok=True)
+
+    mid_text = Path(mid_path).read_text()
+    Path(mid_path).unlink(missing_ok=True)
+
+    clean_text = clean_html_escapes(mid_text)
+    final_text = finalize_markers(clean_text)
+    formatted = format_tags(final_text)
+    out_path = base.with_suffix(".qmd")
+    out_path.write_text(formatted)
+    print(f"Wrote {out_path}")
+    return out_path
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: tex_to_qmd.py file.tex", file=sys.stderr)
+        sys.exit(1)
+    tex2qmd(sys.argv[1])
+
+
+if __name__ == "__main__":
+    main()
+
+
+# Maintain the previous helper name for any existing imports.
+convert_tex_to_qmd = tex2qmd

pydifftools/onewordify.py

@@ -0,0 +1,149 @@
+# again rerun
+from lxml import html, etree
+import os
+from matlablike import *
+from unidecode import unidecode
+import re
+import sys
+
+fp = open(sys.argv[1], "r")
+paragraphcommands_re = re.compile(r"^ *\\(sub)*paragraph{.*}")
+commentline_re = re.compile(r"^ *%")
+beginlatex_re = re.compile(r"^[^#]*\\begin{document}(.*)")
+endlatex_re = re.compile(r"^([^#]*)\\end{document}.*")
+commandstart_re = re.compile(r"(\\[a-zA-Z]+[\[{])")
+word_citation_re = re.compile(r"(\[[0-9 ,]+\][,\.)]*)")
+tex_citation_re = re.compile(r"(.*)(\\cite{[a-zA-Z0-9,_]+}[,\.)]*)(.*)$")
+tex_ref_re = re.compile(r"(.*)(\\c*ref{[a-zA-Z0-9,_:\-]+}[,\.)]*)(.*)$")
+text_list = []
+if sys.argv[1][-4:] == ".tex":
+    latex_file = True
+else:
+    latex_file = False
+found_beginning = False
+start_line = 0
+end_line = 0
+print("opened", sys.argv[1])
+# {{{ pull out just the part between the document text
+j = 0
+for thisline in fp:
+    thisline = thisline.replace("\xa0", " ")  # because word sucks
+    thisline = thisline.replace(
+        "\x93", "``"
+    )  # this and following are just pulled from vim
+    thisline = thisline.replace("\x94", "''")
+    thisline = thisline.replace("\x96", "--")
+    j += 1
+    if latex_file:
+        if not found_beginning:
+            thismatch = beginlatex_re.match(thisline)
+            if thismatch:
+                text_list.append(thismatch.groups()[0].rstrip())
+                found_beginning = True
+                start_line = j + 1
+                print("Found the beginning at line", start_line)
+        else:
+            thismatch = endlatex_re.match(thisline)
+            if thismatch:
+                text_list.append(thismatch.groups()[0].rstrip())
+                print("Found the end")
+                end_line = j
+                print("Found the end at line", end_line)
+        text_list.append(thisline)
+    else:
+        text_list.append(
+            thisline.replace("$$", "")
+        )  # no better place to check for the tex dollar sign double-up
+if end_line == 0:
+    end_line = len(text_list)
+fp.close()
+j = 0
+while j < len(
+    text_list
+):  # first, put citations on their own line, so I can next treat them as special lines
+    thismatch = tex_citation_re.match(text_list[j])
+    othermatch = tex_ref_re.match(text_list[j])
+    if othermatch:
+        thismatch = othermatch
+    if thismatch:
+        text_list.pop(j)
+        text_list.insert(
+            j, thismatch.groups()[2]
+        )  # push on backwards, so it shows up in the right order
+        text_list.insert(
+            j,
+            thismatch.groups()[1].replace(" ", "\n%SPACE%\n")
+            + "%NONEWLINE%\n",
+        )  # since these are "fake" newlines, make sure they don't get broken! -- also to preserve spaces, I'm pre-processing the spacing here
+        text_list.insert(
+            j,
+            thismatch.groups()[0].replace(" ", "\n%SPACE%\n")
+            + "%NONEWLINE%\n",
+        )
+        print(
+            "found citation or reference, broke line:",
+            text_list[j],
+            text_list[j + 1],
+            text_list[j + 2],
+        )
+        print("---")
+        j += 1  # so that we skip the citation we just added
+        end_line += 2  # because we added two lines
+    j += 1
+for j in range(0, len(text_list)):
+    thismatch = paragraphcommands_re.match(text_list[j])
+    if thismatch:
+        text_list[j] = text_list[j].replace(
+            "\n", "%NEWLINE%\n"
+        )  # these lines are protected/preserved from being chopped up, since they are invisible
+        print("found paragraph line:", text_list[j])
+    else:
+        thismatch = tex_citation_re.match(text_list[j])
+        if not thismatch:
+            thismatch = tex_ref_re.match(text_list[j])
+        if thismatch:
+            print("found citation line:", text_list[j])
+        else:
+            text_list[j] = text_list[j].replace("~", "\n~\n")
+            text_list[j] = commandstart_re.sub("\\1\n", text_list[j])
+            text_list[j] = word_citation_re.sub("\n\\1\n", text_list[j])
+            text_list[j] = text_list[j].replace("}", "\n}\n")
+            text_list[j] = text_list[j].replace("]{", "\n]{\n")
+            text_list[j] = text_list[j].replace(" ", "\n%SPACE%\n")
+        if text_list[j][-12:] == "%NONEWLINE%\n":
+            print("trying to drop NONEWLINE going from:")
+            print(text_list[j])
+            text_list[j] = text_list[j][:-12] + "\n"
+            print("to:\n", text_list[j])
+        else:
+            print("line ends in:", text_list[j][-12:])
+            text_list[j] += "%NEWLINE%\n"
+            text_list[j] = text_list[j].replace("\r", "\n%NEWLINE%\n")
+# }}}
+# {{{ write out the result
+outputtext = "".join(text_list)
+outputtext = outputtext.split("\n")
+outputtext = [j for j in outputtext if len(j) > 0]
+if not latex_file:  # easier to just strip the tags here
+    print("this is not a latex file")
+    outputtext = [j for j in outputtext if j != "%SPACE%" and j != "%NEWLINE%"]
+else:
+    print("this is a latex file")
+    outputtex = "".join(
+        text_list[start_line:end_line]
+    )  # up to but not including the end document
+    outputtex = outputtex.split("\n")
+    outputtex = [j for j in outputtex if len(j) > 0]
+    outputtex = [
+        j for j in outputtex if j[0] != "%"
+    ]  # takes care of space and newline as well as tex comments
+newfile = re.sub(r"(.*)(\..*)", r"\1_1word\2", sys.argv[1])
+fp = open(newfile, "w")
+fp.write("\n".join(outputtext))
+fp.close()
+if latex_file:
+    newfile = re.sub(r"(.*)(\..*)", r"\1_1wordstripped\2", sys.argv[1])
+    fp = open(newfile, "w")
+    fp.write("\n".join(outputtex))
+    fp.close()
+# }}}

pydifftools/onewordify_undo.py

@@ -0,0 +1,54 @@
+# again rerun
+from lxml import html, etree
+import os
+from matlablike import *
+from unidecode import unidecode
+import re
+import sys
+
+fp = open(sys.argv[1], "r")
+needsspace_re = re.compile(r'(\w[):;"\-\.,!?}]*) +(["(]*\w)')
+paragraphcommands_re = re.compile(r"^ *\\(sub)*paragraph{.*}")
+commentline_re = re.compile(r"^ *%")
+normalline_re = re.compile(r"^\(%SPACE%\)\|\(%NEWLINE%\)")
+notweird_re = re.compile(r"^(%SPACE%)|(%\[ORIG%)|(%ORIG\]\[NEW%)|(%NEW\]%)")
+text_list = []
+found_beginning = False
+print("opened", sys.argv[1])
+# {{{ pull out just the part between the document text
+for thisline in fp:
+    if (
+        (thisline[:7] == "<<<<<<<")
+        or (thisline[:7] == "=======")
+        or (thisline[:7] == ">>>>>>>")
+    ):
+        text_list.append(
+            "%NEWLINE% %CONFLICT%" + thisline.strip() + "%NEWLINE%"
+        )
+    else:
+        text_list.append(thisline.rstrip())
+fp.close()
+text_list = [x.replace("%NEWLINE%", "\n") for x in text_list]
+# {{{ don't mess with the "special" lines
+for j, thisline in enumerate(text_list):
+    if not notweird_re.match(thisline):
+        if paragraphcommands_re.match(thisline) or commentline_re.match(
+            thisline
+        ):
+            print("found special line '", thisline, "'")
+            text_list[j] = thisline.replace(" ", " %SPACE% ")
+# }}}
+text_list = " ".join(text_list)
+text_list = needsspace_re.sub(r"\1 %SPACE% \2", text_list)
+text_list = needsspace_re.sub(
+    r"\1 %SPACE% \2", text_list
+)  # again to catch the single letter ones
+text_list = text_list.replace(" ", "")
+text_list = text_list.replace("%SPACE%", " ")
+# {{{ write out the result
+newfile = re.sub(r"(.*)(\..*)", r"\1_1wordcollapse\2", sys.argv[1])
+fp = open(newfile, "w")
+outputtext = "".join(text_list)
+fp.write(outputtext)
+fp.close()
+# }}}

pydifftools/outline.py

@@ -0,0 +1,173 @@
+import pickle
+from .doc_contents import doc_contents_class
+import re
+
+from .command_registry import register_command
+
+
+def _write_outline_files(all_contents, basename):
+    # save the reordered data and user-editable outline sidecar
+    with open(f"{basename}_outline.pickle", "wb") as fp:
+        pickle.dump(all_contents, fp)
+    with open(f"{basename}_outline.md", "w", encoding="utf-8") as fp:
+        fp.write(all_contents.outline)
+
+
+def extract_outline(filename):
+    basename = filename.replace(".tex", "")
+    section_re = re.compile(
+        r"\\(paragraph|subparagraph|subsubsection|subsection|section)\{"
+    )
+
+    all_contents = doc_contents_class()
+    bracelevel = 0
+    with open(filename, "r", encoding="utf-8") as fp:
+        for thisline in fp:
+            if bracelevel == 0:
+                thismatch = section_re.match(thisline)
+                if thismatch:
+                    sectype = thismatch.groups()[0]
+                    bracelevel = 1
+                    all_contents += thisline[: thismatch.start()]
+                    escaped = False
+                    thistitle = ""
+                else:
+                    all_contents += thisline
+            if (
+                bracelevel > 0
+            ):  # do this whether continued open brace from previous line,
+                # or if we opened brace on previous
+                for n, j in enumerate(thisline[thismatch.end() :]):
+                    if escaped:
+                        escaped = False
+                    elif j == "\\":
+                        escaped = True
+                    elif j == "{":
+                        bracelevel += 1
+                    elif j == "}":
+                        bracelevel -= 1
+                    if bracelevel > 0:
+                        thistitle += j
+                    else:
+                        all_contents.start_sec(sectype, thistitle)
+                        all_contents += thisline[thismatch.end() + n + 1 :]
+                        break
+                else:  # hit the end of the line without the break
+                    thisline += "\n"
+    _write_outline_files(all_contents, basename)
+
+
+def _reorder_from_outline(targetfile, extension, format_type):
+    # rebuild a file based on the user-adjusted outline list
+    markdownfile = targetfile.replace(extension, "_outline.md")
+    picklefile = targetfile.replace(extension, "_outline.pickle")
+    if not (
+        markdownfile.endswith(".md")
+        and picklefile.endswith(".pickle")
+        and targetfile.endswith(extension)
+    ):
+        raise ValueError("pass 1 argument: target file (output)")
+
+    with open(picklefile, "rb") as fp:
+        all_contents = pickle.load(fp)
+    all_contents.set_format(format_type)
+    with open(markdownfile, "r", encoding="utf-8") as fp:
+        for thisline in fp:
+            all_contents.outline_in_order(thisline.rstrip())
+    with open(targetfile, "w", encoding="utf-8", newline="\n") as fp:
+        fp.write(str(all_contents))
+
+
+@register_command(
+    "Save tex file as outline, with filename_outline.pickle storing content",
+    " and filename_outline.md giving outline.",
+)
+def xo(arguments):
+    assert len(arguments) == 1
+    extract_outline(arguments[0])
+
+
+@register_command(
+    "Save markdown file as outline, with filename_outline.pickle storing"
+    " content",
+    " and filename_outline.md giving outline.",
+)
+def xomd(arguments):
+    assert len(arguments) == 1
+    filename = arguments[0]
+    # read a markdown file and capture headings while keeping content for
+    # reordering
+    basename = filename.replace(".md", "")
+    header_re = re.compile(r"^(#{1,6})\s+(.*)")
+    underline_re = {
+        "section": re.compile(r"^={3,}\s*$"),
+        "subsection": re.compile(r"^-{3,}\s*$"),
+    }
+
+    all_contents = doc_contents_class("markdown")
+    previous_line = None
+    in_code_block = False
+    with open(filename, "r", encoding="utf-8") as fp:
+        for thisline in fp:
+            stripped = thisline.rstrip("\n")
+            if stripped.startswith("```"):
+                in_code_block = not in_code_block
+                all_contents += thisline
+                previous_line = None
+                continue
+            if in_code_block:
+                all_contents += thisline
+                continue
+            thismatch = header_re.match(stripped)
+            if thismatch:
+                hashes, thistitle = thismatch.groups()
+                level = len(hashes)
+                if level == 1:
+                    all_contents.start_sec("section", thistitle.strip())
+                elif level == 2:
+                    all_contents.start_sec("subsection", thistitle.strip())
+                elif level == 3:
+                    all_contents.start_sec("subsubsection", thistitle.strip())
+                elif level == 4:
+                    all_contents.start_sec("paragraph", thistitle.strip())
+                else:
+                    all_contents.start_sec("subparagraph", thistitle.strip())
+                all_contents += "\n"
+                previous_line = None
+                continue
+            if previous_line is not None:
+                if underline_re["section"].match(stripped):
+                    all_contents.start_sec("section", previous_line.strip())
+                    previous_line = None
+                    continue
+                if underline_re["subsection"].match(stripped):
+                    all_contents.start_sec("subsection", previous_line.strip())
+                    previous_line = None
+                    continue
+                all_contents += previous_line + "\n"
+            previous_line = stripped
+        if previous_line:
+            all_contents += previous_line + "\n"
+    _write_outline_files(all_contents, basename)
+
+
+@register_command(
+    "use the modified filename_outline.md to write reordered text",
+    help={"texfile": "TeX file to regenerate from its outline files"},
+)
+def xoreorder(texfile):
+    """Rewrite a TeX file using its saved outline and ordering hints."""
+
+    _reorder_from_outline(texfile, ".tex", "latex")
+
+
+@register_command(
+    "rewrite a markdown file using its saved outline and ordering hints",
+    help={"mdfile": "Markdown file to regenerate from its outline files"},
+)
+def xomdreorder(mdfile):
+    _reorder_from_outline(mdfile, ".md", "markdown")
+
+
+# Provide the previous function name for callers expecting it.
+write_reordered = xoreorder