pal-code 0.1.0__py3-none-any.whl

pal/documents/components/index.py

@@ -0,0 +1,155 @@
+"""Per-session document index on top of the SDK: full text goes to the storage manager's files,
+chunks go to its vector store. Later turns retrieve only the chunks that match the question."""
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from pal.config import DocumentsConfig
+from pal.documents.components.chunker import Chunk, chunk_markdown
+from pal.errors import NotFoundError
+from pal.paths import safe_id
+from pal.tokens import estimate_tokens
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class DocInfo:
+    name: str
+    slug: str
+    tokens: int
+    parts: int
+    outline: list[str] = field(default_factory=list)
+    inline: bool = True   # small enough to send in full on the upload turn
+
+
+@dataclass
+class Hit:
+    doc: str
+    part: int          # 1-based
+    parts: int
+    heading: str
+    text: str
+    score: float | None
+
+
+def _slug(name: str) -> str:
+    return f"{safe_id(name.rsplit('.', 1)[0], default='doc')[:32]}-{hashlib.sha1(name.encode()).hexdigest()[:6]}"  # noqa: S324
+
+
+class DocumentIndex:
+    def __init__(self, sdk, cfg: DocumentsConfig | dict | None = None, count: Callable[[str], int] = estimate_tokens):
+        self.sdk = sdk
+        self.cfg = DocumentsConfig.coerce(cfg)
+        self.count = count
+        self.docs: dict[str, DocInfo] = {}
+
+    # -- indexing --------------------------------------------------------------------------
+    def add(self, name: str, text: str) -> DocInfo:
+        chunks = chunk_markdown(text, self.cfg.chunk_tokens, self.count)
+        slug = _slug(name)
+        total = sum(c.tokens for c in chunks)
+        self.sdk.write(f"docs/{slug}.json", text)
+        for c in chunks:
+            self.sdk.add_knowledge(f"{slug}:{c.index:04d}", c.text,
+                                   {"doc": name, "part": c.index + 1, "parts": len(chunks), "heading": c.heading})
+        outline: list[str] = []
+        for c in chunks:
+            if c.heading and (not outline or not outline[-1].endswith(f" {c.heading}")):
+                outline.append(f"{c.index + 1} {c.heading}")
+        info = DocInfo(name=name, slug=slug, tokens=total, parts=len(chunks), outline=outline[:25],
+                       inline=total <= self.cfg.inline_max_tokens)
+        self.docs[name] = info
+        log.debug("indexed %s: %d tokens, %d parts", name, total, len(chunks))
+        return info
+
+    @property
+    def total_tokens(self) -> int:
+        return sum(d.tokens for d in self.docs.values())
+
+    def _chunks(self, info: DocInfo) -> list[Chunk]:
+        text = self.sdk.read(f"docs/{info.slug}.json")
+        return chunk_markdown(text, self.cfg.chunk_tokens, self.count) if isinstance(text, str) else []
+
+    # -- what the API model sees on the upload turn -----------------------------------------
+    def upload_view(self, name: str, text: str, tools: bool) -> str:
+        """Small documents in full. Large ones as a preview plus an outline."""
+        info = self.docs[name]
+        if info.inline:
+            return text
+        chunks = self._chunks(info)
+        taken: list[str] = []
+        used = 0
+        for c in chunks:
+            if taken and used + c.tokens > self.cfg.preview_tokens:
+                break
+            taken.append(c.text)
+            used += c.tokens
+        rest = info.tokens - used
+        how = ("Use search_documents(query) or read_document(name, part) to read more."
+               if tools else "The parts relevant to each question are attached to it.")
+        outline = ("Outline: " + "; ".join(info.outline) + ". ") if info.outline else ""
+        return "\n\n".join(taken) + f"\n\n[Preview only. {rest:,} more tokens in {info.parts} parts. {outline}{how}]"
+
+    def stub(self, name: str) -> str:
+        """What the history keeps instead of the file body."""
+        i = self.docs[name]
+        return f"[file: {name} | {i.tokens:,} tokens | indexed; relevant parts are attached to each question]"
+
+    # -- retrieval ------------------------------------------------------------------------------------
+    def search(self, query: str, top_k: int | None = None, exclude: set[str] | None = None) -> list[Hit]:
+        if not self.docs or not query.strip():
+            return []
+        k = top_k or self.cfg.retrieve_chunks
+        hits = []
+        for r in self.sdk.search_knowledge(query, top_k=k * 3):
+            score = r.get("score")
+            if score is not None and score < self.cfg.min_score:
+                continue
+            m = r.get("metadata") or {}
+            if m.get("doc") not in self.docs or m.get("doc") in (exclude or ()):
+                continue
+            hits.append(Hit(m["doc"], int(m.get("part", 1)), int(m.get("parts", 1)), m.get("heading", ""),
+                            r["text"], score))
+        return hits[:k]
+
+    def excerpts(self, query: str, budget: int | None = None, exclude: set[str] | None = None) -> tuple[str, int, int]:
+        """(text block, tokens used, number of chunks). Empty when nothing is relevant enough."""
+        budget = budget or self.cfg.budget_tokens
+        blocks, used = [], 0
+        for h in self.search(query, exclude=exclude):
+            body = f"[{h.doc} part {h.part}/{h.parts}{': ' + h.heading if h.heading else ''}]\n{h.text}"
+            cost = self.count(body)
+            if used + cost > budget:
+                continue
+            blocks.append(body)
+            used += cost
+        if not blocks:
+            return "", 0, 0
+        return "<documents>\n" + "\n\n".join(blocks) + "\n</documents>", used, len(blocks)
+
+    def outline_text(self, name: str) -> str:
+        """Every part of a document with its heading and size, so a model can decide which part to read."""
+        match = next((n for n in self.docs if n.lower() == name.strip().lower()), None)
+        if match is None:
+            return f"no document named {name!r}; documents: {', '.join(self.docs) or 'none'}"
+        info = self.docs[match]
+        lines = [f"{info.name}: {info.tokens:,} tokens in {info.parts} parts"]
+        for c in self._chunks(info)[:60]:
+            lines.append(f"part {c.index + 1} ({c.tokens} tokens): {c.heading or '(no heading)'}")
+        if info.parts > 60:
+            lines.append(f"[{info.parts - 60} more parts not shown]")
+        return "\n".join(lines)
+
+    def read_part(self, name: str, part: int) -> str:
+        info = self.docs.get(name) or next((d for n, d in self.docs.items() if n.lower() == name.lower()), None)
+        if info is None:
+            raise NotFoundError(f"no document named {name!r}. Available: {', '.join(self.docs) or 'none'}")
+        chunks = self._chunks(info)
+        if not 1 <= part <= len(chunks):
+            raise NotFoundError(f"{info.name} has parts 1 to {len(chunks)}")
+        c = chunks[part - 1]
+        return f"[{info.name} part {part}/{len(chunks)}{': ' + c.heading if c.heading else ''}]\n{c.text}"

pal/documents/utils/__init__.py

@@ -0,0 +1 @@
+

pal/errors.py

@@ -0,0 +1,40 @@
+"""Exception hierarchy. Everything Pal raises on purpose derives from PalError, so the
+web layer can map it to an HTTP status without string matching and never leaks a traceback."""
+from __future__ import annotations
+
+
+class PalError(Exception):
+    """Base class. `http_status` is what the web layer answers with; `public` says whether the
+    message is safe to show the user as-is."""
+
+    http_status = 500
+    public = False
+
+
+class UserInputError(PalError, ValueError):
+    """The request was wrong: bad file, empty message, unknown option. Message is user-safe."""
+
+    http_status = 400
+    public = True
+
+
+class ConfigError(PalError, ValueError):
+    """Settings are invalid. Raised at startup, so a bad config fails fast and clearly."""
+
+    http_status = 500
+    public = True
+
+
+class ProviderError(PalError):
+    """The API model or the local model failed (network, auth, rate limit, bad response)."""
+
+    http_status = 502
+    public = True
+
+
+class DependencyMissingError(UserInputError):
+    """An optional dependency or system tool (pandoc, graphify) is not installed."""
+
+
+class NotFoundError(UserInputError):
+    http_status = 404

pal/input_converters/__init__.py

@@ -0,0 +1,8 @@
+from pal.input_converters.pipeline import (
+    CONVERTIBLE_EXTENSIONS,
+    ConversionError,
+    ConvertedFile,
+    convert_to_markdown,
+)
+
+__all__ = ["CONVERTIBLE_EXTENSIONS", "ConversionError", "ConvertedFile", "convert_to_markdown"]

pal/input_converters/components/base.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from pal.errors import UserInputError
+
+
+class ConversionError(UserInputError):
+    """A file could not be turned into markdown. A UserInputError: shown to the user as a per-file error."""
+
+
+@dataclass
+class ConvertedFile:
+    name: str
+    text: str        # markdown (or the original text for plain-text files)
+    source_format: str  # "html", "pdf", "docx", ... or "text"
+    converted: bool     # False when the file was already text and passed through
+
+
+def table_to_markdown(rows: list[list[str]]) -> str:
+    """Render rows (first row = header) as a GitHub-style pipe table."""
+    rows = [r for r in rows if any(c.strip() for c in r)]
+    if not rows:
+        return ""
+    width = max(len(r) for r in rows)
+    # drop columns that are empty in every row
+    keep = [i for i in range(width) if any(i < len(r) and r[i].strip() for r in rows)]
+    if not keep:
+        return ""
+
+    def cell(r: list[str], i: int) -> str:
+        v = r[i] if i < len(r) else ""
+        return v.replace("|", "\\|").replace("\r", " ").replace("\n", " ").strip()
+
+    lines = ["| " + " | ".join(cell(rows[0], i) for i in keep) + " |",
+             "| " + " | ".join("---" for _ in keep) + " |"]
+    lines += ["| " + " | ".join(cell(r, i) for i in keep) + " |" for r in rows[1:]]
+    return "\n".join(lines)

pal/input_converters/components/pandoc_converter.py

@@ -0,0 +1,87 @@
+"""HTML, DOCX, ODT, RTF, EPUB and LaTeX -> GitHub-flavoured markdown via pandoc."""
+from __future__ import annotations
+
+import shutil
+import subprocess
+
+from pal.input_converters.components.base import ConversionError
+
+# extension -> pandoc reader
+READERS = {
+    "html": "html", "htm": "html", "xhtml": "html",
+    "docx": "docx", "odt": "odt", "rtf": "rtf",
+    "epub": "epub", "tex": "latex", "latex": "latex",
+}
+
+_TIMEOUT_S = 60
+
+
+def available() -> bool:
+    return shutil.which("pandoc") is not None
+
+
+def docx_without_pandoc(raw: bytes) -> str:
+    """A Word file as markdown without pandoc: headings, lists and tables from the document's XML (Python's standard library only)."""
+    import io
+    import xml.etree.ElementTree as ET  # noqa: S405  (expat in Python 3.10+ limits entity expansion)
+    import zipfile
+    w = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+    try:
+        xml = zipfile.ZipFile(io.BytesIO(raw)).read("word/document.xml")
+    except (zipfile.BadZipFile, KeyError):
+        raise ConversionError("not a valid Word (.docx) file") from None
+    if len(xml) > 50_000_000:
+        raise ConversionError("the Word file is too large to read")
+    body = ET.fromstring(xml).find(f"{w}body")  # noqa: S314
+    if body is None:
+        return ""
+
+    def text_of(el) -> str:
+        return "".join(t.text or "" for t in el.iter(f"{w}t"))
+    out: list[str] = []
+    for el in body:
+        if el.tag == f"{w}p":
+            text = text_of(el).strip()
+            if not text:
+                continue
+            style = el.find(f"{w}pPr/{w}pStyle")
+            name = (style.get(f"{w}val") or "") if style is not None else ""
+            if name.lower().startswith("heading") and name[-1:].isdigit():
+                out.append("#" * min(6, int(name[-1])) + " " + text)
+            elif name.lower() == "title":
+                out.append("# " + text)
+            elif el.find(f"{w}pPr/{w}numPr") is not None or name.lower().startswith("list"):
+                if out and out[-1].startswith("- "):
+                    out[-1] += "\n- " + text          # one list, not a paragraph per item
+                else:
+                    out.append("- " + text)
+            else:
+                out.append(text)
+        elif el.tag == f"{w}tbl":
+            rows = [[text_of(c).strip().replace("|", "\\|") for c in r.iter(f"{w}tc")] for r in el.iter(f"{w}tr")]
+            rows = [r for r in rows if any(r)]
+            if rows:
+                width = max(len(r) for r in rows)
+                rows = [r + [""] * (width - len(r)) for r in rows]
+                out.append("\n".join(["| " + " | ".join(rows[0]) + " |", "|" + " --- |" * width] + ["| " + " | ".join(r) + " |" for r in rows[1:]]))
+    return "\n\n".join(out)
+
+
+def convert(ext: str, raw: bytes) -> str:
+    reader = READERS[ext]
+    if not available() and ext == "docx":
+        return docx_without_pandoc(raw)
+    if not available():
+        raise ConversionError(
+            f".{ext} needs pandoc, which is not installed (apt install pandoc / brew install pandoc)")
+    cmd = ["pandoc", "-f", reader, "-t", "gfm-raw_html", "--wrap=none"]
+    try:
+        p = subprocess.run(cmd, input=raw, capture_output=True, timeout=_TIMEOUT_S)
+    except subprocess.TimeoutExpired:
+        raise ConversionError(f"pandoc timed out after {_TIMEOUT_S}s") from None
+    if p.returncode != 0 and ext == "docx":
+        return docx_without_pandoc(raw)                  # pandoc rejects some Word files that are still readable
+    if p.returncode != 0:
+        err = p.stderr.decode("utf-8", errors="replace").strip().splitlines()
+        raise ConversionError(f"pandoc could not read the file ({err[-1] if err else 'unknown error'})") from None
+    return p.stdout.decode("utf-8", errors="replace")

pal/input_converters/components/pdf_converter.py

@@ -0,0 +1,46 @@
+"""Text-layer PDFs -> markdown. Uses pymupdf4llm (headings, tables) when it is
+installed, otherwise pypdf with one section per page. Scans have no text layer
+and are rejected: there is no OCR here."""
+from __future__ import annotations
+
+import io
+
+from pal.input_converters.components.base import ConversionError
+
+
+def _with_pymupdf(raw: bytes) -> str | None:
+    try:
+        import pymupdf  # noqa: F401
+        import pymupdf4llm
+    except ImportError:
+        return None
+    doc = pymupdf.open(stream=raw, filetype="pdf")
+    try:
+        return pymupdf4llm.to_markdown(doc)
+    finally:
+        doc.close()
+
+
+def _with_pypdf(raw: bytes) -> str:
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        raise ConversionError("PDF support needs pypdf (pip install pypdf)") from None
+    pages = [(p.extract_text() or "").strip() for p in PdfReader(io.BytesIO(raw)).pages]
+    if len(pages) == 1:
+        return pages[0]
+    return "\n\n".join(f"## Page {i}\n\n{t}" for i, t in enumerate(pages, 1) if t)
+
+
+def convert(raw: bytes) -> str:
+    try:
+        text = _with_pymupdf(raw)
+        if text is None or not text.strip():
+            text = _with_pypdf(raw)
+    except ConversionError:
+        raise
+    except Exception as e:
+        raise ConversionError(f"could not read PDF ({e})") from e
+    if not text.strip():
+        raise ConversionError("PDF has no text layer (scanned?); OCR is not supported")
+    return text

pal/input_converters/components/pptx_converter.py

@@ -0,0 +1,52 @@
+"""PowerPoint -> markdown: one `## Slide N` section per slide, speaker notes kept."""
+from __future__ import annotations
+
+import io
+
+from pal.input_converters.components.base import ConversionError, table_to_markdown
+
+
+def _shape_lines(shape, out: list[str]) -> None:
+    if getattr(shape, "shape_type", None) == 6 or hasattr(shape, "shapes"):  # group
+        for s in getattr(shape, "shapes", []):
+            _shape_lines(s, out)
+        return
+    if getattr(shape, "has_table", False) and shape.has_table:
+        rows = [[c.text for c in r.cells] for r in shape.table.rows]
+        md = table_to_markdown(rows)
+        if md:
+            out.append(md)
+        return
+    if getattr(shape, "has_text_frame", False) and shape.has_text_frame:
+        for para in shape.text_frame.paragraphs:
+            t = "".join(r.text for r in para.runs).strip() or para.text.strip()
+            if t:
+                out.append(("  " * min(para.level, 6)) + "- " + t if para.level else t)
+
+
+def convert(raw: bytes) -> str:
+    try:
+        from pptx import Presentation
+    except ImportError:
+        raise ConversionError("PowerPoint support needs python-pptx (pip install python-pptx)") from None
+    try:
+        prs = Presentation(io.BytesIO(raw))
+    except Exception as e:
+        raise ConversionError(f"could not read PPTX ({e})") from e
+    sections = []
+    for i, slide in enumerate(prs.slides, 1):
+        title = ""
+        if slide.shapes.title is not None and slide.shapes.title.has_text_frame:
+            title = slide.shapes.title.text_frame.text.strip()
+        lines: list[str] = []
+        for shape in slide.shapes:
+            if slide.shapes.title is not None and shape.shape_id == slide.shapes.title.shape_id:
+                continue
+            _shape_lines(shape, lines)
+        if slide.has_notes_slide and slide.notes_slide.notes_text_frame is not None:
+            notes = slide.notes_slide.notes_text_frame.text.strip()
+            if notes:
+                lines.append(f"**Notes:** {notes}")
+        head = f"## Slide {i}" + (f": {title}" if title else "")
+        sections.append(head + ("\n\n" + "\n\n".join(lines) if lines else ""))
+    return "\n\n".join(sections)

pal/input_converters/components/xlsx_converter.py

@@ -0,0 +1,38 @@
+"""Excel -> markdown: one `## Sheet: name` section with a pipe table per sheet.
+Formula cells give their cached (last-saved) value."""
+from __future__ import annotations
+
+import io
+
+from pal.input_converters.components.base import ConversionError, table_to_markdown
+
+
+def _fmt(v) -> str:
+    if v is None:
+        return ""
+    if isinstance(v, float) and v.is_integer():
+        return str(int(v))
+    if hasattr(v, "isoformat"):
+        return v.isoformat(sep=" ") if hasattr(v, "hour") and (v.hour or v.minute or v.second) else v.isoformat()[:10]
+    return str(v)
+
+
+def convert(raw: bytes) -> str:
+    try:
+        from openpyxl import load_workbook
+    except ImportError:
+        raise ConversionError("Excel support needs openpyxl (pip install openpyxl)") from None
+    try:
+        wb = load_workbook(io.BytesIO(raw), read_only=True, data_only=True)
+    except Exception as e:
+        raise ConversionError(f"could not read XLSX ({e})") from e
+    sections = []
+    try:
+        for ws in wb.worksheets:
+            rows = [[_fmt(v) for v in row] for row in ws.iter_rows(values_only=True)]
+            table = table_to_markdown(rows)
+            if table:
+                sections.append(f"## Sheet: {ws.title}\n\n{table}")
+    finally:
+        wb.close()
+    return "\n\n".join(sections)

pal/input_converters/pipeline.py

@@ -0,0 +1,72 @@
+"""Attachment -> markdown, before anything reaches an API model.
+
+Text-heavy binary/markup formats are converted to markdown because it costs
+fewer tokens than the source markup and keeps structure (headings, lists,
+tables) that flat extraction loses:
+
+    html/htm/xhtml            pandoc
+    docx, odt, rtf            pandoc
+    epub                      pandoc
+    tex/latex                 pandoc
+    pdf (text layer only)     pymupdf4llm if installed, else pypdf
+    pptx                      python-pptx
+    xlsx                      openpyxl
+
+Anything else is treated as plain text (code, CSV, JSON, .md, ...) and passed
+through unchanged. Binary files that are not on the list are rejected.
+"""
+from __future__ import annotations
+
+from pathlib import PurePath
+
+from pal.input_converters.components import (
+    pandoc_converter,
+    pdf_converter,
+    pptx_converter,
+    xlsx_converter,
+)
+from pal.input_converters.components.base import ConversionError, ConvertedFile
+
+CONVERTIBLE_EXTENSIONS = tuple(sorted(
+    set(pandoc_converter.READERS) | {"pdf", "pptx", "xlsx"}
+))
+
+# Legacy binary formats we cannot convert; say so instead of "binary not supported".
+_LEGACY = {"doc": "docx", "ppt": "pptx", "xls": "xlsx"}
+
+
+def _ext(name: str) -> str:
+    return PurePath(name).suffix.lower().lstrip(".")
+
+
+def convert_to_markdown(name: str, raw: bytes) -> ConvertedFile:
+    """Return the file's content as markdown (or unchanged text). Raises
+    ConversionError (a ValueError) with a user-readable message on failure."""
+    name = name or "file"
+    ext = _ext(name)
+
+    def done(text: str, fmt: str, converted: bool = True) -> ConvertedFile:
+        text = text.replace("\r\n", "\n").strip("\n")
+        if not text.strip():
+            raise ConversionError(f"{name}: no readable text")
+        return ConvertedFile(name=name, text=text, source_format=fmt, converted=converted)
+
+    try:
+        if ext in pandoc_converter.READERS:
+            return done(pandoc_converter.convert(ext, raw), ext)
+        if ext == "pdf":
+            return done(pdf_converter.convert(raw), "pdf")
+        if ext == "pptx":
+            return done(pptx_converter.convert(raw), "pptx")
+        if ext in ("xlsx", "xlsm"):
+            return done(xlsx_converter.convert(raw), "xlsx")
+    except ConversionError as e:
+        raise ConversionError(f"{name}: {e}") from e
+
+    if ext in _LEGACY:
+        raise ConversionError(f"{name}: legacy .{ext} is not supported; save it as .{_LEGACY[ext]} and re-attach")
+    if b"\x00" in raw[:4096]:
+        raise ConversionError(
+            f"{name}: binary files are not supported "
+            f"(text, code, CSV, JSON and {', '.join('.' + e for e in CONVERTIBLE_EXTENSIONS)} only)")
+    return done(raw.decode("utf-8", errors="replace"), "text", converted=False)

pal/input_converters/utils/__init__.py

@@ -0,0 +1 @@
+

pal/input_converters/workspace.py

@@ -0,0 +1,56 @@
+"""Convert a file that is already in a workspace to markdown: the converter as a tool.
+
+Uploads are converted automatically when they arrive. This is the same converter for a file that is already in an agent's workspace
+(an upload kept there, a file saved or generated earlier), asked for on demand. The full markdown is saved next to the original
+under `converted/` so it can be read in parts, and an existing file is never overwritten.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+from pal.input_converters.components.base import ConversionError
+from pal.input_converters.pipeline import CONVERTIBLE_EXTENSIONS, convert_to_markdown
+
+OUT_DIR = "converted"
+
+
+@dataclass
+class WorkspaceConversion:
+    path: str              # the file that was converted
+    text: str              # the whole markdown
+    source_format: str
+    converted: bool        # False when the file was already text and passed through
+    saved: str             # where the markdown was saved ("" if it was not)
+    note: str              # why it was not saved, if so
+
+
+def convertible(path: str) -> bool:
+    return PurePosixPath(path).suffix.lower().lstrip(".") in CONVERTIBLE_EXTENSIONS
+
+
+def convert_workspace_file(path: str, *, read_bytes: Callable[[str], bytes], write_text: Callable[[str, str], object],
+                           exists: Callable[[str], bool], max_chars: int = 500_000) -> WorkspaceConversion:
+    name = PurePosixPath(path).name
+    if not convertible(path):
+        raise ConversionError(f"{name}: only {', '.join(CONVERTIBLE_EXTENSIONS)} files can be converted "
+                              "(text and code files need no conversion)")
+    res = convert_to_markdown(name, read_bytes(path))
+    if len(res.text) > max_chars:
+        raise ConversionError(f"{name}: the markdown is {len(res.text):,} characters, over the {max_chars:,} limit")
+    out = f"{OUT_DIR}/{name}.md"
+    if exists(out):
+        return WorkspaceConversion(path, res.text, res.source_format, res.converted, "", f"{out} already exists and was not overwritten")
+    write_text(out, res.text)
+    return WorkspaceConversion(path, res.text, res.source_format, res.converted, out, "")
+
+
+def describe(r: WorkspaceConversion, preview_chars: int = 2000, indexed: str = "") -> str:
+    """The tool's answer: what happened, then the beginning of the markdown."""
+    head = f"converted {r.path} ({r.source_format} to markdown): {len(r.text):,} characters."
+    head += f" Saved the full text as {r.saved}." if r.saved else f" {r.note}."
+    if indexed:
+        head += " " + indexed
+    cut = r.text[:preview_chars]
+    return head + "\n\n" + cut + ("\n[preview ends; read the rest from the saved file]" if len(r.text) > preview_chars else "")

pal/input_savers/INPUT_RULES.md

@@ -0,0 +1,2 @@
+Edit input. Skip any change altering context. Critical: context must not change.
+Remove: articles, filler, punctuation, pleasantries, hedging, extra whitespace. Paraphrase fewest words same meaning. Never drop not/never/no/only/except: meaning flip worse than tokens saved. Numbers, units exact. Imperative sentences. Instead of "Sure! I'd be happy to help. Issue likely caused by..." write "Bug in auth middleware". One word when one word enough. Never remove negation words. Never change user-pasted or attached file content. If original shorter than result, use original.

pal/input_savers/__init__.py

@@ -0,0 +1,5 @@
+from pal.input_savers.components.base import SaverResult, StepResult
+from pal.input_savers.components.methods import METHOD_ORDER, METHODS
+from pal.input_savers.pipeline import InputSaverPipeline
+
+__all__ = ["InputSaverPipeline", "SaverResult", "StepResult", "METHODS", "METHOD_ORDER"]

pal/input_savers/components/base.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class Method:
+    id: str
+    name: str
+    rule: str  # the one edit the local LLM is allowed to make
+    system: str = ""  # optional full applier system prompt (used by the rules-file mode)
+
+
+@dataclass
+class StepResult:
+    method_id: str
+    name: str
+    status: str               # "kept" | "reverted" | "skipped"
+    reason: str
+    tokens_before: int
+    tokens_after: int         # equals tokens_before when reverted/skipped
+    chunks_total: int = 0
+    chunks_kept: int = 0
+    chunk_notes: list = field(default_factory=list)  # per reverted chunk: why
+    before: str = ""
+    after: str = ""
+
+
+@dataclass
+class SaverResult:
+    original: str
+    reduced: str
+    steps: list
+    tokens_original: int
+    tokens_reduced: int
+    counter: str = ""
+
+    @property
+    def tokens_saved(self) -> int:
+        return max(0, self.tokens_original - self.tokens_reduced)