folioqueue 0.1.0__py3-none-any.whl

folioqueue/__init__.py

@@ -0,0 +1,3 @@
+"""FolioQueue: local, checkpointed document conversion."""
+
+__version__ = "0.1.0"

folioqueue/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

folioqueue/cli.py

@@ -0,0 +1,134 @@
+"""CLI with stable exit codes and a read-only plan command."""
+
+from __future__ import annotations
+
+import argparse
+import codecs
+import json
+import math
+import sys
+from collections import Counter
+from pathlib import Path
+
+from . import __version__
+from .engine import SUPPORTED, Options, plan, run
+from .paths import QueueError
+
+
+def positive(value: str) -> float:
+    number = float(value)
+    if not math.isfinite(number) or not 0 < number <= 1_000_000:
+        raise argparse.ArgumentTypeError("must be a finite positive number, at most 1000000")
+    return number
+
+
+def workers(value: str) -> int:
+    number = int(value)
+    if not 1 <= number <= 16:
+        raise argparse.ArgumentTypeError("must be between 1 and 16")
+    return number
+
+
+def parser() -> argparse.ArgumentParser:
+    root = argparse.ArgumentParser(description="Restartable local document-to-Markdown batches.")
+    root.add_argument("--version", action="version", version=f"folioqueue {__version__}")
+    commands = root.add_subparsers(dest="command", required=True)
+    for command, help_text in (
+        ("convert", "Convert a directory and write reports."),
+        ("plan", "Inspect pending work without writing files."),
+    ):
+        sub = commands.add_parser(command, help=help_text)
+        sub.add_argument("source", type=Path, help="Source directory (scanned recursively).")
+        sub.add_argument(
+            "-o",
+            "--output",
+            type=Path,
+            required=True,
+            help="Separate output directory, outside the source tree.",
+        )
+        sub.add_argument(
+            "--workers", type=workers, default=2, help="Parallel file processes (1–16)."
+        )
+        sub.add_argument(
+            "--timeout", type=positive, default=60, help="Seconds allowed per converter process."
+        )
+        sub.add_argument(
+            "--max-input-mb", type=positive, default=64, help="Per-file input limit in MiB."
+        )
+        sub.add_argument(
+            "--max-output-mb", type=positive, default=32, help="Per-file Markdown limit in MiB."
+        )
+        sub.add_argument(
+            "--encoding", default="utf-8-sig", help="TXT/MD/CSV encoding (default: utf-8-sig)."
+        )
+        sub.add_argument(
+            "--types",
+            default=",".join(sorted(t[1:] for t in SUPPORTED)),
+            help="Comma-separated subset: txt,md,csv,html,htm,docx,pdf.",
+        )
+        sub.add_argument(
+            "--force",
+            action="store_true",
+            help="Reconvert unchanged files; never overwrite edited output.",
+        )
+        sub.add_argument(
+            "--json", action="store_true", help="Print the full machine-readable report."
+        )
+    return root
+
+
+def main(argv: list[str] | None = None) -> int:
+    cli = parser()
+    args = cli.parse_args(argv)
+    try:
+        encoding = codecs.lookup(args.encoding).name
+    except LookupError:
+        cli.error("unknown text encoding")
+    extensions = frozenset("." + part.strip().lower().lstrip(".") for part in args.types.split(","))
+    if not extensions or not extensions <= SUPPORTED:
+        cli.error("--types must contain only supported extensions")
+    options = Options(
+        args.source,
+        args.output,
+        args.workers,
+        args.timeout,
+        max(1, int(args.max_input_mb * 1024 * 1024)),
+        max(1, int(args.max_output_mb * 1024 * 1024)),
+        encoding,
+        extensions,
+        args.force,
+    )
+    try:
+        report = plan(options) if args.command == "plan" else run(options)
+    except KeyboardInterrupt:
+        print(
+            "Interrupted. Completed files are checkpointed; rerun the same command.",
+            file=sys.stderr,
+        )
+        return 130
+    except QueueError as error:
+        print(f"folioqueue: {error}", file=sys.stderr)
+        return 2
+    except OSError:
+        print(
+            "folioqueue: filesystem operation failed; check permissions and disk space.",
+            file=sys.stderr,
+        )
+        return 2
+    if args.json:
+        print(json.dumps(report, ensure_ascii=True, indent=2))
+    elif args.command == "plan":
+        counts = Counter(r.get("action", r["status"]) for r in report["records"])
+        print("Plan: " + ", ".join(f"{n} {s}" for s, n in sorted(counts.items())))
+        print("No files written. Add --json for per-file decisions.")
+    else:
+        print("Finished: " + ", ".join(f"{n} {s}" for s, n in report["summary"].items()))
+        print("Reports: report.html and report.json in the output directory.")
+    return (
+        1
+        if any(
+            r["status"] == "failed" or r.get("action") in {"error", "conflict"}
+            for r in report["records"]
+        )
+        else 0
+    )

folioqueue/engine.py

@@ -0,0 +1,310 @@
+"""Bounded conversion with per-file isolation and conservative output ownership."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import importlib.metadata
+import json
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from . import __version__
+from .paths import QueueError, destination, no_links, roots, scan
+from .report import finish
+from .storage import Ledger, OutputLock, atomic_write, digest
+
+SUPPORTED = {".txt", ".md", ".csv", ".html", ".htm", ".docx", ".pdf"}
+TEXT_TYPES = {".txt", ".md", ".csv"}
+
+
+@dataclass(frozen=True)
+class Options:
+    source: Path
+    output: Path
+    workers: int = 2
+    timeout: float = 60
+    max_input_bytes: int = 64 * 1024 * 1024
+    max_output_bytes: int = 32 * 1024 * 1024
+    encoding: str = "utf-8-sig"
+    extensions: frozenset[str] = frozenset(SUPPORTED)
+    force: bool = False
+
+
+def fingerprint(encoding: str, documents: bool) -> str:
+    environment = []
+    if documents:
+        # Converters are transitive dependencies. Upgrades must invalidate cached output.
+        environment = sorted(
+            {
+                (d.metadata.get("Name", "").lower(), d.version)
+                for d in importlib.metadata.distributions()
+            }
+        )
+    value = [__version__, list(sys.version_info[:2]), encoding, environment]
+    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
+
+
+def owned(target: Path, entry: dict | None) -> bool:
+    if not target.exists():
+        return True
+    if not target.is_file() or not entry:
+        return False
+    return digest(target) in {entry.get("output_sha256"), entry.get("previous_output_sha256")}
+
+
+def prepare(options: Options) -> tuple[Path, Path, Ledger, list[dict], list[dict]]:
+    source, output = roots(options.source, options.output)
+    ledger = Ledger(output, source)
+    files, ignored = scan(source, set(options.extensions))
+    fingerprints = {False: fingerprint(options.encoding, False)}
+    if any(p.suffix.lower() not in TEXT_TYPES for p in files):
+        fingerprints[True] = fingerprint(options.encoding, True)
+    tasks = []
+    for path in files:
+        relative = path.relative_to(source).as_posix()
+        target = destination(output, relative)
+        record = {
+            "source": relative,
+            "output": target.relative_to(output).as_posix(),
+            "status": "planned",
+            "code": "ok",
+            "action": "convert",
+            "fingerprint": fingerprints[path.suffix.lower() not in TEXT_TYPES],
+        }
+        entry = ledger.entries.get(relative)
+        try:
+            if path.stat().st_size > options.max_input_bytes:
+                record.update(action="error", code="input_too_large")
+            else:
+                record["source_sha256"] = digest(path)
+                if not owned(target, entry):
+                    record.update(action="conflict", code="output_conflict")
+                elif (
+                    not options.force
+                    and entry
+                    and target.is_file()
+                    and target.stat().st_size <= options.max_output_bytes
+                    and entry["source_sha256"] == record["source_sha256"]
+                    and entry["fingerprint"] == record["fingerprint"]
+                    and digest(target) == entry["output_sha256"]
+                ):
+                    record.update(action="skip", code="unchanged")
+                    record["output_sha256"] = entry["output_sha256"]
+        except OSError:
+            record.update(action="error", code="io_error")
+        tasks.append(record)
+    # Only absent sources are stale; selecting fewer types does not imply deletion.
+    for relative in ledger.entries:
+        destination(output, relative)  # validate even stale, untrusted ledger keys
+        path = source / Path(relative)
+        if not path.exists():
+            ignored.append({"source": relative, "status": "stale", "code": "source_removed"})
+    return source, output, ledger, tasks, ignored
+
+
+def plan(options: Options) -> dict:
+    _, _, _, tasks, ignored = prepare(options)
+    return {"schema": 1, "version": __version__, "records": tasks + ignored}
+
+
+class Processes:
+    """Track only our direct children so Ctrl+C never waits for all queued timeouts."""
+
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.children: set[subprocess.Popen] = set()
+        self.cancelled = False
+
+    def start(self, command: list[str]) -> subprocess.Popen:
+        with self.lock:
+            if self.cancelled:
+                raise InterruptedError
+            child = subprocess.Popen(
+                command,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                shell=False,
+                creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
+            )
+            self.children.add(child)
+            return child
+
+    def remove(self, child: subprocess.Popen) -> None:
+        with self.lock:
+            self.children.discard(child)
+
+    def cancel(self) -> None:
+        with self.lock:
+            self.cancelled = True
+            for child in self.children:
+                with contextlib.suppress(OSError):
+                    child.kill()
+
+
+def execute(
+    source: Path, scratch: Path, record: dict, options: Options, processes: Processes
+) -> tuple[str, bytes | None]:
+    path = source / Path(record["source"])
+    try:
+        no_links(path)
+        # Snapshot ensures the converter sees precisely the bytes whose hash was planned.
+        with tempfile.TemporaryDirectory(prefix="job-", dir=scratch) as temp:
+            folder = Path(temp)
+            snapshot = folder / ("input" + path.suffix.lower())
+            result, status = folder / "result.md", folder / "status.json"
+            checksum = hashlib.sha256()
+            total = 0
+            with path.open("rb") as src, snapshot.open("wb") as dst:
+                for chunk in iter(lambda: src.read(1024 * 1024), b""):
+                    total += len(chunk)
+                    if total > options.max_input_bytes:
+                        return "input_too_large", None
+                    checksum.update(chunk)
+                    dst.write(chunk)
+            if checksum.hexdigest() != record["source_sha256"]:
+                return "source_changed", None
+            command = [
+                sys.executable,
+                "-m",
+                "folioqueue.worker",
+                str(snapshot),
+                str(result),
+                str(status),
+                options.encoding,
+                str(options.max_output_bytes),
+            ]
+            child = processes.start(command)
+            try:
+                try:
+                    child.wait(timeout=options.timeout)
+                except subprocess.TimeoutExpired:
+                    child.kill()
+                    child.wait()
+                    return "timeout", None
+            finally:
+                processes.remove(child)
+            if not status.is_file() or status.stat().st_size > 1024:
+                return "worker_failed", None
+            details = json.loads(status.read_text(encoding="utf-8"))
+            if details.get("code") != "ok":
+                code = details.get("code")
+                allowed = {
+                    "empty_output",
+                    "output_too_large",
+                    "encoding_error",
+                    "missing_backend",
+                    "archive_limit",
+                    "binary_text",
+                    "conversion_error",
+                }
+                return code if code in allowed else "worker_failed", None
+            if child.returncode != 0 or not result.is_file():
+                return "worker_failed", None
+            if result.stat().st_size > options.max_output_bytes:
+                return "output_too_large", None
+            return "ok", result.read_bytes()
+    except (OSError, QueueError):
+        return "io_error", None
+    except (ValueError, TypeError, AttributeError):
+        return "worker_failed", None
+
+
+def commit(source: Path, output: Path, ledger: Ledger, record: dict, data: bytes) -> str:
+    target = destination(output, record["source"])
+    previous = ledger.entries.get(record["source"])
+    if digest(source / Path(record["source"])) != record["source_sha256"]:
+        return "source_changed"
+    if not owned(target, previous):
+        return "output_conflict"
+    # Write-ahead ownership covers interruption between output replacement and checkpoint.
+    entry = {
+        "source_sha256": record["source_sha256"],
+        "output_sha256": hashlib.sha256(data).hexdigest(),
+        "previous_output_sha256": digest(target) if target.exists() else None,
+        "fingerprint": record["fingerprint"],
+        "status": "pending",
+    }
+    ledger.entries[record["source"]] = entry
+    ledger.save()
+    atomic_write(target, data)
+    entry["status"] = "success"
+    entry["previous_output_sha256"] = None
+    ledger.save()
+    record["output_sha256"] = entry["output_sha256"]
+    return "ok"
+
+
+def run(options: Options) -> dict:
+    started = time.monotonic()
+    started_at = datetime.now(UTC).isoformat()
+    source, output = roots(options.source, options.output)
+    with OutputLock(output):
+        for name in ("report.json", "report.html"):
+            no_links(output / name)
+            if (output / name).exists() and not (output / ".folioqueue/state.json").exists():
+                raise QueueError("Untracked report exists in output. Use a new output directory.")
+        source, output, ledger, tasks, records = prepare(options)
+        ledger.save()
+        scratch = output / ".folioqueue" / "work"
+        no_links(scratch)
+        scratch.mkdir(exist_ok=True)
+        queue = []
+        for record in tasks:
+            action = record.pop("action")
+            if action == "convert":
+                queue.append(record)
+            else:
+                record["status"] = {"skip": "skipped", "error": "failed", "conflict": "failed"}[
+                    action
+                ]
+                records.append(record)
+        processes = Processes()
+        pool = ThreadPoolExecutor(max_workers=options.workers)
+        pending = {}
+        iterator = iter(queue)
+
+        def enqueue() -> None:
+            item = next(iterator, None)
+            if item is not None:
+                pending[pool.submit(execute, source, scratch, item, options, processes)] = item
+
+        try:
+            for _ in range(options.workers):
+                enqueue()
+            while pending:
+                completed, _ = wait(pending, timeout=0.2, return_when=FIRST_COMPLETED)
+                for future in completed:
+                    record = pending.pop(future)
+                    code, data = future.result()
+                    if code == "ok" and data is not None:
+                        try:
+                            code = commit(source, output, ledger, record, data)
+                        except (OSError, QueueError):
+                            code = "io_error"
+                    record.update(status="converted" if code == "ok" else "failed", code=code)
+                    records.append(record)
+                    enqueue()
+        except BaseException:
+            processes.cancel()
+            raise
+        finally:
+            pool.shutdown(wait=True, cancel_futures=True)
+        return finish(
+            output,
+            {
+                "schema": 1,
+                "version": __version__,
+                "started_at": started_at,
+                "duration_seconds": round(time.monotonic() - started, 3),
+                "records": records,
+            },
+        )

folioqueue/paths.py

@@ -0,0 +1,106 @@
+"""Conservative path handling. Symlinks and Windows junctions are not traversed."""
+
+from __future__ import annotations
+
+import os
+import stat
+import unicodedata
+from pathlib import Path
+
+
+class QueueError(Exception):
+    """A user-actionable configuration or filesystem error."""
+
+
+def linked(path: Path) -> bool:
+    try:
+        info = path.lstat()
+    except FileNotFoundError:
+        return False
+    return stat.S_ISLNK(info.st_mode) or bool(
+        getattr(info, "st_file_attributes", 0) & 0x400  # FILE_ATTRIBUTE_REPARSE_POINT
+    )
+
+
+def no_links(path: Path) -> None:
+    for part in (path, *path.parents):
+        if linked(part):
+            raise QueueError("Symlinks and junctions are not supported in input/output paths.")
+
+
+def roots(source: Path, output: Path) -> tuple[Path, Path]:
+    source = Path(os.path.abspath(source))
+    output = Path(os.path.abspath(output))
+    no_links(source)
+    no_links(output)
+    if not source.is_dir():
+        raise QueueError("Source must be an existing directory.")
+    if source == output or source in output.parents or output in source.parents:
+        raise QueueError("Source and output must be separate, non-overlapping directories.")
+    if output.exists() and not output.is_dir():
+        raise QueueError("Output must be a directory.")
+    return source, output
+
+
+def destination(output: Path, relative: str) -> Path:
+    """Never trust paths loaded from a ledger."""
+    parts = relative.split("/")
+    if not parts or any(p in {"", ".", ".."} or "\\" in p or ":" in p for p in parts):
+        raise QueueError("Invalid relative path in the ledger.")
+    target = output / "documents" / Path(*parts[:-1]) / (parts[-1] + ".md")
+    no_links(target)
+    return target
+
+
+def scan(source: Path, extensions: set[str]) -> tuple[list[Path], list[dict]]:
+    files: list[Path] = []
+    ignored: list[dict] = []
+
+    def onerror(error: OSError) -> None:
+        raise QueueError(
+            "A source directory could not be read; no partial scan was accepted."
+        ) from error
+
+    for directory, dirs, names in os.walk(source, followlinks=False, onerror=onerror):
+        base = Path(directory)
+        kept = []
+        for name in sorted(dirs):
+            path = base / name
+            if linked(path) or name.startswith("."):
+                ignored.append(
+                    {
+                        "source": path.relative_to(source).as_posix(),
+                        "status": "ignored",
+                        "code": "link_or_hidden_directory",
+                    }
+                )
+            else:
+                kept.append(name)
+        dirs[:] = kept
+        for name in sorted(names):
+            path = base / name
+            relative = path.relative_to(source).as_posix()
+            if linked(path) or name.startswith(".") or not path.is_file():
+                ignored.append({"source": relative, "status": "ignored", "code": "link_or_hidden"})
+            elif path.suffix.lower() not in extensions:
+                ignored.append(
+                    {"source": relative, "status": "ignored", "code": "unsupported_type"}
+                )
+            else:
+                files.append(path)
+    seen: set[str] = set()
+    output_names: set[str] = set()
+    for path in files:
+        relative = path.relative_to(source).as_posix()
+        key = unicodedata.normalize("NFC", relative).casefold()
+        if key in seen:
+            raise QueueError("Source paths collide under case-insensitive Unicode normalization.")
+        seen.add(key)
+        output_names.add(key + ".md")
+        # Reject names that cannot be safely represented by the ledger on all platforms.
+        destination(Path("output"), relative)
+    for name in output_names:
+        parts = name.split("/")
+        if any("/".join(parts[:index]) in output_names for index in range(1, len(parts))):
+            raise QueueError("An output file would collide with another output's parent directory.")
+    return sorted(files, key=lambda p: p.relative_to(source).as_posix()), ignored

folioqueue/report.py

@@ -0,0 +1,69 @@
+"""Content-free JSON and inert HTML run reports."""
+
+from __future__ import annotations
+
+from collections import Counter
+from html import escape
+from pathlib import Path
+from urllib.parse import quote
+
+from .storage import atomic_write, json_write
+
+HINTS = {
+    "ok": "Converted and checkpointed.",
+    "unchanged": "Input, converter environment and output checksums match.",
+    "output_conflict": "Output is untracked or edited. Move it aside or choose a new output directory.",
+    "timeout": "Conversion exceeded the per-file timeout. Retry with a larger --timeout.",
+    "missing_backend": "Install folioqueue[documents] to convert PDF, DOCX and HTML.",
+    "encoding_error": "Text decoding failed. Set --encoding to the source encoding.",
+    "empty_output": "No text was extracted. Scanned PDFs may require a separate OCR workflow.",
+    "input_too_large": "Input exceeds --max-input-mb.",
+    "output_too_large": "Output exceeds --max-output-mb.",
+    "archive_limit": "DOCX exceeds the archive entry or uncompressed-size limit.",
+    "binary_text": "NUL bytes found in a text file; check its format and encoding.",
+    "conversion_error": "Converter could not process this file. Check validity and supported features.",
+    "worker_failed": "Conversion process ended without a valid result.",
+    "io_error": "A file could not be read or written; check permissions and available space.",
+    "source_changed": "Source changed during the run. Retry when the source is stable.",
+    "unsupported_type": "Extension is outside this run's selected types.",
+    "link_or_hidden": "Symlink, junction, special file or hidden-name file skipped.",
+    "link_or_hidden_directory": "Symlink, junction or hidden-name directory not traversed.",
+    "source_removed": "Source no longer exists in the selected scan. Prior output was retained.",
+}
+
+
+def finish(output: Path, report: dict) -> dict:
+    report["records"].sort(key=lambda item: item["source"])
+    report["summary"] = dict(sorted(Counter(r["status"] for r in report["records"]).items()))
+    rows = []
+    for record in report["records"]:
+        record["message"] = HINTS.get(record["code"], record["code"])
+        label = escape(record["source"])
+        if record.get("output") and record["status"] in {"converted", "skipped"}:
+            label = f'<a href="{quote(record["output"], safe="/")}">{label}</a>'
+        rows.append(
+            f"<tr><td>{label}</td><td>{escape(record['status'])}</td>"
+            f"<td>{escape(record['message'])}</td></tr>"
+        )
+    summary = " · ".join(f"{count} {escape(status)}" for status, count in report["summary"].items())
+    page = f"""<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
+<meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'">
+<title>FolioQueue · Conversion report</title>
+<style>
+:root{{color-scheme:light dark;font:16px/1.55 system-ui,sans-serif;background:#f4f5f0;color:#182827}}
+body{{max-width:1120px;margin:0 auto;padding:48px 24px}}h1{{font-size:40px;letter-spacing:-1.5px;margin:12px 0}}
+.eyebrow{{color:#316d60;letter-spacing:2px;font-size:12px;font-weight:700}}.summary{{font-size:22px;margin:24px 0}}
+.meta,footer{{color:#506560;font-size:14px}}.scroll{{overflow:auto;background:#fff;border:1px solid #d6dfda;border-radius:12px}}
+table{{border-collapse:collapse;width:100%;text-align:left}}th,td{{padding:16px;border-bottom:1px solid #e1e7e3;vertical-align:top}}
+th{{font-size:12px;letter-spacing:1px;text-transform:uppercase;background:#eaf0ec}}td:first-child{{overflow-wrap:anywhere;min-width:180px}}
+a{{color:#17644f}}footer{{margin-top:24px}}@media(prefers-color-scheme:dark){{:root{{background:#14221f;color:#edf3ee}}.scroll{{background:#1c302a;border-color:#385044}}th{{background:#223d32}}th,td{{border-color:#385044}}a,.eyebrow{{color:#94d6b7}}.meta,footer{{color:#afc2b7}}}}
+</style></head><body><div class="eyebrow">FOLIOQUEUE / LOCAL DOCUMENT WORKFLOWS</div>
+<h1>Conversion report</h1><p class="meta">{escape(report["started_at"])} · v{escape(report["version"])} · {report["duration_seconds"]:.2f}s</p>
+<p class="summary">{summary or "No matching files"}</p><div class="scroll"><table>
+<thead><tr><th>Source</th><th>Result</th><th>Details</th></tr></thead><tbody>{"".join(rows)}</tbody></table></div>
+<footer>No document text is embedded in this report. Filenames can still be sensitive. Output links open local Markdown files; their content is untrusted.</footer>
+</body></html>"""
+    json_write(output / "report.json", report)
+    atomic_write(output / "report.html", page.encode("utf-8"))
+    return report

folioqueue/storage.py

@@ -0,0 +1,124 @@
+"""Atomic checkpoints, output ownership and a kernel-released exclusive writer lock."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import os
+import tempfile
+from pathlib import Path
+
+from .paths import QueueError, no_links
+
+
+def digest(path: Path) -> str:
+    no_links(path)
+    result = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            result.update(chunk)
+    return result.hexdigest()
+
+
+def atomic_write(path: Path, data: bytes) -> None:
+    no_links(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    no_links(path)
+    fd, name = tempfile.mkstemp(prefix=".fq-", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(name, path)
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(name)
+
+
+def json_write(path: Path, value: dict) -> None:
+    atomic_write(path, (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode("utf-8"))
+
+
+class Ledger:
+    def __init__(self, output: Path, source: Path):
+        self.path = output / ".folioqueue" / "state.json"
+        no_links(self.path)
+        if self.path.exists():
+            try:
+                self.data = json.loads(self.path.read_text(encoding="utf-8"))
+                if (
+                    self.data["schema"] != 1
+                    or self.data["source_root"] != str(source)
+                    or not isinstance(self.data["entries"], dict)
+                ):
+                    raise ValueError
+                for key, entry in self.data["entries"].items():
+                    if not isinstance(key, str) or not isinstance(entry, dict):
+                        raise ValueError
+                    for field in ("source_sha256", "output_sha256", "fingerprint", "status"):
+                        if not isinstance(entry[field], str):
+                            raise ValueError
+                    for field in ("source_sha256", "output_sha256"):
+                        if len(entry[field]) != 64 or any(
+                            c not in "0123456789abcdef" for c in entry[field]
+                        ):
+                            raise ValueError
+                    if entry["status"] not in {"pending", "success"}:
+                        raise ValueError
+                    previous = entry.get("previous_output_sha256")
+                    if previous is not None and (
+                        not isinstance(previous, str)
+                        or len(previous) != 64
+                        or any(c not in "0123456789abcdef" for c in previous)
+                    ):
+                        raise ValueError
+            except (ValueError, KeyError, TypeError) as error:
+                raise QueueError(
+                    "Ledger is invalid or belongs to another source. Use a new output directory."
+                ) from error
+        else:
+            self.data = {"schema": 1, "source_root": str(source), "entries": {}}
+
+    @property
+    def entries(self) -> dict:
+        return self.data["entries"]
+
+    def save(self) -> None:
+        json_write(self.path, self.data)
+
+
+class OutputLock:
+    """OS advisory lock; remains safe after a process crash (no stale PID guessing)."""
+
+    def __init__(self, output: Path):
+        self.path = output / ".folioqueue" / "writer.lock"
+
+    def __enter__(self):
+        no_links(self.path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.stream = self.path.open("a+b")
+        self.stream.seek(0, os.SEEK_END)
+        if self.stream.tell() == 0:
+            self.stream.write(b"0")
+            self.stream.flush()
+        self.stream.seek(0)
+        try:
+            if os.name == "nt":
+                import msvcrt
+
+                msvcrt.locking(self.stream.fileno(), msvcrt.LK_NBLCK, 1)
+            else:
+                import fcntl
+
+                fcntl.flock(self.stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as error:
+            self.stream.close()
+            raise QueueError(
+                "Another FolioQueue process is using this output directory."
+            ) from error
+        return self
+
+    def __exit__(self, *_):
+        self.stream.close()

folioqueue/worker.py

@@ -0,0 +1,105 @@
+"""One-file subprocess. No cloud clients, external plugins, or shell invocation."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import sys
+import zipfile
+from pathlib import Path
+
+MAX_ARCHIVE_ENTRIES = 10000
+MAX_ARCHIVE_BYTES = 256 * 1024 * 1024
+
+
+def cell(value: str) -> str:
+    return (
+        value.replace("\\", "\\\\")
+        .replace("&", "&")
+        .replace("|", "\\|")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace("\r\n", "<br>")
+        .replace("\n", "<br>")
+        .replace("\r", "<br>")
+    )
+
+
+def convert(path: Path, encoding: str) -> str:
+    if path.suffix.lower() in {".txt", ".md", ".csv"}:
+        text = path.read_text(encoding=encoding)
+        if "\x00" in text:
+            raise ValueError("binary_text")
+        if path.suffix.lower() != ".csv":
+            return text
+        # Fixed comma dialect is explicit; quoted commas/newlines use the stdlib CSV parser.
+        rows = list(csv.reader(io.StringIO(text), strict=True))
+        if not rows:
+            return ""
+        width = max(map(len, rows))
+        if width == 0:
+            return ""
+        lines = [
+            "| " + " | ".join(cell(v) for v in row + [""] * (width - len(row))) + " |"
+            for row in rows
+        ]
+        lines.insert(1, "| " + " | ".join(["---"] * width) + " |")
+        return "\n".join(lines) + "\n"
+    if path.suffix.lower() == ".docx":
+        with zipfile.ZipFile(path) as archive:
+            members = archive.infolist()
+            if (
+                len(members) > MAX_ARCHIVE_ENTRIES
+                or sum(m.file_size for m in members) > MAX_ARCHIVE_BYTES
+            ):
+                raise ValueError("archive_limit")
+    from markitdown import MissingDependencyException, StreamInfo
+    from markitdown.converters import DocxConverter, HtmlConverter, PdfConverter
+
+    converter = {
+        ".pdf": PdfConverter,
+        ".docx": DocxConverter,
+        ".html": HtmlConverter,
+        ".htm": HtmlConverter,
+    }[path.suffix.lower()]()
+    # Use only the intended public converter. Automatic type fallback can otherwise
+    # turn a malformed PDF into a successful plain-text result.
+    try:
+        with path.open("rb") as stream:
+            return converter.convert(stream, StreamInfo(extension=path.suffix.lower())).markdown
+    except MissingDependencyException as error:
+        raise ModuleNotFoundError("Install the documents extra") from error
+
+
+def main() -> int:
+    source, target, status, encoding, limit = sys.argv[1:]
+    result = {}
+    try:
+        markdown = convert(Path(source), encoding)
+        if not markdown.strip():
+            result = {"code": "empty_output"}
+        elif len(markdown.encode("utf-8")) > int(limit):
+            result = {"code": "output_too_large"}
+        else:
+            Path(target).write_text(markdown, encoding="utf-8", newline="\n")
+            result = {"code": "ok"}
+    except UnicodeError:
+        result = {"code": "encoding_error"}
+    except ModuleNotFoundError:
+        result = {"code": "missing_backend"}
+    except ValueError as error:
+        result = {
+            "code": str(error)
+            if str(error) in {"archive_limit", "binary_text"}
+            else "conversion_error"
+        }
+    except Exception:
+        # Do not leak document contents, credentials, or absolute paths through library errors.
+        result = {"code": "conversion_error"}
+    Path(status).write_text(json.dumps(result), encoding="utf-8")
+    return 0 if result["code"] == "ok" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

folioqueue-0.1.0.dist-info/METADATA

@@ -0,0 +1,171 @@
+Metadata-Version: 2.4
+Name: folioqueue
+Version: 0.1.0
+Summary: Restartable local document-to-Markdown batches with a checksum ledger and failure isolation
+Project-URL: Homepage, https://github.com/GokouRuri43/folioqueue
+Project-URL: Issues, https://github.com/GokouRuri43/folioqueue/issues
+Project-URL: Changelog, https://github.com/GokouRuri43/folioqueue/blob/main/CHANGELOG.md
+Author: GokouRuri43
+License-Expression: MIT
+License-File: LICENSE
+Keywords: cli,document-conversion,incremental,markdown,markitdown
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Requires-Python: >=3.11
+Provides-Extra: documents
+Requires-Dist: markitdown[docx,pdf]<0.2,>=0.1.7; extra == 'documents'
+Provides-Extra: test
+Requires-Dist: build<2,>=1; extra == 'test'
+Requires-Dist: pytest-cov<8,>=6; extra == 'test'
+Requires-Dist: pytest<10,>=8; extra == 'test'
+Requires-Dist: ruff<1,>=0.11; extra == 'test'
+Requires-Dist: twine<7,>=6; extra == 'test'
+Description-Content-Type: text/markdown
+
+# FolioQueue
+
+**Restartable local document-to-Markdown batches, with a checksum ledger and inspectable failures.**
+
+[![CI](https://github.com/GokouRuri43/folioqueue/actions/workflows/ci.yml/badge.svg)](https://github.com/GokouRuri43/folioqueue/actions/workflows/ci.yml)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+[简体中文](README.zh-CN.md) · [Design](docs/design.md) · [Validation](docs/validation.md) · [Contributing](CONTRIBUTING.md)
+
+FolioQueue helps maintain a Markdown copy of a **local document collection**. It snapshots each input, converts files in separate processes, checkpoints completed work, and checks both source and output hashes before skipping an unchanged file. A broken document can fail while the rest of the batch finishes.
+
+**Status: 0.1.0 alpha.** See the validation record for what has actually been tested.
+
+## Who is this for?
+
+- **Note / knowledge-base maintainers** folding PDFs, Word and HTML into a Markdown vault, and keeping it in sync as sources change.
+- **Local RAG / LLM corpus builders** who need a reproducible, incremental Markdown mirror of a document folder to index.
+- **Self-hosted archive keepers** who want failures isolated per file and edited output protected, instead of a one-shot bulk convert.
+
+## Why use it?
+
+- **Resume by rerunning.** Completed files are checkpointed individually; failed files are retried on the next run.
+- **Detect real changes.** SHA-256 checksums, rather than timestamps alone, determine whether to convert. Converter dependency changes invalidate document caches.
+- **Protect edits.** Untracked or manually changed output is a conflict, even with `--force`.
+- **Keep names distinct.** `report.pdf` → `documents/report.pdf.md`; `report.docx` → `documents/report.docx.md`.
+- **Contain ordinary failures.** Bounded parallel file processes, per-file timeouts, input/output size checks, and a DOCX expansion preflight.
+- **Inspect every outcome.** Local HTML and JSON reports list converted, skipped, failed, ignored and stale files. Reports omit extracted text and raw converter exceptions.
+- **Plan first.** `plan` shows per-file decisions without creating files.
+
+This is an independently implemented workflow layer. PDF/DOCX/HTML extraction is delegated to [Microsoft MarkItDown](https://github.com/microsoft/markitdown); this project is not affiliated with Microsoft or OpenAI. It does not claim to improve PDF extraction fidelity. [Alternatives and research](docs/research.md).
+
+## Install
+
+Python 3.11 or later is required. Use a virtual environment. On Windows, the Python launcher may be `py` instead of `python`.
+
+```bash
+git clone https://github.com/GokouRuri43/folioqueue.git
+cd folioqueue
+python -m pip install ".[documents]"
+```
+
+`python -m pip install .` installs the dependency-free TXT/Markdown/CSV core. The `documents` extra adds PDF, DOCX and HTML through MarkItDown. A wheel and source archive are also provided in GitHub Releases; the project is **not currently published to PyPI**.
+
+## Try it on the included examples
+
+Run these commands from the repository directory:
+
+```bash
+folioqueue plan examples/documents -o demo-output --json
+folioqueue convert examples/documents -o demo-output
+folioqueue convert examples/documents -o demo-output
+```
+
+The first conversion produces three Markdown files and ignores one unsupported fixture. The second skips all three unchanged files. Open `demo-output/report.html` to inspect the latest run.
+
+For your own collection:
+
+```bash
+folioqueue convert ./documents -o ./markdown-output --workers 2 --timeout 60
+```
+
+On Windows:
+
+```powershell
+folioqueue convert 'C:\My Documents' -o 'C:\Markdown Output' --workers 2
+```
+
+Source and output directories must not overlap. Hidden-name files/directories, symlinks and Windows junctions are skipped. No URL inputs, cloud services, LLM API calls, third-party MarkItDown plugins, or shell commands are used by the conversion workflow. Dependencies must be installed beforehand.
+
+## Supported formats
+
+| Input | Backend | Boundaries |
+| --- | --- | --- |
+| `.txt`, `.md` | Standard library | Strict UTF-8 with optional BOM by default; `--encoding` selects another encoding |
+| `.csv` | Standard library | Comma-separated, first row as header; quoted commas/newlines supported; Markdown cell escaping |
+| `.html`, `.htm` | MarkItDown | Text extraction; no browser, script execution or asset downloading is requested |
+| `.docx` | MarkItDown | Extraction quality follows upstream; no visual layout preservation |
+| `.pdf` | MarkItDown | Text-based PDFs; no OCR, no guarantee of correct reading order or tables |
+
+Empty extracted text is a failure, including a scanned PDF with no text layer. Legacy `.doc`, spreadsheets, slides, images, media, archives and encrypted documents are outside the v0.1 support scope. Source documents remain the authoritative copy.
+
+## Controls and exit codes
+
+```bash
+folioqueue convert ./documents -o ./markdown-output --types txt,csv,docx
+folioqueue convert ./documents -o ./markdown-output --encoding gb18030
+folioqueue convert ./documents -o ./markdown-output --max-input-mb 64 --max-output-mb 32
+folioqueue convert ./documents -o ./markdown-output --force --json
+```
+
+`--workers` is 1–16 (default 2). `--timeout` covers each converter subprocess, including imports, but not scanning, hashing, copying or committing. `--force` reconverts owned, unedited output; it never authorizes overwriting a conflict. Move conflicting output aside or use a new output directory.
+
+| Exit | Meaning |
+| --- | --- |
+| `0` | No failed files / no blocking plan decisions; ignored and stale entries are informational |
+| `1` | At least one failed conversion or planned conflict/error |
+| `2` | Invalid arguments, invalid ledger, overlapping roots, lock contention or a run-level filesystem error |
+| `130` | Interrupted; rerun to continue from completed checkpoints |
+
+Deleted source files are reported as **stale**; their output is retained. A failed updated source can leave the previous Markdown in place. Consumers must check `report.json` and must not assume every existing Markdown file is current. The report describes the most recent completed run, not a global corpus validity certificate.
+
+## Output
+
+```text
+markdown-output/
+  documents/
+    report.pdf.md
+    subfolder/notes.txt.md
+  report.html
+  report.json
+  .folioqueue/
+    state.json
+    writer.lock
+    work/
+```
+
+The private ledger binds an output directory to one absolute source path. Do not edit it or share it as a public artifact. Moving the source requires a new output directory in v0.1. A kernel-managed exclusive writer lock is automatically released on process exit; the lock file may remain and should not be deleted while a run is active.
+
+Interrupted conversions may leave temporary source snapshots in `.folioqueue/work`. Once no run is active, you may remove that directory to reclaim space. A subprocess is **failure isolation, not a security sandbox**; process only trusted documents or use an external sandbox for untrusted inputs. See [SECURITY.md](SECURITY.md).
+
+## Development
+
+```bash
+python -m pip install -e ".[test,documents]"
+python -m pytest --cov=folioqueue --cov-report=term-missing
+python -m ruff check .
+python -m ruff format --check .
+python -m build
+python -m twine check dist/*
+```
+
+The test suite includes synthetic, redistributable PDF/DOCX/HTML fixtures and actual subprocess conversions. CI tests Windows, Linux and macOS. The workflow result, rather than this sentence, is the authority for current pass/fail status.
+
+## Roadmap
+
+1. Collect reproducible reports from document-collection maintainers and improve format diagnostics.
+2. Publish larger, redistributable corpus results and measure overhead, failure behavior and memory use.
+3. Consider an explicit stale-output review command and portable ledger migration after the core behavior is exercised.
+
+Requests for unsupported features belong in issues with a concrete workflow and a minimal non-sensitive sample. No telemetry is collected by FolioQueue.

folioqueue-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+folioqueue/__init__.py,sha256=rAHcO_WKeV-OrjN7mjgEMHGwukpt-WgD4FQXEhv2j1A,82
+folioqueue/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+folioqueue/cli.py,sha256=aeTv0e7bV7YMZ3m2-eD6WAS06JW2UJGpC4Eq4bhx-3w,4735
+folioqueue/engine.py,sha256=UnTBWN1hP2VNopJ1xZq4A-vGfwgOjxE9SqyuLjWW3gA,11914
+folioqueue/paths.py,sha256=NcDid9CyiKvhJFfOH8ECcLGP4aBmGOwvnyMQwAvR7tA,4047
+folioqueue/report.py,sha256=GuEkn5gv4MOg-ujVyuID65Y4Q8NzuF6dHNBmG0VN454,4704
+folioqueue/storage.py,sha256=8WABTpiaudfDvEKcNU8lXaVQO_ROvZFIhPDnyoNYkVM,4363
+folioqueue/worker.py,sha256=jVqWTB0o7dq9O2eieUKKjvlpJ2BHrSeq_job3LE4CBI,3604
+folioqueue-0.1.0.dist-info/METADATA,sha256=axvir8fIdh4yi-foYQhpOjUlagUzwuQCcidqQCxVNCA,9469
+folioqueue-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+folioqueue-0.1.0.dist-info/entry_points.txt,sha256=ndLewl-ba6PVgUpywwBXQmSfYHYvAalsgXHzdwL8OvE,51
+folioqueue-0.1.0.dist-info/licenses/LICENSE,sha256=yLlBV2MXALdOaDm4YOgd4OLnycp8eqeOtdRBWkFHsO8,1068
+folioqueue-0.1.0.dist-info/RECORD,,

folioqueue-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

folioqueue-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+folioqueue = folioqueue.cli:main

folioqueue-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GokouRuri43
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.