repulp 0.1.0__py3-none-any.whl

repulp/__init__.py

@@ -0,0 +1,172 @@
+"""repulp — Parallel batch document conversion, watch mode, and structured extraction."""
+
+from __future__ import annotations
+
+import sys
+import tempfile
+from pathlib import Path
+from typing import Optional, Union
+
+__version__ = "0.1.0"
+
+
+def convert(
+    source: Union[str, Path],
+    clean: bool = True,
+    frontmatter: bool = False,
+    format: str = "md",
+) -> "ConversionResult":
+    """Convert a file, URL, or path to Markdown.
+
+    Args:
+        source: File path, URL (http/https), or "-" for stdin.
+        clean: Post-process the markdown output.
+        frontmatter: Inject YAML frontmatter with metadata.
+        format: Output format - "md", "text", or "json".
+
+    Returns:
+        ConversionResult with markdown content and metadata.
+    """
+    from repulp.converter import convert_file, convert_url, ConversionResult
+    from repulp.fetcher import is_url
+    from repulp.frontmatter import inject_frontmatter
+    from repulp.formatter import format_output
+
+    source_str = str(source)
+
+    if source_str == "-":
+        content = sys.stdin.buffer.read()
+        tmp = tempfile.NamedTemporaryFile(suffix=".html", delete=False)
+        tmp.write(content)
+        tmp.close()
+        try:
+            result = convert_file(Path(tmp.name), clean=clean)
+            result = ConversionResult(
+                source_path=Path("stdin"),
+                markdown=result.markdown,
+                success=result.success,
+                error=result.error,
+            )
+        finally:
+            Path(tmp.name).unlink(missing_ok=True)
+    elif is_url(source_str):
+        result = convert_url(source_str, clean=clean)
+    else:
+        result = convert_file(Path(source_str), clean=clean)
+
+    if result.success and frontmatter:
+        result = ConversionResult(
+            source_path=result.source_path,
+            markdown=inject_frontmatter(result.markdown, source_str),
+            success=True,
+        )
+
+    if result.success and format != "md":
+        result = ConversionResult(
+            source_path=result.source_path,
+            markdown=format_output(result.markdown, format, source_str),
+            success=True,
+        )
+
+    return result
+
+
+def batch(
+    source: Union[str, Path],
+    output_dir: Optional[Union[str, Path]] = None,
+    workers: Optional[int] = None,
+    recursive: bool = False,
+    include: Optional[list[str]] = None,
+    exclude: Optional[list[str]] = None,
+    clean: bool = True,
+    incremental: bool = False,
+) -> "BatchResult":
+    """Convert all supported files in a directory using parallel workers.
+
+    Args:
+        source: Directory to scan.
+        output_dir: Output directory for .md files.
+        workers: Number of parallel workers. None = auto.
+        recursive: Scan subdirectories.
+        include: Glob patterns to include.
+        exclude: Glob patterns to exclude.
+        clean: Post-process markdown.
+        incremental: Skip unchanged files.
+
+    Returns:
+        BatchResult with conversion results and statistics.
+    """
+    from repulp.engine import batch_convert
+
+    return batch_convert(
+        Path(source),
+        workers=workers,
+        recursive=recursive,
+        include=include,
+        exclude=exclude,
+        clean=clean,
+        incremental=incremental,
+    )
+
+
+def extract_tables(
+    source: Union[str, Path],
+    format: str = "dict",
+    clean: bool = True,
+) -> list:
+    """Extract tables from a document as structured data.
+
+    Args:
+        source: File path or URL.
+        format: "dict" (list[dict]), "csv" (strings), "dataframe" (pandas), "markdown" (raw).
+        clean: Post-process markdown before extraction.
+
+    Returns:
+        List of tables in the requested format.
+    """
+    from repulp.extractor import extract_tables_structured
+
+    result = convert(source, clean=clean)
+    if not result.success:
+        return []
+
+    return extract_tables_structured(result.markdown, format=format)
+
+
+def watch(
+    source: Union[str, Path],
+    output_dir: Optional[Union[str, Path]] = None,
+    recursive: bool = True,
+    include: Optional[list[str]] = None,
+    exclude: Optional[list[str]] = None,
+    clean: bool = True,
+    debounce_ms: int = 500,
+    on_change=None,
+    on_command: Optional[str] = None,
+) -> None:
+    """Watch a directory and auto-convert files on change.
+
+    Args:
+        source: Directory to watch.
+        output_dir: Where to write .md files.
+        recursive: Watch subdirectories.
+        include: Glob patterns to include.
+        exclude: Glob patterns to exclude.
+        clean: Post-process markdown.
+        debounce_ms: Debounce interval in milliseconds.
+        on_change: Callback(WatchEvent) after each conversion.
+        on_command: Shell command to run after each conversion.
+    """
+    from repulp.watcher import watch_directory
+
+    watch_directory(
+        Path(source),
+        output_dir=Path(output_dir) if output_dir else None,
+        recursive=recursive,
+        include=include,
+        exclude=exclude,
+        clean=clean,
+        debounce_ms=debounce_ms,
+        on_change=on_change,
+        on_command=on_command,
+    )

repulp/cache.py

@@ -0,0 +1,88 @@
+"""Incremental build cache for repulp.
+
+Tracks SHA256 hashes of source files so that batch_convert() can skip
+files that have not changed since the last successful conversion.
+The cache is persisted as a JSON file (typically `.repulp.cache` inside
+the source directory).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Optional
+
+
+class ConversionCache:
+    """File-hash cache that detects which source files have changed.
+
+    Keys are resolved absolute path strings; values are hex-encoded SHA256
+    digests of the file contents at the time of last successful conversion.
+    """
+
+    def __init__(self, cache_path: Path) -> None:
+        self._cache_path = Path(cache_path)
+        self._hashes: dict[str, str] = {}
+
+        if self._cache_path.exists():
+            try:
+                self._hashes = json.loads(self._cache_path.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError):
+                self._hashes = {}
+
+    @staticmethod
+    def _hash_file(path: Path) -> Optional[str]:
+        """Return the SHA256 hex digest of *path*'s contents, or None if unreadable."""
+        try:
+            data = Path(path).resolve().read_bytes()
+            return hashlib.sha256(data).hexdigest()
+        except (OSError, PermissionError):
+            return None
+
+    def is_changed(self, path: Path) -> bool:
+        """Return True if *path* has changed since the last cached conversion.
+
+        A file is considered changed when:
+        - It has never been cached before.
+        - Its current SHA256 hash differs from the stored hash.
+        - The file no longer exists (hash returns None).
+        """
+        key = str(Path(path).resolve())
+        current_hash = self._hash_file(path)
+        if current_hash is None:
+            return True
+        return self._hashes.get(key) != current_hash
+
+    def mark_converted(self, path: Path) -> None:
+        """Record the current SHA256 hash for *path* as the latest converted state."""
+        key = str(Path(path).resolve())
+        file_hash = self._hash_file(path)
+        if file_hash is not None:
+            self._hashes[key] = file_hash
+
+    def partition(self, files: list[Path]) -> tuple[list[Path], list[Path]]:
+        """Split *files* into (changed, unchanged) based on cached hashes.
+
+        Args:
+            files: List of file paths to check.
+
+        Returns:
+            A 2-tuple of (changed_files, unchanged_files).
+        """
+        changed: list[Path] = []
+        unchanged: list[Path] = []
+        for f in files:
+            if self.is_changed(f):
+                changed.append(f)
+            else:
+                unchanged.append(f)
+        return changed, unchanged
+
+    def save(self) -> None:
+        """Persist the hash cache to disk as JSON."""
+        self._cache_path.parent.mkdir(parents=True, exist_ok=True)
+        self._cache_path.write_text(
+            json.dumps(self._hashes, indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )

repulp/cleaner.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import re
+
+
+def _normalize_blank_lines(text: str) -> str:
+    return re.sub(r"\n{3,}", "\n\n", text)
+
+
+def _strip_trailing_whitespace(text: str) -> str:
+    return "\n".join(line.rstrip() for line in text.split("\n"))
+
+
+def _ensure_heading_spacing(text: str) -> str:
+    lines = text.split("\n")
+    result: list[str] = []
+    for i, line in enumerate(lines):
+        is_heading = bool(re.match(r"^#{1,6}\s", line))
+        if is_heading and i > 0 and result and result[-1] != "":
+            result.append("")
+        result.append(line)
+        if is_heading and i < len(lines) - 1 and lines[i + 1] != "":
+            result.append("")
+    return "\n".join(result)
+
+
+def _fix_table_alignment(text: str) -> str:
+    lines = text.split("\n")
+    i = 0
+    result_lines: list[str] = []
+
+    while i < len(lines):
+        if "|" in lines[i] and i + 1 < len(lines) and re.match(
+            r"^\|[\s\-:|]+\|$", lines[i + 1].strip()
+        ):
+            table_lines: list[str] = []
+            while i < len(lines) and "|" in lines[i]:
+                table_lines.append(lines[i])
+                i += 1
+            result_lines.extend(_format_table(table_lines))
+        else:
+            result_lines.append(lines[i])
+            i += 1
+
+    return "\n".join(result_lines)
+
+
+def _format_table(table_lines: list[str]) -> list[str]:
+    rows: list[list[str]] = []
+    separator_idx = 1
+
+    for line in table_lines:
+        cells = [c.strip() for c in line.strip().strip("|").split("|")]
+        rows.append(cells)
+
+    if len(rows) < 2:
+        return table_lines
+
+    num_cols = max(len(row) for row in rows)
+    for row in rows:
+        while len(row) < num_cols:
+            row.append("")
+
+    col_widths = [
+        max(len(rows[r][c]) for r in range(len(rows)) if r != separator_idx)
+        for c in range(num_cols)
+    ]
+    col_widths = [max(w, 3) for w in col_widths]
+
+    formatted: list[str] = []
+    for r, row in enumerate(rows):
+        if r == separator_idx:
+            cells = ["-" * w for w in col_widths]
+            formatted.append("| " + " | ".join(cells) + " |")
+        else:
+            cells = [row[c].ljust(col_widths[c]) for c in range(num_cols)]
+            formatted.append("| " + " | ".join(cells) + " |")
+
+    return formatted
+
+
+def _strip_artifacts(text: str) -> str:
+    text = text.replace("\x0c", "\n")
+    text = text.replace("\x00", "")
+    text = re.sub(r"[\x01-\x08\x0b\x0e-\x1f\x7f]", "", text)
+    return text
+
+
+def clean_markdown(text: str) -> str:
+    if not text:
+        return ""
+
+    text = _strip_artifacts(text)
+    text = _strip_trailing_whitespace(text)
+    text = _ensure_heading_spacing(text)
+    text = _normalize_blank_lines(text)
+    text = _fix_table_alignment(text)
+    text = text.strip()
+    return text