markitdown-plus 0.2.0__py3-none-any.whl

markitdown_plus/cli.py

@@ -0,0 +1,205 @@
+"""Command-line interface for markitdown-plus."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import traceback
+from pathlib import Path
+
+from .__about__ import __description__, __version__
+from .batch import BatchOptions, parse_extensions, run_batch
+from .chunker import CHUNK_STRATEGIES, chunk_markdown, write_jsonl
+from .cleaner import clean_markdown
+from .converter import PlusConverter
+
+
+def _ensure_file_output_path(output: str | Path, example: str) -> Path:
+    output_path = Path(output)
+    if output_path.exists() and output_path.is_dir():
+        raise ValueError(
+            f"Output path is a directory, not a file: {output_path}. "
+            f"Please specify a file path, for example: {example}"
+        )
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    return output_path
+
+
+def _default_workers() -> int:
+    return max(1, min(4, os.cpu_count() or 1))
+
+
+def _add_common_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("-v", "--verbose", action="store_true", help="Show full traceback on error.")
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="markitdown-plus", description=__description__)
+    parser.add_argument("--version", action="version", version=f"markitdown-plus {__version__}")
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    convert = subparsers.add_parser(
+        "convert",
+        help="Convert one file or a folder to Markdown using Microsoft MarkItDown.",
+    )
+    convert.add_argument("input", help="Input file or directory.")
+    convert.add_argument("-o", "--output", required=True, help="Output directory.")
+    convert.add_argument("-r", "--recursive", action="store_true", help="Scan directories recursively.")
+    convert.add_argument(
+        "--types",
+        help="Comma-separated extensions to include, for example: pdf,docx,pptx,xlsx,html,csv",
+    )
+    convert.add_argument("--clean", action="store_true", help="Clean converted Markdown.")
+    convert.add_argument("--rag", action="store_true", help="Export JSONL chunks for RAG pipelines.")
+    convert.add_argument("--chunk-size", type=int, default=800, help="Target max token estimate per chunk.")
+    convert.add_argument("--overlap", type=int, default=0, help="Word overlap between adjacent chunks.")
+    convert.add_argument("--model", default="gpt4", help="Token estimate model profile: gpt4, claude, gemini, deepseek.")
+    convert.add_argument(
+        "--chunk-strategy",
+        choices=sorted(CHUNK_STRATEGIES),
+        default="heading",
+        help="RAG chunking strategy: heading, fixed, or semantic-lite.",
+    )
+    convert.add_argument("--plugins", action="store_true", help="Enable installed MarkItDown plugins.")
+    convert.add_argument("--dry-run", action="store_true", help="List matching files without converting.")
+    convert.add_argument("--fail-fast", action="store_true", help="Stop at the first conversion error.")
+    convert.add_argument("--quiet", action="store_true", help="Hide progress output.")
+    convert.add_argument("--progress", action="store_true", help="Show tqdm progress bar when available.")
+    convert.add_argument(
+        "--workers",
+        type=int,
+        default=1,
+        help=f"Parallel conversion workers. Use 0 for auto ({_default_workers()}).",
+    )
+    convert.add_argument("--extract-assets", action="store_true", help="Extract DOCX/PPTX/XLSX/HTML image assets when possible.")
+    _add_common_options(convert)
+
+    clean = subparsers.add_parser("clean", help="Clean an existing Markdown file.")
+    clean.add_argument("input", help="Input Markdown file.")
+    clean.add_argument("-o", "--output", required=True, help="Output Markdown file.")
+    _add_common_options(clean)
+
+    chunk = subparsers.add_parser("chunk", help="Chunk an existing Markdown file to JSONL.")
+    chunk.add_argument("input", help="Input Markdown file.")
+    chunk.add_argument("-o", "--output", required=True, help="Output JSONL file.")
+    chunk.add_argument("--chunk-size", type=int, default=800, help="Target max token estimate per chunk.")
+    chunk.add_argument("--overlap", type=int, default=0, help="Word overlap between adjacent chunks.")
+    chunk.add_argument("--model", default="gpt4", help="Token estimate model profile: gpt4, claude, gemini, deepseek.")
+    chunk.add_argument(
+        "--chunk-strategy",
+        choices=sorted(CHUNK_STRATEGIES),
+        default="heading",
+        help="RAG chunking strategy: heading, fixed, or semantic-lite.",
+    )
+    _add_common_options(chunk)
+
+    single = subparsers.add_parser("single", help="Convert one file and write Markdown directly.")
+    single.add_argument("input", help="Input file.")
+    single.add_argument("-o", "--output", required=True, help="Output Markdown file.")
+    single.add_argument("--clean", action="store_true", help="Clean converted Markdown.")
+    single.add_argument("--plugins", action="store_true", help="Enable installed MarkItDown plugins.")
+    _add_common_options(single)
+
+    return parser
+
+
+def command_convert(args: argparse.Namespace) -> int:
+    workers = _default_workers() if args.workers == 0 else max(1, args.workers)
+    options = BatchOptions(
+        input_path=Path(args.input),
+        output_dir=Path(args.output),
+        recursive=args.recursive,
+        extensions=parse_extensions(args.types),
+        clean=args.clean,
+        rag=args.rag,
+        max_tokens=args.chunk_size,
+        overlap=args.overlap,
+        token_model=args.model,
+        chunk_strategy=args.chunk_strategy,
+        enable_plugins=args.plugins,
+        dry_run=args.dry_run,
+        continue_on_error=not args.fail_fast,
+        show_progress=(args.progress or not args.quiet) and not args.dry_run,
+        workers=workers,
+        extract_assets=args.extract_assets,
+    )
+    manifest = run_batch(options)
+    if args.dry_run:
+        print(f"Found: {manifest.total} file(s)")
+        for record in manifest.files:
+            print(record.source_path)
+        print(f"Manifest: {Path(args.output) / 'manifest.json'}")
+        return 0
+    print(f"Converted: {manifest.success}/{manifest.total}")
+    if workers > 1:
+        print(f"Workers: {workers}")
+    if manifest.failed:
+        print(f"Failed: {manifest.failed} file(s). See failed.json or failed.jsonl in {args.output}", file=sys.stderr)
+        return 1
+    print(f"Manifest: {Path(args.output) / 'manifest.json'}")
+    return 0
+
+
+def command_clean(args: argparse.Namespace) -> int:
+    input_path = Path(args.input)
+    output_path = _ensure_file_output_path(args.output, "-o output/clean.md")
+    text = input_path.read_text(encoding="utf-8")
+    output_path.write_text(clean_markdown(text), encoding="utf-8")
+    print(f"Cleaned Markdown: {output_path}")
+    return 0
+
+
+def command_chunk(args: argparse.Namespace) -> int:
+    input_path = Path(args.input)
+    output_path = _ensure_file_output_path(args.output, "-o output/chunks.jsonl")
+    markdown = input_path.read_text(encoding="utf-8")
+    chunks = chunk_markdown(
+        markdown,
+        source=str(input_path),
+        max_tokens=args.chunk_size,
+        overlap=args.overlap,
+        model=args.model,
+        strategy=args.chunk_strategy,
+    )
+    write_jsonl(chunks, output_path)
+    print(f"Chunks: {len(chunks)} -> {output_path}")
+    return 0
+
+
+def command_single(args: argparse.Namespace) -> int:
+    converter = PlusConverter(enable_plugins=args.plugins)
+    markdown = converter.convert_file(args.input)
+    if args.clean:
+        markdown = clean_markdown(markdown)
+    output_path = _ensure_file_output_path(args.output, "-o output/report.md")
+    output_path.write_text(markdown, encoding="utf-8")
+    print(f"Markdown: {output_path}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "convert":
+            return command_convert(args)
+        if args.command == "clean":
+            return command_clean(args)
+        if args.command == "chunk":
+            return command_chunk(args)
+        if args.command == "single":
+            return command_single(args)
+        parser.error(f"Unknown command: {args.command}")
+    except Exception as exc:
+        if getattr(args, "verbose", False):
+            traceback.print_exc()
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

markitdown_plus/converter.py

@@ -0,0 +1,58 @@
+"""Thin wrapper around Microsoft MarkItDown."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from .errors import ConversionError, DependencyError
+
+logger = logging.getLogger(__name__)
+
+
+class PlusConverter:
+    """Convert files to Markdown with Microsoft MarkItDown.
+
+    The wrapper keeps markitdown-plus code isolated from small upstream API
+    changes. It prefers `result.markdown`, then falls back to older aliases.
+    """
+
+    def __init__(self, enable_plugins: bool = False) -> None:
+        try:
+            from markitdown import MarkItDown  # type: ignore
+        except Exception as exc:  # pragma: no cover - depends on environment
+            raise DependencyError(
+                "markitdown package not found. Run: pip install 'markitdown[all]'"
+            ) from exc
+
+        try:
+            self._markitdown = MarkItDown(enable_plugins=enable_plugins)
+        except TypeError:  # pragma: no cover - compatibility fallback
+            if enable_plugins:
+                logger.warning(
+                    "Installed markitdown version does not support enable_plugins. "
+                    "Upgrade with: pip install --upgrade markitdown"
+                )
+            self._markitdown = MarkItDown()
+
+    def convert_file(self, path: str | Path) -> str:
+        """Convert one file and return Markdown text."""
+        source = Path(path)
+        if not source.exists():
+            raise ConversionError(f"File does not exist: {source}")
+        if not source.is_file():
+            raise ConversionError(f"Input is not a file: {source}")
+        if source.stat().st_size == 0:
+            logger.warning("Empty file: %s", source)
+
+        try:
+            result = self._markitdown.convert(str(source))
+        except Exception as exc:
+            raise ConversionError(f"Failed to convert {source}: {exc}") from exc
+
+        markdown = getattr(result, "markdown", None)
+        if markdown is None:
+            markdown = getattr(result, "text_content", None)
+        if markdown is None:
+            markdown = str(result)
+        return str(markdown).strip() + "\n"

markitdown_plus/errors.py

@@ -0,0 +1,13 @@
+"""Custom exceptions used by markitdown-plus."""
+
+
+class MarkItDownPlusError(Exception):
+    """Base exception for markitdown-plus."""
+
+
+class ConversionError(MarkItDownPlusError):
+    """Raised when a file cannot be converted."""
+
+
+class DependencyError(MarkItDownPlusError):
+    """Raised when an optional or required dependency is missing."""

markitdown_plus/manifest.py

@@ -0,0 +1,164 @@
+"""Batch conversion manifest support."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .__about__ import __version__
+
+
+def _drop_none(value: Any) -> Any:
+    """Recursively remove None values from dictionaries/lists for cleaner JSON."""
+    if isinstance(value, dict):
+        return {k: _drop_none(v) for k, v in value.items() if v is not None}
+    if isinstance(value, list):
+        return [_drop_none(item) for item in value]
+    return value
+
+
+@dataclass
+class ManifestRecord:
+    """One file record inside a conversion manifest."""
+
+    source_path: str
+    status: str
+    output_path: str | None = None
+    chunks_path: str | None = None
+    metadata_path: str | None = None
+    error: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-friendly record without meaningless null fields."""
+        return _drop_none(asdict(self))
+
+    def to_json(self) -> str:
+        """Serialize this record as one JSONL row."""
+        return json.dumps(self.to_dict(), ensure_ascii=False)
+
+
+@dataclass
+class Manifest:
+    """A conversion run manifest.
+
+    For very large folders, callers may disable in-memory record storage and
+    stream each record to `manifest-records.jsonl`. The summary counters still
+    stay in memory, while the heavy per-file data is kept on disk.
+    """
+
+    tool: str = "markitdown-plus"
+    version: str = __version__
+    created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    source: str = ""
+    output: str = ""
+    total: int = 0
+    success: int = 0
+    failed: int = 0
+    files: list[ManifestRecord] = field(default_factory=list)
+    files_truncated: bool = False
+    records_path: str | None = None
+    failed_records_path: str | None = None
+    _store_records: bool = field(default=True, repr=False, compare=False)
+
+    def enable_streaming(self, records_path: str | Path, failed_records_path: str | Path | None = None) -> None:
+        """Store per-file records on disk instead of keeping them all in memory."""
+        self._store_records = False
+        self.files_truncated = True
+        self.records_path = str(records_path)
+        if failed_records_path is not None:
+            self.failed_records_path = str(failed_records_path)
+
+    def _add_record(self, record: ManifestRecord) -> ManifestRecord:
+        if self._store_records:
+            self.files.append(record)
+        else:
+            self.files_truncated = True
+
+        self.total += 1
+        if record.status == "success":
+            self.success += 1
+        elif record.status == "failed":
+            self.failed += 1
+        return record
+
+    def add_success(
+        self,
+        source_path: str,
+        output_path: str,
+        chunks_path: str | None = None,
+        metadata_path: str | None = None,
+    ) -> ManifestRecord:
+        return self._add_record(
+            ManifestRecord(
+                source_path=source_path,
+                status="success",
+                output_path=output_path,
+                chunks_path=chunks_path,
+                metadata_path=metadata_path,
+            )
+        )
+
+    def add_failed(self, source_path: str, error: str) -> ManifestRecord:
+        return self._add_record(ManifestRecord(source_path=source_path, status="failed", error=error))
+
+    def _recount(self) -> None:
+        """Recalculate counters when records are edited externally."""
+        self.total = len(self.files)
+        self.success = sum(1 for record in self.files if record.status == "success")
+        self.failed = sum(1 for record in self.files if record.status == "failed")
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = {
+            "tool": self.tool,
+            "version": self.version,
+            "created_at": self.created_at,
+            "source": self.source,
+            "output": self.output,
+            "total": self.total,
+            "success": self.success,
+            "failed": self.failed,
+            "files_truncated": self.files_truncated,
+            "records_path": self.records_path,
+            "failed_records_path": self.failed_records_path,
+            "files": [record.to_dict() for record in self.files],
+        }
+        return _drop_none(payload)
+
+    def failed_records(self) -> list[dict[str, Any]]:
+        return [record.to_dict() for record in self.files if record.status == "failed"]
+
+
+def append_manifest_record(path: str | Path, record: ManifestRecord) -> Path:
+    """Append one manifest record to a JSONL file."""
+    output = Path(path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with output.open("a", encoding="utf-8") as f:
+        f.write(record.to_json() + "\n")
+    return output
+
+
+def write_manifest(manifest: Manifest, output_dir: str | Path) -> tuple[Path, Path | None]:
+    output = Path(output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    manifest_path = output / "manifest.json"
+    manifest_path.write_text(
+        json.dumps(manifest.to_dict(), ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+    failed_path: Path | None = None
+    failed_records = manifest.failed_records()
+    if manifest.failed and failed_records:
+        failed_path = output / "failed.json"
+        failed_path.write_text(
+            json.dumps(failed_records, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+    else:
+        old_failed_path = output / "failed.json"
+        if old_failed_path.exists():
+            old_failed_path.unlink()
+    return manifest_path, failed_path

markitdown_plus/metadata.py

@@ -0,0 +1,97 @@
+"""Metadata helpers for converted documents."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .__about__ import __version__
+
+
+@dataclass
+class FileMetadata:
+    """Basic conversion metadata for one source file."""
+
+    source_path: str
+    output_path: str
+    file_name: str
+    extension: str
+    source_size_bytes: int
+    output_size_bytes: int
+    converted_at: str
+    clean_enabled: bool = False
+    rag_enabled: bool = False
+    extract_assets_enabled: bool = False
+    chunk_strategy: str = "heading"
+    assets_count: int = 0
+    assets: list[dict[str, Any]] = field(default_factory=list)
+    conversion_time_seconds: float = 0.0
+    markitdown_plus_version: str = __version__
+    markitdown_version: str | None = None
+
+    @property
+    def size_bytes(self) -> int:
+        """Backward-compatible alias for v0.1.0 metadata tests/users."""
+        return self.source_size_bytes
+
+    def to_dict(self) -> dict[str, object]:
+        payload = asdict(self)
+        if payload.get("markitdown_version") is None:
+            payload.pop("markitdown_version", None)
+        if not payload.get("assets"):
+            payload.pop("assets", None)
+        return payload
+
+
+def _markitdown_version() -> str | None:
+    try:
+        from importlib.metadata import version
+
+        return version("markitdown")
+    except Exception:
+        return None
+
+
+def build_metadata(
+    source_path: str | Path,
+    output_path: str | Path,
+    *,
+    clean_enabled: bool = False,
+    rag_enabled: bool = False,
+    extract_assets_enabled: bool = False,
+    chunk_strategy: str = "heading",
+    assets: list[dict[str, Any]] | None = None,
+    conversion_time_seconds: float = 0.0,
+) -> FileMetadata:
+    source = Path(source_path)
+    output = Path(output_path)
+    source_size = source.stat().st_size if source.exists() else 0
+    output_size = output.stat().st_size if output.exists() else 0
+    asset_list = assets or []
+    return FileMetadata(
+        source_path=str(source),
+        output_path=str(output),
+        file_name=source.name,
+        extension=source.suffix.lower(),
+        source_size_bytes=source_size,
+        output_size_bytes=output_size,
+        converted_at=datetime.now(timezone.utc).isoformat(),
+        clean_enabled=clean_enabled,
+        rag_enabled=rag_enabled,
+        extract_assets_enabled=extract_assets_enabled,
+        chunk_strategy=chunk_strategy,
+        assets_count=len(asset_list),
+        assets=asset_list,
+        conversion_time_seconds=round(conversion_time_seconds, 6),
+        markitdown_version=_markitdown_version(),
+    )
+
+
+def write_metadata(metadata: FileMetadata, output_path: str | Path) -> Path:
+    output = Path(output_path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(metadata.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
+    return output

markitdown_plus/utils.py

@@ -0,0 +1,52 @@
+"""Small utility helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from pathlib import Path
+
+WINDOWS_UNSAFE_CHARS = r'[<>:"/\\|?*\x00-\x1f]+'
+
+
+def ensure_dir(path: Path) -> Path:
+    """Create a directory if it does not already exist and return it."""
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def safe_stem(path: Path) -> str:
+    """Return a filesystem-safe stem while preserving Unicode names.
+
+    Unlike ASCII-only sanitizers, this keeps Chinese, Japanese, Greek, Cyrillic,
+    Arabic, and other valid filename characters. It only replaces characters that
+    are unsafe on common filesystems, especially Windows.
+    """
+    stem = path.stem.strip() or "document"
+    stem = re.sub(WINDOWS_UNSAFE_CHARS, "-", stem)
+    stem = re.sub(r"-+", "-", stem).strip("-._")
+    return stem or "document"
+
+
+def safe_path_part(value: str) -> str:
+    """Sanitize one relative path component while preserving Unicode text."""
+    cleaned = re.sub(WINDOWS_UNSAFE_CHARS, "-", value.strip())
+    cleaned = re.sub(r"-+", "-", cleaned).strip("-._")
+    return cleaned or "document"
+
+
+def short_hash(value: str, length: int = 8) -> str:
+    """Return a short stable hash for IDs."""
+    return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()[:length]
+
+
+def make_output_path(source: Path, input_root: Path, output_root: Path, suffix: str) -> Path:
+    """Build an output path that preserves relative directory structure."""
+    try:
+        relative = source.relative_to(input_root)
+    except ValueError:
+        relative = Path(source.name)
+
+    clean_parts = [safe_path_part(part) for part in relative.parts]
+    relative_clean = Path(*clean_parts)
+    return output_root / relative_clean.with_suffix(suffix)