likhit 0.1.0__py3-none-any.whl

likhit/__init__.py

@@ -0,0 +1,6 @@
+"""Likhit package."""
+
+from likhit.core import convert, convert_many
+from likhit.version import __version__
+
+__all__ = ["__version__", "convert", "convert_many"]

likhit/cli.py

@@ -0,0 +1,84 @@
+"""CLI entry point for conversion."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from likhit.core import convert_many, derive_convert_output_name
+from likhit.errors import LikhitError, ValidationError
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="likhit",
+        description="Convert born-digital PDFs to editable Markdown.",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    convert_parser = subparsers.add_parser(
+        "convert",
+        help="Convert PDFs to editable Markdown",
+    )
+    convert_parser.add_argument("inputs", nargs="+", help="One or more input PDF files")
+    convert_parser.add_argument("--out", help="Output path for a single input file")
+    convert_parser.add_argument(
+        "--out-dir",
+        help="Directory for generated Markdown files; defaults to current directory",
+    )
+    return parser
+
+
+def _write_convert_outputs(args: argparse.Namespace) -> int:
+    if len(args.inputs) > 1 and args.out:
+        raise ValidationError("--out can only be used with a single input file")
+
+    for input_path in args.inputs:
+        path = Path(input_path)
+        if path.suffix.lower() != ".pdf":
+            raise ValidationError(
+                "Unsupported input format for convert. Only born-digital PDF files are supported."
+            )
+
+    output_dir = Path(args.out_dir or ".")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    results = convert_many(args.inputs)
+
+    existing_names: set[str] = {
+        path.name for path in output_dir.iterdir() if path.is_file()
+    }
+    for source_path, markdown in results:
+        if args.out:
+            destination = Path(args.out)
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            if destination.exists():
+                raise ValidationError(f"Output file already exists: {destination}")
+        else:
+            destination = output_dir / derive_convert_output_name(
+                source_path,
+                existing_names,
+            )
+        destination.write_text(markdown, encoding="utf-8")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command is None:
+        parser.print_help()
+        return 1
+
+    try:
+        if args.command == "convert":
+            return _write_convert_outputs(args)
+        parser.print_help()
+        return 1
+    except LikhitError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

likhit/core.py

@@ -0,0 +1,126 @@
+"""Public conversion entry points."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import re
+
+from likhit.errors import ExtractionError, ValidationError
+from likhit.handlers import CIAAPressReleaseHandler, KanunPatrikaHandler
+from likhit.markitdown_integration import convert_pdf_to_markdown
+from likhit.models import DocumentType, ExtractionResult
+from likhit.renderers import MarkdownRenderer
+from likhit.extractors.font_based import FontBasedStrategy
+
+_KANUN_MARKERS = (
+    "नेपालकानूनपत्रिका",
+    "नेपालकानुनपत्रिका",
+    "निर्णय नं",
+    "ने.का.प.",
+)
+_CIAA_MARKERS = (
+    "अख्तियार दुरुपयोग अनुसन्धान आयोग",
+    "प्रेस विज्ञ",
+    "प्रेस मिज्ञ",
+    "विषय:",
+    "मिषय:",
+    "आयोगकोनिर्णय",
+    "अनुसन्धानबाट पुष्टि भएको",
+)
+
+
+def _metadata_from_options(
+    title: str | None,
+    publication_date: str | None,
+    source_url: str | None,
+) -> dict[str, str | None]:
+    return {
+        "title": title,
+        "publication_date": publication_date,
+        "source_url": source_url,
+    }
+
+
+def _resolve_handler(
+    doc_type: DocumentType,
+) -> CIAAPressReleaseHandler | KanunPatrikaHandler:
+    if doc_type is DocumentType.CIAA_PRESS_RELEASE:
+        return CIAAPressReleaseHandler()
+    if doc_type is DocumentType.KANUN_PATRIKA:
+        return KanunPatrikaHandler()
+    raise ValidationError(f"Unsupported document type: {doc_type.value}")
+
+
+def _normalize_detection_text(text: str) -> str:
+    return re.sub(r"\s+", "", text).casefold()
+
+
+def _detect_document_type(raw_text: str) -> DocumentType | None:
+    normalized = _normalize_detection_text(raw_text)
+
+    if any(
+        _normalize_detection_text(marker) in normalized for marker in _KANUN_MARKERS
+    ):
+        return DocumentType.KANUN_PATRIKA
+    if any(_normalize_detection_text(marker) in normalized for marker in _CIAA_MARKERS):
+        return DocumentType.CIAA_PRESS_RELEASE
+    return None
+
+
+def _render_markdown_without_frontmatter(result: ExtractionResult) -> str:
+    markdown = MarkdownRenderer().render(result).lstrip()
+    if not markdown.startswith("---\n"):
+        return markdown.strip()
+
+    parts = markdown.split("\n---\n", 1)
+    if len(parts) != 2:
+        return markdown.strip()
+    return parts[1].strip()
+
+
+def _convert_with_detected_structure(file_path: str) -> str | None:
+    strategy = FontBasedStrategy()
+    try:
+        raw_document = strategy.extract_text(file_path)
+    except ExtractionError as exc:
+        if str(exc) == "No text content found in document":
+            return None
+        raise
+    doc_type = _detect_document_type(raw_document.raw_text)
+    if doc_type is None:
+        return None
+
+    handler = _resolve_handler(doc_type)
+    result = handler.build_result(
+        raw_document,
+        _metadata_from_options(None, None, None),
+    )
+    return _render_markdown_without_frontmatter(result)
+
+
+def convert(file_path: str) -> str:
+    path = Path(file_path)
+    if path.suffix.lower() != ".pdf":
+        raise ValidationError(
+            "Unsupported input format for convert. Only born-digital PDF files are supported."
+        )
+
+    structured_markdown = _convert_with_detected_structure(file_path)
+    if structured_markdown is not None:
+        return structured_markdown
+    return convert_pdf_to_markdown(file_path)
+
+
+def derive_convert_output_name(source_path: str, existing: set[str]) -> str:
+    base_name = Path(source_path).stem or "document"
+    candidate = f"{base_name}.md"
+    counter = 2
+    while candidate in existing:
+        candidate = f"{base_name}-{counter}.md"
+        counter += 1
+    existing.add(candidate)
+    return candidate
+
+
+def convert_many(file_paths: list[str]) -> list[tuple[str, str]]:
+    return [(file_path, convert(file_path)) for file_path in file_paths]

likhit/errors.py

@@ -0,0 +1,13 @@
+"""Project-specific exceptions."""
+
+
+class LikhitError(Exception):
+    """Base exception for extraction failures."""
+
+
+class ValidationError(LikhitError):
+    """Raised when user input or extracted content is invalid."""
+
+
+class ExtractionError(LikhitError):
+    """Raised when document extraction fails."""

likhit/extractors/__init__.py

@@ -0,0 +1,5 @@
+"""Extraction strategy exports."""
+
+from likhit.extractors.font_based import FontBasedStrategy
+
+__all__ = ["FontBasedStrategy"]

likhit/extractors/base.py

@@ -0,0 +1,45 @@
+"""Shared extraction abstractions."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+from likhit.models import Table
+
+
+@dataclass(slots=True)
+class TextFragment:
+    """A normalized text block plus its page/layout metadata."""
+
+    text: str
+    page_number: int
+    x0: float
+    y0: float
+    x1: float
+    y1: float
+    block_number: int = 0
+    line_number: int = 0
+    gap_before: float | None = None
+
+
+@dataclass(slots=True)
+class RawDocument:
+    """Unstructured text extracted from a document."""
+
+    paragraphs: list[str]
+    raw_text: str
+    fragments: list[TextFragment]
+    tables: list[Table] = field(default_factory=list)
+
+
+class ExtractionStrategy(ABC):
+    """Base interface for extraction strategies."""
+
+    @abstractmethod
+    def extract_text(self, file_path: str, pages: str | None = None) -> RawDocument:
+        """Extract unstructured text from a document."""
+
+    @abstractmethod
+    def extract_tables(self, file_path: str) -> list[Table]:
+        """Extract structured tables from a document."""

likhit/extractors/font_based.py

@@ -0,0 +1,261 @@
+"""Font-based extraction for CIAA documents."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+import re
+from pathlib import Path
+
+import fitz
+
+from likhit.errors import ExtractionError, ValidationError
+from likhit.extractors.base import ExtractionStrategy, RawDocument, TextFragment
+from likhit.extractors.font_classifier import scan_pdf_fonts
+from likhit.extractors.kalimati import (
+    fix_kalimati_cmap,
+    normalize_devanagari_spacing,
+    reorder_devanagari,
+)
+from likhit.extractors.legacy_maps import get_converter
+from likhit.extractors.tables import detect_page_tables, merge_continuation_tables
+from likhit.models import Table
+
+
+PAGE_RANGE_PATTERN = re.compile(r"^\d+(?:-\d+)?$")
+SPAN_GAP_THRESHOLD = 0.75
+
+
+def parse_page_range(spec: str, total_pages: int) -> tuple[int, int]:
+    """Parse a 1-based inclusive page range to 0-based bounds."""
+
+    if not PAGE_RANGE_PATTERN.fullmatch(spec.strip()):
+        raise ValidationError("Invalid page range format. Use format: '1-3' or '5'")
+
+    if "-" in spec:
+        start_text, end_text = spec.split("-", 1)
+        start = int(start_text)
+        end = int(end_text)
+    else:
+        start = end = int(spec)
+
+    if start < 1 or end < 1 or end < start:
+        raise ValidationError("Invalid page range format. Use format: '1-3' or '5'")
+
+    if start > total_pages:
+        raise ValidationError(
+            f"Requested page range starts beyond document length ({total_pages} pages)"
+        )
+
+    end = min(end, total_pages)
+    return start - 1, end - 1
+
+
+def normalize_press_release_paragraph(text: str) -> str:
+    text = text.strip()
+    if not text:
+        return ""
+
+    normalized = text
+    normalized = re.sub(r"\s+", " ", normalized).strip()
+    normalized = re.sub(r"\s+([।,:;])", r"\1", normalized)
+    return normalized
+
+
+def join_words_with_spacing(words: list[str]) -> str:
+    """Reconstruct a line from extracted word tokens."""
+
+    return " ".join(word.strip() for word in words if word.strip())
+
+
+def join_spans_with_layout(
+    spans: list[tuple[float, float, float, float, str]],
+) -> str:
+    """Reconstruct a line from positioned spans without forcing spaces inside words."""
+
+    if not spans:
+        return ""
+
+    parts: list[str] = []
+    previous_x1: float | None = None
+    for x0, _y0, x1, _y1, text in spans:
+        if not text:
+            continue
+        if (
+            previous_x1 is not None
+            and x0 - previous_x1 > SPAN_GAP_THRESHOLD
+            and parts
+            and not parts[-1].endswith((" ", "\t"))
+            and not text.startswith((" ", "\t"))
+        ):
+            parts.append(" ")
+        parts.append(text)
+        previous_x1 = x1
+
+    return "".join(parts)
+
+
+def normalize_extracted_word(text: str) -> str:
+    """Normalize a single extracted token without touching inter-word spacing."""
+
+    normalized = reorder_devanagari(text)
+    normalized = normalize_devanagari_spacing(normalized)
+    return normalized.strip()
+
+
+class FontBasedStrategy(ExtractionStrategy):
+    """Extract text from CIAA PDFs using PyMuPDF blocks."""
+
+    def extract_text(self, file_path: str, pages: str | None = None) -> RawDocument:
+        return self._extract_raw_document(file_path, pages=pages)
+
+    def extract_tables(self, file_path: str) -> list[Table]:
+        return self._extract_raw_document(file_path).tables
+
+    def _extract_raw_document(
+        self,
+        file_path: str,
+        pages: str | None = None,
+    ) -> RawDocument:
+        path = Path(file_path)
+        if path.suffix.lower() != ".pdf":
+            raise ValidationError("Unsupported file format. Please upload a PDF file")
+        if not path.exists():
+            raise ValidationError(f"File not found: {file_path}")
+
+        try:
+            doc = fitz.open(path)
+        except Exception as exc:
+            raise ExtractionError(
+                "Unable to parse PDF. File may be corrupted or encrypted"
+            ) from exc
+
+        try:
+            page_start, page_end = 0, doc.page_count - 1
+            if pages:
+                page_start, page_end = parse_page_range(pages, doc.page_count)
+
+            font_strategies = scan_pdf_fonts(doc)
+            has_broken_cmap = any(
+                strategy == "broken_cmap" for strategy in font_strategies.values()
+            )
+            needs_reorder = False
+            if has_broken_cmap:
+                doc, needs_reorder = fix_kalimati_cmap(doc)
+            paragraphs: list[str] = []
+            fragments: list[TextFragment] = []
+            tables: list[Table] = []
+            table_index = 0
+
+            for page_index in range(page_start, page_end + 1):
+                page = doc[page_index]
+                page_dict = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)
+                lines_by_key: dict[
+                    tuple[int, int], list[tuple[float, float, float, float, str]]
+                ] = defaultdict(list)
+                for block_number, block in enumerate(page_dict["blocks"]):
+                    if "lines" not in block:
+                        continue
+                    for line_number, line in enumerate(block["lines"]):
+                        for span in line["spans"]:
+                            text = self._convert_span_text(
+                                str(span["text"]),
+                                str(span["font"]),
+                                font_strategies,
+                                needs_reorder,
+                            )
+                            if not text:
+                                continue
+                            x0, y0, x1, y1 = span["bbox"]
+                            lines_by_key[(block_number, line_number)].append(
+                                (
+                                    float(x0),
+                                    float(y0),
+                                    float(x1),
+                                    float(y1),
+                                    text,
+                                )
+                            )
+
+                page_fragments: list[TextFragment] = []
+                previous_y1: float | None = None
+                for (block_number, line_number), line_words in sorted(
+                    lines_by_key.items(),
+                    key=lambda item: (
+                        round(min(piece[1] for piece in item[1]), 2),
+                        min(piece[0] for piece in item[1]),
+                    ),
+                ):
+                    ordered_words = sorted(line_words, key=lambda piece: piece[0])
+                    line_text = join_spans_with_layout(ordered_words)
+                    paragraph = normalize_press_release_paragraph(line_text)
+                    if not paragraph:
+                        previous_y1 = None
+                        continue
+
+                    x0 = min(piece[0] for piece in ordered_words)
+                    y0 = min(piece[1] for piece in ordered_words)
+                    x1 = max(piece[2] for piece in ordered_words)
+                    y1 = max(piece[3] for piece in ordered_words)
+                    gap_before = None
+                    if previous_y1 is not None:
+                        gap_before = y0 - previous_y1
+                    previous_y1 = y1
+
+                    fragment = TextFragment(
+                        text=paragraph,
+                        page_number=page_index + 1,
+                        x0=x0,
+                        y0=y0,
+                        x1=x1,
+                        y1=y1,
+                        block_number=block_number,
+                        line_number=line_number,
+                        gap_before=gap_before,
+                    )
+                    paragraphs.append(paragraph)
+                    page_fragments.append(fragment)
+
+                fragments.extend(page_fragments)
+                page_tables = detect_page_tables(page, page_fragments, table_index)
+                tables.extend(page_tables)
+                table_index += len(page_tables)
+
+            raw_text = "\n\n".join(paragraphs).strip()
+            if not raw_text:
+                raise ExtractionError("No text content found in document")
+
+            return RawDocument(
+                paragraphs=paragraphs,
+                raw_text=raw_text,
+                fragments=fragments,
+                tables=merge_continuation_tables(tables),
+            )
+        except (ExtractionError, ValidationError):
+            raise
+        except Exception as exc:
+            raise ExtractionError(
+                f"Failed to extract text from PDF: {path.name}"
+            ) from exc
+        finally:
+            doc.close()
+
+    def _convert_span_text(
+        self,
+        text: str,
+        font_name: str,
+        font_strategies: dict[str, str],
+        needs_reorder: bool,
+    ) -> str:
+        base = font_name.split("+", 1)[-1] if "+" in font_name else font_name
+        strategy = font_strategies.get(base, "correct")
+
+        if strategy == "legacy_remap":
+            converter = get_converter(font_name)
+            if converter is not None:
+                return converter(text)
+            return text
+
+        if strategy == "broken_cmap" and needs_reorder:
+            text = reorder_devanagari(text)
+            text = normalize_devanagari_spacing(text)
+        return text

likhit/extractors/font_classifier.py

@@ -0,0 +1,50 @@
+"""Font strategy classification for extraction."""
+
+from __future__ import annotations
+
+import logging
+
+import fitz
+
+from . import legacy_maps
+
+logger = logging.getLogger(__name__)
+
+_KNOWN_BROKEN_CMAP = {"kalimati"}
+
+
+def classify_font(font_name: str, font_type: str) -> str:
+    """Classify a PDF font into an extraction strategy."""
+
+    del font_type
+
+    base = font_name.split("+", 1)[-1] if "+" in font_name else font_name
+    base_lower = base.lower().strip()
+
+    if legacy_maps.is_legacy_font(font_name):
+        return "legacy_remap"
+
+    for name in _KNOWN_BROKEN_CMAP:
+        if name in base_lower:
+            return "broken_cmap"
+
+    return "correct"
+
+
+def scan_pdf_fonts(doc: fitz.Document) -> dict[str, str]:
+    """Scan all PDF fonts and return a strategy per unique base font name."""
+
+    font_strategies: dict[str, str] = {}
+
+    for page_index in range(doc.page_count):
+        page = doc[page_index]
+        for font_info in page.get_fonts(full=True):
+            _xref, _ext, font_type, name, _encoding = font_info[:5]
+            base = name.split("+", 1)[-1] if "+" in name else name
+            if base in font_strategies:
+                continue
+            strategy = classify_font(name, font_type)
+            font_strategies[base] = strategy
+            logger.info("Font '%s' (type=%s) -> %s", base, font_type, strategy)
+
+    return font_strategies