pysfi 0.1.6__py3-none-any.whl → 0.1.10__py3-none-any.whl

sfi/docscan/docscan.py

@@ -0,0 +1,1145 @@
+"""Scan documents and extract text, images, and metadata with certain rules."""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import csv
+import html
+import json
+import logging
+import re
+import sys
+import threading
+import time
+import xml.etree.ElementTree as ET
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable
+
+try:
+    import fitz  # PyMuPDF
+except ImportError:
+    fitz = None
+
+try:
+    from docx import Document
+except ImportError:
+    Document = None
+
+try:
+    from openpyxl import load_workbook
+except ImportError:
+    load_workbook = None
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+try:
+    import pytesseract
+except ImportError:
+    pytesseract = None
+
+try:
+    import odf.opendocument as odf_odt  # ODT support
+except ImportError:
+    odf_odt = None
+
+try:
+    import ebooklib  # EPUB support
+    from ebooklib import epub
+except ImportError:
+    ebooklib = None
+
+try:
+    import markdown  # Markdown to text
+except ImportError:
+    markdown = None
+
+try:
+    import pypdf  # Alternative PDF library
+except ImportError:
+    pypdf = None
+
+# Language support imports
+try:
+    from sfi.docscan.lang.eng import ENGLISH_DEFAULTS as EN_TRANSLATIONS
+    from sfi.docscan.lang.zhcn import TRANSLATIONS as ZH_TRANSLATIONS
+except ImportError:
+    try:
+        from lang.eng import ENGLISH_DEFAULTS as EN_TRANSLATIONS
+        from lang.zhcn import TRANSLATIONS as ZH_TRANSLATIONS
+    except ImportError:
+        # Fallback translations if import fails
+        ZH_TRANSLATIONS = {}
+        EN_TRANSLATIONS = {}
+
+# Global language setting
+USE_CHINESE = True  # Default to Chinese
+
+
+def t(key: str, **kwargs) -> str:
+    """Get translated text for the given key.
+
+    Args:
+        key: Translation key
+        **kwargs: Arguments for string formatting
+
+    Returns:
+        Translated text
+    """
+    text = ZH_TRANSLATIONS.get(key, key) if USE_CHINESE else EN_TRANSLATIONS.get(key, key)
+
+    # Format with kwargs if provided
+    if kwargs:
+        with contextlib.suppress(KeyError, ValueError):
+            text = text.format(**kwargs)
+    return text
+
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+cwd = Path.cwd()
+
+
+class Rule:
+    """Represents a scanning rule with optimized pattern matching."""
+
+    def __init__(self, rule_data: dict[str, Any]):
+        """Initialize rule from dictionary."""
+        self.name = rule_data.get("name", "")
+        self.pattern = rule_data.get("pattern", "")
+        self.is_regex = rule_data.get("regex", False)
+        self.case_sensitive = rule_data.get("case_sensitive", False)
+        self.context_lines = rule_data.get("context_lines", 3)
+        self.description = rule_data.get("description", "")
+
+        if self.is_regex:
+            flags = 0 if self.case_sensitive else re.IGNORECASE
+            try:
+                # Use re.ASCII for faster matching when possible
+                self.compiled_pattern = re.compile(self.pattern, flags | re.ASCII)
+            except re.error as e:
+                logger.warning(t("invalid_regex_pattern", pattern=self.pattern, error=e))
+                self.compiled_pattern = None
+        else:
+            self.compiled_pattern = None
+
+    def search(self, text: str) -> list[dict[str, Any]]:
+        """Search for pattern in text and return matches."""
+        if not text or not self.pattern:
+            return []
+
+        matches = []
+        lines = text.split("\n")
+
+        if self.is_regex and self.compiled_pattern:
+            # Regex search
+            for line_num, line in enumerate(lines, 1):
+                for match in self.compiled_pattern.finditer(line):
+                    matches.append({
+                        "type": "regex",
+                        "line_number": line_num,
+                        "match": match.group(),
+                        "start": match.start(),
+                        "end": match.end(),
+                        "context": self._get_context(lines, line_num - 1),
+                    })
+        else:
+            # Simple text search
+            search_text = self.pattern if self.case_sensitive else self.pattern.lower()
+            for line_num, line in enumerate(lines, 1):
+                compare_line = line if self.case_sensitive else line.lower()
+                start = 0
+                while True:
+                    pos = compare_line.find(search_text, start)
+                    if pos == -1:
+                        break
+                    matches.append({
+                        "type": "text",
+                        "line_number": line_num,
+                        "match": line[pos : pos + len(self.pattern)],
+                        "start": pos,
+                        "end": pos + len(self.pattern),
+                        "context": self._get_context(lines, line_num - 1),
+                    })
+                    start = pos + 1
+
+        return matches
+
+    def _get_context(self, lines: list[str], line_index: int) -> list[str]:
+        """Get context lines around a match."""
+        start = max(0, line_index - self.context_lines)
+        end = min(len(lines), line_index + self.context_lines + 1)
+        return lines[start:end]
+
+
+class DocumentScanner:
+    """High-performance document scanner with multi-format support."""
+
+    def __init__(
+        self,
+        input_dir: Path,
+        rules: list[Rule],
+        file_types: list[str],
+        use_pdf_ocr: bool = False,
+        use_process_pool: bool = False,
+        batch_size: int = 50,
+    ):
+        """Initialize scanner with input directory and rules.
+
+        Args:
+            input_dir: Directory containing documents to scan
+            rules: List of scanning rules
+            file_types: List of file extensions to scan
+            use_pdf_ocr: Use OCR for PDF files
+            use_process_pool: Use process pool instead of thread pool for CPU-intensive tasks
+            batch_size: Number of files to process in each batch
+        """
+        self.input_dir = Path(input_dir)
+        self.rules = rules
+        self.file_types = file_types
+        self.use_pdf_ocr = use_pdf_ocr
+        self.use_process_pool = use_process_pool
+        self.batch_size = batch_size
+        self.results = []
+        self.paused = False
+        self.paused_event = threading.Event()
+        self.paused_event.set()  # Initially not paused
+        self.stopped = False
+        self._progress_callback = None
+        self._executor = None  # Keep reference to executor for forced shutdown
+
+    def set_progress_callback(self, callback: Callable[[int, int], None]) -> None:
+        """Set callback function for progress updates.
+
+        Args:
+            callback: Function to call with progress (current, total)
+        """
+        self._progress_callback = callback
+
+    def pause(self) -> None:
+        """Pause the scanning process."""
+        self.paused = True
+        self.paused_event.clear()
+
+    def resume(self) -> None:
+        """Resume the scanning process."""
+        self.paused = False
+        self.paused_event.set()
+        logger.info(t("scan_resumed"))
+
+    def stop(self) -> None:
+        """Stop the scanning process."""
+        self.stopped = True
+        self.paused_event.set()  # Ensure thread can exit
+        logger.info(t("scan_stopped"))
+
+    def is_paused(self) -> bool:
+        """Check if the scanner is paused."""
+        return self.paused
+
+    def is_stopped(self) -> bool:
+        """Check if the scanner is stopped."""
+        return self.stopped
+
+    def scan(self, threads: int = 4, show_progress: bool = False) -> dict[str, Any]:
+        """Scan all documents in input directory.
+
+        Args:
+            threads: Number of worker threads/processes
+            show_progress: Show progress bar
+
+        Returns:
+            Dictionary containing scan results
+        """
+        self.stopped = False
+        self.paused = False
+        self.paused_event.set()
+
+        logger.info(t("scanning_directory", directory=str(self.input_dir)))
+        files = self._collect_files()
+        logger.info(t("found_files_to_scan", count=len(files)))
+
+        results = {
+            "scan_info": {
+                "input_directory": str(self.input_dir),
+                "scan_time": datetime.now().isoformat(),
+                "file_types_scanned": self.file_types,
+                "total_files": len(files),
+                "rules_count": len(self.rules),
+                "use_pdf_ocr": self.use_pdf_ocr,
+                "use_process_pool": self.use_process_pool,
+            },
+            "rules": [{"name": r.name, "pattern": r.pattern, "is_regex": r.is_regex} for r in self.rules],
+            "matches": [],
+        }
+
+        # Scan files in parallel
+        processed = 0
+        executor_class = ProcessPoolExecutor if self.use_process_pool else ThreadPoolExecutor
+        executor = executor_class(max_workers=threads)
+        self._executor = executor  # Keep reference for forced shutdown
+
+        try:
+            # Submit futures one by one to respect pause state
+            submitted_futures = []
+            was_paused = False  # Track previous pause state
+
+            for file in files:
+                # Check if stopped before submitting more tasks
+                if self.stopped:
+                    logger.info(t("scan_stopped_before_submitting_tasks"))
+                    break
+
+                # Wait if paused before submitting new tasks
+                while self.paused:
+                    # Log when entering paused state
+                    if not was_paused:
+                        logger.info(t("scan_paused"))
+                        was_paused = True
+
+                    self.paused_event.wait(0.1)
+                    if self.stopped:
+                        logger.info(t("scan_stopped_while_paused"))
+                        break
+
+                # Log when exiting paused state
+                if was_paused and not self.paused:
+                    logger.info(t("scan_resumed"))
+                    was_paused = False
+
+                if self.stopped:
+                    break
+
+                future = executor.submit(self._scan_file_with_pause_check, file)
+                submitted_futures.append(future)
+
+            # Process completed futures
+            for future in as_completed(submitted_futures):
+                # Check if stopped before processing this future
+                if self.stopped:
+                    logger.info(t("scan_stopped_by_user_canceling_tasks"))
+                    # Cancel all remaining futures
+                    for f in submitted_futures:
+                        if not f.done():
+                            f.cancel()
+                    break
+
+                # Wait if paused before processing result
+                while self.paused:
+                    # Log when entering paused state
+                    if not was_paused:
+                        logger.info(t("scan_paused"))
+                        was_paused = True
+
+                    self.paused_event.wait(0.1)
+                    if self.stopped:
+                        logger.info(t("scan_stopped_while_paused"))
+                        break
+
+                # Log when exiting paused state
+                if was_paused and not self.paused:
+                    logger.info(t("scan_resumed"))
+                    was_paused = False
+
+                if self.stopped:
+                    break
+
+                try:
+                    file_result = future.result(timeout=1.0)  # Short timeout to allow quick stop
+                    if file_result and file_result["matches"]:
+                        results["matches"].append(file_result)
+                        logger.info(t("found_matches_in_file", file_name=Path(file_result.get("file_path", "")).name))
+                except TimeoutError:
+                    logger.warning(t("task_timeout_scan_may_be_stopping"))
+                    if self.stopped:
+                        break
+                except Exception as e:
+                    if not self.stopped:
+                        logger.error(t("error_scanning_file", error=e))
+
+                processed += 1
+
+                # Report progress
+                if show_progress and processed % 10 == 0:
+                    logger.info(t("progress_report", processed=processed, total=len(files)))
+
+                # Call progress callback if set
+                if self._progress_callback:
+                    self._progress_callback(processed, len(files))
+
+        finally:
+            # Force shutdown if stopped
+            if self.stopped and self._executor:
+                logger.info(t("force_shutting_down_executor"))
+                if sys.version_info >= (3, 9):
+                    self._executor.shutdown(wait=False, cancel_futures=True)
+                else:
+                    self._executor.shutdown(wait=False)
+            else:
+                self._executor.shutdown(wait=True)
+            self._executor = None
+
+        results["scan_info"]["files_with_matches"] = len(results["matches"])
+        results["scan_info"]["files_processed"] = processed
+        results["stopped"] = self.stopped
+
+        if self.stopped:
+            logger.info(t("scan_stopped_processed_files", processed=processed))
+        else:
+            logger.info(t("scan_complete_found_matches", matches_count=len(results["matches"])))
+
+        return results
+
+    def _scan_file_with_pause_check(self, file_path: Path) -> dict[str, Any]:
+        """Scan a single file with pause check."""
+        # Check if stopped before processing
+        if self.stopped:
+            return {}
+
+        # Check if paused before processing
+        while self.paused:
+            self.paused_event.wait(0.1)
+            if self.stopped:
+                return {}
+
+        return self._scan_file(file_path)
+
+    def _collect_files(self) -> list[Path]:
+        """Collect all files matching the specified types."""
+        files = []
+        image_extensions = ["jpg", "jpeg", "png", "gif", "bmp", "tiff"]
+
+        for ext in self.file_types:
+            # If extension is an image format and OCR is not enabled, skip
+            if ext.lower() in image_extensions and not self.use_pdf_ocr:
+                continue
+            files.extend(self.input_dir.rglob(f"*.{ext.lower()}"))
+            files.extend(self.input_dir.rglob(f"*.{ext.upper()}"))
+        return list(set(files))  # Remove duplicates
+
+    def _scan_file(self, file_path: Path) -> dict[str, Any]:
+        """Scan a single file and return matches."""
+        # Check if stopped before starting
+        if self.stopped:
+            return {}
+
+        # Check if paused before starting
+        while self.paused:
+            self.paused_event.wait(0.1)
+            if self.stopped:
+                return {}
+
+        file_start_time = time.perf_counter()
+        ext = file_path.suffix.lower().lstrip(".")
+        text = ""
+        metadata = {}
+
+        try:
+            # Check if stopped before extraction
+            if self.stopped:
+                return {}
+
+            # Check if paused before extraction
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    return {}
+
+            # Route to appropriate extractor
+            if ext == "pdf":
+                text, metadata = self._extract_pdf(file_path)
+            elif ext == "odt":
+                text, metadata = self._extract_odt(file_path)
+            elif ext == "rtf":
+                text, metadata = self._extract_rtf(file_path)
+            elif ext == "epub":
+                text, metadata = self._extract_epub(file_path)
+            elif ext == "csv":
+                text, metadata = self._extract_csv(file_path)
+            elif ext == "xml":
+                text, metadata = self._extract_xml(file_path)
+            elif ext == "html" or ext == "htm":
+                text, metadata = self._extract_html(file_path)
+            elif ext == "md":
+                text, metadata = self._extract_markdown(file_path)
+            elif ext in ["docx", "doc"]:
+                text, metadata = self._extract_docx(file_path)
+            elif ext in ["xlsx", "xls"]:
+                text, metadata = self._extract_xlsx(file_path)
+            elif ext in ["pptx", "ppt"]:
+                text, metadata = self._extract_pptx(file_path)
+            elif ext in ["jpg", "jpeg", "png", "gif", "bmp", "tiff"]:
+                # Only extract images if OCR is enabled
+                if self.use_pdf_ocr:  # Using the same flag for consistency
+                    text, metadata = self._extract_image(file_path)
+                else:
+                    return {}  # Skip image files if OCR is disabled
+            else:
+                text, metadata = self._extract_text(file_path)
+
+            # Check if stopped after extraction
+            if self.stopped:
+                return {}
+
+            # Check if paused after extraction
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    return {}
+
+        except Exception as e:
+            logger.warning(t("could_not_extract_text_from_file", file_path=file_path, error=e))
+            return {}
+
+        processing_time = time.perf_counter() - file_start_time
+
+        if not text:
+            return {}
+
+        # Apply all rules with stop check
+        file_matches = []
+        for rule in self.rules:
+            if self.stopped:
+                return {}
+            # Check if paused before each rule
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    return {}
+            rule_matches = rule.search(text)
+            if rule_matches:
+                for match in rule_matches:
+                    match["rule_name"] = rule.name
+                    match["rule_description"] = rule.description
+                file_matches.extend(rule_matches)
+
+        if not file_matches:
+            return {}
+
+        # Add processing time to metadata
+        metadata["processing_time_seconds"] = round(processing_time, 3)
+
+        logger.info(
+            t(
+                "processed_file_info",
+                file_name=file_path.name,
+                ext=ext,
+                time=round(processing_time, 3),
+                matches_count=len(file_matches),
+            )
+        )
+
+        return {
+            "file_path": str(file_path),
+            "file_type": ext,
+            "file_size": file_path.stat().st_size,
+            "metadata": metadata,
+            "matches": file_matches,
+        }
+
+    def _extract_pdf(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PDF file with fallback."""
+        # Try PyMuPDF first (faster)
+        if fitz is not None:
+            try:
+                return self._extract_pdf_fitz(file_path)
+            except Exception as e:
+                logger.warning(t("pymupdf_failed_for_file", file_name=file_path.name, error=e))
+
+        # Fallback to pypdf
+        if pypdf is not None:
+            try:
+                return self._extract_pdf_pypdf(file_path)
+            except Exception as e:
+                logger.error(t("pypdf_also_failed_for_file", file_name=file_path.name, error=e))
+                return "", {}
+
+        logger.warning(t("no_pdf_library_installed"))
+        return "", {}
+
+    def _extract_pdf_fitz(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PDF using PyMuPDF (fastest method)."""
+        if not fitz:
+            logger.warning(t("pymupdf_not_installed"))
+            return "", {}
+
+        doc = None
+        try:
+            doc = fitz.open(str(file_path))
+            if doc.page_count == 0:
+                logger.warning(t("no_pages_found_in_file", file_path=file_path))
+                return "", {}
+            if not doc.metadata:
+                logger.warning(t("no_metadata_found_in_file", file_path=file_path))
+                return "", {}
+
+            text_parts = []
+            metadata = {
+                "page_count": doc.page_count,
+                "title": doc.metadata.get("title", ""),
+                "author": doc.metadata.get("author", ""),
+                "subject": doc.metadata.get("subject", ""),
+                "creator": doc.metadata.get("creator", ""),
+            }
+
+            if self.use_pdf_ocr and pytesseract and Image:
+                # OCR for image-based PDFs
+                import io
+
+                for page_num, page in enumerate(doc, 1):  # pyright: ignore[reportArgumentType]
+                    # Check if stopped before processing each page
+                    if self.stopped:
+                        doc.close()
+                        return "", {}
+
+                    # Check if paused before processing each page
+                    while self.paused:
+                        self.paused_event.wait(0.1)
+                        if self.stopped:
+                            doc.close()
+                            return "", {}
+
+                    pix = page.get_pixmap()
+                    img_data = pix.tobytes("png")
+                    image = Image.open(io.BytesIO(img_data))
+                    text = pytesseract.image_to_string(image)
+                    text_parts.append(f"[Page {page_num}]\n{text}")
+            else:
+                # Extract text directly (faster)
+                for page_num, page in enumerate(doc, 1):  # pyright: ignore[reportArgumentType]
+                    # Check if stopped before processing each page
+                    if self.stopped:
+                        doc.close()
+                        return "", {}
+
+                    # Check if paused before processing each page
+                    while self.paused:
+                        self.paused_event.wait(0.1)
+                        if self.stopped:
+                            doc.close()
+                            return "", {}
+
+                    text = page.get_text()
+                    text_parts.append(f"[Page {page_num}]\n{text}")
+
+            doc.close()
+            return "\n\n".join(text_parts), metadata
+        except Exception as e:
+            if doc:
+                doc.close()
+            logger.warning(t("pymupdf_error_trying_fallback", file_path=file_path, error=e))
+            # Re-raise to trigger fallback to pypdf
+            raise
+
+    def _extract_pdf_pypdf(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PDF using pypdf (fallback method)."""
+        if not pypdf:
+            logger.warning(t("pypdf_not_installed_skipping_extraction"))
+            return "", {}
+
+        text_parts = []
+        metadata = {}
+        try:
+            with open(file_path, "rb") as f:
+                pdf_reader = pypdf.PdfReader(f)
+
+                if not pdf_reader.metadata:
+                    logger.warning(t("no_metadata_found_in_file", file_path=file_path))
+                    return "", {}
+
+                metadata = {
+                    "page_count": len(pdf_reader.pages),
+                    "title": pdf_reader.metadata.get("/Title", ""),
+                    "author": pdf_reader.metadata.get("/Author", ""),
+                }
+
+                for page_num, page in enumerate(pdf_reader.pages, 1):
+                    # Check if stopped before processing each page
+                    if self.stopped:
+                        return "", {}
+
+                    # Check if paused before processing each page
+                    while self.paused:
+                        self.paused_event.wait(0.1)
+                        if self.stopped:
+                            return "", {}
+
+                    text = page.extract_text()
+                    text_parts.append(f"[Page {page_num}]\n{text}")
+
+        except Exception as e:
+            logger.warning(t("error_extracting_pdf_with_pypdf", error=e))
+            return "", {}
+
+        return "\n\n".join(text_parts), metadata
+
+    def _extract_odt(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from ODT (OpenDocument Text) file."""
+        if odf_odt is None:
+            logger.warning(t("odfpy_not_installed_skipping_extraction"))
+            return "", {}
+
+        try:
+            doc = odf_odt.load(file_path)
+            text = doc.textual_content  # pyright: ignore[reportAttributeAccessIssue]
+
+            metadata = {
+                "format": "ODT",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_odt", error=e))
+            return "", {}
+
+    def _extract_rtf(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from RTF (Rich Text Format) file."""
+        try:
+            with open(file_path, "rb") as f:
+                content = f.read()
+
+            # Simple RTF text extraction (removes control words)
+            text = ""
+            i = 0
+            while i < len(content):
+                if content[i] == ord("\\") and i + 1 < len(content):
+                    if content[i + 1] in [ord("'"), ord("*"), ord("\\")]:
+                        i += 2
+                        continue
+                    # Skip control words
+                    while (
+                        i < len(content)
+                        and content[i] != ord(" ")
+                        and content[i] != ord("{")
+                        and content[i] != ord("}")
+                    ):
+                        i += 1
+                elif content[i] >= 32 and content[i] <= 126:  # Printable ASCII
+                    text += chr(content[i])
+                i += 1
+
+            metadata = {
+                "format": "RTF",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_rtf", error=e))
+            return "", {}
+
+    def _extract_epub(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from EPUB (ebook) file."""
+        if ebooklib is None:
+            logger.warning(t("ebooklib_not_installed_skipping_extraction"))
+            return "", {}
+
+        try:
+            book = epub.read_epub(file_path)
+            text_parts = []
+
+            # Extract text from all items
+            for item in book.get_items():
+                # Check if stopped before processing each item
+                if self.stopped:
+                    return "", {}
+
+                # Check if paused before processing each item
+                while self.paused:
+                    self.paused_event.wait(0.1)
+                    if self.stopped:
+                        return "", {}
+
+                if item.get_type() == ebooklib.ITEM_DOCUMENT:  # pyright: ignore[reportAttributeAccessIssue]
+                    # Remove HTML tags
+                    html_content = item.get_content().decode("utf-8")  # pyright: ignore[reportAttributeAccessIssue]
+                    import re
+
+                    text = re.sub(r"<[^>]+>", " ", html_content)
+                    text = html.unescape(text)
+                    text_parts.append(text)
+
+            metadata = {
+                "title": book.get_metadata("DC", "title")[0][0] if book.get_metadata("DC", "title") else "",  # pyright: ignore[reportAttributeAccessIssue]
+                "author": book.get_metadata("DC", "creator")[0][0] if book.get_metadata("DC", "creator") else "",  # pyright: ignore[reportAttributeAccessIssue]
+                "format": "EPUB",
+            }
+
+            return "\n\n".join(text_parts), metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_epub", error=e))
+            return "", {}
+
+    def _extract_csv(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from CSV file."""
+        try:
+            text_parts = []
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                reader = csv.reader(f)
+                for row in reader:
+                    # Check if stopped periodically during row processing
+                    if self.stopped:
+                        return "", {}
+
+                    # Check if paused periodically during row processing
+                    while self.paused:
+                        self.paused_event.wait(0.1)
+                        if self.stopped:
+                            return "", {}
+
+                    row_text = " | ".join(str(cell) for cell in row)
+                    text_parts.append(row_text)
+
+            metadata = {
+                "format": "CSV",
+            }
+
+            return "\n".join(text_parts), metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_csv", error=e))
+            return "", {}
+
+    def _extract_xml(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from XML file."""
+        try:
+            tree = ET.parse(file_path)
+            root = tree.getroot()
+
+            # Extract all text content
+            text_parts = [elem.text for elem in root.iter() if elem.text and elem.text.strip()]
+            text = "\n".join(text_parts)
+
+            metadata = {
+                "format": "XML",
+                "root_tag": root.tag,
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_xml", error=e))
+            return "", {}
+
+    def _extract_html(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from HTML file."""
+        try:
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                html_content = f.read()
+
+            # Remove HTML tags
+            import re
+
+            text = re.sub(r"<[^>]+>", " ", html_content)
+            text = html.unescape(text)
+            text = re.sub(r"\s+", " ", text).strip()
+
+            metadata = {
+                "format": "HTML",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_html", error=e))
+            return "", {}
+
+    def _extract_markdown(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from Markdown file."""
+        try:
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                content = f.read()
+
+            if markdown:
+                # Convert Markdown to HTML then extract text
+                html_content = markdown.markdown(content)  # pyright: ignore[reportAttributeAccessIssue]
+                import re
+
+                text = re.sub(r"<[^>]+>", " ", html_content)
+                text = html.unescape(text)
+                text = re.sub(r"\s+", " ", text).strip()
+            else:
+                # Simple Markdown processing
+                text = content
+
+            metadata = {
+                "format": "Markdown",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(t("error_extracting_markdown", error=e))
+            return "", {}
+
+    def _extract_docx(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from DOCX file."""
+        if Document is None:
+            logger.warning(t("python_docx_not_installed_skipping_extraction"))
+            return "", {}
+
+        doc = Document(str(file_path))
+        text_parts = []
+
+        for paragraph in doc.paragraphs:
+            # Check if stopped periodically during paragraph processing
+            if self.stopped:
+                return "", {}
+
+            # Check if paused periodically during paragraph processing
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    return "", {}
+
+            text_parts.append(paragraph.text)
+
+        # Extract tables
+        for table in doc.tables:
+            # Check if stopped before processing each table
+            if self.stopped:
+                return "", {}
+
+            # Check if paused before processing each table
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    return "", {}
+
+            for row in table.rows:
+                row_text = " | ".join(cell.text for cell in row.cells)
+                text_parts.append(row_text)
+
+        metadata = {
+            "paragraph_count": len(doc.paragraphs),
+            "table_count": len(doc.tables),
+        }
+
+        return "\n".join(text_parts), metadata
+
+    def _extract_xlsx(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from XLSX file."""
+        if load_workbook is None:
+            logger.warning(t("openpyxl_not_installed_skipping_extraction"))
+            return "", {}
+
+        wb = load_workbook(file_path, read_only=True, data_only=True)
+        text_parts = []
+
+        for sheet_name in wb.sheetnames:
+            # Check if stopped before processing each sheet
+            if self.stopped:
+                wb.close()
+                return "", {}
+
+            # Check if paused before processing each sheet
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    wb.close()
+                    return "", {}
+
+            sheet = wb[sheet_name]
+            text_parts.append(f"[Sheet: {sheet_name}]")
+            for row in sheet.iter_rows(values_only=True):
+                # Check if stopped periodically during row processing
+                if self.stopped:
+                    wb.close()
+                    return "", {}
+
+                # Check if paused periodically during row processing
+                while self.paused:
+                    self.paused_event.wait(0.1)
+                    if self.stopped:
+                        wb.close()
+                        return "", {}
+
+                row_text = " | ".join(str(cell) if cell is not None else "" for cell in row)
+                if row_text.strip():
+                    text_parts.append(row_text)
+
+        metadata = {
+            "sheet_count": len(wb.sheetnames),
+            "sheets": wb.sheetnames,
+        }
+
+        wb.close()
+        return "\n".join(text_parts), metadata
+
+    def _extract_pptx(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PPTX file."""
+        try:
+            from pptx import Presentation
+        except ImportError:
+            logger.warning(t("python_pptx_not_installed_skipping_extraction"))
+            return "", {}
+
+        prs = Presentation(str(file_path))
+        text_parts = []
+
+        for slide_num, slide in enumerate(prs.slides, 1):
+            # Check if stopped before processing each slide
+            if self.stopped:
+                return "", {}
+
+            # Check if paused before processing each slide
+            while self.paused:
+                self.paused_event.wait(0.1)
+                if self.stopped:
+                    return "", {}
+
+            text_parts.append(f"[Slide {slide_num}]")
+            for shape in slide.shapes:
+                if hasattr(shape, "text"):
+                    text_parts.append(shape.text)  # pyright: ignore[reportAttributeAccessIssue]
+
+        metadata = {
+            "slide_count": len(prs.slides),
+        }
+
+        return "\n".join(text_parts), metadata
+
+    def _extract_image(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from image file using OCR."""
+        if Image is None or pytesseract is None:
+            logger.warning(t("pillow_or_tesseract_not_installed_skipping_ocr"))
+            return "", {}
+
+        try:
+            img = Image.open(file_path)
+            text = pytesseract.image_to_string(img)
+
+            metadata = {
+                "format": img.format,
+                "mode": img.mode,
+                "size": img.size,
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(t("could_not_perform_ocr_on_file", file_path=file_path, error=e))
+            return "", {}
+
+    def _extract_text(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from plain text file."""
+        encodings = ["utf-8", "latin-1", "cp1252", "utf-16"]
+
+        for encoding in encodings:
+            try:
+                with open(file_path, encoding=encoding, errors="ignore") as f:
+                    text = f.read()
+                return text, {"encoding": encoding}
+            except UnicodeDecodeError:
+                continue
+
+        return "", {}
+
+
+def main():
+    """Main entry point for document scanner."""
+    # 首先解析语言参数，但不使用翻译
+    temp_parser = argparse.ArgumentParser(add_help=False)
+    temp_parser.add_argument("--lang", choices=["en", "zh"], default="zh")
+    temp_args, _ = temp_parser.parse_known_args()
+
+    # 设置语言
+    global USE_CHINESE
+    USE_CHINESE = temp_args.lang == "zh"
+
+    parser = argparse.ArgumentParser(description=t("document_scanner_description"))
+    parser.add_argument("input", type=str, nargs="?", default=str(cwd), help=t("input_directory_help"))
+    parser.add_argument("-r", "--rules", type=str, default="rules.json", help=t("rules_file_help"))
+    parser.add_argument("--recursive", action="store_true", help=t("recursive_help"))
+    parser.add_argument(
+        "-f",
+        "--file-types",
+        help=t("file_types_help"),
+        default="pdf,docx,xlsx,pptx,txt,odt,rtf,epub,csv,xml,html,md,jpg,jpeg,png,gif,bmp,tiff",
+    )
+    parser.add_argument("--use-pdf-ocr", help=t("use_pdf_ocr_help"), action="store_true")
+    parser.add_argument(
+        "--use-process-pool",
+        help=t("use_process_pool_help"),
+        action="store_true",
+    )
+    parser.add_argument(
+        "-b",
+        "--batch-size",
+        help=t("batch_size_help"),
+        default=50,
+        type=int,
+    )
+    parser.add_argument("-t", "--threads", help=t("threads_help"), default=4, type=int)
+    parser.add_argument("--progress", help=t("progress_help"), action="store_true")
+    parser.add_argument("-v", "--verbose", help=t("verbose_help"), action="store_true")
+
+    # 添加语言参数
+    parser.add_argument("--lang", help=t("language_help"), choices=["en", "zh"], default="zh")
+
+    args = parser.parse_args()
+
+    # 再次确认语言设置（以防万一用户在完整参数中改变了语言）
+    USE_CHINESE = args.lang == "zh"
+
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    t0 = time.perf_counter()
+    # Validate input directory
+    input_dir = Path(args.input)
+    if not input_dir.exists() or not input_dir.is_dir():
+        logger.error(t("input_directory_does_not_exist", input_dir=args.input))
+        return
+    logger.info(t("scanning_directory", directory=str(input_dir)))
+
+    # Load rules file
+    rules_file = Path(args.rules)
+    if not rules_file.exists() or not rules_file.is_file():
+        rule_files_in_input_dir = list(input_dir.glob("rules*.json"))
+
+        if rule_files_in_input_dir:
+            rules_file = rule_files_in_input_dir[0]
+        else:
+            logger.error(t("rules_file_does_not_exist_alt", rules_file=args.rules))
+            return
+    logger.info(t("using_rules_file", rules_file=str(rules_file)))
+
+    try:
+        with open(rules_file, encoding="utf-8") as f:
+            rules_data = json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(t("invalid_json_in_rules_file", error=e))
+        return
+
+    # Parse rules
+    rules = []
+    if isinstance(rules_data, list):
+        rules = [Rule(rule) for rule in rules_data]
+    elif isinstance(rules_data, dict) and "rules" in rules_data:
+        rules = [Rule(rule) for rule in rules_data["rules"]]
+    else:
+        logger.error(t("invalid_rules_format"))
+        return
+
+    if not rules:
+        logger.error(t("no_valid_rules_found"))
+        return
+
+    # Parse file types
+    file_types = [ft.strip() for ft in args.file_types.split(",")]
+
+    # Create scanner and run scan
+    scanner = DocumentScanner(input_dir, rules, file_types, args.use_pdf_ocr, args.use_process_pool, args.batch_size)
+    results = scanner.scan(threads=args.threads, show_progress=args.progress)
+
+    # Save results to JSON file in input directory
+    output_file = input_dir / f"scan_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+
+    logger.info(t("results_saved_to", path=str(output_file)))
+    logger.info(t("total_time_elapsed", time=round(time.perf_counter() - t0, 2)))
+
+
+if __name__ == "__main__":
+    main()