pysfi 0.1.7__py3-none-any.whl

sfi/docscan/docscan.py

@@ -0,0 +1,841 @@
+"""Scan documents and extract text, images, and metadata with certain rules."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import html
+import json
+import logging
+import re
+import threading
+import time
+import xml.etree.ElementTree as ET
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable
+
+try:
+    import fitz  # PyMuPDF
+except ImportError:
+    fitz = None
+
+try:
+    from docx import Document
+except ImportError:
+    Document = None
+
+try:
+    from openpyxl import load_workbook
+except ImportError:
+    load_workbook = None
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+try:
+    import pytesseract
+except ImportError:
+    pytesseract = None
+
+try:
+    import odf.opendocument as odf_odt  # ODT support
+except ImportError:
+    odf_odt = None
+
+try:
+    import ebooklib  # EPUB support
+    from ebooklib import epub
+except ImportError:
+    ebooklib = None
+
+try:
+    import markdown  # Markdown to text
+except ImportError:
+    markdown = None
+
+try:
+    import pypdf  # Alternative PDF library
+except ImportError:
+    pypdf = None
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+cwd = Path.cwd()
+
+
+class Rule:
+    """Represents a scanning rule with optimized pattern matching."""
+
+    def __init__(self, rule_data: dict[str, Any]):
+        """Initialize rule from dictionary."""
+        self.name = rule_data.get("name", "")
+        self.pattern = rule_data.get("pattern", "")
+        self.is_regex = rule_data.get("regex", False)
+        self.case_sensitive = rule_data.get("case_sensitive", False)
+        self.context_lines = rule_data.get("context_lines", 3)
+        self.description = rule_data.get("description", "")
+
+        if self.is_regex:
+            flags = 0 if self.case_sensitive else re.IGNORECASE
+            try:
+                # Use re.ASCII for faster matching when possible
+                self.compiled_pattern = re.compile(self.pattern, flags | re.ASCII)
+            except re.error as e:
+                logger.warning(f"Invalid regex pattern '{self.pattern}': {e}")
+                self.compiled_pattern = None
+        else:
+            self.compiled_pattern = None
+
+    def search(self, text: str) -> list[dict[str, Any]]:
+        """Search for pattern in text and return matches."""
+        if not text or not self.pattern:
+            return []
+
+        matches = []
+        lines = text.split("\n")
+
+        if self.is_regex and self.compiled_pattern:
+            # Regex search
+            for line_num, line in enumerate(lines, 1):
+                for match in self.compiled_pattern.finditer(line):
+                    matches.append({
+                        "type": "regex",
+                        "line_number": line_num,
+                        "match": match.group(),
+                        "start": match.start(),
+                        "end": match.end(),
+                        "context": self._get_context(lines, line_num - 1),
+                    })
+        else:
+            # Simple text search
+            search_text = self.pattern if self.case_sensitive else self.pattern.lower()
+            for line_num, line in enumerate(lines, 1):
+                compare_line = line if self.case_sensitive else line.lower()
+                start = 0
+                while True:
+                    pos = compare_line.find(search_text, start)
+                    if pos == -1:
+                        break
+                    matches.append({
+                        "type": "text",
+                        "line_number": line_num,
+                        "match": line[pos : pos + len(self.pattern)],
+                        "start": pos,
+                        "end": pos + len(self.pattern),
+                        "context": self._get_context(lines, line_num - 1),
+                    })
+                    start = pos + 1
+
+        return matches
+
+    def _get_context(self, lines: list[str], line_index: int) -> list[str]:
+        """Get context lines around a match."""
+        start = max(0, line_index - self.context_lines)
+        end = min(len(lines), line_index + self.context_lines + 1)
+        return lines[start:end]
+
+
+class DocumentScanner:
+    """High-performance document scanner with multi-format support."""
+
+    def __init__(
+        self,
+        input_dir: Path,
+        rules: list[Rule],
+        file_types: list[str],
+        use_pdf_ocr: bool = False,
+        use_process_pool: bool = False,
+        batch_size: int = 50,
+    ):
+        """Initialize scanner with input directory and rules.
+
+        Args:
+            input_dir: Directory containing documents to scan
+            rules: List of scanning rules
+            file_types: List of file extensions to scan
+            use_pdf_ocr: Use OCR for PDF files
+            use_process_pool: Use process pool instead of thread pool for CPU-intensive tasks
+            batch_size: Number of files to process in each batch
+        """
+        self.input_dir = Path(input_dir)
+        self.rules = rules
+        self.file_types = file_types
+        self.use_pdf_ocr = use_pdf_ocr
+        self.use_process_pool = use_process_pool
+        self.batch_size = batch_size
+        self.results = []
+        self.paused = False
+        self.paused_event = threading.Event()
+        self.paused_event.set()  # Initially not paused
+        self.stopped = False
+        self._progress_callback = None
+
+    def set_progress_callback(self, callback: Callable[[int, int], None]) -> None:
+        """Set callback function for progress updates.
+
+        Args:
+            callback: Function to call with progress (current, total)
+        """
+        self._progress_callback = callback
+
+    def pause(self) -> None:
+        """Pause the scanning process."""
+        self.paused = True
+        self.paused_event.clear()
+        logger.info("Scan paused")
+
+    def resume(self) -> None:
+        """Resume the scanning process."""
+        self.paused = False
+        self.paused_event.set()
+        logger.info("Scan resumed")
+
+    def stop(self) -> None:
+        """Stop the scanning process."""
+        self.stopped = True
+        self.paused_event.set()  # Ensure thread can exit
+        logger.info("Scan stopped")
+
+    def is_paused(self) -> bool:
+        """Check if the scanner is paused."""
+        return self.paused
+
+    def is_stopped(self) -> bool:
+        """Check if the scanner is stopped."""
+        return self.stopped
+
+    def scan(self, threads: int = 4, show_progress: bool = False) -> dict[str, Any]:
+        """Scan all documents in input directory.
+
+        Args:
+            threads: Number of worker threads/processes
+            show_progress: Show progress bar
+
+        Returns:
+            Dictionary containing scan results
+        """
+        self.stopped = False
+        self.paused = False
+        self.paused_event.set()
+
+        logger.info(f"Scanning directory: {self.input_dir}")
+        files = self._collect_files()
+        logger.info(f"Found {len(files)} files to scan")
+
+        results = {
+            "scan_info": {
+                "input_directory": str(self.input_dir),
+                "scan_time": datetime.now().isoformat(),
+                "file_types_scanned": self.file_types,
+                "total_files": len(files),
+                "rules_count": len(self.rules),
+                "use_pdf_ocr": self.use_pdf_ocr,
+                "use_process_pool": self.use_process_pool,
+            },
+            "rules": [{"name": r.name, "pattern": r.pattern, "is_regex": r.is_regex} for r in self.rules],
+            "matches": [],
+        }
+
+        # Scan files in parallel
+        processed = 0
+        executor_class = ProcessPoolExecutor if self.use_process_pool else ThreadPoolExecutor
+        with executor_class(max_workers=threads) as executor:
+            future_to_file = {executor.submit(self._scan_file_with_pause_check, file): file for file in files}
+
+            for future in as_completed(future_to_file):
+                # Check if stopped before processing this future
+                if self.stopped:
+                    logger.info("Scan stopped by user, cancelling remaining tasks...")
+                    # Cancel all remaining futures
+                    for f in future_to_file:
+                        if not f.done():
+                            f.cancel()
+                    break
+
+                # Wait if paused
+                while self.paused:
+                    time.sleep(0.1)
+                    if self.stopped:
+                        logger.info("Scan stopped while paused")
+                        break
+
+                file_path = future_to_file[future]
+                try:
+                    file_result = future.result()
+                    if file_result and file_result["matches"]:
+                        results["matches"].append(file_result)
+                        logger.info(f"Found matches in: {file_path.name}")
+                except Exception as e:
+                    logger.error(f"Error scanning {file_path}: {e}")
+
+                processed += 1
+
+                # Report progress
+                if show_progress and processed % 10 == 0:
+                    logger.info(f"Progress: {processed}/{len(files)} files processed")
+
+                # Call progress callback if set
+                if self._progress_callback:
+                    self._progress_callback(processed, len(files))
+
+        results["scan_info"]["files_with_matches"] = len(results["matches"])
+        results["scan_info"]["files_processed"] = processed
+        results["stopped"] = self.stopped
+
+        if self.stopped:
+            logger.info(f"Scan stopped. Processed {processed} files")
+        else:
+            logger.info(f"Scan complete. Found matches in {len(results['matches'])} files")
+
+        return results
+
+    def _scan_file_with_pause_check(self, file_path: Path) -> dict[str, Any]:
+        """Scan a single file with pause check."""
+        # Check if stopped before processing
+        if self.stopped:
+            return {}
+
+        return self._scan_file(file_path)
+
+    def _collect_files(self) -> list[Path]:
+        """Collect all files matching the specified types."""
+        files = []
+        for ext in self.file_types:
+            files.extend(self.input_dir.rglob(f"*.{ext.lower()}"))
+            files.extend(self.input_dir.rglob(f"*.{ext.upper()}"))
+        return list(set(files))  # Remove duplicates
+
+    def _scan_file(self, file_path: Path) -> dict[str, Any]:
+        """Scan a single file and return matches."""
+        file_start_time = time.perf_counter()
+        ext = file_path.suffix.lower().lstrip(".")
+        text = ""
+        metadata = {}
+
+        try:
+            # Route to appropriate extractor
+            if ext == "pdf":
+                text, metadata = self._extract_pdf(file_path)
+            elif ext == "odt":
+                text, metadata = self._extract_odt(file_path)
+            elif ext == "rtf":
+                text, metadata = self._extract_rtf(file_path)
+            elif ext == "epub":
+                text, metadata = self._extract_epub(file_path)
+            elif ext == "csv":
+                text, metadata = self._extract_csv(file_path)
+            elif ext == "xml":
+                text, metadata = self._extract_xml(file_path)
+            elif ext == "html" or ext == "htm":
+                text, metadata = self._extract_html(file_path)
+            elif ext == "md":
+                text, metadata = self._extract_markdown(file_path)
+            elif ext in ["docx", "doc"]:
+                text, metadata = self._extract_docx(file_path)
+            elif ext in ["xlsx", "xls"]:
+                text, metadata = self._extract_xlsx(file_path)
+            elif ext in ["pptx", "ppt"]:
+                text, metadata = self._extract_pptx(file_path)
+            elif ext in ["jpg", "jpeg", "png", "gif", "bmp", "tiff"]:
+                text, metadata = self._extract_image(file_path)
+            else:
+                text, metadata = self._extract_text(file_path)
+        except Exception as e:
+            logger.warning(f"Could not extract text from {file_path}: {e}")
+            return {}
+
+        processing_time = time.perf_counter() - file_start_time
+
+        if not text:
+            return {}
+
+        # Apply all rules
+        file_matches = []
+        for rule in self.rules:
+            rule_matches = rule.search(text)
+            if rule_matches:
+                for match in rule_matches:
+                    match["rule_name"] = rule.name
+                    match["rule_description"] = rule.description
+                file_matches.extend(rule_matches)
+
+        if not file_matches:
+            return {}
+
+        # Add processing time to metadata
+        metadata["processing_time_seconds"] = round(processing_time, 3)
+
+        logger.info(f"Processed {file_path.name} ({ext}) in {processing_time:.3f}s - {len(file_matches)} matches found")
+
+        return {
+            "file_path": str(file_path),
+            "file_type": ext,
+            "file_size": file_path.stat().st_size,
+            "metadata": metadata,
+            "matches": file_matches,
+        }
+
+    def _extract_pdf(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PDF file with fallback."""
+        if fitz is not None:
+            return self._extract_pdf_fitz(file_path)
+        elif pypdf is not None:
+            return self._extract_pdf_pypdf(file_path)
+        else:
+            logger.warning("No PDF library installed (pymupdf or pypdf)")
+            return "", {}
+
+    def _extract_pdf_fitz(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PDF using PyMuPDF (fastest method)."""
+        if not fitz:
+            logger.warning("PyMuPDF not installed")
+            return "", {}
+
+        doc = fitz.open(str(file_path))
+        if doc.page_count == 0:
+            logger.warning(f"No pages found in {file_path}")
+            return "", {}
+        if not doc.metadata:
+            logger.warning(f"No metadata found in {file_path}")
+            return "", {}
+
+        text_parts = []
+        metadata = {
+            "page_count": doc.page_count,
+            "title": doc.metadata.get("title", ""),
+            "author": doc.metadata.get("author", ""),
+            "subject": doc.metadata.get("subject", ""),
+            "creator": doc.metadata.get("creator", ""),
+        }
+
+        if self.use_pdf_ocr and pytesseract and Image:
+            # OCR for image-based PDFs
+            import io
+
+            for page_num, page in enumerate(doc, 1):  # pyright: ignore[reportArgumentType]
+                pix = page.get_pixmap()
+                img_data = pix.tobytes("png")
+                image = Image.open(io.BytesIO(img_data))
+                text = pytesseract.image_to_string(image)
+                text_parts.append(f"[Page {page_num}]\n{text}")
+        else:
+            # Extract text directly (faster)
+            for page_num, page in enumerate(doc, 1):  # pyright: ignore[reportArgumentType]
+                text = page.get_text()
+                text_parts.append(f"[Page {page_num}]\n{text}")
+
+        doc.close()
+        return "\n\n".join(text_parts), metadata
+
+    def _extract_pdf_pypdf(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PDF using pypdf (fallback method)."""
+        if not pypdf:
+            logger.warning("pypdf not installed, skipping PDF extraction")
+            return "", {}
+
+        text_parts = []
+        metadata = {}
+        try:
+            with open(file_path, "rb") as f:
+                pdf_reader = pypdf.PdfReader(f)
+
+                if not pdf_reader.metadata:
+                    logger.warning(f"No metadata found in {file_path}")
+                    return "", {}
+
+                metadata = {
+                    "page_count": len(pdf_reader.pages),
+                    "title": pdf_reader.metadata.get("/Title", ""),
+                    "author": pdf_reader.metadata.get("/Author", ""),
+                }
+
+                for page_num, page in enumerate(pdf_reader.pages, 1):
+                    text = page.extract_text()
+                    text_parts.append(f"[Page {page_num}]\n{text}")
+
+        except Exception as e:
+            logger.warning(f"Error extracting PDF with pypdf: {e}")
+            return "", {}
+
+        return "\n\n".join(text_parts), metadata
+
+    def _extract_odt(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from ODT (OpenDocument Text) file."""
+        if odf_odt is None:
+            logger.warning("odfpy not installed, skipping ODT extraction")
+            return "", {}
+
+        try:
+            doc = odf_odt.load(file_path)
+            text = doc.textual_content  # pyright: ignore[reportAttributeAccessIssue]
+
+            metadata = {
+                "format": "ODT",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(f"Error extracting ODT: {e}")
+            return "", {}
+
+    def _extract_rtf(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from RTF (Rich Text Format) file."""
+        try:
+            with open(file_path, "rb") as f:
+                content = f.read()
+
+            # Simple RTF text extraction (removes control words)
+            text = ""
+            i = 0
+            while i < len(content):
+                if content[i] == ord("\\") and i + 1 < len(content):
+                    if content[i + 1] in [ord("'"), ord("*"), ord("\\")]:
+                        i += 2
+                        continue
+                    # Skip control words
+                    while (
+                        i < len(content)
+                        and content[i] != ord(" ")
+                        and content[i] != ord("{")
+                        and content[i] != ord("}")
+                    ):
+                        i += 1
+                elif content[i] >= 32 and content[i] <= 126:  # Printable ASCII
+                    text += chr(content[i])
+                i += 1
+
+            metadata = {
+                "format": "RTF",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(f"Error extracting RTF: {e}")
+            return "", {}
+
+    def _extract_epub(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from EPUB (ebook) file."""
+        if ebooklib is None:
+            logger.warning("ebooklib not installed, skipping EPUB extraction")
+            return "", {}
+
+        try:
+            book = epub.read_epub(file_path)
+            text_parts = []
+
+            # Extract text from all items
+            for item in book.get_items():
+                if item.get_type() == ebooklib.ITEM_DOCUMENT:  # pyright: ignore[reportAttributeAccessIssue]
+                    # Remove HTML tags
+                    html_content = item.get_content().decode("utf-8")  # pyright: ignore[reportAttributeAccessIssue]
+                    import re
+
+                    text = re.sub(r"<[^>]+>", " ", html_content)
+                    text = html.unescape(text)
+                    text_parts.append(text)
+
+            metadata = {
+                "title": book.get_metadata("DC", "title")[0][0] if book.get_metadata("DC", "title") else "",  # pyright: ignore[reportAttributeAccessIssue]
+                "author": book.get_metadata("DC", "creator")[0][0] if book.get_metadata("DC", "creator") else "",  # pyright: ignore[reportAttributeAccessIssue]
+                "format": "EPUB",
+            }
+
+            return "\n\n".join(text_parts), metadata
+        except Exception as e:
+            logger.warning(f"Error extracting EPUB: {e}")
+            return "", {}
+
+    def _extract_csv(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from CSV file."""
+        try:
+            text_parts = []
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                reader = csv.reader(f)
+                for row in reader:
+                    row_text = " | ".join(str(cell) for cell in row)
+                    text_parts.append(row_text)
+
+            metadata = {
+                "format": "CSV",
+            }
+
+            return "\n".join(text_parts), metadata
+        except Exception as e:
+            logger.warning(f"Error extracting CSV: {e}")
+            return "", {}
+
+    def _extract_xml(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from XML file."""
+        try:
+            tree = ET.parse(file_path)
+            root = tree.getroot()
+
+            # Extract all text content
+            text_parts = [elem.text for elem in root.iter() if elem.text and elem.text.strip()]
+            text = "\n".join(text_parts)
+
+            metadata = {
+                "format": "XML",
+                "root_tag": root.tag,
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(f"Error extracting XML: {e}")
+            return "", {}
+
+    def _extract_html(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from HTML file."""
+        try:
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                html_content = f.read()
+
+            # Remove HTML tags
+            import re
+
+            text = re.sub(r"<[^>]+>", " ", html_content)
+            text = html.unescape(text)
+            text = re.sub(r"\s+", " ", text).strip()
+
+            metadata = {
+                "format": "HTML",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(f"Error extracting HTML: {e}")
+            return "", {}
+
+    def _extract_markdown(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from Markdown file."""
+        try:
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                content = f.read()
+
+            if markdown:
+                # Convert Markdown to HTML then extract text
+                html_content = markdown.markdown(content)  # pyright: ignore[reportAttributeAccessIssue]
+                import re
+
+                text = re.sub(r"<[^>]+>", " ", html_content)
+                text = html.unescape(text)
+                text = re.sub(r"\s+", " ", text).strip()
+            else:
+                # Simple Markdown processing
+                text = content
+
+            metadata = {
+                "format": "Markdown",
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(f"Error extracting Markdown: {e}")
+            return "", {}
+
+    def _extract_docx(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from DOCX file."""
+        if Document is None:
+            logger.warning("python-docx not installed, skipping DOCX extraction")
+            return "", {}
+
+        doc = Document(str(file_path))
+        text_parts = []
+
+        for paragraph in doc.paragraphs:
+            text_parts.append(paragraph.text)
+
+        # Extract tables
+        for table in doc.tables:
+            for row in table.rows:
+                row_text = " | ".join(cell.text for cell in row.cells)
+                text_parts.append(row_text)
+
+        metadata = {
+            "paragraph_count": len(doc.paragraphs),
+            "table_count": len(doc.tables),
+        }
+
+        return "\n".join(text_parts), metadata
+
+    def _extract_xlsx(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from XLSX file."""
+        if load_workbook is None:
+            logger.warning("openpyxl not installed, skipping XLSX extraction")
+            return "", {}
+
+        wb = load_workbook(file_path, read_only=True, data_only=True)
+        text_parts = []
+
+        for sheet_name in wb.sheetnames:
+            sheet = wb[sheet_name]
+            text_parts.append(f"[Sheet: {sheet_name}]")
+            for row in sheet.iter_rows(values_only=True):
+                row_text = " | ".join(str(cell) if cell is not None else "" for cell in row)
+                if row_text.strip():
+                    text_parts.append(row_text)
+
+        metadata = {
+            "sheet_count": len(wb.sheetnames),
+            "sheets": wb.sheetnames,
+        }
+
+        return "\n".join(text_parts), metadata
+
+    def _extract_pptx(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from PPTX file."""
+        try:
+            from pptx import Presentation
+        except ImportError:
+            logger.warning("python-pptx not installed, skipping PPTX extraction")
+            return "", {}
+
+        prs = Presentation(str(file_path))
+        text_parts = []
+
+        for slide_num, slide in enumerate(prs.slides, 1):
+            text_parts.append(f"[Slide {slide_num}]")
+            for shape in slide.shapes:
+                if hasattr(shape, "text"):
+                    text_parts.append(shape.text)  # pyright: ignore[reportAttributeAccessIssue]
+
+        metadata = {
+            "slide_count": len(prs.slides),
+        }
+
+        return "\n".join(text_parts), metadata
+
+    def _extract_image(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from image file using OCR."""
+        if Image is None or pytesseract is None:
+            logger.warning("PIL or pytesseract not installed, skipping image OCR")
+            return "", {}
+
+        try:
+            img = Image.open(file_path)
+            text = pytesseract.image_to_string(img)
+
+            metadata = {
+                "format": img.format,
+                "mode": img.mode,
+                "size": img.size,
+            }
+
+            return text, metadata
+        except Exception as e:
+            logger.warning(f"Could not perform OCR on {file_path}: {e}")
+            return "", {}
+
+    def _extract_text(self, file_path: Path) -> tuple[str, dict[str, Any]]:
+        """Extract text from plain text file."""
+        encodings = ["utf-8", "latin-1", "cp1252", "utf-16"]
+
+        for encoding in encodings:
+            try:
+                with open(file_path, encoding=encoding, errors="ignore") as f:
+                    text = f.read()
+                return text, {"encoding": encoding}
+            except UnicodeDecodeError:
+                continue
+
+        return "", {}
+
+
+def main():
+    """Main entry point for document scanner."""
+    parser = argparse.ArgumentParser(
+        description="Scan documents and extract text, images, and metadata with high performance"
+    )
+    parser.add_argument("input", type=str, nargs="?", default=str(cwd), help="Input directory")
+    parser.add_argument("-r", "--rules", type=str, default="rules.json", help="Rules file (JSON)")
+    parser.add_argument("--recursive", action="store_true", help="Scan files recursively")
+    parser.add_argument(
+        "-f",
+        "--file-types",
+        help="File types to scan (comma-separated)",
+        default="pdf,docx,xlsx,pptx,txt,odt,rtf,epub,csv,xml,html,md",
+    )
+    parser.add_argument("--use-pdf-ocr", help="Use PDF OCR for image-based PDFs", action="store_true")
+    parser.add_argument(
+        "--use-process-pool",
+        help="Use process pool instead of thread pool (better for CPU-intensive tasks)",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-b",
+        "--batch-size",
+        help="Number of files to process in each batch",
+        default=50,
+        type=int,
+    )
+    parser.add_argument("-t", "--threads", help="Number of threads for parallel scanning", default=4, type=int)
+    parser.add_argument("--progress", help="Show progress bar", action="store_true")
+    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    args = parser.parse_args()
+
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    t0 = time.perf_counter()
+    # Validate input directory
+    input_dir = Path(args.input)
+    if not input_dir.exists() or not input_dir.is_dir():
+        logger.error(f"Input directory does not exist: {args.input}")
+        return
+    logger.info(f"Scanning directory: {input_dir}...")
+
+    # Load rules file
+    rules_file = Path(args.rules)
+    if not rules_file.exists() or not rules_file.is_file():
+        rule_files_in_input_dir = list(input_dir.glob("rules*.json"))
+
+        if rule_files_in_input_dir:
+            rules_file = rule_files_in_input_dir[0]
+        else:
+            logger.error(f"Rules file does not exist: {args.rules}")
+            return
+    logger.info(f"Using rules file: {rules_file}")
+
+    try:
+        with open(rules_file, encoding="utf-8") as f:
+            rules_data = json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(f"Invalid JSON in rules file: {e}")
+        return
+
+    # Parse rules
+    rules = []
+    if isinstance(rules_data, list):
+        rules = [Rule(rule) for rule in rules_data]
+    elif isinstance(rules_data, dict) and "rules" in rules_data:
+        rules = [Rule(rule) for rule in rules_data["rules"]]
+    else:
+        logger.error("Invalid rules format. Expected a list or dict with 'rules' key")
+        return
+
+    if not rules:
+        logger.error("No valid rules found")
+        return
+
+    # Parse file types
+    file_types = [ft.strip() for ft in args.file_types.split(",")]
+
+    # Create scanner and run scan
+    scanner = DocumentScanner(input_dir, rules, file_types, args.use_pdf_ocr, args.use_process_pool, args.batch_size)
+    results = scanner.scan(threads=args.threads, show_progress=args.progress)
+
+    # Save results to JSON file in input directory
+    output_file = input_dir / f"scan_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+
+    logger.info(f"Results saved to: {output_file}")
+    logger.info(f"Total time elapsed: {time.perf_counter() - t0:.2f}s")
+
+
+if __name__ == "__main__":
+    main()