gptmed 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

gptmed/data_preparation/text/case_normalizer.py

@@ -0,0 +1,66 @@
+"""
+Case normalization strategy - converts text case
+"""
+
+from typing import Any, Dict
+
+from .base_strategy import TextPreprocessingStrategy
+
+
+class CaseNormalizer(TextPreprocessingStrategy):
+    """
+    Handles text case conversion (lowercase, uppercase, title case)
+    """
+    
+    def __init__(self, mode: str = 'lower'):
+        """
+        Initialize case normalizer
+        
+        Args:
+            mode: Case conversion mode ('lower', 'upper', 'title', 'sentence')
+        """
+        if mode not in ('lower', 'upper', 'title', 'sentence'):
+            raise ValueError(f"Invalid mode: {mode}. Use 'lower', 'upper', 'title', or 'sentence'")
+        
+        self.mode = mode
+        self.stats = {
+            'mode_used': mode,
+            'conversions_applied': 0,
+        }
+    
+    def process(self, text: str) -> str:
+        """
+        Normalize text case
+        
+        Args:
+            text: Text to normalize
+            
+        Returns:
+            Case-normalized text
+        """
+        if not text:
+            return text
+        
+        original = text
+        
+        if self.mode == 'lower':
+            text = text.lower()
+        elif self.mode == 'upper':
+            text = text.upper()
+        elif self.mode == 'title':
+            text = text.title()
+        elif self.mode == 'sentence':
+            text = text[0].upper() + text[1:].lower() if len(text) > 0 else text
+        
+        if text != original:
+            self.stats['conversions_applied'] += 1
+        
+        return text
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get conversion statistics"""
+        return self.stats.copy()
+    
+    def reset_stats(self) -> None:
+        """Reset statistics"""
+        self.stats['conversions_applied'] = 0

gptmed/data_preparation/text/pdf_processor.py

@@ -0,0 +1,302 @@
+"""
+PDF processing strategy - extracts text from PDFs and exports to CSV
+
+Supports parallel processing for batch operations using concurrent.futures
+for efficient handling of multiple PDF files.
+"""
+
+import csv
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
+
+from .base_strategy import TextPreprocessingStrategy
+
+
+logger = logging.getLogger(__name__)
+
+
+class PDFProcessor(TextPreprocessingStrategy):
+    """
+    Processes PDF files: extracts text and exports to CSV
+    
+    Supports both sequential and parallel batch processing for efficient
+    handling of multiple PDF files using ThreadPoolExecutor.
+    """
+    
+    def __init__(self, max_workers: int = 4, use_threading: bool = True):
+        """
+        Initialize PDF processor
+        
+        Args:
+            max_workers: Maximum number of parallel workers (default: 4)
+            use_threading: Use ThreadPoolExecutor (True) or ProcessPoolExecutor (False)
+                          Threading is preferred for I/O-bound PDF extraction
+        """
+        self.PyPDF2 = None
+        self._import_pdf_library()
+        
+        self.max_workers = max_workers
+        self.use_threading = use_threading
+        
+        self.stats = {
+            'pdfs_processed': 0,
+            'total_text_extracted': 0,
+            'csv_exports': 0,
+            'parallel_jobs_completed': 0,
+            'processing_errors': 0,
+        }
+        self.last_extracted_texts: Dict[str, str] = {}
+    
+    def _import_pdf_library(self) -> None:
+        """Import PyPDF2 library, raise error if not available"""
+        try:
+            import PyPDF2
+            self.PyPDF2 = PyPDF2
+        except ImportError:
+            raise ImportError(
+                "PyPDF2 is required for PDF processing. "
+                "Install it with: pip install gptmed[data-preparation]"
+            )
+    
+    def process(self, text: str) -> str:
+        """
+        This strategy is not designed for streaming text processing.
+        Use extract_text_from_pdf() or batch_process_pdfs() instead.
+        
+        Args:
+            text: Not used for PDF processing
+            
+        Returns:
+            Empty string (PDFs are processed via dedicated methods)
+        """
+        return ""
+    
+    def extract_text_from_pdf(self, pdf_path: str) -> str:
+        """
+        Extract text from a single PDF file
+        
+        Args:
+            pdf_path: Path to PDF file
+            
+        Returns:
+            Extracted text from PDF
+            
+        Raises:
+            FileNotFoundError: If PDF file doesn't exist
+            Exception: If PDF processing fails
+        """
+        pdf_path = Path(pdf_path)
+        
+        if not pdf_path.exists():
+            raise FileNotFoundError(f"PDF file not found: {pdf_path}")
+        
+        try:
+            text_content = []
+            with open(pdf_path, 'rb') as file:
+                pdf_reader = self.PyPDF2.PdfReader(file)
+                
+                for page_num in range(len(pdf_reader.pages)):
+                    page = pdf_reader.pages[page_num]
+                    text_content.append(page.extract_text())
+            
+            full_text = '\n'.join(text_content)
+            self.last_extracted_texts[str(pdf_path)] = full_text
+            self.stats['pdfs_processed'] += 1
+            self.stats['total_text_extracted'] += len(full_text)
+            
+            return full_text
+        
+        except Exception as e:
+            raise Exception(f"Failed to process PDF {pdf_path}: {str(e)}")
+    
+    def batch_process_pdfs(
+        self, 
+        pdf_dir: str, 
+        output_format: str = 'dict',
+        parallel: bool = True,
+        max_workers: Optional[int] = None,
+    ) -> Any:
+        """
+        Process multiple PDFs from a directory
+        
+        Args:
+            pdf_dir: Directory containing PDF files
+            output_format: Format to return ('dict' or 'list')
+            parallel: Whether to use parallel processing (default: True)
+            max_workers: Override default max workers for this batch operation
+            
+        Returns:
+            Dictionary or list of {filename: text} pairs
+        """
+        pdf_dir = Path(pdf_dir)
+        
+        if not pdf_dir.is_dir():
+            raise NotADirectoryError(f"Directory not found: {pdf_dir}")
+        
+        pdf_files = list(pdf_dir.glob('*.pdf'))
+        
+        if not pdf_files:
+            logger.warning(f"No PDF files found in {pdf_dir}")
+            return {} if output_format == 'dict' else []
+        
+        if parallel and len(pdf_files) > 1:
+            return self._batch_process_parallel(
+                pdf_files, 
+                output_format, 
+                max_workers or self.max_workers
+            )
+        else:
+            return self._batch_process_sequential(pdf_files, output_format)
+    
+    def _batch_process_sequential(self, pdf_files: List[Path], output_format: str) -> Any:
+        """
+        Process PDFs sequentially (original method)
+        
+        Args:
+            pdf_files: List of PDF file paths
+            output_format: Format to return ('dict' or 'list')
+            
+        Returns:
+            Dictionary or list of {filename: text} pairs
+        """
+        results = {}
+        
+        for pdf_file in pdf_files:
+            try:
+                text = self.extract_text_from_pdf(str(pdf_file))
+                results[pdf_file.name] = text
+            except Exception as e:
+                error_msg = f"Error: {str(e)}"
+                results[pdf_file.name] = error_msg
+                self.stats['processing_errors'] += 1
+                logger.error(f"Error processing {pdf_file.name}: {str(e)}")
+        
+        if output_format == 'list':
+            return [{'filename': k, 'text': v} for k, v in results.items()]
+        
+        return results
+    
+    def _batch_process_parallel(
+        self, 
+        pdf_files: List[Path], 
+        output_format: str,
+        max_workers: int,
+    ) -> Any:
+        """
+        Process PDFs in parallel using ThreadPoolExecutor
+        
+        Args:
+            pdf_files: List of PDF file paths
+            output_format: Format to return ('dict' or 'list')
+            max_workers: Maximum number of concurrent workers
+            
+        Returns:
+            Dictionary or list of {filename: text} pairs
+        """
+        results = {}
+        executor_class = ThreadPoolExecutor if self.use_threading else ProcessPoolExecutor
+        
+        logger.info(f"Starting parallel PDF processing with {max_workers} workers")
+        
+        with executor_class(max_workers=max_workers) as executor:
+            # Submit all tasks
+            future_to_file = {
+                executor.submit(self.extract_text_from_pdf, str(pdf_file)): pdf_file 
+                for pdf_file in pdf_files
+            }
+            
+            # Process completed tasks as they finish
+            for i, future in enumerate(as_completed(future_to_file), 1):
+                pdf_file = future_to_file[future]
+                
+                try:
+                    text = future.result()
+                    results[pdf_file.name] = text
+                    self.stats['parallel_jobs_completed'] += 1
+                    logger.debug(f"[{i}/{len(pdf_files)}] Completed: {pdf_file.name}")
+                    
+                except Exception as e:
+                    error_msg = f"Error: {str(e)}"
+                    results[pdf_file.name] = error_msg
+                    self.stats['processing_errors'] += 1
+                    logger.error(f"Error processing {pdf_file.name}: {str(e)}")
+        
+        logger.info(f"Parallel processing complete: {len(results)} files processed")
+        
+        if output_format == 'list':
+            return [{'filename': k, 'text': v} for k, v in results.items()]
+        
+        return results
+    
+    def export_to_csv(self, texts: Dict[str, str], output_file: str) -> None:
+        """
+        Export text data to CSV with filename and text content
+        
+        Args:
+            texts: Dictionary of {filename: text_content}
+            output_file: Path to output CSV file
+        """
+        output_path = Path(output_file)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        try:
+            with open(output_path, 'w', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                writer.writerow(['filename', 'text_content'])
+                
+                for filename, text in texts.items():
+                    writer.writerow([filename, text])
+            
+            self.stats['csv_exports'] += 1
+        
+        except Exception as e:
+            raise Exception(f"Failed to export CSV: {str(e)}")
+    
+    def export_to_csv_detailed(self, texts: Dict[str, str], output_file: str) -> None:
+        """
+        Export text data to CSV with detailed statistics
+        
+        Args:
+            texts: Dictionary of {filename: text_content}
+            output_file: Path to output CSV file
+        """
+        output_path = Path(output_file)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        try:
+            with open(output_path, 'w', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                writer.writerow([
+                    'filename', 'text_content', 'word_count', 
+                    'char_count', 'sentence_count'
+                ])
+                
+                for filename, text in texts.items():
+                    word_count = len(text.split())
+                    char_count = len(text)
+                    sentence_count = len([s for s in text.split('.') if s.strip()])
+                    
+                    writer.writerow([
+                        filename, text, word_count, char_count, sentence_count
+                    ])
+            
+            self.stats['csv_exports'] += 1
+        
+        except Exception as e:
+            raise Exception(f"Failed to export detailed CSV: {str(e)}")
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get PDF processing statistics"""
+        return self.stats.copy()
+    
+    def reset_stats(self) -> None:
+        """Reset statistics"""
+        self.stats = {
+            'pdfs_processed': 0,
+            'total_text_extracted': 0,
+            'csv_exports': 0,
+            'parallel_jobs_completed': 0,
+            'processing_errors': 0,
+        }

gptmed/data_preparation/text/pipeline.py

@@ -0,0 +1,333 @@
+"""
+Complete PDF → Tokens Pipeline
+
+Orchestrates the full preprocessing pipeline:
+1. Extract text from PDFs (in-memory)
+2. Preprocess text (in-memory)
+3. Tokenize (saves merged_tokens.jsonl only)
+
+This is the main entry point for generating training data.
+
+Usage:
+    python3 pipeline.py \
+        --input-dir ./pdfs \
+        --output-dir ./output \
+        --tokenizer-method huggingface \
+        --tokenizer-model gpt2 \
+        --workers 4
+"""
+
+import sys
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Dict, List, Any
+from dataclasses import asdict
+import argparse
+import importlib.util
+
+# Adjust path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+# Load batch_pdf_to_jsonl
+spec1 = importlib.util.spec_from_file_location("batch_pdf_to_jsonl", Path(__file__).parent / "batch_pdf_to_jsonl.py")
+batch_module = importlib.util.module_from_spec(spec1)
+spec1.loader.exec_module(batch_module)
+PDFBatchProcessor = batch_module.PDFBatchProcessor
+PDFRecord = batch_module.PDFRecord
+
+# Load preprocess_jsonl
+spec2 = importlib.util.spec_from_file_location("preprocess_jsonl", Path(__file__).parent / "preprocess_jsonl.py")
+preprocess_module = importlib.util.module_from_spec(spec2)
+spec2.loader.exec_module(preprocess_module)
+ComprehensiveJSONLPreprocessor = preprocess_module.ComprehensiveJSONLPreprocessor
+FullPreprocessedRecord = preprocess_module.FullPreprocessedRecord
+
+# Load tokenize_jsonl
+spec3 = importlib.util.spec_from_file_location("tokenize_jsonl", Path(__file__).parent / "tokenize_jsonl.py")
+tokenize_module = importlib.util.module_from_spec(spec3)
+spec3.loader.exec_module(tokenize_module)
+ParallelJSONLTokenizer = tokenize_module.ParallelJSONLTokenizer
+SimplifiedTokenizedRecord = tokenize_module.SimplifiedTokenizedRecord
+
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+class EndToEndPipeline:
+    """Complete PDF to training data pipeline"""
+    
+    def __init__(
+        self,
+        input_dir: str = "./pdfs",
+        output_dir: str = "./output",
+        tokenizer_method: str = "huggingface",
+        tokenizer_model: str = "gpt2",
+        workers: int = 4,
+        case_mode: str = "lower",
+        remove_stopwords: bool = False,
+        remove_punctuation: bool = False,
+    ):
+        """
+        Initialize pipeline
+        
+        Args:
+            input_dir: Directory containing PDFs
+            output_dir: Output directory for final results
+            tokenizer_method: Tokenization method (huggingface/custom/sentencepiece)
+            tokenizer_model: Tokenizer model name
+            workers: Number of parallel workers
+            case_mode: Case normalization (lower/upper/title/sentence)
+            remove_stopwords: Whether to remove stopwords
+            remove_punctuation: Whether to remove punctuation
+        """
+        self.input_dir = Path(input_dir)
+        self.output_dir = Path(output_dir)
+        self.tokenizer_method = tokenizer_method
+        self.tokenizer_model = tokenizer_model
+        self.workers = workers
+        self.case_mode = case_mode
+        self.remove_stopwords = remove_stopwords
+        self.remove_punctuation = remove_punctuation
+        
+        # Create output directory
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.logger = logging.getLogger(self.__class__.__name__)
+    
+    def step1_extract_pdfs(self) -> List[PDFRecord]:
+        """Step 1: Extract text from PDFs in-memory"""
+        self.logger.info("\n" + "="*70)
+        self.logger.info("STEP 1: PDF EXTRACTION")
+        self.logger.info("="*70)
+        
+        processor = PDFBatchProcessor(
+            input_dir=str(self.input_dir),
+            output_dir=str(self.output_dir),
+            max_workers=self.workers,
+        )
+        
+        result = processor.process()
+        records = result.get('records', [])
+        
+        self.logger.info(f"\n✓ Extracted {len(records)} PDF records")
+        return records
+    
+    def step2_preprocess_text(self, records: List[PDFRecord]) -> List[FullPreprocessedRecord]:
+        """Step 2: Preprocess text in-memory"""
+        self.logger.info("\n" + "="*70)
+        self.logger.info("STEP 2: TEXT PREPROCESSING")
+        self.logger.info("="*70)
+        
+        # Create a dummy temp file just to initialize the preprocessor
+        # We'll manually preprocess records using its methods
+        temp_input = self.output_dir / "_temp_input.jsonl"
+        with open(temp_input, 'w') as f:
+            f.write("{}\n")
+        
+        try:
+            processor = ComprehensiveJSONLPreprocessor(
+                input_file=str(temp_input),
+                output_file=None,
+                case_mode=self.case_mode,
+                remove_stopwords=self.remove_stopwords,
+                remove_punctuation=self.remove_punctuation,
+            )
+        finally:
+            temp_input.unlink()
+        
+        preprocessed_records = []
+        
+        # Convert PDFRecord to dict format expected by preprocess_record
+        for record in records:
+            record_dict = {
+                'filename': record.filename,
+                'text': record.text,
+                'word_count': record.word_count,
+            }
+            
+            # Preprocess the record
+            preprocessed = processor.preprocess_record(record_dict)
+            preprocessed_records.append(preprocessed)
+        
+        self.logger.info(f"✓ Preprocessed {len(preprocessed_records)} records")
+        
+        # Save preprocessed output
+        output_file = self.output_dir / "full_preprocessed.jsonl"
+        with open(output_file, 'w', encoding='utf-8') as f:
+            for record in preprocessed_records:
+                json.dump(asdict(record), f, ensure_ascii=False)
+                f.write('\n')
+        
+        self.logger.info(f"✓ Saved: {output_file.name}")
+        
+        return preprocessed_records
+    
+    def step3_tokenize(self, preprocessed_records: List[FullPreprocessedRecord]) -> Dict[str, Any]:
+        """Step 3: Tokenize preprocessed text"""
+        self.logger.info("\n" + "="*70)
+        self.logger.info("STEP 3: TOKENIZATION")
+        self.logger.info("="*70)
+        
+        # Save preprocessed records to temporary JSONL for tokenizer to consume
+        temp_file = self.output_dir / "_temp_preprocessed.jsonl"
+        with open(temp_file, 'w', encoding='utf-8') as f:
+            for record in preprocessed_records:
+                json.dump(asdict(record), f, ensure_ascii=False)
+                f.write('\n')
+        
+        # Initialize tokenizer
+        tokens_dir = self.output_dir / "tokens"
+        tokenizer = ParallelJSONLTokenizer(
+            input_file=str(temp_file),
+            output_dir=str(tokens_dir),
+            method=self.tokenizer_method,
+            model_name=self.tokenizer_model,
+            workers=self.workers,
+        )
+        
+        # Tokenize
+        result = tokenizer.process()
+        
+        # Clean up temporary file
+        temp_file.unlink()
+        
+        return result
+    
+    def run(self) -> Dict[str, Any]:
+        """Execute full pipeline"""
+        start_time = time.time()
+        
+        self.logger.info("\n")
+        self.logger.info("╔" + "="*68 + "╗")
+        self.logger.info("║" + " "*15 + "END-TO-END PDF → TOKENS PIPELINE" + " "*21 + "║")
+        self.logger.info("╚" + "="*68 + "╝")
+        
+        try:
+            # Step 1: Extract PDFs
+            pdf_records = self.step1_extract_pdfs()
+            
+            if not pdf_records:
+                self.logger.error("No PDF records extracted. Exiting.")
+                return {'status': 'failure', 'message': 'No PDFs extracted'}
+            
+            # Step 2: Preprocess
+            preprocessed_records = self.step2_preprocess_text(pdf_records)
+            
+            if not preprocessed_records:
+                self.logger.error("No records preprocessed. Exiting.")
+                return {'status': 'failure', 'message': 'Preprocessing failed'}
+            
+            # Step 3: Tokenize
+            tokenization_result = self.step3_tokenize(preprocessed_records)
+            
+            total_time = time.time() - start_time
+            
+            # Final summary
+            self.logger.info("\n" + "="*70)
+            self.logger.info("PIPELINE COMPLETE")
+            self.logger.info("="*70)
+            self.logger.info(f"\nFinal Outputs:")
+            self.logger.info(f"  1. {self.output_dir}/full_preprocessed.jsonl (cleaned text)")
+            self.logger.info(f"  2. {self.output_dir}/tokens/merged_tokens.jsonl (training tokens)")
+            self.logger.info(f"\nTotal Time: {total_time:.2f}s")
+            self.logger.info(f"="*70 + "\n")
+            
+            return {
+                'status': 'success',
+                'total_time': total_time,
+                'pdf_extraction': {
+                    'records': len(pdf_records),
+                },
+                'preprocessing': {
+                    'records': len(preprocessed_records),
+                },
+                'tokenization': tokenization_result,
+                'output_dir': str(self.output_dir),
+            }
+            
+        except Exception as e:
+            self.logger.error(f"Pipeline failed: {str(e)}")
+            import traceback
+            traceback.print_exc()
+            return {'status': 'failure', 'error': str(e)}
+
+
+def main():
+    """Main entry point"""
+    parser = argparse.ArgumentParser(
+        description='Complete PDF to training tokens pipeline'
+    )
+    parser.add_argument(
+        '--input-dir',
+        default='./pdfs',
+        help='Input directory containing PDFs (default: ./pdfs)'
+    )
+    parser.add_argument(
+        '--output-dir',
+        default='./output',
+        help='Output directory for final results (default: ./output)'
+    )
+    parser.add_argument(
+        '--tokenizer-method',
+        default='huggingface',
+        choices=['huggingface', 'custom', 'sentencepiece'],
+        help='Tokenization method (default: huggingface)'
+    )
+    parser.add_argument(
+        '--tokenizer-model',
+        default='gpt2',
+        help='Tokenizer model name (default: gpt2)'
+    )
+    parser.add_argument(
+        '--workers',
+        type=int,
+        default=4,
+        help='Number of parallel workers (default: 4)'
+    )
+    parser.add_argument(
+        '--case-mode',
+        default='lower',
+        choices=['lower', 'upper', 'title', 'sentence'],
+        help='Case normalization mode (default: lower)'
+    )
+    parser.add_argument(
+        '--remove-stopwords',
+        action='store_true',
+        help='Remove common stopwords during preprocessing'
+    )
+    parser.add_argument(
+        '--remove-punctuation',
+        action='store_true',
+        help='Remove punctuation during preprocessing'
+    )
+    
+    args = parser.parse_args()
+    
+    # Create and run pipeline
+    pipeline = EndToEndPipeline(
+        input_dir=args.input_dir,
+        output_dir=args.output_dir,
+        tokenizer_method=args.tokenizer_method,
+        tokenizer_model=args.tokenizer_model,
+        workers=args.workers,
+        case_mode=args.case_mode,
+        remove_stopwords=args.remove_stopwords,
+        remove_punctuation=args.remove_punctuation,
+    )
+    
+    result = pipeline.run()
+    
+    # Exit with appropriate code
+    return 0 if result['status'] == 'success' else 1
+
+
+if __name__ == '__main__':
+    exit_code = main()
+    sys.exit(exit_code)