gptmed 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

gptmed/data_preparation/text/text_statistics.py

@@ -0,0 +1,137 @@
+"""
+Text statistics extraction strategy
+"""
+
+import re
+from typing import Any, Dict
+
+from .base_strategy import TextPreprocessingStrategy
+
+
+class TextStatistics(TextPreprocessingStrategy):
+    """
+    Analyzes and extracts statistics from text
+    """
+    
+    def __init__(self):
+        """Initialize text statistics analyzer"""
+        self.stats = {
+            'analyses_performed': 0,
+        }
+        self.last_analysis: Dict[str, Any] = {}
+    
+    def process(self, text: str) -> str:
+        """
+        Analyze text and return original text unchanged
+        (statistics are stored in get_stats())
+        
+        Args:
+            text: Text to analyze
+            
+        Returns:
+            Original text unchanged
+        """
+        self.analyze(text)
+        return text
+    
+    def analyze(self, text: str) -> Dict[str, Any]:
+        """
+        Extract comprehensive statistics from text
+        
+        Args:
+            text: Text to analyze
+            
+        Returns:
+            Dictionary of statistics
+        """
+        if not text:
+            self.last_analysis = self._empty_analysis()
+            return self.last_analysis
+        
+        # Count basic metrics
+        word_count = len(text.split())
+        char_count = len(text)
+        char_count_no_spaces = len(text.replace(' ', ''))
+        line_count = len(text.split('\n'))
+        
+        # Sentence count (approximate)
+        sentence_count = len([s for s in re.split(r'[.!?]+', text) if s.strip()])
+        
+        # Average metrics
+        avg_word_length = char_count_no_spaces / word_count if word_count > 0 else 0
+        avg_sentence_length = word_count / sentence_count if sentence_count > 0 else 0
+        
+        # Unique words
+        words = text.lower().split()
+        unique_words = len(set(words))
+        
+        # Vocabulary richness
+        vocabulary_richness = unique_words / word_count if word_count > 0 else 0
+        
+        # Punctuation counts
+        punctuation_count = len([c for c in text if c in '.,!?;:'])
+        
+        # Number extraction
+        numbers = re.findall(r'\d+', text)
+        number_count = len(numbers)
+        
+        # Capitalized words
+        capitalized_words = len([w for w in text.split() if w and w[0].isupper()])
+        
+        self.last_analysis = {
+            'word_count': word_count,
+            'char_count': char_count,
+            'char_count_no_spaces': char_count_no_spaces,
+            'line_count': line_count,
+            'sentence_count': sentence_count,
+            'avg_word_length': round(avg_word_length, 2),
+            'avg_sentence_length': round(avg_sentence_length, 2),
+            'unique_words': unique_words,
+            'vocabulary_richness': round(vocabulary_richness, 3),
+            'punctuation_count': punctuation_count,
+            'number_count': number_count,
+            'capitalized_words': capitalized_words,
+        }
+        
+        self.stats['analyses_performed'] += 1
+        return self.last_analysis
+    
+    def _empty_analysis(self) -> Dict[str, Any]:
+        """Return empty analysis dictionary"""
+        return {
+            'word_count': 0,
+            'char_count': 0,
+            'char_count_no_spaces': 0,
+            'line_count': 0,
+            'sentence_count': 0,
+            'avg_word_length': 0.0,
+            'avg_sentence_length': 0.0,
+            'unique_words': 0,
+            'vocabulary_richness': 0.0,
+            'punctuation_count': 0,
+            'number_count': 0,
+            'capitalized_words': 0,
+        }
+    
+    def get_text_stats(self, text: str) -> Dict[str, Any]:
+        """
+        Convenience method to analyze text and return stats immediately
+        
+        Args:
+            text: Text to analyze
+            
+        Returns:
+            Dictionary of statistics
+        """
+        return self.analyze(text)
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get analyzer statistics"""
+        result = self.stats.copy()
+        result['last_analysis'] = self.last_analysis.copy()
+        return result
+    
+    def reset_stats(self) -> None:
+        """Reset statistics"""
+        self.stats = {'analyses_performed': 0}
+        self.last_analysis = {}

gptmed/data_preparation/text/tokenize_jsonl.py

@@ -0,0 +1,376 @@
+"""
+Parallel Tokenization Pipeline
+
+Takes preprocessed JSONL and generates tokens for model training.
+Supports parallel processing with ThreadPoolExecutor for efficient batch processing.
+
+Supports multiple tokenization methods:
+1. Hugging Face Transformers (AutoTokenizer)
+2. SentencePiece
+3. Built-in Tokenizer from data_preparation module
+
+Parallel Processing:
+- Automatically scales based on dataset size
+- ThreadPoolExecutor for I/O-bound tokenization
+- Separate outputs for individual documents and merged files
+
+Usage:
+    python3 tokenize_jsonl.py \
+        --input-file ./output/full_preprocessed.jsonl \
+        --method huggingface \
+        --model gpt2 \
+        --workers 4 \
+        --output-dir ./output/tokens
+"""
+
+import sys
+import json
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Any, Tuple
+from dataclasses import dataclass, asdict
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import os
+
+# Adjust path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from gptmed.data_preparation.text import Tokenizer as CustomTokenizer
+
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SimplifiedTokenizedRecord:
+    """Simplified tokenized record - only final tokens"""
+    filename: str
+    tokens: List[int]  # Token IDs
+    token_count: int
+    tokenizer_method: str
+    status: str
+
+
+class ParallelJSONLTokenizer:
+    """Parallel tokenization for preprocessed JSONL files"""
+    
+    def __init__(
+        self,
+        input_file: str,
+        output_dir: str = None,
+        method: str = 'huggingface',
+        model_name: Optional[str] = None,
+        workers: Optional[int] = None,
+    ):
+        """
+        Initialize parallel tokenizer
+        
+        Args:
+            input_file: Input preprocessed JSONL file
+            output_dir: Output directory for tokens
+            method: Tokenization method (custom/huggingface/sentencepiece)
+            model_name: Model name for huggingface/sentencepiece
+            workers: Number of worker threads (auto-detect if None)
+        """
+        self.input_file = Path(input_file)
+        
+        if output_dir is None:
+            output_dir = self.input_file.parent / 'tokens'
+        
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.method = method
+        self.model_name = model_name
+        
+        # Auto-detect workers based on CPU count
+        if workers is None:
+            workers = max(2, min(4, os.cpu_count() or 2))
+        self.workers = workers
+        
+        self.logger = logging.getLogger(self.__class__.__name__)
+        
+        # Initialize tokenizer based on method
+        self._initialize_tokenizer()
+    
+    def _initialize_tokenizer(self):
+        """Initialize tokenizer based on method"""
+        if self.method == 'custom':
+            self.tokenizer = CustomTokenizer(mode='word')
+            self.logger.info("Initialized custom tokenizer (word mode)")
+        
+        elif self.method == 'huggingface':
+            try:
+                from transformers import AutoTokenizer
+                if self.model_name is None:
+                    self.model_name = 'gpt2'
+                self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+                self.logger.info(f"Initialized Hugging Face tokenizer ({self.model_name})")
+            except ImportError:
+                raise ImportError("Install transformers: pip install transformers")
+        
+        elif self.method == 'sentencepiece':
+            try:
+                import sentencepiece as spm
+                if self.model_name is None:
+                    raise ValueError("SentencePiece requires model_name parameter")
+                self.tokenizer = spm.SentencePieceProcessor(model_file=self.model_name)
+                self.logger.info(f"Initialized SentencePiece tokenizer ({self.model_name})")
+            except ImportError:
+                raise ImportError("Install sentencepiece: pip install sentencepiece")
+        
+        else:
+            raise ValueError(f"Unknown tokenization method: {self.method}")
+    
+    def load_jsonl(self) -> List[Dict[str, Any]]:
+        """Load preprocessed JSONL"""
+        if not self.input_file.exists():
+            raise FileNotFoundError(f"Input file not found: {self.input_file}")
+        
+        records = []
+        with open(self.input_file, 'r', encoding='utf-8') as f:
+            for line_num, line in enumerate(f, 1):
+                try:
+                    record = json.loads(line.strip())
+                    records.append(record)
+                except json.JSONDecodeError as e:
+                    self.logger.warning(f"Line {line_num}: Invalid JSON - {str(e)}")
+        
+        self.logger.info(f"Loaded {len(records)} records from {self.input_file.name}")
+        return records
+    
+    def _tokenize_with_custom(self, text: str) -> List[str]:
+        """Tokenize using custom tokenizer"""
+        return self.tokenizer.tokenize(text)
+    
+    def _tokenize_with_huggingface(self, text: str) -> List[int]:
+        """Tokenize using Hugging Face tokenizer - returns token IDs"""
+        return self.tokenizer.encode(text, add_special_tokens=False)
+    
+    def _tokenize_with_sentencepiece(self, text: str) -> List[int]:
+        """Tokenize using SentencePiece - returns token IDs"""
+        return self.tokenizer.encode(text)
+    
+    def tokenize_record(self, record: Dict[str, Any], index: int = 0) -> Tuple[SimplifiedTokenizedRecord, str]:
+        """
+        Tokenize a single record
+        
+        Returns:
+            Tuple of (TokenizedRecord, filename_clean)
+        """
+        try:
+            filename = record.get('filename', f'document_{index}')
+            text = record.get('text', '')
+            
+            if not text or not isinstance(text, str) or len(text.strip()) < 10:
+                return (
+                    SimplifiedTokenizedRecord(
+                        filename=filename,
+                        tokens=[],
+                        token_count=0,
+                        tokenizer_method=self.method,
+                        status='empty_or_invalid_text'
+                    ),
+                    filename
+                )
+            
+            # Tokenize based on method
+            if self.method == 'custom':
+                tokens = self._tokenize_with_custom(text)
+                # Convert to integers if needed
+                if tokens and isinstance(tokens[0], str):
+                    tokens = list(range(len(tokens)))  # Fallback
+            elif self.method == 'huggingface':
+                tokens = self._tokenize_with_huggingface(text)
+            elif self.method == 'sentencepiece':
+                tokens = self._tokenize_with_sentencepiece(text)
+            else:
+                tokens = []
+            
+            return (
+                SimplifiedTokenizedRecord(
+                    filename=filename,
+                    tokens=tokens,
+                    token_count=len(tokens),
+                    tokenizer_method=self.method,
+                    status='success'
+                ),
+                filename
+            )
+            
+        except Exception as e:
+            self.logger.error(f"Error tokenizing {record.get('filename', 'unknown')}: {str(e)}")
+            return (
+                SimplifiedTokenizedRecord(
+                    filename=record.get('filename', f'document_{index}'),
+                    tokens=[],
+                    token_count=0,
+                    tokenizer_method=self.method,
+                    status=f'error: {str(e)}'
+                ),
+                record.get('filename', f'document_{index}')
+            )
+    
+    def save_merged_jsonl(self, records: List[SimplifiedTokenizedRecord]) -> bool:
+        """Save all tokenized records to merged file"""
+        try:
+            output_file = self.output_dir / 'merged_tokens.jsonl'
+            
+            with open(output_file, 'w', encoding='utf-8') as f:
+                for record in records:
+                    json.dump(asdict(record), f, ensure_ascii=False)
+                    f.write('\n')
+            
+            self.logger.info(f"✓ Saved {len(records)} merged tokens to {output_file.name}")
+            return True
+        except Exception as e:
+            self.logger.error(f"Error saving merged JSONL: {str(e)}")
+            return False
+    
+    def process(self) -> Dict[str, Any]:
+        """Main parallel tokenization pipeline"""
+        start_time = time.time()
+        
+        self.logger.info("="*70)
+        self.logger.info("Parallel Tokenization Pipeline")
+        self.logger.info("="*70)
+        self.logger.info(f"Input file: {self.input_file}")
+        self.logger.info(f"Output directory: {self.output_dir}")
+        self.logger.info(f"Method: {self.method}")
+        self.logger.info(f"Workers: {self.workers}")
+        if self.model_name:
+            self.logger.info(f"Model: {self.model_name}")
+        self.logger.info("="*70)
+        
+        # Load records
+        records = self.load_jsonl()
+        
+        if not records:
+            self.logger.warning("No records to tokenize")
+            return {'status': 'failure', 'total_records': 0}
+        
+        # Parallel tokenization
+        self.logger.info(f"\nTokenizing {len(records)} records with {self.workers} workers...")
+        tokenized_records = []
+        
+        with ThreadPoolExecutor(max_workers=self.workers) as executor:
+            # Submit all tasks
+            futures = {
+                executor.submit(self.tokenize_record, record, idx): idx
+                for idx, record in enumerate(records)
+            }
+            
+            # Collect results as they complete
+            completed = 0
+            for future in as_completed(futures):
+                try:
+                    tokenized, filename = future.result()
+                    tokenized_records.append(tokenized)
+                    
+                    completed += 1
+                    if completed % max(1, len(records) // 10) == 0:
+                        self.logger.info(f"Progress: {completed}/{len(records)} records tokenized")
+                
+                except Exception as e:
+                    self.logger.error(f"Error processing record: {str(e)}")
+        
+        # Save merged file
+        self.logger.info(f"\nSaving merged tokenized records...")
+        saved = self.save_merged_jsonl(tokenized_records)
+        
+        # Calculate statistics
+        successful = [r for r in tokenized_records if r.status == 'success']
+        failed = [r for r in tokenized_records if r.status != 'success']
+        
+        total_time = time.time() - start_time
+        
+        # Statistics
+        total_tokens = sum(r.token_count for r in successful)
+        avg_tokens_per_file = total_tokens / len(successful) if successful else 0
+        
+        # Print summary
+        self.logger.info(f"\n" + "="*70)
+        self.logger.info(f"Tokenization Summary")
+        self.logger.info(f"="*70)
+        self.logger.info(f"Total records: {len(records)}")
+        self.logger.info(f"Successfully tokenized: {len(successful)}")
+        self.logger.info(f"Failed: {len(failed)}")
+        self.logger.info(f"\nToken Statistics:")
+        self.logger.info(f"  Total tokens: {total_tokens:,}")
+        self.logger.info(f"  Average tokens per file: {avg_tokens_per_file:.1f}")
+        self.logger.info(f"\nOutput File:")
+        self.logger.info(f"  {self.output_dir}/merged_tokens.jsonl")
+        self.logger.info(f"\nTotal time: {total_time:.2f}s")
+        self.logger.info(f"Throughput: {len(records)/total_time:.2f} records/sec")
+        self.logger.info(f"="*70)
+        
+        return {
+            'status': 'success' if saved else 'failure',
+            'input_file': str(self.input_file),
+            'output_dir': str(self.output_dir),
+            'total_records': len(records),
+            'successful': len(successful),
+            'failed': len(failed),
+            'total_tokens': total_tokens,
+            'average_tokens_per_record': avg_tokens_per_file,
+            'tokenizer_method': self.method,
+            'workers_used': self.workers,
+            'total_time': total_time,
+            'throughput': len(records) / total_time,
+        }
+
+
+
+
+if __name__ == '__main__':
+    import argparse
+    
+    parser = argparse.ArgumentParser(
+        description='Parallel tokenize preprocessed JSONL files'
+    )
+    parser.add_argument(
+        '--input-file',
+        default='./output/full_preprocessed.jsonl',
+        help='Input preprocessed JSONL file'
+    )
+    parser.add_argument(
+        '--output-dir',
+        default=None,
+        help='Output directory for tokens (auto-generated if not specified)'
+    )
+    parser.add_argument(
+        '--method',
+        default='huggingface',
+        choices=['custom', 'huggingface', 'sentencepiece'],
+        help='Tokenization method (default: huggingface)'
+    )
+    parser.add_argument(
+        '--model',
+        default='gpt2',
+        help='Model name for huggingface/sentencepiece (default: gpt2)'
+    )
+    parser.add_argument(
+        '--workers',
+        type=int,
+        default=None,
+        help='Number of worker threads (auto-detect if not specified)'
+    )
+    
+    args = parser.parse_args()
+    
+    tokenizer = ParallelJSONLTokenizer(
+        input_file=args.input_file,
+        output_dir=args.output_dir,
+        method=args.method,
+        model_name=args.model,
+        workers=args.workers,
+    )
+    
+    result = tokenizer.process()
+    sys.exit(0 if result['status'] == 'success' else 1)

gptmed/data_preparation/text/tokenizer.py

@@ -0,0 +1,74 @@
+"""
+Tokenization strategy - converts text to tokens
+"""
+
+import re
+from typing import Any, Dict, List
+
+from .base_strategy import TextPreprocessingStrategy
+
+
+class Tokenizer(TextPreprocessingStrategy):
+    """
+    Tokenizes text into words or sentences
+    """
+    
+    def __init__(self, mode: str = 'word'):
+        """
+        Initialize tokenizer
+        
+        Args:
+            mode: Tokenization mode ('word' or 'sentence')
+        """
+        if mode not in ('word', 'sentence'):
+            raise ValueError(f"Invalid mode: {mode}. Use 'word' or 'sentence'")
+        
+        self.mode = mode
+        self.stats = {
+            'mode': mode,
+            'tokens_created': 0,
+        }
+        self.last_tokens: List[str] = []
+    
+    def process(self, text: str) -> str:
+        """
+        Tokenize text and return as space-separated tokens
+        
+        Args:
+            text: Text to tokenize
+            
+        Returns:
+            Space-separated tokens
+        """
+        tokens = self.tokenize(text)
+        self.stats['tokens_created'] = len(tokens)
+        return ' '.join(tokens)
+    
+    def tokenize(self, text: str) -> List[str]:
+        """
+        Tokenize text into list of tokens
+        
+        Args:
+            text: Text to tokenize
+            
+        Returns:
+            List of tokens
+        """
+        if self.mode == 'word':
+            tokens = text.split()
+        elif self.mode == 'sentence':
+            # Split on sentence-ending punctuation
+            tokens = re.split(r'[.!?]+', text)
+            tokens = [s.strip() for s in tokens if s.strip()]
+        
+        self.last_tokens = tokens
+        self.stats['tokens_created'] = len(tokens)
+        return tokens
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get tokenization statistics"""
+        return self.stats.copy()
+    
+    def reset_stats(self) -> None:
+        """Reset statistics"""
+        self.stats['tokens_created'] = 0

gptmed/data_preparation/text/unicode_normalizer.py

@@ -0,0 +1,61 @@
+"""
+Unicode normalization strategy - handles unicode characters and accents
+"""
+
+import unicodedata
+from typing import Any, Dict
+
+from .base_strategy import TextPreprocessingStrategy
+
+
+class UnicodeNormalizer(TextPreprocessingStrategy):
+    """
+    Normalizes unicode characters, removes diacritics and combining marks
+    """
+    
+    def __init__(self, form: str = 'NFD'):
+        """
+        Initialize normalizer
+        
+        Args:
+            form: Unicode normalization form (NFD, NFC, NFKD, NFKC)
+                  NFD is decomposed form (default)
+                  NFC is composed form
+        """
+        self.form = form
+        self.stats = {
+            'characters_removed': 0,
+            'form_used': form,
+        }
+    
+    def process(self, text: str) -> str:
+        """
+        Normalize unicode text
+        
+        Args:
+            text: Text to normalize
+            
+        Returns:
+            Normalized text
+        """
+        # Normalize unicode (NFD - decomposed form removes accents)
+        normalized = unicodedata.normalize(self.form, text)
+        
+        # Remove combining marks (accents, diacritics)
+        cleaned = ''.join(
+            ch for ch in normalized 
+            if unicodedata.category(ch) != 'Mn'
+        )
+        
+        chars_removed = len(text) - len(cleaned)
+        self.stats['characters_removed'] += chars_removed
+        
+        return cleaned
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get normalization statistics"""
+        return self.stats.copy()
+    
+    def reset_stats(self) -> None:
+        """Reset statistics"""
+        self.stats['characters_removed'] = 0

{gptmed-0.6.0.dist-info → gptmed-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gptmed
-Version: 0.6.0
+Version: 0.7.0
 Summary: A lightweight GPT-based language model framework for training custom question-answering models on any domain
 Author-email: Sanjog Sigdel <sigdelsanjog@gmail.com>, Sanjog Sigdel <sanjog.sigdel@ku.edu.np>
 License-Expression: MIT
@@ -51,6 +51,7 @@ Requires-Dist: librosa>=0.10.0; extra == "data-preparation"
 Requires-Dist: soundfile>=0.12.0; extra == "data-preparation"
 Requires-Dist: opencv-python>=4.5.0; extra == "data-preparation"
 Requires-Dist: numpy>=1.24.0; extra == "data-preparation"
+Requires-Dist: PyPDF2>=3.0.0; extra == "data-preparation"
 Dynamic: license-file
 
 # GptMed 🤖