gptmed 0.5.5__py3-none-any.whl → 0.7.0__py3-none-any.whl

gptmed/data_preparation/video/__init__.py

@@ -0,0 +1,415 @@
+"""
+Video data preprocessing and cleaning module
+
+Handles video frame extraction, resizing, quality checks, and metadata extraction
+"""
+
+import logging
+from typing import Any, Dict, List, Optional, Tuple
+from pathlib import Path
+import json
+
+from ..base import BaseDataPreprocessor, PreprocessingConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+class VideoPreprocessor(BaseDataPreprocessor):
+    """
+    Video preprocessing with frame extraction, resizing, and validation
+    
+    Features:
+        - Video format validation
+        - Frame extraction at specified intervals
+        - Resolution resizing
+        - Frame rate conversion
+        - Bitrate analysis
+        - Duration validation
+        - Metadata extraction
+        - Codec detection
+        - Corruption detection
+        - Thumbnail generation
+    """
+    
+    def __init__(
+        self,
+        config: Optional[PreprocessingConfig] = None,
+        target_fps: int = 30,
+        target_resolution: Tuple[int, int] = (640, 480),
+        preserve_aspect_ratio: bool = True,
+        min_duration: float = 1.0,  # seconds
+        max_duration: Optional[float] = None,  # seconds
+        min_width: int = 320,
+        min_height: int = 240,
+        frame_extraction_interval: Optional[int] = None,  # Extract every N frames
+        extract_frames: bool = False,
+        supported_formats: Optional[List[str]] = None,
+    ):
+        """
+        Initialize video preprocessor
+        
+        Args:
+            config: PreprocessingConfig instance
+            target_fps: Target frames per second
+            target_resolution: Target resolution (width, height)
+            preserve_aspect_ratio: Whether to preserve aspect ratio
+            min_duration: Minimum video duration in seconds
+            max_duration: Maximum video duration in seconds
+            min_width: Minimum video width
+            min_height: Minimum video height
+            frame_extraction_interval: Extract every N frames (None = no extraction)
+            extract_frames: Whether to extract frames to disk
+            supported_formats: List of supported video formats
+        """
+        if config is None:
+            config = PreprocessingConfig(
+                input_path="./data/raw/videos",
+                output_path="./data/processed/videos",
+                data_type="video"
+            )
+        
+        super().__init__(config)
+        
+        self.target_fps = target_fps
+        self.target_resolution = target_resolution
+        self.preserve_aspect_ratio = preserve_aspect_ratio
+        self.min_duration = min_duration
+        self.max_duration = max_duration
+        self.min_width = min_width
+        self.min_height = min_height
+        self.frame_extraction_interval = frame_extraction_interval
+        self.extract_frames = extract_frames
+        self.supported_formats = supported_formats or ['mp4', 'avi', 'mov', 'mkv', 'flv', 'wmv']
+        
+        self._import_video_library()
+    
+    def _import_video_library(self):
+        """Attempt to import opencv and other video libraries"""
+        self.opencv_available = False
+        self.ffmpeg_available = False
+        
+        try:
+            import cv2
+            self.cv2 = cv2
+            self.opencv_available = True
+        except ImportError:
+            self.logger.warning(
+                "OpenCV not available. Install with: pip install opencv-python"
+            )
+        
+        try:
+            import subprocess
+            result = subprocess.run(['ffmpeg', '-version'], capture_output=True)
+            self.ffmpeg_available = result.returncode == 0
+        except:
+            self.logger.warning(
+                "ffmpeg not available. Install from: https://ffmpeg.org/download.html"
+            )
+    
+    def validate(self, data: Any) -> bool:
+        """
+        Validate video input
+        
+        Args:
+            data: Video file path (str)
+            
+        Returns:
+            True if valid, False otherwise
+        """
+        if not self.opencv_available:
+            self.logger.error("OpenCV is required for video processing")
+            return False
+        
+        try:
+            if not isinstance(data, str):
+                self.logger.warning(f"Invalid video type: {type(data)}")
+                return False
+            
+            video_path = Path(data)
+            if not video_path.exists():
+                self.logger.warning(f"Video file not found: {data}")
+                return False
+            
+            if not any(str(video_path).lower().endswith(f) for f in self.supported_formats):
+                self.logger.warning(f"Unsupported format: {data}")
+                return False
+            
+            # Try to open video
+            cap = self.cv2.VideoCapture(str(video_path))
+            if not cap.isOpened():
+                self.logger.warning(f"Cannot open video: {data}")
+                return False
+            
+            # Check properties
+            fps = cap.get(self.cv2.CAP_PROP_FPS)
+            width = int(cap.get(self.cv2.CAP_PROP_FRAME_WIDTH))
+            height = int(cap.get(self.cv2.CAP_PROP_FRAME_HEIGHT))
+            frame_count = int(cap.get(self.cv2.CAP_PROP_FRAME_COUNT))
+            
+            cap.release()
+            
+            # Validate dimensions
+            if width < self.min_width or height < self.min_height:
+                self.logger.warning(f"Video resolution too small: {width}x{height}")
+                return False
+            
+            # Validate duration
+            duration = frame_count / fps if fps > 0 else 0
+            if duration < self.min_duration:
+                self.logger.warning(f"Video too short: {duration:.2f}s")
+                return False
+            
+            if self.max_duration and duration > self.max_duration:
+                self.logger.warning(f"Video too long: {duration:.2f}s")
+                return False
+            
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"Video validation error: {str(e)}")
+            return False
+    
+    def clean(self, video_path: str) -> Any:
+        """
+        Clean video data (basic validation)
+        
+        Args:
+            video_path: Path to video file
+            
+        Returns:
+            OpenCV VideoCapture object or None
+        """
+        try:
+            cap = self.cv2.VideoCapture(str(video_path))
+            if not cap.isOpened():
+                raise Exception(f"Cannot open video: {video_path}")
+            
+            return cap
+            
+        except Exception as e:
+            self.logger.error(f"Video cleaning error: {str(e)}")
+            return None
+    
+    def normalize(self, video_cap: Any) -> Any:
+        """
+        Normalize video properties
+        
+        Args:
+            video_cap: OpenCV VideoCapture object
+            
+        Returns:
+            VideoCapture with normalized properties
+        """
+        # Note: OpenCV doesn't allow changing FPS on the fly
+        # Normalization happens during frame extraction
+        return video_cap
+    
+    def extract_frames(
+        self,
+        video_path: str,
+        output_dir: str,
+        sample_rate: int = 1,
+    ) -> List[str]:
+        """
+        Extract frames from video
+        
+        Args:
+            video_path: Path to video file
+            output_dir: Directory to save frames
+            sample_rate: Extract every Nth frame
+            
+        Returns:
+            List of extracted frame paths
+        """
+        if not self.opencv_available:
+            self.logger.error("OpenCV is required")
+            return []
+        
+        try:
+            Path(output_dir).mkdir(parents=True, exist_ok=True)
+            
+            cap = self.cv2.VideoCapture(str(video_path))
+            frame_count = 0
+            extracted_count = 0
+            extracted_frames = []
+            
+            while True:
+                ret, frame = cap.read()
+                if not ret:
+                    break
+                
+                if frame_count % sample_rate == 0:
+                    # Resize frame
+                    resized = self._resize_frame(frame)
+                    
+                    frame_path = Path(output_dir) / f"frame_{extracted_count:06d}.jpg"
+                    self.cv2.imwrite(str(frame_path), resized)
+                    extracted_frames.append(str(frame_path))
+                    extracted_count += 1
+                
+                frame_count += 1
+            
+            cap.release()
+            self.logger.info(f"Extracted {extracted_count} frames from video")
+            
+            return extracted_frames
+            
+        except Exception as e:
+            self.logger.error(f"Frame extraction error: {str(e)}")
+            return []
+    
+    def _resize_frame(self, frame: Any) -> Any:
+        """
+        Resize a single frame while preserving aspect ratio
+        
+        Args:
+            frame: OpenCV frame
+            
+        Returns:
+            Resized frame
+        """
+        if self.preserve_aspect_ratio:
+            h, w = frame.shape[:2]
+            scale = min(
+                self.target_resolution[0] / w,
+                self.target_resolution[1] / h
+            )
+            new_w = int(w * scale)
+            new_h = int(h * scale)
+            resized = self.cv2.resize(frame, (new_w, new_h), interpolation=self.cv2.INTER_LANCZOS4)
+            
+            # Pad to target size
+            top = (self.target_resolution[1] - new_h) // 2
+            bottom = self.target_resolution[1] - new_h - top
+            left = (self.target_resolution[0] - new_w) // 2
+            right = self.target_resolution[0] - new_w - left
+            
+            padded = self.cv2.copyMakeBorder(
+                resized, top, bottom, left, right,
+                self.cv2.BORDER_CONSTANT, value=[0, 0, 0]
+            )
+            return padded
+        else:
+            return self.cv2.resize(frame, self.target_resolution, interpolation=self.cv2.INTER_LANCZOS4)
+    
+    def get_video_stats(self, video_path: str) -> Dict[str, Any]:
+        """
+        Get statistics about video
+        
+        Args:
+            video_path: Path to video file
+            
+        Returns:
+            Dictionary with video statistics
+        """
+        try:
+            if not self.opencv_available:
+                return {}
+            
+            cap = self.cv2.VideoCapture(str(video_path))
+            
+            fps = cap.get(self.cv2.CAP_PROP_FPS)
+            width = int(cap.get(self.cv2.CAP_PROP_FRAME_WIDTH))
+            height = int(cap.get(self.cv2.CAP_PROP_FRAME_HEIGHT))
+            frame_count = int(cap.get(self.cv2.CAP_PROP_FRAME_COUNT))
+            
+            cap.release()
+            
+            duration = frame_count / fps if fps > 0 else 0
+            
+            stats = {
+                'file': str(video_path),
+                'width': width,
+                'height': height,
+                'fps': float(fps),
+                'frame_count': frame_count,
+                'duration_seconds': float(duration),
+                'resolution': f"{width}x{height}",
+                'file_size_bytes': Path(video_path).stat().st_size,
+                'aspect_ratio': width / height if height > 0 else 0,
+            }
+            
+            return stats
+            
+        except Exception as e:
+            self.logger.error(f"Error getting video stats: {str(e)}")
+            return {}
+    
+    def batch_process_directory(
+        self,
+        input_dir: str,
+        output_dir: Optional[str] = None,
+        extract_frames: bool = False,
+        frame_sample_rate: int = 30,  # Extract every 30th frame
+    ) -> Dict[str, Any]:
+        """
+        Process all videos in a directory
+        
+        Args:
+            input_dir: Input directory path
+            output_dir: Output directory path
+            extract_frames: Whether to extract frames
+            frame_sample_rate: Sample rate for frame extraction
+            
+        Returns:
+            Processing results
+        """
+        if not self.opencv_available:
+            self.logger.error("OpenCV is required")
+            return {'error': 'OpenCV not available'}
+        
+        output_dir = output_dir or self.config.output_path
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        
+        input_path = Path(input_dir)
+        results = []
+        
+        for video_file in input_path.iterdir():
+            if video_file.suffix.lower()[1:] not in self.supported_formats:
+                continue
+            
+            try:
+                # Validate
+                if not self.validate(str(video_file)):
+                    self.stats['skipped'] += 1
+                    continue
+                
+                # Process video
+                cap = self.clean(str(video_file))
+                if cap is None:
+                    raise Exception("Failed to open video")
+                
+                normalized = self.normalize(cap)
+                cap.release()
+                
+                # Extract frames if requested
+                frame_list = []
+                if extract_frames:
+                    frames_dir = Path(output_dir) / video_file.stem / "frames"
+                    frame_list = self.extract_frames(
+                        str(video_file),
+                        str(frames_dir),
+                        sample_rate=frame_sample_rate
+                    )
+                
+                self.stats['output_count'] += 1
+                
+                results.append({
+                    'file': str(video_file),
+                    'status': 'success',
+                    'frames_extracted': len(frame_list),
+                    'stats': self.get_video_stats(str(video_file))
+                })
+                
+            except Exception as e:
+                self.logger.error(f"Error processing {video_file}: {str(e)}")
+                self.stats['errors'] += 1
+                results.append({
+                    'file': str(video_file),
+                    'status': 'error',
+                    'error': str(e)
+                })
+        
+        self.logger.info(f"Processed {self.stats['output_count']} videos")
+        return {'results': results, 'stats': self.get_statistics()}

{gptmed-0.5.5.dist-info → gptmed-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gptmed
-Version: 0.5.5
+Version: 0.7.0
 Summary: A lightweight GPT-based language model framework for training custom question-answering models on any domain
 Author-email: Sanjog Sigdel <sigdelsanjog@gmail.com>, Sanjog Sigdel <sanjog.sigdel@ku.edu.np>
 License-Expression: MIT
@@ -45,6 +45,13 @@ Requires-Dist: matplotlib>=3.5.0; extra == "xai"
 Requires-Dist: seaborn>=0.12.0; extra == "xai"
 Requires-Dist: captum>=0.6.0; extra == "xai"
 Requires-Dist: scikit-learn>=1.0.0; extra == "xai"
+Provides-Extra: data-preparation
+Requires-Dist: pillow>=9.0.0; extra == "data-preparation"
+Requires-Dist: librosa>=0.10.0; extra == "data-preparation"
+Requires-Dist: soundfile>=0.12.0; extra == "data-preparation"
+Requires-Dist: opencv-python>=4.5.0; extra == "data-preparation"
+Requires-Dist: numpy>=1.24.0; extra == "data-preparation"
+Requires-Dist: PyPDF2>=3.0.0; extra == "data-preparation"
 Dynamic: license-file
 
 # GptMed 🤖

{gptmed-0.5.5.dist-info → gptmed-0.7.0.dist-info}/RECORD

@@ -9,6 +9,26 @@ gptmed/data/__init__.py,sha256=iAHeakB5pBAd7MkmarPPY0UKS9bTaO_winLZ23Y2O90,54
 gptmed/data/parsers/__init__.py,sha256=BgVzXuZgeE5DUCC4SzN7vflL40wQ4Q4_4DmJ1Y43_nw,211
 gptmed/data/parsers/medquad_parser.py,sha256=g3QCRiVBdcq8RdyuYH_qKFrHgU5KkHY59WfWxUwspP0,7974
 gptmed/data/parsers/text_formatter.py,sha256=tVmnDBT54BbxX9BPKMXSPzzLmM39frDxKRKuz_HoRag,4072
+gptmed/data_preparation/__init__.py,sha256=FlrFDgTe64dzblCodV0vvpxbFsvZg3ewycp8yj7gxTo,988
+gptmed/data_preparation/base.py,sha256=UnkdTE3rZd2LFRLqgvErhhIYpJlYQRPHcWfCaziKj9E,5360
+gptmed/data_preparation/cli.py,sha256=MUUEMmZE6_FLU-xEZlxmwuVm61C-kbpP-pknieMe5vU,13570
+gptmed/data_preparation/audio/__init__.py,sha256=SAzX2jmjnEXlPWB59u_EF2wEA4ENL3Z4Pz0qsitWWR4,11871
+gptmed/data_preparation/image/__init__.py,sha256=GLVmPvEEeBpXFz1FahTn94PgMKvBJUzgS7hmYQ2Qp0U,9871
+gptmed/data_preparation/text/__init__.py,sha256=zUINytRdxHn4BwhN0nD7QkmjBVJfGmrrN6d6fxNqBik,11470
+gptmed/data_preparation/text/base_strategy.py,sha256=12L946oWKwQmp2P9Yq40F0cVU6KS6XBmNUMZLGmuo5s,634
+gptmed/data_preparation/text/batch_pdf_to_jsonl.py,sha256=f0loZuq5_SNvFDtB1aeiCG1ykTPL0fqTTRX45RtbBjQ,8628
+gptmed/data_preparation/text/case_normalizer.py,sha256=WefsM2dSRq39R3MjcFL-aQmCEYJvqEbrnyKxjjOGs0s,1768
+gptmed/data_preparation/text/pdf_processor.py,sha256=4J1crK51_cgu6HfCwsOwHEiYPquwueRhgFAfGYGvhDA,10545
+gptmed/data_preparation/text/pipeline.py,sha256=Jj999_kSxFwNe1nGICk_ALOJpjH1cRYeDkutQr8c8Ak,11583
+gptmed/data_preparation/text/preprocess_jsonl.py,sha256=HIXSJjlL6QanFlhG2i7o1Xr1fpYoUy3iQkwpU5Oj1CE,16229
+gptmed/data_preparation/text/punctuation_handler.py,sha256=zPfuHR0cQIIc8HCLWA7zFsI86O-fo_7oUkCPq-GkmH4,1957
+gptmed/data_preparation/text/stopword_remover.py,sha256=FGf8utJ11gAjX2DqpVsa_u7ir-DHr1ybBTc9lu47wO0,2670
+gptmed/data_preparation/text/text_cleaner.py,sha256=vrvt4yGK77AleDmWwRJKmflPbOYuh66AK2QStn9xHcY,2234
+gptmed/data_preparation/text/text_statistics.py,sha256=-DqW4221wt_Jc1erEeoacNMcQVpzLjyO9o9ElSUMICU,4223
+gptmed/data_preparation/text/tokenize_jsonl.py,sha256=54MImqrSln9aLNhi5PHEJwgXhuAIqByvSPxE3CCu854,13801
+gptmed/data_preparation/text/tokenizer.py,sha256=ukweQTR3jjUvV_gYVkDYKAtEBkES9pf8oJjh4DA252k,1958
+gptmed/data_preparation/text/unicode_normalizer.py,sha256=Ck5NcGggNePpMNSr-8_0YOrIo3ZgdtlBlC2fZPo3JDY,1656
+gptmed/data_preparation/video/__init__.py,sha256=nKinO9FuPbjl7oJlJcbUe2qCmShuUMpZEcuRW32k_8w,14310
 gptmed/framework/__init__.py,sha256=TlzM7NS_n0KQnm9PQTJRrb5pEb6rBXC1pqGPhbSO_bQ,25
 gptmed/framework/cli/__init__.py,sha256=oBUmoaWLCjFs3_aod-hcMCcC11UP4t4SohDnZ7Sdmx0,729
 gptmed/framework/cli/__main__.py,sha256=rLBZjEi695ZgOW8pypqpg2kLgtcDhrI_9_QcrUO3WkU,103
@@ -46,9 +66,9 @@ gptmed/training/utils.py,sha256=pJxCwneNr2STITIYwIDCxRzIICDFOxOMzK8DT7ck2oQ,5651
 gptmed/utils/__init__.py,sha256=XuMhIqOXF7mjnog_6Iky-hSbwvFb0iK42B4iDUpgi0U,44
 gptmed/utils/checkpoints.py,sha256=jPKJtO0YRZieGmpwqotgDkBzd__s_raDxS1kLpfjBJE,7113
 gptmed/utils/logging.py,sha256=7dJc1tayMxCBjFSDXe4r9ACUTpoPTTGsJ0UZMTqZIDY,5303
-gptmed-0.5.5.dist-info/licenses/LICENSE,sha256=v2spsd7N1pKFFh2G8wGP_45iwe5S0DYiJzG4im8Rupc,1066
-gptmed-0.5.5.dist-info/METADATA,sha256=-7Drfeaxy2SWQ3nlnHqwP4pj1q9bsrmTxe99pIWMePk,13842
-gptmed-0.5.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-gptmed-0.5.5.dist-info/entry_points.txt,sha256=ZZeYg2kOQuHHvRvQYRvq5L-RpClnBMHSpUom9DxQW0c,145
-gptmed-0.5.5.dist-info/top_level.txt,sha256=mhyEq3rG33t21ziJz5w3TPgx0RjPf4zXMNUx2JTiNmE,7
-gptmed-0.5.5.dist-info/RECORD,,
+gptmed-0.7.0.dist-info/licenses/LICENSE,sha256=v2spsd7N1pKFFh2G8wGP_45iwe5S0DYiJzG4im8Rupc,1066
+gptmed-0.7.0.dist-info/METADATA,sha256=7tYxcCOQEd5RCsIRgEa7VbNk_FdNjTMdmZfUj6-pFdk,14236
+gptmed-0.7.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+gptmed-0.7.0.dist-info/entry_points.txt,sha256=AFrr1BzQTo-kc4cYE-uq4qcpUq1AzMyHDKojXb5xXa0,197
+gptmed-0.7.0.dist-info/top_level.txt,sha256=mhyEq3rG33t21ziJz5w3TPgx0RjPf4zXMNUx2JTiNmE,7
+gptmed-0.7.0.dist-info/RECORD,,

{gptmed-0.5.5.dist-info → gptmed-0.7.0.dist-info}/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
+data-preparation = gptmed.data_preparation.cli:main
 gptmed = gptmed.framework.cli:main
 gptmed-generate = gptmed.inference.generator:main
 gptmed-train = gptmed.training.train:main