text2rel 0.1__py3-none-any.whl

text2rel/cleaner.py

@@ -0,0 +1,909 @@
+
+import os
+import sys
+import json
+import numpy as np
+import pandas as pd
+import re
+from bs4 import BeautifulSoup
+import ast
+import string
+from pathlib import Path
+from text2rel.utils import insert_jumpjump
+from text2rel.io import _load_inventory, _save_inventory
+from typing import Dict
+
+
+
+class HTMLCleaner:
+    def __init__(self, json_path: str, html_directory: str = None, extension: str = ".htm"):
+        """
+        Initializes the cleaner. If the JSON doesn't exist but a directory is given, it builds the inventory.
+        """
+        if not json_path:
+            raise ValueError("json_path parameter is required. Please provide a path to your JSON inventory file. It can be simply and empty JSON file")
+        if not html_directory:
+            raise ValueError("html_directory parameter is required. Please provide a path to your HTML files directory.")
+        self.json_path = Path(json_path)
+
+        if not self.json_path.exists():
+            if html_directory is None:
+                raise FileNotFoundError(f"JSON not found at {json_path} and no directory given to create one.")
+            self._create_inventory(html_directory, extension)
+            print(f"Created new inventory at {json_path}")
+
+        self.inventory = _load_inventory(self.json_path)
+
+    #Creates the inventory from a given directory of HTML files.
+    def _create_inventory(self, directory: str, extension: str = ".htm"):
+        directory = Path(directory)
+        if not directory.exists():
+            raise FileNotFoundError(f"Directory {directory} does not exist.")
+
+        data = []
+        for root, _, files in os.walk(directory):
+            for file in files:
+                if file.endswith(extension):
+                    data.append({
+                        "file_path": str(Path(root) / file),
+                        "relevant_beginning": None, #Needs to be manually set by the user.
+                        "relevant_end": None, #Needs to be manually set by the user.
+                        "font_usage": [] #Gets updated during cleaning.
+                    })
+
+        inventory = {"files": data}
+        with open(self.json_path, 'w', encoding='utf-8') as f:
+            json.dump(inventory, f, indent=4)
+
+    #Lists all files in the inventory with their index for easy reference.
+    def list_files(self):
+        """
+        Prints all file paths with their corresponding index for easy reference.
+        """
+        print("Inventory File Index:")
+        for idx, file_info in enumerate(self.inventory['files']):
+            print(f"{idx}: {Path(file_info['file_path']).name}")
+
+    #Allowing the user to set the relevant beginning and end for each file, the files can be accessed by filename
+    def set_relevant_bounds(self, filename: str, beginning, end):
+        for file in self.inventory['files']:
+            if file['file_path'].endswith(filename):
+                file['relevant_beginning'] = beginning
+                file['relevant_end'] = end
+                _save_inventory(self.json_path,self.inventory)
+                return
+        raise ValueError(f"File {filename} not found in inventory.")
+    
+    def analyze_html_fonts(self, file_index: int):
+        """
+        Analyzes the HTML content of a file's relevant section to extract font usage statistics.
+        Parameters:
+        - file_index (int): Index of the file in the inventory
+        Updates:
+        - Updates the `font_usage` field in the JSON for the selected file.
+        """
+        file_info = self.inventory['files'][file_index]
+        html_path = Path(file_info['file_path'])
+
+        if not html_path.exists():
+            print(f"File not found: {html_path}")
+            return
+
+        if file_info['relevant_beginning'] is None or file_info['relevant_end'] is None:
+            print(f"relevant_beginning and relevant_end must be set first.")
+            return
+
+        with open(html_path, 'r', encoding='utf-8') as f:
+            html_content = f.read()
+
+        relevant_text = html_content[file_info['relevant_beginning']:file_info['relevant_end']-1]
+        soup = BeautifulSoup(relevant_text, 'html.parser')
+
+        # Match font classes like font1, font12, etc.
+        pattern = re.compile(r'font\d{1,2}')
+        class_stats = {}
+
+        for tag in soup.find_all(class_=pattern):
+            class_list = tag.get('class', [])
+            for cls_name in class_list:
+                if not pattern.match(cls_name):
+                    continue
+                text = tag.get_text(strip=True)
+                if cls_name in class_stats:
+                    class_stats[cls_name]['total_length'] += len(text)
+                    class_stats[cls_name]['count'] += 1
+                else:
+                    class_stats[cls_name] = {'total_length': len(text), 'count': 1}
+
+        # Format and save to inventory
+        font_usage = []
+        for cls, stats in class_stats.items():
+            font_usage.append({
+                'class': cls,
+                'instances': stats['count'],
+                'average_length': stats['total_length'] / stats['count'],
+                'usage': ""  # Still to be set by user
+            })
+
+        file_info['font_usage'] = font_usage
+        _save_inventory(self.json_path, self.inventory)
+        #Display the fonts to the user
+        df = pd.DataFrame(font_usage)
+        print(df)
+        print(f"Font usage statistics for {Path(file_info['file_path']).name} updated.")
+
+
+        #Display the results as a DataFrame
+        
+
+    def classify_fonts_usage(self, index: int, usages: dict):
+        """
+        Classifies font usage for a specific file in the inventory.
+
+        Parameters:
+        - index (int): The index of the file in the inventory.
+        - usages (dict): Dictionary of font classes to usage labels (e.g., {'font3': 'MainText'}).
+        """
+
+        if 0 <= index < len(self.inventory['files']):
+            file_info = self.inventory['files'][index]
+            for font, usage_type in usages.items():
+                for font_entry in file_info.get('font_usage', []):
+                    if font_entry['class'] == font:
+                        font_entry['usage'] = usage_type
+                        break
+
+            _save_inventory(self.json_path,self.inventory)
+            print("Updated font usage for file index", index)
+        else:
+            print(f"Invalid index: {index}")
+    
+        #Remove all the
+
+    #Converts the inventory into a pandas DataFrame for easy analysis if the user desires it.
+
+    def to_dataframe(self):
+        """
+        Converts the inventory into a pandas DataFrame.
+
+        Returns:
+        - pd.DataFrame: Columns include File, RelevantBeginning, RelevantEnd, class, instances, average_length, Usage.
+        """
+        records = []
+
+        for file_info in self.inventory['files']:
+            for font_entry in file_info.get('font_usage', []):
+                records.append({
+                    'File': file_info['file_path'],
+                    'RelevantBeginning': file_info['relevant_beginning'],
+                    'RelevantEnd': file_info['relevant_end'],
+                    'class': font_entry['class'],
+                    'instances': font_entry['instances'],
+                    'average_length': font_entry['average_length'],
+                    'Usage': font_entry['usage']
+                })
+                
+    def selecting_text_chunks(self):
+        json_path = self.json_path
+
+        for file_info in self.inventory['files']:
+            html_file_path = file_info['file_path']
+
+            # Skip files without bounds or font usage set
+            if file_info.get('relevant_beginning') is None or file_info.get('relevant_end') is None:
+                continue
+            if not any(entry.get('usage') == 'MainText' for entry in file_info.get('font_usage', [])):
+                continue
+
+            try:
+                with open(html_file_path, 'r', encoding='utf-8') as f:
+                    html_content = f.read()
+            except FileNotFoundError:
+                print(f"File not found: {html_file_path}")
+                continue
+
+            print(f"Processing: {Path(html_file_path).name}")
+
+            # Slice relevant section, based on the markings in relevant beginning and relevant end
+            relevant_text = html_content[file_info['relevant_beginning']:file_info['relevant_end']]
+            soup = BeautifulSoup(relevant_text, 'html.parser')
+
+            # Step 1: Remove (decompose) elements with 'Rest' usage - they are not needed
+            for font_entry in file_info.get('font_usage', []):
+                if font_entry.get('usage') == 'Rest':
+                    class_name = font_entry['class']
+                    for el in soup.find_all(class_=class_name):
+                        el.decompose()
+
+            # Step 2: Insert JumPJumP before Title classes
+            title_count = 0
+            for font_entry in file_info.get('font_usage', []):
+                if font_entry.get('usage') == 'Title':
+                    class_name = font_entry['class']
+                    for el in soup.find_all(class_=class_name):
+                        el.insert_before(soup.new_string("JumPJumP"))
+                        title_count += 1
+            
+            # Step 3: Extract ALL remaining text (includes JumPJumP markers and titles)
+            text = soup.get_text()
+            
+            print(f"   → {title_count} title elements found (JumPJumP inserted before each)")
+
+
+            # Clean up the text
+            text = re.sub(r'\n+', '\n', text)
+            text = re.sub(r'[ ]+', ' ', text)
+
+            excluded_punctuation = re.escape(string.punctuation.replace('.', ''))
+            text = re.sub(
+                fr"([a-záéíóúü]) ?([{excluded_punctuation}])?\n([{excluded_punctuation}])? ?([a-záéíóúü])",
+                r'\1 \4',
+                text,
+                flags=re.IGNORECASE
+            )
+
+            # Count JumPJumP from titles only
+            jumps_from_titles = text.count("JumPJumP")
+
+            # Note: Length-based chopping is handled in apply_file_specific_rules() 
+            # based on the LengthChop flag in the protocol CSV
+
+            # Merge short sections unnecessarily split by jumps
+            while True:
+                new_text = re.sub(r'JumPJumP(.{1,500})JumPJumP', r'JumPJumP\1', text, flags=re.DOTALL)
+                if new_text == text:
+                    break
+
+                text = new_text
+
+            # Consolidate multiple consecutive JumPJumP markers into one
+            text = re.sub(r'(\n*JumPJumP\n*)+', '\nJumPJumP\n', text)
+
+            # Split by marker and embed directly in JSON
+            text_sections = [t.strip() for t in re.split(r'\n*JumPJumP\n*', text) if t.strip()]
+            file_info['cleaned_subsections'] = text_sections
+            
+            print(f"   → JumPJumP from titles: {jumps_from_titles}")
+            print(f"   → Initial subsections: {len(text_sections)}")
+
+        # Save updated JSON
+        _save_inventory(json_path, self.inventory)
+
+        print("Cleaned text added to each file entry in the JSON.")
+
+
+###_____________REMOVE WHEN EVERYTHING REGARDING FILE-SPECIFIC RULES IS FULLY TESTED AND WORKING________________####
+    # def apply_cleaning_rules(self, file_index, rule_keys: list = None, multiline: bool = False):
+    #     """
+    #     Applies selected regex cleaning rules from cleaning_rules.py to each subsection.
+
+    #     Parameters:
+    #     - file_index (int): The index of the file in the inventory
+    #     - rule_keys (list[str]): List of rule category keys to apply. 
+    #                         If None or empty, no rules will be applied.
+    #     - multiline (bool): Whether to apply rules across multiple lines using re.DOTALL
+
+    #     Returns:
+    #     - list[str]: List of cleaned text sections
+    #     """
+    #     # Get the file info from inventory
+    #     file_info = self.inventory['files'][file_index]
+
+    #     # Skip if no rules specified
+    #     if not rule_keys:
+    #         return file_info["cleaned_subsections"]
+
+    #     cleaned_sections = []
+
+    #     # Process each subsection
+    #     for section in file_info["cleaned_subsections"]:
+    #         cleaned_text = section
+
+    #         # Apply each rule category
+    #         for key in rule_keys:
+    #             rules = CLEANING_RULES.get(key, [])
+    #             print(f"Applying rules for {key}")
+                
+    #             # Apply each pattern/replacement pair in the rule category
+    #             for pattern, repl in rules:
+    #                 if multiline:
+    #                     print(f"Applying multiline rule: {pattern} -> {repl}")
+    #                     # Use re.DOTALL to match across multiple lines
+    #                     cleaned_text = re.sub(pattern, repl, cleaned_text, flags=re.DOTALL)
+    #                 else:
+    #                     # Use default behavior (single line)
+    #                     cleaned_text = re.sub(pattern, repl, cleaned_text)
+
+    #         cleaned_sections.append(cleaned_text)
+
+    #     # Update the cleaned subsections in the inventory
+    #     file_info["cleaned_subsections"] = cleaned_sections
+
+    #     # Save changes to JSON
+    #     with open(self.json_path, 'w', encoding='utf-8') as f:
+    #         json.dump(self.inventory, f, indent=4, ensure_ascii=False)
+
+    #     return cleaned_sections
+
+    def apply_file_specific_rules(self, file_index: int):
+        """
+        Applies file-specific regex rules from the attached protocol CSV.
+        """
+        if not hasattr(self, "protocol_df"):
+            print("No protocol attached. Use attach_protocol() first.")
+            return
+
+        file_info = self.inventory["files"][file_index]
+        filename = Path(file_info["file_path"]).name
+
+        # Find matching protocol entry by filename
+        matches = self.protocol_df[
+            self.protocol_df['HTML_Location'].apply(lambda p: Path(p).name == filename)
+        ]
+
+        if len(matches) == 0:
+            print(f"No protocol entry found for file: {filename}")
+            return
+        elif len(matches) > 1:
+            print(f"Multiple entries found for {filename}. Using the first match.")
+
+        match = matches.iloc[0]
+
+        # Parse the rules dictionary from the CSV
+        try:
+            rules = ast.literal_eval(match['patterns_replacements'])
+            if not isinstance(rules, dict):
+                raise ValueError("Parsed rules are not a dictionary")
+        except Exception as e:
+            print(f"Error parsing rules for {filename}: {e}")
+            return
+
+        # Skip if no rules or empty rules
+        if not rules or rules == {'': ''}:
+            return
+
+        # Get flags from protocol
+        length_chop = bool(int(match.get('LengthChop', 0)))
+        use_dotall = bool(int(match.get('LongReplacement', 0)))
+
+        # Get current subsections
+        text_sections = file_info.get("cleaned_subsections", [])
+        if not text_sections:
+            print(f"⚠ No cleaned_subsections found for: {filename} (skipping)")
+            return
+
+        print(f"Processing: {filename}")
+        print(f"   → {len(text_sections)} subsection(s) to process")
+        print(f"   → Rules to apply:")
+        for p, r in rules.items():
+            print(f"      '{p[:50]}...' → '{r[:30]}...'" if len(p) > 50 else f"      '{p}' → '{r}'")
+        if length_chop:
+            print(f"   → LengthChop enabled (will insert JumPJumP every 10k chars)")
+
+        cleaned_sections = []
+        total_modified = 0
+        total_regex_matches = 0
+
+        for text in text_sections:
+            original_text = text
+            section_matches = 0
+            
+            # Apply replacements and count matches
+            for pattern, repl in rules.items():
+                if pattern:  # Skip empty patterns
+                    matches_before = len(re.findall(pattern, text, flags=re.DOTALL if use_dotall else 0))
+                    section_matches += matches_before
+                    text = re.sub(pattern, repl, text, flags=re.DOTALL if use_dotall else 0)
+            
+            total_regex_matches += section_matches
+
+            # Track if any changes were made
+            if text != original_text:
+                total_modified += 1
+
+            # Insert jump markers for long sections (if LengthChop=1)
+            if length_chop:
+                text = insert_jumpjump(text, max_chars=10000)
+
+            # Consolidate multiple consecutive JumPJumP markers into one
+            text = re.sub(r'(\n*JumPJumP\n*)+', '\nJumPJumP\n', text)
+            
+            # Merge short sections (less than 500 chars) that were unnecessarily split
+            while True:
+                new_text = re.sub(r'JumPJumP(.{1,500})JumPJumP', r'JumPJumP\1', text, flags=re.DOTALL)
+                if new_text == text:
+                    break
+                text = new_text
+            
+            # Re-split by JumPJumP in case regex patterns added new markers
+            sub_sections = [t.strip() for t in re.split(r'\n*JumPJumP\n*', text) if t.strip()]
+            cleaned_sections.extend(sub_sections)
+
+        # Update the inventory with cleaned sections
+        file_info["cleaned_subsections"] = cleaned_sections
+        _save_inventory(self.json_path, self.inventory)
+        
+        print(f"   {total_regex_matches} regex matches applied to {total_modified}/{len(text_sections)} subsection(s)")
+        print(f"   Final subsections after re-split: {len(cleaned_sections)}")
+
+    def attach_protocol(self, csv_path: str):
+        """
+        Load and cache the protocol CSV that contains file-specific cleaning rules.
+        """
+        self.protocol_df = pd.read_csv(csv_path)
+        print(f"Protocol CSV loaded with {len(self.protocol_df)} entries.")
+
+    def export_cleaned_text(self, output_path: str = None, include_empty: bool = False):
+        """
+        Exports cleaned subsections to a separate JSON file for downstream use.
+        
+        Parameters:
+        - output_path (str): Path to save the cleaned JSON. 
+                            If None, uses '<original_name>_cleaned.json'
+        - include_empty (bool): Whether to include files with no cleaned_subsections
+        
+        Returns:
+        - str: Path to the exported file
+        """
+        if output_path is None:
+            # Generate default name from inventory path
+            stem = self.json_path.stem
+            output_path = self.json_path.parent / f"{stem}_cleaned.json"
+        else:
+            output_path = Path(output_path)
+        
+        cleaned_data = {
+            "source_inventory": str(self.json_path),
+            "exported_at": pd.Timestamp.now().isoformat(),
+            "files": []
+        }
+        
+        files_with_content = 0
+        total_subsections = 0
+        
+        for file_info in self.inventory["files"]:
+            subsections = file_info.get("cleaned_subsections", [])
+            
+            # Skip empty files unless include_empty is True
+            if not subsections and not include_empty:
+                continue
+            
+            cleaned_entry = {
+                "file_path": file_info["file_path"],
+                "filename": Path(file_info["file_path"]).name,
+                "subsection_count": len(subsections),
+                "cleaned_subsections": subsections
+            }
+            
+            cleaned_data["files"].append(cleaned_entry)
+            
+            if subsections:
+                files_with_content += 1
+                total_subsections += len(subsections)
+        
+        # Save to file
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(cleaned_data, f, indent=4, ensure_ascii=False)
+        
+        print(f"Exported cleaned text to: {output_path}")
+        print(f"   → {files_with_content} file(s) with content")
+        print(f"   → {total_subsections} total subsection(s)")
+        
+        return str(output_path)
+
+    def load_fonts_from_csv(self, csv_path: str):
+        """
+        Load font classifications from a comprehensive CSV file (like 231228_ProtocolForPreliminaryCleaning.csv)
+        and automatically assign them to all files in the inventory.
+        
+        This replaces the manual font assignment workflow with automated loading from the original protocol.
+        
+        Parameters:
+        - csv_path (str): Path to the CSV file containing font classifications
+        
+        Returns:
+        - dict: Statistics about the font assignment process
+        """
+        print("Loading font classification data from CSV...")
+        df_fonts = pd.read_csv(csv_path)
+        print(f"Loaded {len(df_fonts)} font classification records")
+        
+        # Statistics tracking
+        stats = {
+            'files_processed': 0,
+            'fonts_assigned': 0,
+            'files_matched': 0,
+            'files_not_found': [],
+            'font_usage_summary': {}
+        }
+        
+        # Group by file path for easier processing
+        print("\nGrouping font data by file...")
+        font_groups = df_fonts.groupby('File')
+        
+        # Process each file in the inventory
+        print("\nAssigning fonts to inventory files...")
+        
+        for file_idx, file_info in enumerate(self.inventory['files']):
+            file_path = file_info['file_path']
+            filename = Path(file_path).name
+            
+            print(f"Processing file {file_idx}: {filename}")
+            
+            # Find matching entries in the CSV (try both full path and filename)
+            matching_fonts = None
+            
+            # First try exact path match
+            if file_path in font_groups.groups:
+                matching_fonts = font_groups.get_group(file_path)
+            else:
+                # Try to find by filename in the CSV paths
+                for csv_path_key in font_groups.groups.keys():
+                    if Path(csv_path_key).name == filename:
+                        matching_fonts = font_groups.get_group(csv_path_key)
+                        print(f"  Matched by filename: {Path(csv_path_key).name}")
+                        break
+            
+            if matching_fonts is not None:
+                # Create font usage entries
+                font_usage = []
+                
+                for _, row in matching_fonts.iterrows():
+                    font_entry = {
+                        'class': row['class'],
+                        'instances': int(row['instances']),
+                        'average_length': float(row['average_length']),
+                        'usage': row['Usage']  # MainText, Title, Footnotes, Table, Rest
+                    }
+                    font_usage.append(font_entry)
+                    
+                    # Updateno, it has  usage summary statistics
+                    usage_type = row['Usage']
+                    if usage_type not in stats['font_usage_summary']:
+                        stats['font_usage_summary'][usage_type] = 0
+                    stats['font_usage_summary'][usage_type] += 1
+                
+                # Assign the font usage to the file
+                file_info['font_usage'] = font_usage
+                
+                # Also assign relevant bounds if available in CSV
+                if not matching_fonts.empty:
+                    first_row = matching_fonts.iloc[0]
+                    if pd.notna(first_row['RelevantBeginning']) and pd.notna(first_row['RelevantEnd']):
+                        file_info['relevant_beginning'] = int(first_row['RelevantBeginning'])
+                        file_info['relevant_end'] = int(first_row['RelevantEnd'])
+                        print(f"  Set bounds: {first_row['RelevantBeginning']} - {first_row['RelevantEnd']}")
+                
+                stats['files_matched'] += 1
+                stats['fonts_assigned'] += len(font_usage)
+                print(f"  Assigned {len(font_usage)} font classifications")
+                
+            else:
+                stats['files_not_found'].append(filename)
+                print(f"  Warning: No font data found for {filename}")
+            
+            stats['files_processed'] += 1
+        
+        # Save the updated inventory
+        print(f"\nSaving updated inventory to {self.json_path}...")
+        _save_inventory(self.json_path, self.inventory)
+        
+        # Print summary statistics
+        print("\n" + "="*50)
+        print("FONT ASSIGNMENT SUMMARY")
+        print("="*50)
+        print(f"Files processed: {stats['files_processed']}")
+        print(f"Files matched with font data: {stats['files_matched']}")
+        print(f"Total fonts assigned: {stats['fonts_assigned']}")
+        print(f"Files without font data: {len(stats['files_not_found'])}")
+        
+        if stats['files_not_found']:
+            print("\nFiles not found in CSV:")
+            for filename in stats['files_not_found']:
+                print(f"  - {filename}")
+        
+        print("\nFont usage distribution:")
+        for usage_type, count in stats['font_usage_summary'].items():
+            print(f"  {usage_type}: {count} fonts")
+        
+        return stats
+
+    def get_font_assignment_sample(self, csv_path: str, file_index: int = None):
+        """
+        Generate a sample assignment dictionary for a specific file, useful for verification.
+        
+        Parameters:
+        - csv_path (str): Path to the CSV file
+        - file_index (int): Index of file in inventory (if None, uses first file as example)
+        
+        Returns:
+        - dict: Assignment dictionary in the format used by classify_fonts_usage()
+        """
+        df_fonts = pd.read_csv(csv_path)
+        
+        if file_index is not None:
+            file_path = self.inventory['files'][file_index]['file_path']
+            filename = Path(file_path).name
+            file_fonts = df_fonts[df_fonts['File'].apply(lambda p: Path(p).name == filename)]
+        else:
+            # Use first file as example
+            first_file = df_fonts['File'].iloc[0]
+            file_fonts = df_fonts[df_fonts['File'] == first_file]
+            print(f"Showing example for: {Path(first_file).name}")
+        
+        assignment_dict = {}
+        for _, row in file_fonts.iterrows():
+            assignment_dict[row['class']] = row['Usage']
+        
+        return assignment_dict
+
+
+    def inspect_chunks(self, file_index, show_content=True, outlier_threshold=0.95, max_content_length=500):
+        """
+        Inspect chunks after text processing to identify outliers and patterns.
+        
+        Args:
+            file_index (int): Index of the file to inspect
+            show_content (bool): Whether to show actual chunk content
+            outlier_threshold (float): Percentile threshold for outlier detection (0.95 = top/bottom 5%)
+            max_content_length (int): Maximum characters to show in content preview
+        """
+        if file_index >= len(self.inventory['files']):
+            print(f"Error: File index {file_index} out of range")
+            return
+        
+        file_info = self.inventory['files'][file_index]
+        
+        # Check if chunks exist
+        if 'cleaned_subsections' not in file_info or not file_info['cleaned_subsections']:
+            print(f"No chunks found for file index {file_index}. Run selecting_text_chunks() first.")
+            return
+        
+        chunks = file_info['cleaned_subsections']
+        chunk_lengths = [len(chunk) for chunk in chunks]
+        
+        # Basic statistics
+        total_chunks = len(chunks)
+        avg_length = sum(chunk_lengths) / total_chunks
+        median_length = sorted(chunk_lengths)[total_chunks // 2]
+        min_length = min(chunk_lengths)
+        max_length = max(chunk_lengths)
+        std_dev = (sum((x - avg_length) ** 2 for x in chunk_lengths) / total_chunks) ** 0.5
+        
+        # Print summary
+        print(f"\nChunk Statistics for file: \"{file_info['file_path'].split('/')[-1]}\"")
+        print("=" * 60)
+        print(f"Total chunks: {total_chunks}")
+        print(f"Average length: {avg_length:,.0f} characters")
+        print(f"Median length: {median_length:,.0f} characters")
+        print(f"Shortest chunk: {min_length:,} characters")
+        print(f"Longest chunk: {max_length:,} characters")
+        print(f"Standard deviation: {std_dev:,.0f} characters")
+        
+        # Length distribution
+        print(f"\nLength Distribution:")
+        bins = [(0, 500), (500, 2000), (2000, 5000), (5000, float('inf'))]
+        for min_len, max_len in bins:
+            if max_len == float('inf'):
+                count = sum(1 for length in chunk_lengths if length > min_len)
+                label = f"> {min_len:,} chars:"
+            else:
+                count = sum(1 for length in chunk_lengths if min_len <= length < max_len)
+                label = f"{min_len:,}-{max_len:,}:"
+            
+            percentage = (count / total_chunks) * 100
+            print(f"  {label:<15} {count:3d} chunks ({percentage:5.1f}%)")
+        
+        if not show_content:
+            return
+        
+        # Find outliers
+        sorted_indices = sorted(range(len(chunk_lengths)), key=lambda i: chunk_lengths[i])
+        
+        # Bottom outliers
+        bottom_count = max(1, int((1 - outlier_threshold) * total_chunks))
+        shortest_indices = sorted_indices[:bottom_count]
+        
+        # Top outliers  
+        top_count = max(1, int((1 - outlier_threshold) * total_chunks))
+        longest_indices = sorted_indices[-top_count:]
+        
+        # Show shortest chunks
+        print(f"\n=== SHORTEST CHUNKS (bottom {((1-outlier_threshold)*100):.0f}%) ===")
+        for i, chunk_idx in enumerate(shortest_indices):
+            chunk = chunks[chunk_idx]
+            print(f"\nChunk {chunk_idx}: {len(chunk):,} characters")
+            preview = chunk[:max_content_length]
+            if len(chunk) > max_content_length:
+                preview += "..."
+            print(f"Content: '{preview}'")
+        
+        # Show longest chunks
+        print(f"\n=== LONGEST CHUNKS (top {((1-outlier_threshold)*100):.0f}%) ===")
+        for i, chunk_idx in enumerate(longest_indices):
+            chunk = chunks[chunk_idx]
+            print(f"\nChunk {chunk_idx}: {len(chunk):,} characters")
+            preview = chunk[:max_content_length]
+            if len(chunk) > max_content_length:
+                preview += "..."
+            print(f"Content: '{preview}'")
+
+
+    def show_chunk(self, file_index, chunk_index, full_content=False, context_chars=100):
+        """
+        Display specific chunk content with optional context.
+        
+        Args:
+            file_index (int): Index of the file
+            chunk_index (int): Index of the chunk to display
+            full_content (bool): Whether to show the full chunk or truncated version
+            context_chars (int): Number of characters to show before/after if not full_content
+        """
+        if file_index >= len(self.inventory['files']):
+            print(f"Error: File index {file_index} out of range")
+            return
+        
+        file_info = self.inventory['files'][file_index]
+        
+        # Check if chunks exist
+        if 'cleaned_subsections' not in file_info or not file_info['cleaned_subsections']:
+            print(f"No chunks found for file index {file_index}. Run selecting_text_chunks() first.")
+            return
+        
+        chunks = file_info['cleaned_subsections']
+        
+        if chunk_index >= len(chunks):
+            print(f"Error: Chunk index {chunk_index} out of range. File has {len(chunks)} chunks.")
+            return
+        
+        chunk = chunks[chunk_index]
+        
+        print(f"\nChunk {chunk_index} from file: \"{file_info['file_path'].split('/')[-1]}\"")
+        print(f"Length: {len(chunk):,} characters")
+        print("=" * 60)
+        
+        if full_content:
+            print(chunk)
+        else:
+            # Show truncated content with context
+            if len(chunk) <= context_chars * 2:
+                print(chunk)
+            else:
+                start_part = chunk[:context_chars]
+                end_part = chunk[-context_chars:]
+                middle_chars = len(chunk) - (context_chars * 2)
+                print(f"{start_part}")
+                print(f"\n... [{middle_chars:,} characters hidden] ...\n")
+                print(f"{end_part}")
+
+
+    def get_chunks_summary(self, file_index=None):
+        """
+        Get a quick summary of chunks across all files or a specific file.
+        
+        Args:
+            file_index (int, optional): Index of specific file. If None, summarizes all files.
+        
+        Returns:
+            dict: Summary statistics
+        """
+        if file_index is not None:
+            # Summary for specific file
+            if file_index >= len(self.inventory['files']):
+                return {"error": f"File index {file_index} out of range"}
+            
+            file_info = self.inventory['files'][file_index]
+            if 'cleaned_subsections' not in file_info or not file_info['cleaned_subsections']:
+                return {"error": "No chunks found for this file"}
+            
+            chunks = file_info['cleaned_subsections']
+            chunk_lengths = [len(chunk) for chunk in chunks]
+            
+            return {
+                "file_path": file_info['file_path'].split('/')[-1],
+                "total_chunks": len(chunks),
+                "total_characters": sum(chunk_lengths),
+                "avg_length": sum(chunk_lengths) / len(chunks),
+                "min_length": min(chunk_lengths),
+                "max_length": max(chunk_lengths),
+            }
+        
+        else:
+            # Summary for all files
+            all_summaries = []
+            for i, file_info in enumerate(self.inventory['files']):
+                if 'cleaned_subsections' in file_info and file_info['cleaned_subsections']:
+                    summary = self.get_chunks_summary(i)
+                    summary['file_index'] = i
+                    all_summaries.append(summary)
+            
+            return {
+                "files_with_chunks": len(all_summaries),
+                "total_files": len(self.inventory['files']),
+                "files": all_summaries
+            }
+
+     #FUNCTIONS USED FOR GPT-BASED EXTRACTION   
+     # 
+    def _save_extraction_results(self, extracted_data: Dict, output_path: str, file_index: int):
+        """Save extraction results to file with metadata"""
+        file_info = self.inventory['files'][file_index]
+        
+        result_data = {
+            "file_index": file_index,
+            "filename": Path(file_info['file_path']).name,
+            "extraction_timestamp": pd.Timestamp.now().isoformat(),
+            "total_chunks_processed": len(file_info.get('cleaned_subsections', [])),
+            "extracted_data": extracted_data
+        }
+        
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(result_data, f, indent=2, ensure_ascii=False)
+    def is_file_ready_for_llm(self, file_index: int) -> bool:
+        """Check if a file is ready for LLM processing"""
+        if file_index >= len(self.inventory['files']):
+            return False
+        
+        file_info = self.inventory['files'][file_index]
+        
+        has_bounds = 'relevant_beginning' in file_info and 'relevant_end' in file_info
+        has_fonts = any(font.get('usage') for font in file_info.get('fonts', {}).values()) if 'fonts' in file_info else any(font.get('usage') for font in file_info.get('font_usage', []))
+        has_chunks = len(file_info.get('cleaned_subsections', [])) > 0
+        
+        return has_bounds and has_fonts and has_chunks
+
+    def process_with_llm(self, file_index: int, 
+                    llm_processor: 'LLMProcessor',
+                        extraction_type: str = "relations",
+                        output_path: str = None) -> Dict:
+        """
+        Process cleaned subsections with LLM for information extraction
+        """
+        if file_index >= len(self.inventory['files']):
+            raise IndexError(f"File index {file_index} out of range")
+        
+        file_info = self.inventory['files'][file_index]
+        
+        if 'cleaned_subsections' not in file_info or not file_info['cleaned_subsections']:
+            raise ValueError(f"No cleaned subsections found. Run selecting_text_chunks() first.")
+        
+        if not llm_processor:
+            raise ValueError("LLM processor is required")
+        
+        # Use the correct method signature
+        extracted_data = llm_processor.extract_information(
+            file_info['cleaned_subsections'],
+            extraction_schema=None,  # Will use defaults
+            extraction_type=extraction_type
+        )
+        
+        # Save results
+        if output_path:
+            self._save_extraction_results(extracted_data, output_path, file_index)
+        
+        return extracted_data
+
+    def batch_process_with_llm(self, llm_processor: 'LLMProcessor',
+                            extraction_type: str = "relations",
+                            output_dir: str = "llm_results") -> Dict:
+        """
+        Process all files with cleaned subsections using LLM
+        """
+        results = {}
+        output_path = Path(output_dir)
+        output_path.mkdir(exist_ok=True)
+        
+        for idx, file_info in enumerate(self.inventory['files']):
+            if 'cleaned_subsections' in file_info and file_info['cleaned_subsections']:
+                try:
+                    file_results = self.process_with_llm(
+                        idx, llm_processor, extraction_type,
+                        output_path / f"file_{idx}_results.json"
+                    )
+                    results[idx] = file_results
+                    print(f"✓ Processed file {idx}: {Path(file_info['file_path']).name}")
+                except Exception as e:
+                    print(f"✗ Error processing file {idx}: {e}")
+                    results[idx] = {"error": str(e)}
+        
+        return results