BibexPy 2.0.0__py3-none-any.whl

bibex_core/modules/ml_utils.py

@@ -0,0 +1,332 @@
+import pandas as pd
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.multioutput import MultiOutputClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import MultiLabelBinarizer
+import re
+from colorama import Fore, Style
+
+# NLTK OPSIYONELDIR. Import veya veri (punkt/stopwords) yoksa builtin fallback'lere
+# dusulur; bu, cevrimdisi makinede ve ilk-calistirmada cokmemeyi garanti eder.
+# Veri indirme import-zamaninda DEGIL, yalniz ilk gercek kullanimda ve sessizce
+# denenir (paket import edilince ag erisimi/indirme tetiklenmez).
+try:
+    import nltk  # type: ignore
+except Exception:  # pragma: no cover
+    nltk = None
+
+# NLTK yoksa kullanilan builtin Ingilizce stopword listesi.
+_FALLBACK_STOPWORDS = frozenset((
+    "a about above after again against all am an and any are aren't as at be because been "
+    "before being below between both but by can cannot could couldn't did didn't do does "
+    "doesn't doing don't down during each few for from further had hadn't has hasn't have "
+    "haven't having he her here hers herself him himself his how i if in into is isn't it "
+    "its itself just me more most my myself no nor not now of off on once only or other our "
+    "ours ourselves out over own same she should shouldn't so some such than that the their "
+    "theirs them themselves then there these they this those through to too under until up "
+    "very was wasn't we were weren't what when where which while who whom why will with won't "
+    "would wouldn't you your yours yourself yourselves"
+).split())
+
+_nltk_ready = None  # None=denenmedi; True/False=sonuc (bir kez denenir)
+
+
+def _ensure_nltk() -> bool:
+    """punkt+stopwords mevcut mu? Ilk cagride sessizce indirmeyi dener.
+    Basarisizsa (cevrimdisi/izinsiz) False doner ve builtin fallback kullanilir."""
+    global _nltk_ready
+    if _nltk_ready is not None:
+        return _nltk_ready
+    if nltk is None:
+        _nltk_ready = False
+        return _nltk_ready
+    try:
+        try:
+            nltk.data.find('tokenizers/punkt')
+            nltk.data.find('corpora/stopwords')
+        except LookupError:
+            for pkg in ('punkt', 'punkt_tab', 'stopwords'):
+                try:
+                    nltk.download(pkg, quiet=True)
+                except Exception:
+                    pass
+            nltk.data.find('tokenizers/punkt')
+            nltk.data.find('corpora/stopwords')
+        _nltk_ready = True
+    except Exception:
+        _nltk_ready = False
+    return _nltk_ready
+
+
+def _tokenize(text: str) -> list:
+    """NLTK varsa word_tokenize; yoksa basit regex tokenizer."""
+    if _ensure_nltk():
+        try:
+            from nltk.tokenize import word_tokenize
+            return word_tokenize(text)
+        except Exception:
+            pass
+    return re.findall(r"[a-z]+", text)
+
+
+def _stopword_set() -> set:
+    """NLTK varsa Ingilizce stopword'leri; yoksa builtin fallback."""
+    if _ensure_nltk():
+        try:
+            from nltk.corpus import stopwords
+            return set(stopwords.words('english'))
+        except Exception:
+            pass
+    return set(_FALLBACK_STOPWORDS)
+
+
+def preprocess_text(text):
+    """Preprocess text data for ML"""
+    if pd.isna(text):
+        return ""
+    text = str(text).lower()
+    text = re.sub(r'[^a-zA-Z\s]', ' ', text)
+    text = ' '.join(text.split())
+    tokens = _tokenize(text)
+    stop_words = _stopword_set()
+    tokens = [token for token in tokens if token not in stop_words]
+    return ' '.join(tokens)
+
+def prepare_training_data(df, text_columns=['TI', 'AB']):
+    """Prepare text data for training"""
+    # Combine and preprocess text columns
+    combined_text = df[text_columns].fillna('').agg(' '.join, axis=1)
+    processed_text = combined_text.apply(preprocess_text)
+    
+    return processed_text
+
+def train_keyword_model(train_df, text_columns=['TI', 'AB']):
+    """Train model for keyword prediction"""
+    # Prepare text data
+    X = prepare_training_data(train_df, text_columns)
+    
+    # Prepare keywords data
+    mlb = MultiLabelBinarizer()
+    keywords = train_df['DE'].fillna('').str.split(';')
+    keywords = [[kw.strip() for kw in kws if kw.strip()] for kws in keywords]
+    y = mlb.fit_transform(keywords)
+    
+    # Create and train TF-IDF vectorizer
+    vectorizer = TfidfVectorizer(max_features=5000)
+    X_tfidf = vectorizer.fit_transform(X)
+    
+    # Train model
+    model = MultiOutputClassifier(RandomForestClassifier(n_estimators=100))
+    model.fit(X_tfidf, y)
+    
+    return vectorizer, model, mlb
+
+def train_subject_model(train_df, text_columns=['TI', 'AB']):
+    """Train model for subject category prediction"""
+    # Prepare text data
+    X = prepare_training_data(train_df, text_columns)
+    
+    # Prepare subject categories data
+    mlb = MultiLabelBinarizer()
+    subjects = train_df['SC'].fillna('').str.split(';')
+    subjects = [[subj.strip() for subj in subjs if subj.strip()] for subjs in subjects]
+    y = mlb.fit_transform(subjects)
+    
+    # Create and train TF-IDF vectorizer
+    vectorizer = TfidfVectorizer(max_features=5000)
+    X_tfidf = vectorizer.fit_transform(X)
+    
+    # Train model
+    model = MultiOutputClassifier(RandomForestClassifier(n_estimators=100))
+    model.fit(X_tfidf, y)
+    
+    return vectorizer, model, mlb
+
+def predict_keywords(text, vectorizer, model, mlb, threshold=0.3):
+    """Predict keywords for given text"""
+    # Preprocess text
+    processed_text = preprocess_text(text)
+    
+    # Transform text using vectorizer
+    X_tfidf = vectorizer.transform([processed_text])
+    
+    # Predict probabilities
+    y_pred_proba = model.predict_proba(X_tfidf)
+    
+    # Get keywords above threshold
+    keywords = []
+    for i, proba in enumerate(zip(*[estimator.predict_proba(X_tfidf)[0] for estimator in model.estimators_])):
+        if max(proba) >= threshold:
+            keywords.append(mlb.classes_[i])
+    
+    return '; '.join(keywords) if keywords else None
+
+def predict_subjects(text, vectorizer, model, mlb, threshold=0.3):
+    """Predict subject categories for given text"""
+    # Preprocess text
+    processed_text = preprocess_text(text)
+    
+    # Transform text using vectorizer
+    X_tfidf = vectorizer.transform([processed_text])
+    
+    # Predict probabilities
+    y_pred_proba = model.predict_proba(X_tfidf)
+    
+    # Get subjects above threshold
+    subjects = []
+    for i, proba in enumerate(zip(*[estimator.predict_proba(X_tfidf)[0] for estimator in model.estimators_])):
+        if max(proba) >= threshold:
+            subjects.append(mlb.classes_[i])
+    
+    return '; '.join(subjects) if subjects else None
+
+def train_keywords_plus_model(train_df, text_columns=['TI', 'AB']):
+    """Train model for Keywords Plus prediction"""
+    # Prepare text data
+    X = prepare_training_data(train_df, text_columns)
+    
+    # Prepare Keywords Plus data
+    mlb = MultiLabelBinarizer()
+    keywords_plus = train_df['ID'].fillna('').str.split(';')
+    keywords_plus = [[kw.strip() for kw in kws if kw.strip()] for kws in keywords_plus]
+    y = mlb.fit_transform(keywords_plus)
+    
+    # Create and train TF-IDF vectorizer
+    vectorizer = TfidfVectorizer(max_features=5000)
+    X_tfidf = vectorizer.fit_transform(X)
+    
+    # Train model
+    model = MultiOutputClassifier(RandomForestClassifier(n_estimators=100))
+    model.fit(X_tfidf, y)
+    
+    return vectorizer, model, mlb
+
+def predict_keywords_plus(text, vectorizer, model, mlb, threshold=0.3):
+    """Predict Keywords Plus for given text"""
+    # Preprocess text
+    processed_text = preprocess_text(text)
+    
+    # Transform text using vectorizer
+    X_tfidf = vectorizer.transform([processed_text])
+    
+    # Predict probabilities
+    y_pred_proba = model.predict_proba(X_tfidf)
+    
+    # Get keywords above threshold
+    keywords_plus = []
+    for i, proba in enumerate(zip(*[estimator.predict_proba(X_tfidf)[0] for estimator in model.estimators_])):
+        if max(proba) >= threshold:
+            keywords_plus.append(mlb.classes_[i])
+    
+    return '; '.join(keywords_plus) if keywords_plus else None
+
+def enrich_metadata_ml(df):
+    """Enrich metadata using ML models"""
+    print("\nStarting ML-based metadata enrichment...")
+    
+    # Create copies of the dataframe for training
+    train_df_keywords = df[df['DE'].notna()].copy()
+    train_df_keywords_plus = df[df['ID'].notna()].copy()  # New training set for Keywords Plus
+    train_df_subjects = df[df['SC'].notna()].copy()
+    
+    # Create a copy of the input dataframe
+    enriched_df = df.copy()
+    
+    # Initialize counters for ID field
+    original_empty_id = df['ID'].isna().sum()
+    
+    # Train models if enough training data is available
+    if len(train_df_keywords) > 10:
+        print(f"\n{Fore.CYAN}Training keyword prediction model...{Style.RESET_ALL}")
+        print(f"Using {len(train_df_keywords)} records for training")
+        kw_vectorizer, kw_model, kw_mlb = train_keyword_model(train_df_keywords)
+        
+        # Predict missing keywords
+        print(f"\n{Fore.CYAN}Predicting missing keywords...{Style.RESET_ALL}")
+        mask = enriched_df['DE'].isna()
+        total = mask.sum()
+        completed = 0
+        
+        for idx in enriched_df[mask].index:
+            text = ' '.join([
+                str(enriched_df.loc[idx, 'TI']) if pd.notna(enriched_df.loc[idx, 'TI']) else '',
+                str(enriched_df.loc[idx, 'AB']) if pd.notna(enriched_df.loc[idx, 'AB']) else ''
+            ])
+            if text.strip():
+                predicted_keywords = predict_keywords(text, kw_vectorizer, kw_model, kw_mlb)
+                if predicted_keywords:
+                    enriched_df.loc[idx, 'DE'] = predicted_keywords
+            completed += 1
+            if completed % 10 == 0:
+                print(f"Progress: {completed}/{total} records processed", end='\r')
+        print()  # New line after progress
+    
+    # Train Keywords Plus model if enough training data is available
+    if len(train_df_keywords_plus) > 10:
+        print(f"\n{Fore.CYAN}Training Keywords Plus prediction model...{Style.RESET_ALL}")
+        print(f"Using {len(train_df_keywords_plus)} records for training")
+        kw_plus_vectorizer, kw_plus_model, kw_plus_mlb = train_keywords_plus_model(train_df_keywords_plus)
+        
+        # Predict missing Keywords Plus
+        print(f"\n{Fore.CYAN}Predicting missing Keywords Plus...{Style.RESET_ALL}")
+        mask = enriched_df['ID'].isna()
+        total = mask.sum()
+        completed = 0
+        
+        for idx in enriched_df[mask].index:
+            text = ' '.join([
+                str(enriched_df.loc[idx, 'TI']) if pd.notna(enriched_df.loc[idx, 'TI']) else '',
+                str(enriched_df.loc[idx, 'AB']) if pd.notna(enriched_df.loc[idx, 'AB']) else ''
+            ])
+            if text.strip():
+                predicted_keywords_plus = predict_keywords_plus(text, kw_plus_vectorizer, kw_plus_model, kw_plus_mlb)
+                if predicted_keywords_plus:
+                    enriched_df.loc[idx, 'ID'] = predicted_keywords_plus
+            completed += 1
+            if completed % 10 == 0:
+                print(f"Progress: {completed}/{total} records processed", end='\r')
+        print()  # New line after progress
+    
+    # Train models if enough training data is available
+    if len(train_df_subjects) > 10:
+        print(f"\n{Fore.CYAN}Training subject category prediction model...{Style.RESET_ALL}")
+        print(f"Using {len(train_df_subjects)} records for training")
+        subj_vectorizer, subj_model, subj_mlb = train_subject_model(train_df_subjects)
+        
+        # Predict missing subject categories
+        print(f"\n{Fore.CYAN}Predicting missing subject categories...{Style.RESET_ALL}")
+        mask = enriched_df['SC'].isna()
+        total = mask.sum()
+        completed = 0
+        
+        for idx in enriched_df[mask].index:
+            text = ' '.join([
+                str(enriched_df.loc[idx, 'TI']) if pd.notna(enriched_df.loc[idx, 'TI']) else '',
+                str(enriched_df.loc[idx, 'AB']) if pd.notna(enriched_df.loc[idx, 'AB']) else ''
+            ])
+            if text.strip():
+                predicted_subjects = predict_subjects(text, subj_vectorizer, subj_model, subj_mlb)
+                if predicted_subjects:
+                    enriched_df.loc[idx, 'SC'] = predicted_subjects
+                    enriched_df.loc[idx, 'WC'] = predicted_subjects  # Copy to WC as well
+            completed += 1
+            if completed % 10 == 0:
+                print(f"Progress: {completed}/{total} records processed", end='\r')
+        print()  # New line after progress
+    
+    # Calculate enrichment statistics
+    stats = {
+        'total_records': len(df),
+        'original_empty_keywords': df['DE'].isna().sum(),
+        'original_empty_subjects': df['SC'].isna().sum(),
+        'original_empty_id': original_empty_id,
+        'enriched_empty_keywords': enriched_df['DE'].isna().sum(),
+        'enriched_empty_subjects': enriched_df['SC'].isna().sum(),
+        'enriched_empty_id': enriched_df['ID'].isna().sum(),
+        'keywords_filled': df['DE'].isna().sum() - enriched_df['DE'].isna().sum(),
+        'subjects_filled': df['SC'].isna().sum() - enriched_df['SC'].isna().sum(),
+        'id_filled': original_empty_id - enriched_df['ID'].isna().sum()
+    }
+    
+    return enriched_df, stats 

bibex_core/modules/post_process.py

@@ -0,0 +1,50 @@
+import pandas as pd
+import os
+
+def copy_cr_raw_to_cr(file_path: str) -> bool:
+    """
+    Copies CR_raw column to CR column
+    
+    Args:
+        file_path (str): Path to Excel file
+        
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    try:
+        # Read Excel file
+        df = pd.read_excel(file_path)
+        
+        # If CR_raw column exists and CR column is empty or doesn't exist
+        if 'CR_raw' in df.columns:
+            df['CR'] = df['CR_raw']
+            
+            # Save changes
+            df.to_excel(file_path, index=False)
+            print(f"\nCR_raw column copied to CR: {os.path.basename(file_path)}")
+            return True
+            
+    except Exception as e:
+        print(f"\nError: Failed to process file {file_path}: {str(e)}")
+    return False
+
+def process_merged_files(result_dir: str) -> None:
+    """
+    Copies CR_raw column to CR column in Merged_Bib.xlsx and 
+    Merged_API_Enriched_Bib.xlsx files in Cell_Files folder
+    
+    Args:
+        result_dir (str): Path to result directory
+    """
+    cell_files_dir = os.path.join(result_dir, "Cell_Files")
+    
+    if os.path.exists(cell_files_dir):
+        # Files to process
+        files_to_process = [
+            os.path.join(cell_files_dir, "Merged_Bib.xlsx"),
+            os.path.join(cell_files_dir, "Merged_API_Enriched_Bib.xlsx")
+        ]
+        
+        for file_path in files_to_process:
+            if os.path.exists(file_path):
+                copy_cr_raw_to_cr(file_path) 

bibex_core/modules/stats_utils.py

@@ -0,0 +1,216 @@
+import pandas as pd
+from datetime import datetime
+
+def generate_detailed_statistics(wos_df: pd.DataFrame, scopus_df: pd.DataFrame, merged_df: pd.DataFrame) -> dict:
+    """Generate detailed statistics"""
+    # Basic statistics
+    basic_stats = {
+        'General Statistics': {
+            'Total WoS Records': len(wos_df),
+            'Total Scopus Records': len(scopus_df),
+            'Merged Records': len(merged_df),
+            'Merge Rate': f"{(len(merged_df) / (len(wos_df) + len(scopus_df)) * 100):.2f}%"
+        }
+    }
+
+    # Column statistics
+    column_stats = {
+        'Column Statistics': {
+            'WoS Column Count': len(wos_df.columns),
+            'Scopus Column Count': len(scopus_df.columns),
+            'Merged Column Count': len(merged_df.columns),
+            'Common Column Count': len(set(wos_df.columns) & set(scopus_df.columns))
+        }
+    }
+
+    # Data quality statistics
+    quality_stats = {
+        'Data Quality': {
+            'WoS Empty Cell Rate': f"{(wos_df.isna().sum().sum() / (len(wos_df) * len(wos_df.columns)) * 100):.2f}%",
+            'Scopus Empty Cell Rate': f"{(scopus_df.isna().sum().sum() / (len(scopus_df) * len(scopus_df.columns)) * 100):.2f}%",
+            'Merged Empty Cell Rate': f"{(merged_df.isna().sum().sum() / (len(merged_df) * len(merged_df.columns)) * 100):.2f}%"
+        }
+    }
+
+    # Year-based statistics
+    if 'Year' in merged_df.columns:
+        year_stats = merged_df['Year'].value_counts().sort_index().to_dict()
+        year_distribution = {'Year Distribution': year_stats}
+    else:
+        year_distribution = {'Year Distribution': {'Year information not found': 0}}
+
+    return {**basic_stats, **column_stats, **quality_stats, **year_distribution}
+
+def generate_metadata_statistics(df: pd.DataFrame) -> pd.DataFrame:
+    """Generate quality statistics for metadata fields"""
+    metadata_desc = {
+        'AU': 'Author',
+        'DT': 'Document Type',
+        'PY': 'Publication Year',
+        'TI': 'Title',
+        'TC': 'Total Citation',
+        'SO': 'Journal',
+        'AB': 'Abstract',
+        'C1': 'Affiliation',
+        'DI': 'DOI',
+        'RP': 'Corresponding Author',
+        'ID': 'Keywords Plus',
+        'WC': 'Science Categories',
+        'DE': 'Keywords',
+        'LA': 'Language',
+        'CR': 'Cited References'
+    }
+    
+    stats = []
+    total_docs = len(df)
+    
+    for field, desc in metadata_desc.items():
+        if field in df.columns:
+            missing = df[field].isna().sum() + df[field].eq('').sum()
+            missing_pct = (missing / total_docs) * 100
+            
+            # Determine status based on missing percentage
+            if missing_pct == 0:
+                status = 'Excellent'
+            elif missing_pct < 1:
+                status = 'Very Good'
+            elif missing_pct < 5:
+                status = 'Good'
+            elif missing_pct < 20:
+                status = 'Acceptable'
+            elif missing_pct < 50:
+                status = 'Poor'
+            elif missing_pct < 90:
+                status = 'Critical'
+            else:
+                status = 'Completely Missing'
+            
+            stats.append({
+                'Field': field,
+                'Description': desc,
+                'Missing Count': missing,
+                'Missing %': round(missing_pct, 2),
+                'Status': status
+            })
+    
+    return pd.DataFrame(stats)
+
+def generate_metadata_comparison(simple_df: pd.DataFrame, enhanced_df: pd.DataFrame) -> pd.DataFrame:
+    """Compare metadata statistics between two merge methods"""
+    metadata_desc = {
+        'AU': 'Author',
+        'DT': 'Document Type',
+        'PY': 'Publication Year',
+        'TI': 'Title',
+        'TC': 'Total Citation',
+        'SO': 'Journal',
+        'AB': 'Abstract',
+        'C1': 'Affiliation',
+        'DI': 'DOI',
+        'RP': 'Corresponding Author',
+        'ID': 'Keywords Plus',
+        'WC': 'Science Categories',
+        'DE': 'Keywords',
+        'LA': 'Language',
+        'CR': 'Cited References'
+    }
+    
+    comparison_stats = []
+    
+    for field, desc in metadata_desc.items():
+        if field in simple_df.columns and field in enhanced_df.columns:
+            simple_missing = simple_df[field].isna().sum() + simple_df[field].eq('').sum()
+            enhanced_missing = enhanced_df[field].isna().sum() + enhanced_df[field].eq('').sum()
+            
+            simple_missing_pct = (simple_missing / len(simple_df)) * 100
+            enhanced_missing_pct = (enhanced_missing / len(enhanced_df)) * 100
+            
+            # Calculate improvement rate
+            improvement = simple_missing_pct - enhanced_missing_pct
+            improvement_pct = (improvement / simple_missing_pct * 100) if simple_missing_pct > 0 else 0
+            
+            # Determine status based on enhanced method results
+            if enhanced_missing_pct == 0:
+                status = 'Excellent'
+            elif enhanced_missing_pct < 1:
+                status = 'Very Good'
+            elif enhanced_missing_pct < 5:
+                status = 'Good'
+            elif enhanced_missing_pct < 20:
+                status = 'Acceptable'
+            elif enhanced_missing_pct < 50:
+                status = 'Poor'
+            elif enhanced_missing_pct < 90:
+                status = 'Critical'
+            else:
+                status = 'Completely Missing'
+            
+            comparison_stats.append({
+                'Field': field,
+                'Description': desc,
+                'Simple Missing': simple_missing,
+                'Simple Missing %': round(simple_missing_pct, 2),
+                'Enhanced Missing': enhanced_missing,
+                'Enhanced Missing %': round(enhanced_missing_pct, 2),
+                'Improvement': round(improvement, 2),
+                'Improvement %': round(improvement_pct, 2),
+                'Status': status
+            })
+    
+    return pd.DataFrame(comparison_stats)
+
+def compare_merge_methods(simple_stats: dict, enhanced_stats: dict, 
+                        simple_df: pd.DataFrame, enhanced_df: pd.DataFrame) -> dict:
+    """Compare two merge methods and provide detailed analysis"""
+    
+    # 1. Basic Comparison Metrics
+    basic_comparison = {
+        'Record Counts and Rates': {
+            'Simple Merge Records': simple_stats['Total Records'],
+            'Enhanced Merge Records': enhanced_stats['Total Records'],
+            'Record Difference': enhanced_stats['Total Records'] - simple_stats['Total Records'],
+            'Improvement Rate': f"{((enhanced_stats['Total Records'] - simple_stats['Total Records']) / simple_stats['Total Records'] * 100):.2f}%"
+        }
+    }
+
+    # 2. Data Quality Metrics
+    simple_empty = simple_df.isna().sum().sum()
+    enhanced_empty = enhanced_df.isna().sum().sum()
+    simple_total = len(simple_df) * len(simple_df.columns)
+    enhanced_total = len(enhanced_df) * len(enhanced_df.columns)
+    
+    quality_metrics = {
+        'Data Quality Metrics': {
+            'Simple Merge Completeness': f"{(1 - simple_empty / simple_total) * 100:.2f}%",
+            'Enhanced Merge Completeness': f"{(1 - enhanced_empty / enhanced_total) * 100:.2f}%",
+            'Empty Cell Improvement Rate': f"{((simple_empty/simple_total - enhanced_empty/enhanced_total) / (simple_empty/simple_total) * 100):.2f}%",
+            'Total Improved Cells': simple_empty - enhanced_empty
+        }
+    }
+
+    # 3. Field-Based Detailed Analysis
+    field_improvements = {}
+    for col in enhanced_df.columns:
+        if col in simple_df.columns:
+            simple_empty = simple_df[col].isna().sum()
+            enhanced_empty = enhanced_df[col].isna().sum()
+            simple_unique = simple_df[col].nunique()
+            enhanced_unique = enhanced_df[col].nunique()
+            
+            if simple_empty != enhanced_empty or simple_unique != enhanced_unique:
+                improvement = {
+                    'Simple Empty Cells': simple_empty,
+                    'Enhanced Empty Cells': enhanced_empty,
+                    'Empty Cell Improvement': simple_empty - enhanced_empty,
+                    'Improvement Rate': f"{((simple_empty - enhanced_empty) / simple_empty * 100):.2f}%" if simple_empty > 0 else "0%",
+                    'Simple Unique Values': simple_unique,
+                    'Enhanced Unique Values': enhanced_unique,
+                    'Enrichment': enhanced_unique - simple_unique
+                }
+                field_improvements[col] = improvement
+
+    return {
+        'Basic Comparison': basic_comparison,
+        'Data Quality': quality_metrics,
+        'Field-Based Improvements': field_improvements
+    }