BibexPy 1.0.0__tar.gz

bibexpy-1.0.0/BibexPy.egg-info/PKG-INFO

@@ -0,0 +1,119 @@
+Metadata-Version: 2.1
+Name: BibexPy
+Version: 1.0.0
+Summary: A Python library for advanced bibliography management and export.
+Home-page: https://github.com/bcankara/BibexPy
+Author: bcankara
+Author-email: info@bibexpy.com
+License: MIT
+Project-URL: Bug Tracker, https://github.com/bcankara/BibexPy/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: python-dotenv==1.0.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: openpyxl>=3.1.2
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: scipy>=1.11.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: colorama>=0.4.6
+Requires-Dist: xlrd>=2.0.1
+Requires-Dist: xlsxwriter>=3.1.0
+Requires-Dist: typing-extensions>=4.7.0
+
+# BibexPy
+
+**Harmonizing the Bibliometric Symphony of Scopus and Web of Science**
+
+BibexPy is a Python-based software designed to streamline bibliometric data integration, deduplication, metadata enrichment, and format conversion. It simplifies the preparation of high-quality datasets for advanced analyses by automating traditionally manual and error-prone tasks.
+
+## Features
+
+- **DOI-Based Deduplication and Merging**: Identifies and removes duplicate entries while enriching metadata by merging complementary records.
+- **API-Driven Metadata Enrichment**: Completes missing fields such as abstracts, keywords, and affiliations using APIs like Unpaywall and Semantic Scholar.
+- **Format Conversion**: Generates outputs compatible with VosViewer and Biblioshiny for easy analysis.
+- **Command-Line Interface (CLI)**: Offers user-friendly interaction with minimal setup requirements.
+
+# Prerequisites
+
+## Required Python Version
+- Python ≥ 3.9.0
+  
+**Libraries**:
+  - `python-dotenv==1.0.0`
+  - `pandas>=2.0.0`
+  - `openpyxl>=3.1.2`
+  - `numpy>=1.24.0`
+  - `requests>=2.31.0`
+  - `scikit-learn>=1.3.0`
+  - `scipy>=1.11.0`
+  - `tqdm>=4.65.0`
+  - `xlrd>=2.0.1`
+  - `xlsxwriter>=3.1.0`
+  - `colorama >= 0.4.6`
+  - `typing-extensions >= 4.7.0`
+
+
+## Installation
+
+1. **Clone the Repository**  
+   ```bash
+   git clone https://github.com/bcankara/BibexPy.git
+   ```
+2. **Navigate to the Directory**  
+   ```bash
+   cd BibexPy
+   ```
+3. **Install Dependencies**  
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+4. **(Optional) Virtual Environment Setup**  
+   ```bash
+   python -m venv venv
+   source venv/bin/activate  # Mac/Linux
+   venv\Scripts\activate     # Windows
+   ```
+
+## Usage
+
+1. **Run the Application**  
+   ```bash
+   python DataProcessor.py
+   ```
+
+2. **Follow the Workflow**  
+   - Select your project.
+   - Upload Scopus (`.csv`) and Web of Science (`.txt`) files.
+   - Choose to enrich metadata (optional).
+   - Review and save the processed datasets.
+
+## Outputs
+
+BibexPy generates the following outputs:
+- Unified datasets (`Prefix_Bib.xlsx`).
+- VosViewer-compatible files (`Prefix_Vos.txt`).
+- Statistical summaries for dataset quality and completeness.
+
+## Documentation
+
+For detailed documentation and examples, visit: [BibexPy Documentation](http://bibexpy.com/doc)
+
+## Support
+
+For questions or feedback, contact:  
+📧 **info@bibexpy.com**
+
+## License
+
+BibexPy is licensed under the **GNU General Public License (GPL)**. See the LICENSE file for details.
+
+---
+
+Enhance your bibliometric research with BibexPy, making data preparation efficient, reliable, and analysis-ready!

bibexpy-1.0.0/BibexPy.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+DataProcessor.py
+LICENSE
+README.md
+setup.py
+BibexPy.egg-info/PKG-INFO
+BibexPy.egg-info/SOURCES.txt
+BibexPy.egg-info/dependency_links.txt
+BibexPy.egg-info/entry_points.txt
+BibexPy.egg-info/requires.txt
+BibexPy.egg-info/top_level.txt

bibexpy-1.0.0/BibexPy.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

bibexpy-1.0.0/BibexPy.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+bibexpy = DataProcessor:main

bibexpy-1.0.0/BibexPy.egg-info/requires.txt

@@ -0,0 +1,12 @@
+python-dotenv==1.0.0
+pandas>=2.0.0
+openpyxl>=3.1.2
+numpy>=1.24.0
+requests>=2.31.0
+scikit-learn>=1.3.0
+scipy>=1.11.0
+tqdm>=4.65.0
+colorama>=0.4.6
+xlrd>=2.0.1
+xlsxwriter>=3.1.0
+typing-extensions>=4.7.0

bibexpy-1.0.0/BibexPy.egg-info/top_level.txt

@@ -0,0 +1 @@
+DataProcessor

bibexpy-1.0.0/DataProcessor.py

@@ -0,0 +1,360 @@
+import os
+import sys
+import time
+import threading
+import pandas as pd
+from dotenv import load_dotenv
+from Main.modules import (
+    ensure_dir,
+    find_files,
+    find_data_folders,
+    save_statistics,
+    save_comprehensive_statistics,
+    merge_databases_simple,
+    merge_databases_enhanced,
+    compare_merge_methods,
+    create_result_folder
+)
+from datetime import datetime
+
+class ProcessIndicator:
+    """Process indicator for showing progress"""
+    busy = False
+    delay = 0.1
+    
+    @staticmethod
+    def indicator():
+        while 1:
+            for cursor in '|/-\\':
+                yield cursor
+    
+    def __init__(self, description="Processing"):
+        self.description = description
+        self.generator = self.indicator()
+        self.busy = False
+        self.visible = False
+    
+    def write_next(self):
+        with self._screen_lock:
+            if not self.visible:
+                sys.stdout.write(f"\r{self.description:<50}")
+                self.visible = True
+            sys.stdout.write(next(self.generator))
+            sys.stdout.flush()
+            sys.stdout.write('\b')
+    
+    def remove_indicator(self, cleanup=False):
+        with self._screen_lock:
+            if self.visible:
+                sys.stdout.write('\b')
+                self.visible = False
+                if cleanup:
+                    sys.stdout.write(f"\r{self.description:<50}[Completed]\n")
+                    sys.stdout.flush()
+    
+    def indicator_task(self):
+        while self.busy:
+            self.write_next()
+            time.sleep(self.delay)
+    
+    def __enter__(self):
+        if sys.stdout.isatty():
+            self._screen_lock = threading.Lock()
+            self.busy = True
+            self.thread = threading.Thread(target=self.indicator_task)
+            self.thread.start()
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if sys.stdout.isatty():
+            self.busy = False
+            time.sleep(self.delay)
+            self.remove_indicator(cleanup=True)
+
+def process_wos_data(input_file: str, output_file: str) -> tuple[bool, pd.DataFrame, dict]:
+    """Process WoS data"""
+    try:
+        from Main.wos2xlsx import save_to_excel
+        success = save_to_excel(input_file, output_file)
+        
+        if success:
+            with ProcessIndicator("Converting WoS data to Excel format") as indicator:
+                df = pd.read_excel(output_file)
+                stats = {
+                    'Record Count': len(df),
+                    'Column Count': len(df.columns),
+                    'Non-Empty Columns': df.count().to_dict()
+                }
+                return True, df, stats
+        return False, None, {}
+    except Exception as e:
+        sys.stderr.write(f"\nError: {str(e)}\n")
+        return False, None, {}
+
+def process_scopus_data(input_file: str, output_file: str) -> tuple[bool, pd.DataFrame, dict]:
+    """Process Scopus data"""
+    try:
+        from Main.scp2xlsx import save_to_excel
+        success = save_to_excel(input_file, output_file)
+        
+        if success:
+            with ProcessIndicator("Converting Scopus data to Excel format") as indicator:
+                df = pd.read_excel(output_file)
+                stats = {
+                    'Record Count': len(df),
+                    'Column Count': len(df.columns),
+                    'Non-Empty Columns': df.count().to_dict()
+                }
+                return True, df, stats
+        return False, None, {}
+    except Exception as e:
+        sys.stderr.write(f"\nError: {str(e)}\n")
+        return False, None, {}
+
+def merge_txt_files(data_dir: str) -> str:
+    """Merge all txt files in data directory into wos_raw.txt in merged_raw folder"""
+    txt_files = find_files(data_dir, "txt")
+    if not txt_files:
+        raise ValueError("No WoS files (txt) found in Data folder.")
+    
+    # Create merged_raw directory if not exists
+    merged_raw_dir = os.path.join(data_dir, "merged_raw")
+    os.makedirs(merged_raw_dir, exist_ok=True)
+    
+    output_file = os.path.join(merged_raw_dir, "wos_raw.txt")
+    with open(output_file, 'w', encoding='utf-8') as outfile:
+        for txt_file in txt_files:
+            with open(txt_file, 'r', encoding='utf-8') as infile:
+                outfile.write(infile.read())
+                outfile.write('\n')  # Add newline between files
+    
+    return output_file
+
+def merge_csv_files(data_dir: str) -> str:
+    """Merge all csv files in data directory into scp_raw.csv in merged_raw folder"""
+    csv_files = find_files(data_dir, "csv")
+    if not csv_files:
+        raise ValueError("No Scopus files (csv) found in Data folder.")
+    
+    # Create merged_raw directory if not exists
+    merged_raw_dir = os.path.join(data_dir, "merged_raw")
+    os.makedirs(merged_raw_dir, exist_ok=True)
+    
+    output_file = os.path.join(merged_raw_dir, "scp_raw.csv")
+    
+    if len(csv_files) == 1:
+        # If only one CSV file, just copy it
+        import shutil
+        shutil.copy2(csv_files[0], output_file)
+    else:
+        # Read and combine all CSV files
+        all_data = []
+        for csv_file in csv_files:
+            df = pd.read_csv(csv_file, encoding='utf-8')
+            all_data.append(df)
+        
+        # Concatenate all dataframes
+        combined_df = pd.concat(all_data, ignore_index=True)
+        
+        # Save combined data
+        combined_df.to_csv(output_file, index=False, encoding='utf-8')
+    
+    return output_file
+
+def main():
+    try:
+        # Load environment variables
+        load_dotenv()
+        
+        print("Database Merge Tool")
+        print("------------------")
+        
+        # Find project folders
+        workspace_dir = "Workspace"
+        if not os.path.exists(workspace_dir):
+            os.makedirs(workspace_dir)
+        
+        project_folders = [d for d in os.listdir(workspace_dir) if os.path.isdir(os.path.join(workspace_dir, d))]
+        
+        if not project_folders:
+            sys.stderr.write("\nError: No project folder found in Workspace.\n")
+            return
+        
+        print("\nProject Folders:")
+        for i, folder in enumerate(project_folders, 1):
+            print(f"{i}. {folder}")
+        
+        # Project selection
+        while True:
+            try:
+                choice = int(input("\nSelect project number: "))
+                if 1 <= choice <= len(project_folders):
+                    project_dir = os.path.join(workspace_dir, project_folders[choice-1])
+                    break
+                else:
+                    sys.stderr.write("Error: Invalid selection.\n")
+            except ValueError:
+                sys.stderr.write("Error: Invalid input. Please enter a number.\n")
+        
+        # Create Data directory if not exists
+        data_dir = os.path.join(project_dir, "Data")
+        if not os.path.exists(data_dir):
+            os.makedirs(data_dir)
+        
+        print(f"\nSelected Project: {project_folders[choice-1]}")
+        
+        # Create unique analysis directory
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        analysis_dir = os.path.join(project_dir, f"Analysis_{timestamp}")
+        os.makedirs(analysis_dir, exist_ok=True)
+        
+        # Create subfolders
+        text_files_dir = os.path.join(analysis_dir, "Text_Files")
+        cell_files_dir = os.path.join(analysis_dir, "Cell_Files")
+        os.makedirs(text_files_dir, exist_ok=True)
+        os.makedirs(cell_files_dir, exist_ok=True)
+        
+        # Set output files in analysis directory
+        wos_output = os.path.join(cell_files_dir, "WoS.xlsx")
+        scopus_output = os.path.join(cell_files_dir, "Scopus.xlsx")
+        merged_bib = os.path.join(cell_files_dir, "Merged_Bib.xlsx")
+        merged_vos = os.path.join(text_files_dir, "Merged_Vos.txt")
+        api_enriched_bib = os.path.join(cell_files_dir, "Merged_API_Enriched_Bib.xlsx")
+        api_enriched_vos = os.path.join(text_files_dir, "Merged_API_Enriched_Vos.txt")
+        stats_excel = os.path.join(analysis_dir, "Statistics.xlsx")
+        api_log = os.path.join(analysis_dir, "Api_Log.txt")
+        api_updates = os.path.join(analysis_dir, "Api_Update.xlsx")
+        
+        # Check if merged_raw directory exists and contains required files
+        merged_raw_dir = os.path.join(data_dir, "merged_raw")
+        wos_raw = os.path.join(merged_raw_dir, "wos_raw.txt")
+        scp_raw = os.path.join(merged_raw_dir, "scp_raw.csv")
+        
+        use_existing = False
+        if os.path.exists(merged_raw_dir) and os.path.exists(wos_raw) and os.path.exists(scp_raw):
+            while True:
+                choice = input("\nPreviously merged raw files found. What would you like to do?\n"
+                             "1. Use previously merged files\n"
+                             "2. Perform new merge\n"
+                             "Your choice (1/2): ")
+                if choice in ['1', '2']:
+                    use_existing = (choice == '1')
+                    break
+                else:
+                    sys.stderr.write("Invalid choice. Please enter 1 or 2.\n")
+        
+        if not use_existing:
+            if os.path.exists(merged_raw_dir):
+                print("\nDeleting old merged files...")
+                import shutil
+                shutil.rmtree(merged_raw_dir)
+            
+            # Merge txt files into wos_raw.txt
+            print("\nMerging process in progress...")
+            try:
+                wos_input = merge_txt_files(data_dir)
+                print(f"WoS files merged: {os.path.basename(wos_input)}")
+            except ValueError as e:
+                sys.stderr.write(f"\nError: {str(e)}\n")
+                return
+            
+            # Merge csv files into scp_raw.csv
+            print("\nMerging Scopus files...")
+            try:
+                scopus_input = merge_csv_files(data_dir)
+                print(f"Scopus files merged: {os.path.basename(scopus_input)}")
+            except ValueError as e:
+                sys.stderr.write(f"\nError: {str(e)}\n")
+                return
+        else:
+            print("\nUsing existing merged files...")
+            wos_input = wos_raw
+            scopus_input = scp_raw
+        
+        # Process WoS data
+        print("\nProcessing WoS data...")
+        wos_success, wos_df, wos_stats = process_wos_data(wos_input, wos_output)
+        if not wos_success:
+            sys.stderr.write("\nError: Failed to process WoS data.\n")
+            return
+        print("WoS data processing completed.")
+        
+        # Process Scopus data
+        print("\nProcessing Scopus data...")
+        scopus_success, scopus_df, scopus_stats = process_scopus_data(scopus_input, scopus_output)
+        if not scopus_success:
+            sys.stderr.write("\nError: Failed to process Scopus data.\n")
+            return
+        print("Scopus data processing completed.")
+        
+        # Get Scopus API key from environment variable
+        scopus_api_key = os.getenv('SCOPUS_API_KEY')
+        unpaywall_email = os.getenv('UNPAYWALL_EMAIL')
+
+        if not scopus_api_key:
+            print("\nWarning: Scopus API key not found in .env file.")
+            print("You can add it to the .env file to enable Scopus metadata enrichment.")
+            print("Continuing with other data sources...")
+
+        if not unpaywall_email:
+            print("\nWarning: Unpaywall email not found in .env file.")
+            print("You can add it to the .env file to enable Unpaywall metadata enrichment.")
+            print("Continuing with other data sources...")
+            
+        # Enhanced merge
+        print("\nMerging process in progress...")
+        enhanced_success, enhanced_stats, enhanced_df = merge_databases_enhanced(
+            wos_df, 
+            scopus_df, 
+            merged_bib, 
+            scopus_api_key=scopus_api_key,
+            unpaywall_email=unpaywall_email,
+            result_dir=analysis_dir
+        )
+
+        print("\nConverting data to VosViewer format...")
+        from Main.xlsx2vos import convert_excel_to_wos
+        convert_excel_to_wos(merged_bib, merged_vos)
+        print("VosViewer conversion completed.")
+        
+        if enhanced_success:
+            # Save statistics
+            print("\nPreparing statistics...")
+            all_stats = {
+                'WoS Statistics': wos_stats,
+                'Scopus Statistics': scopus_stats,
+                'Merge Statistics': enhanced_stats
+            }
+            
+            save_comprehensive_statistics(all_stats, wos_df, scopus_df, enhanced_df, stats_excel)
+            print("Statistics completed.")
+            
+            # Check and convert API enriched file
+            if os.path.exists(api_enriched_bib):
+                print("\nConverting API Enriched data to VosViewer format...")
+                convert_excel_to_wos(api_enriched_bib, api_enriched_vos)
+                print("API Enriched VosViewer conversion completed.")
+            
+            print("\nProcess completed successfully.")
+            print(f"\nAnalysis results saved to: {analysis_dir}")
+            print("Generated Files:")
+            print(f"1. WoS Data: {os.path.basename(wos_output)}")
+            print(f"2. Scopus Data: {os.path.basename(scopus_output)}")
+            print(f"3. Merged Data (Biblioshiny): {os.path.basename(merged_bib)}")
+            print(f"4. Merged Data (VosViewer): {os.path.basename(merged_vos)}")
+            print(f"5. Statistics: {os.path.basename(stats_excel)}")
+            print(f"6. API Log: {os.path.basename(api_log)}")
+            if os.path.exists(api_updates):
+                print(f"7. API Updates: {os.path.basename(api_updates)}")
+            if os.path.exists(api_enriched_bib):
+                print(f"8. API Enriched Data (Biblioshiny): {os.path.basename(api_enriched_bib)}")
+                print(f"9. API Enriched Data (VosViewer): {os.path.basename(api_enriched_vos)}")
+        else:
+            sys.stderr.write("\nError: Merge process failed.\n")
+    
+    except Exception as e:
+        print(f"\nAn unexpected error occurred: {str(e)}")
+    finally:
+        print("\nProgram terminating...")
+
+if __name__ == "__main__":
+    main()