pyeasyphd 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

pyeasyphd/tools/compare/compare_bibs.py

@@ -1,234 +0,0 @@
-import copy
-import os
-import re
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-from pyadvtools import standard_path, transform_to_data_list
-
-from ...bib.bibtexparser import Block, Library
-from ...main import PythonRunBib, PythonWriters
-from ..experiments_base import obtain_local_abbr_paths
-
-ARXIV_BIORXIV = ["arxiv", "biorxiv", "ssrn"]
-
-
-def obtain_local_abbr_paths_for_abbr(options: dict, path_spidered_bibs: str, path_spidering_bibs: str) -> List[str]:
-    path_spidered_bibs = standard_path(path_spidered_bibs)
-    path_spidering_bibs = standard_path(path_spidering_bibs)
-
-    path_abbrs = []
-    path_abbrs.extend(obtain_local_abbr_paths(os.path.join(path_spidered_bibs, "Journals"), options))
-    path_abbrs.extend(obtain_local_abbr_paths(os.path.join(path_spidered_bibs, "Conferences"), options))
-    path_abbrs.extend(obtain_local_abbr_paths(os.path.join(path_spidering_bibs, "spider_j"), options))
-    path_abbrs.extend(obtain_local_abbr_paths(os.path.join(path_spidering_bibs, "spider_c"), options))
-
-    if options.get("include_early_access", True):
-        path_abbrs.extend(obtain_local_abbr_paths(os.path.join(path_spidering_bibs, "spider_j_e"), options))
-
-    path_abbrs = [p for p in path_abbrs if os.path.basename(p).lower() not in ARXIV_BIORXIV]
-    return path_abbrs
-
-
-def compare_bibs_with_local(
-    original_data: Union[List[str], str],
-    path_spidered_bibs: str,
-    path_spidering_bibs: str,
-    path_output: str,
-    options: Dict[str, Any],
-) -> None:
-    path_output = standard_path(path_output)
-
-    # generate for original data
-    _options = {}
-    _options.update(options)
-    _python_bib = PythonRunBib(_options)
-    data_list = transform_to_data_list(original_data, ".bib")
-    library = _python_bib.parse_to_single_standard_library(data_list)
-    original_entry_keys = [entry.key for entry in library.entries]
-
-    # generate dict for abbr key entry
-    abbr_key_entries_dict, not_in_local_entries = generate_abbr_key_entry_dict(library, options)
-
-    # compare with local bibs
-    tuple_entries = _compare_with_local(abbr_key_entries_dict, path_spidered_bibs, path_spidering_bibs, options)
-    searched_entries, not_searched_entries, duplicate_original_entries, duplicate_searched_entries = tuple_entries
-    not_in_local_entries.extend(not_searched_entries)
-
-    # write with sorting blocks according to original cite keys
-    _options = {}
-    _options["is_sort_entry_fields"] = True  # default is True
-    _options["is_sort_blocks"] = True  # default is True
-    _options["sort_entries_by_cite_keys"] = original_entry_keys
-    _python_write = PythonWriters(_options)
-    _python_write.write_to_file(searched_entries, "in_local_entries.bib", "w", path_output, False)
-    _python_write.write_to_file(not_in_local_entries, "not_in_local_entries.bib", "w", path_output, False)
-
-    # write without sorting blocks
-    _options = {}
-    _options["is_sort_entry_fields"] = True  # default is True
-    _options["is_sort_blocks"] = False  # default is True
-    _python_write = PythonWriters(_options)
-    _python_write.write_to_file(duplicate_original_entries, "duplicate_original_entries.bib", "w", path_output, False)
-    _python_write.write_to_file(duplicate_searched_entries, "duplicate_searched_entries.bib", "w", path_output, False)
-    return None
-
-
-def generate_abbr_key_entry_dict(library: Library, options: Dict[str, Any]):
-    _options = {}
-    _options["is_standardize_bib"] = True  # default is True
-    _options["choose_abbr_zotero_save"] = "save"  # default is "save"
-    _options["function_common_again"] = True  # default is True
-    _options["generate_entry_cite_keys"] = True  # default is False
-    _options.update(options)
-    _python_bib = PythonRunBib(_options)
-
-    abbr_key_entries_dict, not_in_local_entries = {}, []
-    for entry in library.entries:
-        flag = False
-
-        if ("title" in entry) and (entry["title"].strip()) and ("year" in entry) and (entry["year"].strip()):
-            temp_library = _python_bib.parse_to_single_standard_library(copy.deepcopy(Library([entry])))
-            if len(entries := temp_library.entries) == 1:
-
-                # article and inproceedings
-                temps = entries[0].key.split("_")
-                if (len(temps) == 3) and temps[0].lower() in ["j", "c"]:
-                    abbr_key_entries_dict.setdefault(temps[1], {}).update({entry.key: entry})
-                    flag = True
-
-                # misc (arXiv, bioRxiv, and ssrn)
-                elif (len(temps) == 2) and temps[0].lower() in ARXIV_BIORXIV:
-                    abbr_key_entries_dict.setdefault(temps[0], {}).update({entry.key: entry})
-                    flag = True
-
-        if not flag:
-            not_in_local_entries.append(entry)
-    return abbr_key_entries_dict, not_in_local_entries
-
-
-def _compare_with_local(
-    abbr_key_entries_dict: Dict[str, Dict[str, Block]],
-    local_path_spidered_bibs: str,
-    local_path_spidering_bibs: str,
-    options: Dict[str, Any],
-) -> Tuple[List[Block], List[Block], List[Block], List[Block]]:
-    # compare with local bibs
-    searched_entries, not_searched_entries, duplicate_original_entries, duplicate_searched_entries = [], [], [], []
-    for abbr, old_key_entries_dict in abbr_key_entries_dict.items():
-
-        options_ = {}
-        options_.update(options)
-        if abbr.lower() not in ARXIV_BIORXIV:
-            options_["include_abbr_list"] = [abbr]
-        path_abbrs = obtain_local_abbr_paths_for_abbr(options_, local_path_spidered_bibs, local_path_spidering_bibs)
-
-        new_key_entries_dict = {}
-        for path_abbr in path_abbrs:
-            if len(data_list := transform_to_data_list(path_abbr, ".bib")) == 0:
-                continue
-
-            print("*" * 9 + f" Compare in {f'{os.sep}'.join(path_abbr.split(os.sep)[-3:])} for {abbr} " + "*" * 9)
-
-            _options = {}
-            _options["is_standardize_bib"] = False  # default is True
-            _options["choose_abbr_zotero_save"] = "save"  # default is "save"
-            _options["function_common_again"] = False  # default is True
-            _options["function_common_again_abbr"] = False  # default is True
-            _options["function_common_again_zotero"] = False  # default is True
-            _options["function_common_again_save"] = False  # default is True
-            _options["generate_entry_cite_keys"] = False  # default is False
-            _options.update(options)
-            _library = PythonRunBib(_options).parse_to_single_standard_library(data_list)
-            for key, entry in old_key_entries_dict.items():
-                for _entry in _library.entries:
-                    if check_equal_for_entry(entry, _entry, ["title"], abbr):
-                        new_key_entries_dict.setdefault(key, []).append(copy.deepcopy(_entry))
-
-        print()
-
-        for key, entry in old_key_entries_dict.items():
-            entries = new_key_entries_dict.get(key, [])
-            if (length := len(entries)) == 1:
-                entries[0].key = key
-                searched_entries.extend(entries)
-            elif length == 0:
-                not_searched_entries.append(entry)
-            else:
-                for i, _entry in enumerate(entries):
-                    _entry.key = key + "-a" * i
-                duplicate_original_entries.append(entry)
-                duplicate_searched_entries.extend(entries)
-
-    return searched_entries, not_searched_entries, duplicate_original_entries, duplicate_searched_entries
-
-
-def check_equal_for_entry(original_entry, new_entry, compare_field_list: List[str], abbr: Optional[str] = None):
-    a_list, b_list = [original_entry.entry_type.lower()], [new_entry.entry_type.lower()]
-    if (abbr is not None) and (abbr.lower() in ARXIV_BIORXIV):
-        a_list, b_list = [], []
-
-    regex_title = re.compile(r"\\href{(.*)}{(.*)}")
-    for field in compare_field_list:
-        x = original_entry[field].lower().strip() if field in original_entry else ""
-        y = new_entry[field].lower().strip() if field in new_entry else ""
-
-        if field == "title":
-            if mch := regex_title.search(x):
-                x = mch.group(2)
-            if mch := regex_title.search(y):
-                y = mch.group(2)
-
-        a_list.append(x)
-        b_list.append(y)
-
-    a_list = [re.sub(r"\W", "", a) for a in a_list]
-    b_list = [re.sub(r"\W", "", b) for b in b_list]
-    if "_".join(a_list) == "_".join(b_list):
-        return True
-    return False
-
-
-def compare_bibs_with_zotero(
-    zotero_bib: Union[List[str], str],
-    download_bib: Union[List[str], str],
-    path_output: str,
-    options: Dict[str, Any],
-) -> None:
-    path_output = standard_path(path_output)
-
-    # for zotero bib
-    _options = {}
-    _options.update(options)
-    _options["generate_entry_cite_keys"] = False  # default is False
-    _python_bib = PythonRunBib(_options)
-    data_list = transform_to_data_list(zotero_bib, ".bib")
-    zotero_library = _python_bib.parse_to_single_standard_library(data_list)
-
-    # for download bib
-    _options = {}
-    _options.update(options)
-    _options["generate_entry_cite_keys"] = True  # default is False
-    _python_bib = PythonRunBib(_options)
-    data_list = transform_to_data_list(download_bib, ".bib")
-    download_library = _python_bib.parse_to_single_standard_library(data_list)
-
-    # compare download bib and zotero bib
-    only_in_download_entries, in_download_and_zotero_entries = [], []
-    for download_entry in download_library.entries:
-        flag = False
-        for zotero_entry in zotero_library.entries:
-            if check_equal_for_entry(zotero_entry, download_entry, ["title"], None):
-                in_download_and_zotero_entries.append(download_entry)
-                flag = True
-                break
-
-        if not flag:
-            only_in_download_entries.append(download_entry)
-
-    # write
-    _options = {}
-    _options.update(options)
-    _python_write = PythonWriters(_options)
-    _python_write.write_to_file(only_in_download_entries, "only_in_download.bib", "w", path_output, False)
-    _python_write.write_to_file(in_download_and_zotero_entries, "in_download_and_zotero.bib", "w", path_output, False)
-    return None

pyeasyphd/tools/experiments_base.py

@@ -1,203 +0,0 @@
-import copy
-import os
-from typing import Any, Dict, List
-
-from pyadvtools import sort_int_str
-
-from ..bib.bibtexparser import Entry
-
-
-def obtain_local_abbr_paths(path_storage: str, options: dict) -> List[str]:
-    """Get all local abbreviation paths from the storage directory.
-
-    Scans the specified storage directory and returns paths to all abbreviation
-    subdirectories that match the inclusion/exclusion criteria in options.
-
-    Parameters
-    ----------
-    path_storage : str
-        Root directory containing publisher/abbreviation folders
-    options : dict
-        Configuration dictionary containing:
-            - include_publisher_list: List of publishers to include
-            - exclude_publisher_list: List of publishers to exclude
-            - include_abbr_list: List of abbreviations to include
-            - exclude_abbr_list: List of abbreviations to exclude
-
-    Returns
-    -------
-    List[str]
-        List of full paths to all matching abbreviation directories
-    """
-    abbr_list = []
-    if not os.path.exists(path_storage):
-        return []
-
-    publisher_abbr_dict = generate_standard_publisher_abbr_options_dict(path_storage, options)
-    for publisher in publisher_abbr_dict:
-        for abbr in publisher_abbr_dict[publisher]:
-            abbr_list.append(os.path.join(path_storage, publisher, abbr))
-    return abbr_list
-
-
-def generate_standard_publisher_abbr_options_dict(
-    path_storage: str, options: Dict[str, Any]
-) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    """Generate a nested dictionary of publisher/abbreviation options.
-
-    Creates a hierarchical dictionary structure representing all publishers
-    and their abbreviations that match the inclusion/exclusion criteria.
-
-    Parameters
-    ----------
-    path_storage : str
-        Root directory containing publisher/abbreviation folders
-    options : Dict[str, Any]
-        Configuration options including inclusion/exclusion lists
-
-    Returns
-    -------
-    Dict[str, Dict[str, Dict[str, Any]]]
-        Nested dictionary structure:
-            - Top level: Publisher names
-            - Middle level: Abbreviation names
-            - Inner level: Copy of options dictionary
-    """
-    if not os.path.exists(path_storage):
-        return {}
-
-    # First scan directory structure to find all publishers and abbreviations
-    publisher_abbr_dict: Dict[str, List[str]] = {}
-    publishers = [f for f in os.listdir(path_storage) if os.path.isdir(os.path.join(path_storage, f))]
-    for p in publishers:
-        path_p = os.path.join(path_storage, p)
-        publisher_abbr_dict.update({p: [f for f in os.listdir(path_p) if os.path.isdir(os.path.join(path_p, f))]})
-
-    # Apply inclusion/exclusion filters to publishers
-    publisher_list = in_not_in_list(
-            list(publisher_abbr_dict.keys()),
-            options.get("include_publisher_list", []),
-            options.get("exclude_publisher_list", []),
-    )
-
-    # Build the nested options dictionary structure
-    publisher_abbr_options_dict: Dict[str, Dict[str, Dict[str, Any]]] = {}
-    for publisher in sort_int_str(publisher_list):
-
-        # Apply inclusion/exclusion filters to abbreviations
-        abbr_list = in_not_in_list(
-                publisher_abbr_dict[publisher],
-                options.get("include_abbr_list", []),
-                options.get("exclude_abbr_list", []),
-        )
-
-        # Create nested structure with copied options
-        for abbr_standard in sort_int_str(abbr_list):
-            publisher_abbr_options_dict.setdefault(publisher, {}).setdefault(abbr_standard, copy.deepcopy(options))
-    return publisher_abbr_options_dict
-
-
-def in_not_in_list(original: List[str], in_list: List[str], out_list: List[str]):
-    """Filter a list based on inclusion and exclusion criteria.
-
-    Parameters
-    ----------
-    original : List[str]
-        Original list to filter
-    in_list : List[str]
-        List of items to include (case-insensitive)
-    out_list : List[str]
-        List of items to exclude (case-insensitive)
-
-    Returns
-    -------
-    List[str]
-        Filtered list containing only items that:
-            - Are in in_list (if in_list is not empty)
-            - Are not in out_list
-    """
-    if in_list := [o.lower() for o in in_list]:
-        original = [o for o in original if o.lower() in in_list]
-    if out_list := [o.lower() for o in out_list]:
-        original = [o for o in original if o.lower() not in out_list]
-    return original
-
-
-def generate_readme(
-    j_conf_abbr: str,
-    entry_type: str,
-    year_volume_number_month_entry_dict: Dict[str, Dict[str, Dict[str, Dict[str, List[Entry]]]]],
-) -> List[str]:
-    """Generate a README markdown file summarizing bibliography entries.
-
-    Creates a formatted markdown table showing publication statistics
-    organized by year, volume, number, and month.
-
-    Parameters
-    ----------
-    j_conf_abbr : str
-        Journal/conference abbreviation for the title
-    entry_type : str
-        Type of bibliography entries (article, inproceedings, etc.)
-    year_volume_number_month_entry_dict : Dict[str, Dict[str, Dict[str, Dict[str, List[Entry]]]]]
-        Nested dictionary structure containing entries organized by:
-            - Year
-            - Volume
-            - Number
-            - Month
-
-    Returns
-    -------
-    List[str]
-        Lines of the generated markdown file, or empty list if no valid entries
-    """
-    # Configuration for different entry types
-    entry_type_list = ["article", "inproceedings", "misc"]
-    filed_key_list = ["journal", "booktitle", "publisher"]
-
-    # Determine which field to display based on entry type
-    field_key = ""
-    if (entry_type := entry_type.lower()) in entry_type_list:
-        field_key = filed_key_list[entry_type_list.index(entry_type)]
-
-    def extract_journal_booktitle(entries: List[Entry], field_key: str) -> List[str]:
-        """Extract unique journal/booktitle values from entries."""
-        if field_key:
-            contents = []
-            for entry in entries:
-                value = entry[field_key] if field_key in entry else ""
-                contents.append(value)
-            return sorted(set(contents), key=contents.index)
-        return []
-
-    # Initialize markdown content with header
-    readme = [
-        f"# {j_conf_abbr}-{entry_type.title()}\n\n",
-        f"|Name|Year|Papers|{field_key.title()}|\n",
-        "|-|-|-|-|\n"
-    ]
-
-    # Process each hierarchical level to build the table
-    for year in year_volume_number_month_entry_dict:
-        for volume in year_volume_number_month_entry_dict[year]:
-            for number in year_volume_number_month_entry_dict[year][volume]:
-                for month in year_volume_number_month_entry_dict[year][volume][number]:
-
-                    # Generate filename components
-                    file_name = ""
-                    for i, j in zip(["", "Vol.", "No.", "Month"], [j_conf_abbr, volume, number, month]):
-                        if j.lower().strip() in ["volume", "number", "month"]:
-                            j = ""
-                        file_name += (i + j + "-") * (len(j.strip()) >= 1)
-
-                    # Count papers and get journal/booktitle info
-                    number_paper = len(temp := year_volume_number_month_entry_dict[year][volume][number][month])
-                    j_b = extract_journal_booktitle(temp, field_key)
-
-                    # Add table row
-                    readme.append(f'|{file_name[:-1]}|{year}|{number_paper}|{"; ".join(j_b)}|' + "\n")
-
-    # Only return content if we have more than just the header
-    if len(readme) > 3:
-        return readme
-    return []

pyeasyphd/tools/format_save_bibs.py

@@ -1,178 +0,0 @@
-import copy
-import math
-import os
-import re
-from typing import Any, Dict, List, Union
-
-from pyadvtools import (
-    IterateCombineExtendDict,
-    read_list,
-    sort_int_str,
-    standard_path,
-    write_list,
-)
-
-from ..bib.bibtexparser import Block, Library
-from ..main import PythonRunBib, PythonWriters
-from ..tools.experiments_base import generate_readme
-
-
-def format_entries_for_abbr_zotero_save(
-    j_conf_abbr: str,
-    path_output: str,
-    original_data: Union[List[str], str, Library],
-    combine_year_length: int = 1,
-    default_year_list: List[str] = [],
-    write_flag_bib: str = "w",
-    check_bib_exist: bool = False,
-    write_flag_readme: str = "w",
-    check_md_exist: bool = False,
-    options: Dict[str, Any] = {},
-) -> None:
-    """Format bibliography entries and organize them by year and type.
-
-    Processes bibliography data and organizes it into separate files by entry type and year,
-    generating both BibTeX files and README documentation.
-
-    Parameters
-    ----------
-    j_conf_abbr : str
-        Journal/conference abbreviation used for naming output files
-    path_output : str
-        Output directory path for processed files
-    original_data : Union[List[str], str, Library]
-        Input bibliography data in various formats (list of strings, file path, or Library object)
-    combine_year_length : int, optional
-        Number of years to combine in each output file, by default 1
-    default_year_list : List[str], optional
-        Specific years to process (if empty, processes all years), by default []
-    write_flag_bib : str, optional
-        Write mode for BibTeX files ("w" for write, "a" for append), by default "w"
-    check_bib_exist : bool, optional
-        Whether to check if BibTeX files exist before writing, by default False
-    write_flag_readme : str, optional
-        Write mode for README files ("w" for write, "a" for append), by default "w"
-    check_md_exist : bool, optional
-        Whether to check if README files exist before writing, by default False
-    options : Dict[str, Any], optional
-        Additional processing options, by default {}
-
-    Returns
-    -------
-    None
-    """
-    path_output = standard_path(path_output)
-
-    # Set up processing options
-    _options = {}
-    _options.update(options)
-    _options["is_sort_entry_fields"] = True  # Force field sorting
-    _options["is_sort_blocks"] = True  # Force block sorting
-    _options["sort_entries_by_field_keys_reverse"] = False  # Sort in ascending order
-
-    # Initialize helper classes
-    _python_bib = PythonRunBib(_options)
-
-    _options["empty_entry_cite_keys"] = True  # Allow empty citation keys
-    _python_writer = PythonWriters(_options)
-
-    # Organize entries by type, year, volume, number, and month
-    entry_type_year_volume_number_month_entry_dict = _python_bib.parse_to_nested_entries_dict(original_data)
-
-    # Process each entry type separately
-    for entry_type in entry_type_year_volume_number_month_entry_dict:
-
-        # Filter years if specified
-        year_dict = entry_type_year_volume_number_month_entry_dict[entry_type]
-        year_list = sort_int_str(list(year_dict.keys()))
-        if default_year_list:
-            year_list = [y for y in year_list if y in default_year_list]
-        year_dict = {year: year_dict[year] for year in year_list}
-
-        # Save bibliography files grouped by years
-        path_write = os.path.join(path_output, entry_type.lower(), "bib")
-        for i in range(math.ceil(len(year_list) / combine_year_length)):
-
-            # Determine year range for this file
-            start_year_index = i * combine_year_length
-            end_year_index = min([(i + 1) * combine_year_length, len(year_list)])
-            combine_year = year_list[start_year_index:end_year_index]
-
-            # Create subset dictionary for these years
-            new_year_dict = {year: year_dict[year] for year in combine_year}
-            entries: List[Block] = IterateCombineExtendDict().dict_update(copy.deepcopy(new_year_dict))
-
-            # Generate filename based on year range
-            name = f"{j_conf_abbr}_{combine_year[0]}"
-            if len(combine_year) > 1:
-                name += f"_{combine_year[-1]}"
-            name += ".bib"
-
-            # Write the bibliography file
-            _python_writer.write_to_file(entries, name, write_flag_bib, path_write, check_bib_exist)
-
-        # Generate and save README documentation
-        path_write = os.path.join(path_output, entry_type.lower())
-        readme_md = generate_readme(j_conf_abbr, entry_type, year_dict)
-
-        # Handle append mode for README
-        if re.search("a", write_flag_readme):
-            old_readme_md = [re.sub(r"[ ]+", "", line) for line in read_list("README.md", "r", path_write)]
-            readme_md = readme_md[3:] if old_readme_md else readme_md
-            readme_md = [line for line in readme_md if re.sub(r"[ ]+", "", line) not in old_readme_md]
-
-        write_list(readme_md, "README.md", write_flag_readme, path_write, check_md_exist)
-
-
-def generate_statistic_information(path_output: str) -> None:
-    """Generate statistical information from bibliography files.
-
-    Processes all BibTeX files in the directory tree and extracts key information
-    (DOIs and URLs) into CSV files for analysis.
-
-    Parameters
-    ----------
-    path_output : str
-        Root directory containing BibTeX files to process
-
-    Returns
-    -------
-    None
-    """
-    # Find all BibTeX files in the directory tree
-    full_files = []
-    for root, _, files in os.walk(path_output):
-        full_files.extend([os.path.join(root, f) for f in files if f.endswith(".bib")])
-
-    # Configure processing options
-    _options = {
-        "is_standardize_bib": False,  # Skip standardization, default is True
-        "choose_abbr_zotero_save": "save",  # Use save format, default is "save"
-        "function_common_again": False,  # Skip reprocessing, default is True
-        "function_common_again_abbr": False,  # Skip abbreviation reprocessing, default is True
-        "function_common_again_zotero": False,  # Skip Zotero reprocessing, default is True
-        "function_common_again_save": False,  # Skip save format reprocessing, default is True
-        "is_sort_entry_fields": False,  # Skip field sorting
-        "is_sort_blocks": False,  # Skip block sorting
-    }
-    _python_bib = PythonRunBib(_options)
-
-    # Process each BibTeX file
-    for f in full_files:
-        informations = []
-        library = _python_bib.parse_to_single_standard_library(f)
-
-        # Extract DOI or URL for each entry
-        for entry in library.entries:
-            flag = ""
-            if not flag:
-                flag = entry["doi"] if "doi" in entry else ""
-            if not flag:
-                flag = entry["url"] if "url" in entry else ""
-            informations.append(flag + "\n")
-
-        # Write information to CSV file
-        csv_path = f.replace(".bib", ".csv").replace(f"{os.sep}bib{os.sep}", f"{os.sep}url{os.sep}")
-        write_list(informations, csv_path, "w", None, False)
-
-    return None