pyeasyphd 0.0.2__py3-none-any.whl

pyeasyphd/tools/search/search_core.py

@@ -0,0 +1,326 @@
+import copy
+import os
+import re
+import shutil
+from typing import Any, Dict, List, Tuple
+
+from pyadvtools import (
+    IterateCombineExtendDict,
+    IterateUpdateDict,
+    combine_content_in_list,
+    pairwise_combine_in_list,
+    read_list,
+    sort_int_str,
+    standard_path,
+    write_list,
+)
+
+from ...bib.bibtexparser import Library
+from ...main import BasicInput, PythonRunBib
+from .search_base import SearchInitialResult
+from .search_writers import WriteAbbrCombinedResults
+from .utils import keywords_type_for_title, switch_keywords_list, switch_keywords_type
+
+
+class SearchResultsCore(BasicInput):
+    """Generate tex, md, html, and pdf.
+
+    Args:
+        path_storage (str): the path of storage `abbr`
+        path_output (str): the path of output `abbr`
+        path_separate (str): the path of separate `abbr`
+        j_conf_abbr (str): the abbreviation of journal or conference
+        options (dict): options
+
+    Attributes:
+        path_storage (str): the path of storage
+        path_output (str): the path of output
+        path_separate (str): the path of separate
+        j_conf_abbr (str): the abbreviation of journal or conference
+
+        is_standard_bib_file_name (bool = True): whether the bib file name is standard
+        keywords_type_list (List[str] = []): keywords type list
+        keywords_dict (dict = {}): keywords dict
+        delete_redundant_files (bool = True): delete redundant files
+        generate_basic_md (bool = False): generate basic md
+        generate_beauty_md (bool = False): generate beauty md
+        generate_complex_md (bool = True): generate complex md
+        generate_tex (bool = False): generate tex
+        first_field_second_keywords (bool = True): first field second keywords
+        deepcopy_library_for_every_field (bool = False): deepcopy library for every field
+        deepcopy_library_for_every_keywords (bool = False): deepcopy library for every keywords
+    """
+
+    def __init__(
+        self, path_storage: str, path_output: str, path_separate: str, j_conf_abbr: str, options: Dict[str, Any]
+    ) -> None:
+        super().__init__(options)
+        self.path_storage: str = standard_path(path_storage)
+        self.path_output: str = standard_path(path_output)
+        self.path_separate: str = standard_path(path_separate)
+        self.j_conf_abbr: str = j_conf_abbr
+
+        # for bib
+        # Whether the bib file name is standard, such as `TEVC_2023.bib`.
+        self.is_standard_bib_file_name: bool = options.get("is_standard_bib_file_name", True)  # TEVC_2023.bib
+
+        # for search
+        self.keywords_dict = options.get("default_keywords_dict", {})
+        if temp := options.get("keywords_dict", []):
+            self.keywords_dict = temp
+
+        if keywords_type_list := options.get("keywords_type_list", []):
+            self.keywords_dict = {k: v for k, v in self.keywords_dict.items() if k in keywords_type_list}
+
+        self.keywords_dict = {switch_keywords_type(k): v for k, v in self.keywords_dict.items()}
+
+        self.search_field_list = options.get("default_search_field_list", ["title", "abstract"])
+        if temp := options.get("search_field_list", []):
+            self.search_field_list = temp
+
+        # for pandoc
+        self.delete_redundant_files: bool = options.get("delete_redundant_files", True)
+
+        # for md
+        self.generate_basic_md: bool = options.get("generate_basic_md", False)
+        self.generate_beauty_md: bool = options.get("generate_beauty_md", False)
+        self.generate_complex_md: bool = options.get("generate_complex_md", True)
+
+        # for tex
+        self.generate_tex = options.get("generate_tex", False)
+
+        # for search
+        self.first_field_second_keywords = options.get("first_field_second_keywords", True)
+        self.deepcopy_library_for_every_field = options.get("deepcopy_library_for_every_field", False)
+        self.deepcopy_library_for_every_keywords = options.get("deepcopy_library_for_every_keywords", False)
+
+        # for bib
+        self._python_bib = PythonRunBib(options)
+
+    def optimize(self, search_year_list: List[str] = []) -> Dict[str, Dict[str, Dict[str, Dict[str, int]]]]:
+        search_year_list = list(set([str(i) for i in search_year_list]))
+
+        data_list = self._obtain_full_files_data(self.path_storage, "bib", search_year_list)
+
+        entry_type_keyword_type_keyword_field_number_dict = self.optimize_core(data_list, search_year_list)
+        return entry_type_keyword_type_keyword_field_number_dict
+
+    def _obtain_full_files_data(self, path_storage: str, extension: str, search_year_list: List[str] = []) -> List[str]:
+        regex = None
+        if self.is_standard_bib_file_name and search_year_list:
+            regex = re.compile(f'({"|".join(search_year_list)})')
+
+        file_list = []
+        for root, _, files in os.walk(path_storage, topdown=True):
+            files = [f for f in files if f.endswith(f".{extension}")]
+
+            if regex:
+                files = [f for f in files if regex.search(f)]
+
+            file_list.extend([os.path.join(root, f) for f in files])
+
+        return combine_content_in_list([read_list(f, "r") for f in sort_int_str(file_list)], None)
+
+    def optimize_core(self, data_list: List[str], search_year_list) -> Dict[str, Dict[str, Dict[str, Dict[str, int]]]]:
+        print("\n" + "*" * 9 + f" Search in {self.j_conf_abbr} " + "*" * 9)
+
+        entry_type_year_volume_number_month_entry_dict = self._python_bib.parse_to_nested_entries_dict(data_list)
+
+        # generate standard bib and output
+        entry_type_keyword_type_keyword_field_number_dict: Dict[str, Dict[str, Dict[str, Dict[str, int]]]] = {}
+        for entry_type in entry_type_year_volume_number_month_entry_dict:
+
+            # obtain search years
+            year_list = list(entry_type_year_volume_number_month_entry_dict[entry_type].keys())
+            if search_year_list:
+                year_list = [y for y in year_list if y in search_year_list]
+            year_list = sort_int_str(year_list, reverse=True)
+            if not year_list:
+                print("year_list is empty.")
+                continue
+
+            # output prefix
+            output_prefix = "-".join([self.j_conf_abbr, year_list[-1], year_list[0]])
+
+            # generate paths
+            p_origin = os.path.join(self.path_output, entry_type, f"{output_prefix}-Origin")
+            p_separate = os.path.join(self.path_separate, entry_type)
+            p_combine = os.path.join(self.path_output, entry_type, f"{output_prefix}-Combine")
+
+            # obtain library
+            new_dict = {year: entry_type_year_volume_number_month_entry_dict[entry_type][year] for year in year_list}
+            entries = IterateCombineExtendDict().dict_update(new_dict)
+            library = Library(entries)
+
+            # search, generate and save
+            keyword_type_keyword_field_number_dict = {}
+            for keywords_type in self.keywords_dict:
+                library = copy.deepcopy(library)
+
+                if self.first_field_second_keywords:
+                    keyword_field_number_dict = self._optimize_fields_keyword(
+                        keywords_type, library, output_prefix, p_origin, p_separate, p_combine
+                    )
+                else:
+                    keyword_field_number_dict = self._optimize_keywords_field(
+                        keywords_type, library, output_prefix, p_origin, p_separate, p_combine
+                    )
+                keyword_type_keyword_field_number_dict.update({keywords_type: keyword_field_number_dict})
+
+            # collect results
+            entry_type_keyword_type_keyword_field_number_dict.setdefault(entry_type, {}).update(
+                keyword_type_keyword_field_number_dict
+            )
+
+        return entry_type_keyword_type_keyword_field_number_dict
+
+    def _optimize_fields_keyword(self, keywords_type, library, output_prefix, p_origin, p_separate, p_combine):
+        no_search_library = library
+
+        keyword_field_number_dict_ = {}
+        for field in self.search_field_list:
+            keyword_field_number_dict, no_search_library = self.core_optimize(
+                [field],
+                keywords_type,
+                no_search_library,
+                output_prefix,
+                p_origin,
+                p_separate,
+                p_combine,
+            )
+
+            if self.deepcopy_library_for_every_field:
+                no_search_library = copy.deepcopy(library)
+
+            temp = keyword_field_number_dict
+            keyword_field_number_dict_ = IterateUpdateDict().dict_update(keyword_field_number_dict_, temp)
+        return keyword_field_number_dict_
+
+    def _optimize_keywords_field(self, keywords_type, library, output_prefix, p_origin, p_separate, p_combine):
+        no_search_library = library
+
+        keyword_field_number_dict, no_search_library = self.core_optimize(
+            self.search_field_list,
+            keywords_type,
+            no_search_library,
+            output_prefix,
+            p_origin,
+            p_separate,
+            p_combine,
+        )
+        return keyword_field_number_dict
+
+    def core_optimize(
+        self,
+        search_field_list: List[str],
+        keywords_type,
+        library: Library,
+        output_prefix: str,
+        p_origin: str,
+        p_separate: str,
+        p_combine: str,
+    ) -> Tuple[Dict[str, Dict[str, int]], Library]:
+        error_pandoc_md_md: List[str] = []
+        save_field_data_dict: Dict[str, List[List[str]]] = {}
+        keyword_field_number_dict: Dict[str, Dict[str, int]] = {}
+
+        no_search_library = library
+        for keywords_list in self.keywords_dict[keywords_type]:
+
+            print(f"{output_prefix}-{keywords_type}-search-{keywords_list}")
+            keywords_list_list, combine_keyword = switch_keywords_list(keywords_list)
+
+            # for initial results
+            error_md, field_data_dict, field_number_dict, no_search_library = SearchInitialResult(
+                copy.deepcopy(self.options)
+            ).main(
+                search_field_list,
+                p_origin,
+                no_search_library,
+                keywords_type,
+                keywords_list_list,
+                combine_keyword,
+                output_prefix,
+                p_separate,
+            )
+
+            if self.deepcopy_library_for_every_keywords:
+                no_search_library = copy.deepcopy(library)
+
+            # collect error parts
+            error_pandoc_md_md.extend(error_md)
+
+            # collect data
+            for field in field_data_dict:
+                temp = pairwise_combine_in_list(save_field_data_dict.get(field, []), field_data_dict[field], "\n")
+                save_field_data_dict.update({field: temp})
+
+            # collect number
+            keyword_field_number_dict.update({combine_keyword: field_number_dict})
+
+        kws_type = keywords_type_for_title(keywords_type)
+        flag = "-".join(search_field_list)
+
+        # for error parts in pandoc markdown to markdown
+        if error_pandoc_md_md:
+            error_pandoc_md_md.insert(0, f"# Error in pandoc md to md for {kws_type}\n\n")
+            write_list(error_pandoc_md_md, rf"{flag}_{output_prefix}_error_pandoc_md_md.md", "a", p_combine)
+
+        # combine part
+        # for combined results
+        error_pandoc_md_pdf, error_pandoc_md_html = WriteAbbrCombinedResults(copy.deepcopy(self.options)).main(
+            search_field_list, keywords_type, save_field_data_dict, p_combine
+        )
+
+        # for error parts in pandoc markdown to pdf
+        if error_pandoc_md_pdf:
+            error_pandoc_md_pdf.insert(0, f"# Error in pandoc md to pdf for {kws_type}\n\n")
+            write_list(error_pandoc_md_pdf, rf"{flag}_{output_prefix}_error_pandoc_md_pdf.md", "a", p_combine)
+
+        # for error parts in pandoc markdown to html
+        if error_pandoc_md_html:
+            error_pandoc_md_html.insert(0, f"# Error in pandoc md to html for {kws_type}\n\n")
+            write_list(error_pandoc_md_html, rf"{flag}_{output_prefix}_error_pandoc_md_html.md", "a", p_combine)
+
+        # delete redundant files
+        if self.delete_redundant_files:
+            self.delete_files(keywords_type, p_origin, p_separate, p_combine)
+
+        return keyword_field_number_dict, no_search_library
+
+    def delete_files(self, keywords_type: str, p_origin: str, p_separate: str, p_combine: str) -> None:
+        """Delete some redundant files."""
+        # for initial tex md bib
+        if os.path.exists(p_origin):
+            shutil.rmtree(p_origin)
+
+        # for separate keywords
+        delete_folder_list = []
+        if not self.generate_basic_md:
+            delete_folder_list.append("basic")
+        if not self.generate_beauty_md:
+            delete_folder_list.append("beauty")
+        if not self.generate_complex_md:
+            delete_folder_list.append("complex")
+
+        for d in delete_folder_list:
+            for field in self.search_field_list:
+                path_delete = os.path.join(p_separate, rf"{keywords_type}/{field}-md-{d}")
+                if os.path.exists(path_delete):
+                    shutil.rmtree(path_delete)
+
+        # for combine
+        delete_folder_list = ["md"]
+        if not self.generate_basic_md:
+            delete_folder_list.append("md-basic")
+        if not self.generate_beauty_md:
+            delete_folder_list.append("md-beauty")
+        if not self.generate_complex_md:
+            delete_folder_list.append("md-complex")
+        if not self.generate_tex:
+            delete_folder_list.extend(["tex", "tex-subsection"])
+
+        for d in delete_folder_list:
+            path_delete = os.path.join(p_combine, f"{d}")
+            if os.path.exists(path_delete):
+                shutil.rmtree(path_delete)

pyeasyphd/tools/search/search_keywords.py

@@ -0,0 +1,227 @@
+import copy
+import os
+import re
+from typing import Any, Dict, List
+
+from pyadvtools import (
+    generate_nested_dict,
+    read_list,
+    standard_path,
+    write_list,
+)
+
+from ...main import PandocMdTo
+from ...utils.utils import html_head, html_style, html_tail
+from ..experiments_base import generate_standard_publisher_abbr_options_dict
+from .data import obtain_search_keywords
+from .search_core import SearchResultsCore
+from .utils import extract_information, temp_html_style
+
+
+class Searchkeywords(object):
+    """Search.
+
+    Args:
+        path_storage (str): the path of storage journals or conferences
+        path_output (str): the path of output journals or conferences
+        options (dict): options
+
+    Attributes:
+        path_storage (str): the path of storage
+        path_output (str): the path of output
+        options (dict): options
+
+        search_year_list (List[str] = []): search year list
+    """
+
+    def __init__(self, path_storage: str, path_output: str, options: Dict[str, Any]) -> None:
+        self.path_storage = standard_path(path_storage)
+        self.path_output = standard_path(path_output)
+
+        options_ = {}
+        options_["display_one_line_reference_note"] = True  # default is False
+        options_["is_standardize_bib"] = False  # default is True
+        options_["choose_abbr_zotero_save"] = "save"  # default is "save"
+        options_["function_common_again"] = True  # default is True
+        options_["function_common_again_abbr"] = False  # default is True
+        options_["function_common_again_zotero"] = False  # default is True
+        options_["function_common_again_save"] = False  # default is True
+        options_["is_sort_entry_fields"] = True  # default is False
+        options_["is_sort_entries_by_field_keys"] = True  # default is False
+        options_["sort_entries_by_field_keys_reverse"] = True  # default is False
+        options_["generate_entry_cite_keys"] = True  # default is False
+
+        options_["default_keywords_dict"] = obtain_search_keywords()
+        options_["default_search_field_list"] = ["title", "abstract"]
+        options_.update(options)
+        self.options = options_
+
+        self.search_year_list = options.get("search_year_list", [])
+        self._path_separate = self.path_output + "-Separate"
+
+        self._path_statistic = self.path_output + "-Statistics"
+        self._path_combine = self.path_output + "-Combine"
+
+    def run(self) -> None:
+        all_dict = {}
+        publisher_abbr_dict = generate_standard_publisher_abbr_options_dict(self.path_storage, self.options)
+        for publisher in publisher_abbr_dict:
+            for abbr in publisher_abbr_dict[publisher]:
+                options = publisher_abbr_dict[publisher][abbr]
+
+                path_storage = os.path.join(self.path_storage, publisher, abbr)
+                path_output = os.path.join(self.path_output, publisher, abbr)
+                entry_type_keyword_type_keyword_field_number_dict = SearchResultsCore(
+                    path_storage, path_output, self._path_separate, abbr, options
+                ).optimize(copy.deepcopy(self.search_year_list))
+
+                all_dict.update({abbr: entry_type_keyword_type_keyword_field_number_dict})
+
+        if not self.options.get("print_on_screen", False):
+            extract_information(all_dict, self._path_statistic)
+
+            print()
+            self._generate_bib_html_for_publisher(publisher_abbr_dict, "bib")
+            print()
+            self._generate_bib_html_for_publisher(publisher_abbr_dict, "html")
+            self._generate_link_to_bib_html_for_combine()
+
+            print()
+            self._pandoc_md_to_html_in_path_separate()
+            self._generate_link_to_html_bib_for_separate()
+
+        return None
+
+    def _extract_files(
+        self, publisher_abbr_dict: dict, ext: str = "html"
+    ) -> Dict[str, Dict[str, Dict[str, Dict[str, List[str]]]]]:
+        data_dict = {}
+        for publisher in publisher_abbr_dict:
+            for abbr in publisher_abbr_dict[publisher]:
+                p = os.path.join(self.path_output, publisher, abbr)
+                if not os.path.exists(p):
+                    continue
+
+                for entry_type in [f for f in os.listdir(p) if os.path.isdir(os.path.join(p, f))]:
+                    if not (folders := [f for f in os.listdir(os.path.join(p, entry_type)) if "combine" in f.lower()]):
+                        continue
+
+                    for root, _, files in os.walk(os.path.join(p, entry_type, folders[0])):
+                        for file in [f for f in files if f.endswith(ext)]:
+                            (
+                                data_dict.setdefault(file, {})
+                                .setdefault(entry_type, {})
+                                .setdefault(publisher, {})
+                                .setdefault(abbr, [])
+                                .append(os.path.join(root, file))
+                            )
+        return data_dict
+
+    def _generate_bib_html_for_publisher(self, publisher_abbr_dict, ext: str = "html") -> None:
+        data_dict = self._extract_files(publisher_abbr_dict, ext)
+        for file in data_dict:
+            basename = file.split(".")[0]
+            for entry_type in data_dict[file]:
+                for publisher in data_dict[file][entry_type]:
+
+                    print(f"Generate {ext} for `{publisher}-{entry_type}-{basename}`")
+                    data_list = []
+                    for abbr in data_dict[file][entry_type][publisher]:
+                        for i in range(ll := len(data_dict[file][entry_type][publisher][abbr])):
+                            full_file = data_dict[file][entry_type][publisher][abbr][i]
+                            temp_data_list = read_list(full_file, "r", None)
+                            if ext == "html":
+                                if mch := re.search(r"(<h3.*)</body>", "".join(temp_data_list), re.DOTALL):
+                                    temp_data_list = mch.group(1).splitlines(keepends=True)
+
+                            data_list.extend(temp_data_list)
+                            if i < (ll - 1):
+                                data_list.append("\n")
+                        data_list.append("\n")
+
+                    p = os.path.join(self._path_combine, entry_type, publisher, ext)
+                    if ext == "html":
+                        data_list_ = [html_head.format(basename)]
+                        data_list_.extend(html_style)
+                        data_list_.append(f'<h2 id="{publisher.upper()}">{publisher.upper()}</h2>\n')
+                        data_list_.extend(data_list)
+                        data_list_.append(html_tail)
+                        write_list(data_list_, f"{basename}.{ext}", "w", p, False)
+
+                    else:
+                        write_list(data_list, f"{basename}.{ext}", "w", p, False)
+        return None
+
+    def _generate_link_to_bib_html_for_combine(self) -> None:
+        nested_dict = generate_nested_dict(self._path_combine)
+
+        for entry_type in nested_dict:
+            data_dict = {}
+            for publisher in nested_dict[entry_type]:
+                for ext in nested_dict[entry_type][publisher]:
+                    if ext == "html":
+                        for file in nested_dict[entry_type][publisher][ext]:
+                            data_dict.setdefault(publisher, []).append(file)
+
+                    if ext == "bib":
+                        for file in nested_dict[entry_type][publisher][ext]:
+                            if not re.search(r"\-zotero", file):
+                                continue
+
+                            data_dict.setdefault(publisher, []).append(file)
+
+            data_list = self._html_format(entry_type, data_dict, "Publishers", "combine")
+            write_list(data_list, f"{entry_type.lower()}_links.html", "w", self._path_combine, False)
+        return None
+
+    def _pandoc_md_to_html_in_path_separate(self) -> None:
+        mds = []
+        for root, _, files in os.walk(self._path_separate):
+            mds.extend([os.path.join(root, f) for f in files if f.endswith(".md")])
+
+        for full_md in mds:
+            print(f"pandoc md to html for `{full_md.split(self._path_separate)[-1]}`")
+            full_html = full_md.replace("-md", "-html").replace(".md", ".html")
+            PandocMdTo({}).pandoc_md_to_html(full_md, full_html, None, None, True)
+
+    def _generate_link_to_html_bib_for_separate(self) -> None:
+        for entry_type in (nested_dict := generate_nested_dict(self._path_separate)):
+            data_dict = {}
+            for keywords_type in nested_dict[entry_type]:
+                for ext in nested_dict[entry_type][keywords_type]:
+                    if not re.search(r"(\-html\-|\-bib\-zotero)", ext):
+                        continue
+
+                    for file in nested_dict[entry_type][keywords_type][ext]:
+                        data_dict.setdefault(os.path.basename(file).split(".")[0], []).append(file)
+
+            data_list = self._html_format(entry_type, data_dict, "Keywords",  "separate")
+            write_list(data_list, f"{entry_type.lower()}_links.html", "w", self._path_separate, False)
+        return None
+
+    @staticmethod
+    def _html_format(entry_type, data_dict, name_flag, index):
+        data_list = [html_head.format(f"{entry_type.title()} Links"), temp_html_style]
+        data_list.append('\n<table border="1">\n')
+        data_list.append(f"<caption>{entry_type.title()} Links</caption>\n")
+
+        data_list.extend(["<thead>\n", "<tr>\n", f"<th>{name_flag}</th>\n", "</tr>\n", "</thead>\n"])
+
+        x = '<td><a href="{}" target="_blank">{}</a></td>\n'
+        data_list.append("<tbody>\n")
+        for name in data_dict:
+            data_list.append("<tr>\n")
+            data_list.append(f"<td>{name}</td>\n")
+
+            for f in data_dict[name]:
+                if index == "combine":
+                    data_list.append(x.format(f, f.split("-")[0].split("/")[-1].title() + ":" + f.split(".")[-1]))
+                elif index == "separate":
+                    data_list.append(x.format(f, f.split("/")[-2].split("-")[0].title() + ":" + f.split(".")[-1]))
+
+            data_list.append("</tr>\n")
+        data_list.append("</tbody>\n")
+
+        data_list.append("</table>\n")
+        data_list.append(html_tail)
+        return data_list