pyformatjson 0.0.1__tar.gz → 0.0.2__tar.gz

{pyformatjson-0.0.1 → pyformatjson-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyformatjson
-Version: 0.0.1
+Version: 0.0.2
 Summary: pyformatjson
 License: GPL-3.0-or-later
 Keywords: Python,json
@@ -8,9 +8,14 @@ Author: NextAI
 Author-email: nextartifintell@gmail.com
 Maintainer: NextAI
 Maintainer-email: nextartifintell@gmail.com
-Requires-Python: >=3.13
+Requires-Python: >=3.8
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries :: Python Modules

pyformatjson-0.0.2/pyformatjson/__init__.py

@@ -0,0 +1,17 @@
+"""PyFormatJSON: A Python library for formatting and processing JSON data for conferences and journals.
+
+This package provides utilities for processing conference and journal data from JSON files,
+generating markdown documentation, and creating various formatted outputs for academic
+publication management.
+
+Modules:
+    core: Core functionality for data processing and JSON manipulation
+    tools: Utility tools for data generation and markdown writing
+
+Classes:
+    GenerateDataDict: Generates data dictionaries from JSON input
+    WriteDataToMd: Writes publication data to Markdown files
+
+Functions:
+    main_generate_md_files: Main function to generate markdown files for conferences and journals
+"""

pyformatjson-0.0.2/pyformatjson/core/_base.py

@@ -0,0 +1,223 @@
+# coding=utf-8
+
+import os
+import re
+from typing import List
+
+
+def split_text_by_length(text, max_length=120) -> List[str]:
+    """Split text into lines of specified maximum length.
+
+    This function breaks long text into multiple lines, ensuring each line
+    does not exceed the specified maximum length. It attempts to break at
+    word boundaries when possible.
+
+    Args:
+        text (str): The input text to be split into lines.
+        max_length (int, optional): Maximum length for each line. Defaults to 120.
+
+    Returns:
+        List[str]: A list of text lines, each not exceeding max_length characters.
+
+    Example:
+        >>> split_text_by_length("This is a very long text that needs to be split", 20)
+        ['This is a very long', 'text that needs to be', 'split']
+    """
+    lines = []
+    while text:
+        if len(text) <= max_length:
+            lines.append(text)
+            break
+
+        split_pos = text.rfind(" ", 0, max_length + 1)
+        if split_pos == -1:
+            split_pos = max_length
+
+        line = text[:split_pos]
+        lines.append(line)
+
+        text = text[split_pos:]
+
+    new_lines = []
+    for line in lines:
+        new_lines.append(line)
+    return new_lines
+
+
+def split_data_list(split_pattern: str, data_list: List[str], last_next: str = "next") -> List[str]:
+    """Split data list according to the split pattern.
+
+    This function splits each string in the data list using the provided regex pattern
+    and reconstructs the data based on the last_next parameter. The pattern must use
+    capturing parentheses to define split points.
+
+    Args:
+        split_pattern (str): Regular expression pattern for splitting. Must use capturing
+            parentheses, e.g., r"(\n)" for newline splits.
+        data_list (List[str]): List of strings to be split and processed.
+        last_next (str, optional): Determines how to handle split parts. "next" places
+            the split character at the beginning of the next part, "last" places it at
+            the end of the current part. Defaults to "next".
+
+    Returns:
+        List[str]: New list of processed strings with empty strings filtered out.
+
+    Raises:
+        re.error: If the split_pattern is not a valid regular expression.
+
+    Example:
+        >>> split_data_list(r"(\n)", ["line1\nline2", "line3\nline4"], "next")
+        ['line1', 'line2', 'line3', 'line4']
+    """
+    new_data_list = []
+    for line in data_list:
+        split_list = re.split(split_pattern, line)
+        list_one = split_list[0 : len(split_list) : 2]
+        list_two = split_list[1 : len(split_list) : 2]
+
+        temp = []
+        if last_next == "next":
+            list_two.insert(0, "")
+            temp = [list_two[i] + list_one[i] for i in range(len(list_one))]
+        if last_next == "last":
+            list_two.append("")
+            temp = [list_one[i] + list_two[i] for i in range(len(list_one))]
+        new_data_list.extend(temp)
+    new_data_list = [line for line in new_data_list if line.strip()]
+    return new_data_list
+
+
+def standardize_path(path_input: str) -> str:
+    """Standardize and ensure a directory path exists.
+
+    This function expands environment variables and user home directory references
+    in the path, then creates the directory if it doesn't exist.
+
+    Args:
+        path_input (str): The input path to be standardized and created.
+
+    Returns:
+        str: The standardized absolute path.
+
+    Example:
+        >>> standardize_path("~/Documents/data")
+        '/Users/username/Documents/data'
+    """
+    path_input = os.path.expandvars(os.path.expanduser(path_input))
+    if not os.path.exists(path_input):
+        os.makedirs(path_input)
+    return path_input
+
+
+def sort_strings_with_embedded_numbers(s: str) -> List[str]:
+    """Split string into pieces for natural sorting with embedded numbers.
+
+    This function splits a string into pieces where numbers are converted to integers
+    for proper natural sorting (e.g., "item2" comes before "item10").
+
+    Args:
+        s (str): The string to be split into sortable pieces.
+
+    Returns:
+        List[str]: List of string pieces with numbers converted to integers.
+
+    Example:
+        >>> sort_strings_with_embedded_numbers("item10")
+        ['item', 10]
+    """
+    re_digits = re.compile(r"(\d+)")
+    pieces = re_digits.split(s)
+    pieces[1::2] = map(int, pieces[1::2])
+    return pieces
+
+
+def sort_int_str(str_int: List[str], reverse: bool = False) -> List[str]:
+    """Sort list of strings with embedded numbers naturally.
+
+    This function sorts a list of strings using natural sorting that handles
+    embedded numbers correctly (e.g., "item2" comes before "item10").
+
+    Args:
+        str_int (List[str]): List of strings to be sorted.
+        reverse (bool, optional): If True, sorts in descending order. Defaults to False.
+
+    Returns:
+        List[str]: Sorted list of strings.
+
+    Example:
+        >>> sort_int_str(["item10", "item2", "item1"])
+        ['item1', 'item2', 'item10']
+    """
+    return sorted(str_int, key=sort_strings_with_embedded_numbers, reverse=reverse)
+
+
+class IterateSortDict(object):
+    """A class for recursively sorting dictionary keys with natural sorting.
+
+    This class provides methods to sort dictionary keys recursively, handling
+    nested dictionaries and using natural sorting for strings with embedded numbers.
+
+    Attributes:
+        reverse (bool): If True, sorts keys in descending order. Defaults to False.
+
+    Example:
+        >>> sorter = IterateSortDict(reverse=False)
+        >>> data = {"item10": {"sub2": 1, "sub1": 2}, "item2": 3}
+        >>> sorted_data = sorter.dict_update(data)
+    """
+
+    def __init__(self, reverse: bool = False) -> None:
+        """Initialize the IterateSortDict instance.
+
+        Args:
+            reverse (bool, optional): If True, sorts keys in descending order.
+                Defaults to False.
+        """
+        self.reverse = reverse
+
+    def dict_update(self, old):
+        """Update and sort a dictionary recursively.
+
+        This method sorts the dictionary keys and recursively processes
+        any nested dictionaries.
+
+        Args:
+            old (dict): The dictionary to be sorted and updated.
+
+        Returns:
+            dict: The updated dictionary with sorted keys at all levels.
+        """
+        old = self.dict_sort_iteration(old)
+        old = self.dict_sort(old)
+        return old
+
+    def dict_sort_iteration(self, old: dict):
+        """Recursively sort nested dictionaries.
+
+        This method iterates through the dictionary and recursively sorts
+        any nested dictionary values.
+
+        Args:
+            old (dict): The dictionary to be processed recursively.
+
+        Returns:
+            dict: The dictionary with nested dictionaries sorted.
+        """
+        for key in old:
+            if isinstance(old[key], dict):
+                old[key] = self.dict_update(old[key])
+        return old
+
+    def dict_sort(self, old: dict):
+        """Sort dictionary keys using natural sorting.
+
+        This method sorts the top-level keys of the dictionary using
+        natural sorting that handles embedded numbers.
+
+        Args:
+            old (dict): The dictionary whose keys are to be sorted.
+
+        Returns:
+            dict: A new dictionary with sorted keys.
+        """
+        return {k: old[k] for k in sort_int_str(list(old.keys()), self.reverse)}

{pyformatjson-0.0.1 → pyformatjson-0.0.2}/pyformatjson/core/update_json.py

@@ -8,7 +8,24 @@ from ._base import split_data_list, split_text_by_length
 
 
 def load_json_data(path_json: str, filename: str) -> Dict:
-    """Load JSON data from file."""
+    """Load JSON data from a specified file.
+
+    This function attempts to load JSON data from a file located in the specified
+    directory. If the file doesn't exist or there's an error loading it, an empty
+    dictionary is returned.
+
+    Args:
+        path_json (str): Directory path containing the JSON file.
+        filename (str): Name of the JSON file (without .json extension).
+
+    Returns:
+        Dict: The loaded JSON data as a dictionary, or empty dict if file not found
+            or error occurs.
+
+    Example:
+        >>> load_json_data("/data", "conferences")
+        {"publisher1": {"conferences": {...}}}
+    """
     try:
         file_path = os.path.join(path_json, f"{filename}.json")
         if not os.path.exists(file_path):
@@ -23,15 +40,25 @@ def load_json_data(path_json: str, filename: str) -> Dict:
 
 
 def update_json_file(path_root: str, conferences_or_journals: str) -> Dict[str, Any]:
-    """
-    Update and format JSON file containing conference/journal data.
+    """Update and format JSON file containing conference/journal data.
+
+    This function loads JSON data, processes and formats text fields by splitting
+    long text into appropriate lengths, checks for duplicate abbreviations, and
+    saves the updated data back to the file.
 
     Args:
-        path_root (str): Root directory path
-        conferences_or_journals (str): Type of publication ('conferences' or 'journals')
+        path_root (str): Root directory path containing the JSON file.
+        conferences_or_journals (str): Type of publication ('conferences' or 'journals').
 
     Returns:
-        Dict[str, Any]: Processed JSON data
+        Dict[str, Any]: Processed JSON data dictionary.
+
+    Raises:
+        ValueError: If duplicate abbreviations are found in the data.
+
+    Example:
+        >>> update_json_file("/data", "conferences")
+        {"publisher1": {"conferences": {"conf1": {...}}}}
     """
     # Load Json Data
     json_dict = load_json_data(path_root, conferences_or_journals)
@@ -68,15 +95,21 @@ def update_json_file(path_root: str, conferences_or_journals: str) -> Dict[str,
 
 
 def _check_duplicate_abbr(json_dict: Dict[str, Any], conferences_or_journals: str) -> None:
-    """
-    Check for duplicate abbreviations in the data.
+    """Check for duplicate abbreviations in the data.
+
+    This function validates that there are no duplicate abbreviations within
+    the same publication type across all publishers.
 
     Args:
-        json_dict: JSON data dictionary
-        conferences_or_journals (str): Type of publication ('conferences' or 'journals')
+        json_dict (Dict[str, Any]): JSON data dictionary containing publication information.
+        conferences_or_journals (str): Type of publication ('conferences' or 'journals').
 
     Raises:
-        ValueError: If duplicate abbreviations are found
+        ValueError: If duplicate abbreviations are found in the data.
+
+    Example:
+        >>> _check_duplicate_abbr(data, "conferences")
+        # Raises ValueError if "ICML" appears twice in conferences
     """
     abbr_list = []
 

pyformatjson-0.0.2/pyformatjson/main.py

@@ -0,0 +1,155 @@
+# coding=utf-8
+
+import json
+import os
+from typing import Optional
+
+from .core._base import standardize_path
+from .core.update_json import load_json_data, update_json_file
+from .tools.generate_dict import GenerateDataDict
+from .tools.write_dict import WriteDataToMd
+
+
+def main_generate_md_files(
+    path_json: str,
+    path_output_md: str,
+    path_output_simplified_json: str,
+    path_spidered_bibs: Optional[str] = None,
+    for_vue: bool = True,
+    conferences_or_journals: Optional[str] = None,
+    keywords_category_name: str = "",
+) -> None:
+    """Generate markdown files for conferences and journals.
+
+    This function processes JSON data containing conference and journal information,
+    generates various markdown documentation files, and creates simplified JSON outputs.
+    It supports both conference and journal processing with customizable keyword categories.
+
+    Args:
+        path_json (str): Path to the input JSON data file containing publication information.
+        path_output_md (str): Output directory path where markdown files will be saved.
+        path_output_simplified_json (str): Output directory path for simplified JSON files.
+        path_spidered_bibs (Optional[str], optional): Directory containing crawled BibTeX files.
+            Defaults to None.
+        for_vue (bool, optional): Whether to generate Vue.js-compatible format for date calculations.
+            Defaults to True.
+        conferences_or_journals (Optional[str], optional): Specify 'conferences' or 'journals' to
+            process only one type, or None to process both. Defaults to None.
+        keywords_category_name (str, optional): The category name for keywords filtering.
+            Defaults to "".
+
+    Returns:
+        None: This function does not return a value.
+
+    Raises:
+        FileNotFoundError: If the input JSON file or required directories are not found.
+        ValueError: If there are issues with the data format or duplicate abbreviations.
+
+    Example:
+        >>> main_generate_md_files(
+        ...     path_json="/data/publications.json",
+        ...     path_output_md="/output/markdown",
+        ...     path_output_simplified_json="/output/json",
+        ...     for_vue=True,
+        ...     conferences_or_journals="conferences"
+        ... )
+    """
+    # Standardize all paths
+    path_json = standardize_path(path_json)
+    path_output_md = standardize_path(path_output_md)
+    path_output_simplified_json = standardize_path(path_output_simplified_json)
+
+    path_spidered_bibs = standardize_path(path_spidered_bibs) if path_spidered_bibs else ""
+
+    # Process keyword category name and load data
+    keywords_category_name = keywords_category_name.lower().strip() if keywords_category_name else ""
+    category_prefix = f"{keywords_category_name}_" if keywords_category_name else ""
+    keywords_list = load_json_data(path_json, "keywords").get(f"{category_prefix}keywords", [])
+
+    # Validate data availability
+    if not keywords_list or not keywords_category_name:
+        keywords_list, keywords_category_name = [], ""
+
+    # Process both conferences and journals
+    for cj, ia in zip(["conferences", "journals"], ["inproceedings", "article"]):
+        # Skip if specific type requested and doesn't match
+        if conferences_or_journals and conferences_or_journals.lower() != cj:
+            continue
+
+        # Update JSON data
+        json_dict = update_json_file(path_json, cj)
+        if not json_dict:
+            continue
+
+        # Simplify JSON data
+        simplify_json(json_dict, cj, path_output_simplified_json)
+
+        # Generate data dictionaries
+        path_spidered_cj = os.path.join(path_spidered_bibs, cj.title())
+        generater = GenerateDataDict(cj, ia, json_dict, for_vue, path_spidered_cj)
+        publisher_meta_dict, publisher_abbr_meta_dict, keyword_abbr_meta_dict = generater.generate()
+        if not (publisher_meta_dict and publisher_abbr_meta_dict and keyword_abbr_meta_dict):
+            continue
+
+        # Initialize writer and save all markdown files
+        _path_output = os.path.join(path_output_md, f"{cj.title()}")
+        save_data = WriteDataToMd(
+            cj, ia, publisher_meta_dict, publisher_abbr_meta_dict, keyword_abbr_meta_dict, _path_output
+        )
+        # Save various documentation files
+        save_data.save_introductions()
+        save_data.save_categories(keywords_category_name, keywords_list)
+        save_data.save_categories_separate_keywords()
+
+        save_data.save_publishers()
+        save_data.save_publishers_separate_abbrs()
+
+        save_data.save_statistics(keywords_category_name, keywords_list)
+        save_data.save_statistics_separate_abbrs()
+
+    return None
+
+
+def simplify_json(json_dict, cj: str, output_dir: str) -> None:
+    """Simplify JSON dictionary by extracting only essential fields.
+
+    This function creates a simplified version of the JSON dictionary containing
+    only the names_abbr and names_full fields for each publisher and publication type.
+    The simplified data is saved to a new JSON file in the specified output directory.
+
+    Args:
+        json_dict (dict): The original JSON dictionary containing publication data.
+        cj (str): The type of publication, either 'conferences' or 'journals'.
+        output_dir (str): Directory path where the simplified JSON file will be saved.
+
+    Returns:
+        None: This function does not return a value.
+
+    Note:
+        The function creates a new JSON file named '{cj}.json' in the output directory.
+        Only the 'names_abbr' and 'names_full' fields are preserved in the simplified version.
+
+    Example:
+        >>> simplify_json(
+        ...     json_dict=publication_data,
+        ...     cj="conferences",
+        ...     output_dir="/output/simplified"
+        ... )
+    """
+    new_json_dict = {}
+    for publisher in json_dict:
+        for abbr in json_dict[publisher][cj.lower()]:
+            names_abbr = json_dict[publisher][cj.lower()][abbr].get("names_abbr", [])
+            names_full = json_dict[publisher][cj.lower()][abbr].get("names_full", [])
+
+            new_json_dict.setdefault(publisher, {}).setdefault(cj.lower(), {}).setdefault(abbr, {}).update(
+                {"names_abbr": names_abbr, "names_full": names_full}
+            )
+
+    # Save updated JSON
+    if new_json_dict:
+        path_file = os.path.join(output_dir, f"{cj}.json")
+        with open(path_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(new_json_dict, indent=4, sort_keys=True, ensure_ascii=True))
+
+    return None