ckparser 0.1__tar.gz → 0.2.1__tar.gz

{ckparser-0.1/ckparser.egg-info → ckparser-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ckparser
-Version: 0.1
+Version: 0.2.1
 Summary: Parse Paradox Jomini data files to Python/JSON and revert them back experimentally.
 Author: Marc Debureaux (debnet)
 License-Expression: MIT
@@ -132,10 +132,10 @@ Parse a directory recursively and write JSON output:
 python -m ckparser my_mod/common --output output
 ```
 
-Include comments in the parsed output:
+Include comments and/or dates in the parsed output:
 
 ```bash
-python -m ckparser my_mod/common --comments
+python -m ckparser my_mod/common --comments --dates
 ```
 
 Revert a JSON file back to Jomini format:
@@ -412,6 +412,14 @@ With `comments=True` or `--comments`, the parser attempts to preserve them in an
 
 Comment preservation should be considered practical rather than perfectly lossless.
 
+### Dates
+
+By default, dates are not converted in Python `datetime.date` objects in parsing. 
+
+While convenient, Jomini date format (`Y[YYY].M[M].D[D]`) does not enforce validation and can be invalid.
+
+With `dates=True` or `--dates`, the parser attempts to convert these dates if possible and keeps invalid dates as text.
+
 ### Localization parsing
 
 `ckparser` also includes a dedicated parser for Paradox localization files:
@@ -510,9 +518,9 @@ Paradox scripting languages include many practical exceptions, formatting varian
 
 ### Parsing
 
-* `parse_text(text, return_text_on_error=False, comments=False, filename=None, is_global=False)`
-* `parse_file(path, output_dir=None, encoding="utf_8_sig", base_dir=None, save=False, comments=False, is_global=False, patch=None)`
-* `parse_all_files(path, output_dir=None, encoding="utf_8_sig", keep_data=False, save=False, comments=False, variables_first=True)`
+* `parse_text(text, return_text_on_error=False, comments=False, dates=False, filename=None, is_global=False)`
+* `parse_file(path, output_dir=None, encoding="utf_8_sig", base_dir=None, save=False, comments=False, dates=False, is_global=False, patch=None)`
+* `parse_all_files(path, output_dir=None, encoding="utf_8_sig", keep_data=False, save=False, comments=False, dates=False, variables_first=True)`
 
 ### Reversion
 

{ckparser-0.1 → ckparser-0.2.1}/README.md

@@ -101,10 +101,10 @@ Parse a directory recursively and write JSON output:
 python -m ckparser my_mod/common --output output
 ```
 
-Include comments in the parsed output:
+Include comments and/or dates in the parsed output:
 
 ```bash
-python -m ckparser my_mod/common --comments
+python -m ckparser my_mod/common --comments --dates
 ```
 
 Revert a JSON file back to Jomini format:
@@ -381,6 +381,14 @@ With `comments=True` or `--comments`, the parser attempts to preserve them in an
 
 Comment preservation should be considered practical rather than perfectly lossless.
 
+### Dates
+
+By default, dates are not converted in Python `datetime.date` objects in parsing. 
+
+While convenient, Jomini date format (`Y[YYY].M[M].D[D]`) does not enforce validation and can be invalid.
+
+With `dates=True` or `--dates`, the parser attempts to convert these dates if possible and keeps invalid dates as text.
+
 ### Localization parsing
 
 `ckparser` also includes a dedicated parser for Paradox localization files:
@@ -479,9 +487,9 @@ Paradox scripting languages include many practical exceptions, formatting varian
 
 ### Parsing
 
-* `parse_text(text, return_text_on_error=False, comments=False, filename=None, is_global=False)`
-* `parse_file(path, output_dir=None, encoding="utf_8_sig", base_dir=None, save=False, comments=False, is_global=False, patch=None)`
-* `parse_all_files(path, output_dir=None, encoding="utf_8_sig", keep_data=False, save=False, comments=False, variables_first=True)`
+* `parse_text(text, return_text_on_error=False, comments=False, dates=False, filename=None, is_global=False)`
+* `parse_file(path, output_dir=None, encoding="utf_8_sig", base_dir=None, save=False, comments=False, dates=False, is_global=False, patch=None)`
+* `parse_all_files(path, output_dir=None, encoding="utf_8_sig", keep_data=False, save=False, comments=False, dates=False, variables_first=True)`
 
 ### Reversion
 

{ckparser-0.1 → ckparser-0.2.1/ckparser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ckparser
-Version: 0.1
+Version: 0.2.1
 Summary: Parse Paradox Jomini data files to Python/JSON and revert them back experimentally.
 Author: Marc Debureaux (debnet)
 License-Expression: MIT
@@ -132,10 +132,10 @@ Parse a directory recursively and write JSON output:
 python -m ckparser my_mod/common --output output
 ```
 
-Include comments in the parsed output:
+Include comments and/or dates in the parsed output:
 
 ```bash
-python -m ckparser my_mod/common --comments
+python -m ckparser my_mod/common --comments --dates
 ```
 
 Revert a JSON file back to Jomini format:
@@ -412,6 +412,14 @@ With `comments=True` or `--comments`, the parser attempts to preserve them in an
 
 Comment preservation should be considered practical rather than perfectly lossless.
 
+### Dates
+
+By default, dates are not converted in Python `datetime.date` objects in parsing. 
+
+While convenient, Jomini date format (`Y[YYY].M[M].D[D]`) does not enforce validation and can be invalid.
+
+With `dates=True` or `--dates`, the parser attempts to convert these dates if possible and keeps invalid dates as text.
+
 ### Localization parsing
 
 `ckparser` also includes a dedicated parser for Paradox localization files:
@@ -510,9 +518,9 @@ Paradox scripting languages include many practical exceptions, formatting varian
 
 ### Parsing
 
-* `parse_text(text, return_text_on_error=False, comments=False, filename=None, is_global=False)`
-* `parse_file(path, output_dir=None, encoding="utf_8_sig", base_dir=None, save=False, comments=False, is_global=False, patch=None)`
-* `parse_all_files(path, output_dir=None, encoding="utf_8_sig", keep_data=False, save=False, comments=False, variables_first=True)`
+* `parse_text(text, return_text_on_error=False, comments=False, dates=False, filename=None, is_global=False)`
+* `parse_file(path, output_dir=None, encoding="utf_8_sig", base_dir=None, save=False, comments=False, dates=False, is_global=False, patch=None)`
+* `parse_all_files(path, output_dir=None, encoding="utf_8_sig", keep_data=False, save=False, comments=False, dates=False, variables_first=True)`
 
 ### Reversion
 

{ckparser-0.1 → ckparser-0.2.1}/ckparser.py

@@ -28,6 +28,7 @@ Author: Marc Debureaux (debnet)
 Project: https://github.com/debnet/ckparser
 License: MIT
 """
+
 import argparse
 import ast
 import colorsys
@@ -40,7 +41,7 @@ import re
 import time
 
 # Script version
-__version__ = "0.1"
+__version__ = "0.2.1"
 
 # Logger (because logging is awesome)
 logger = logging.getLogger(__name__)
@@ -52,6 +53,38 @@ except ImportError:
     detect = None
     logger.warning("chardet not installed, encoding detection is disabled!")
 
+
+class JominiJSONEncoder(json.JSONEncoder):
+    # JSON specific encoder for dates
+    def default(self, obj):
+        if isinstance(obj, datetime.date):
+            return f"{obj.year}.{obj.month}.{obj.day}"
+        return super().default(obj)
+
+
+def jomini_object_hook(obj):
+    # JSON specific decoder for dates
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if isinstance(value, str) and regex_date.fullmatch(value):
+                obj[key] = convert_date(value) or value
+            elif isinstance(value, list):
+                obj[key] = jomini_object_hook(value)
+    elif isinstance(obj, list):
+        for index in range(len(obj)):
+            item = obj[index]
+            if isinstance(item, str) and regex_date.fullmatch(item):
+                obj[index] = convert_date(item) or item
+            elif isinstance(item, list):
+                obj[index] = jomini_object_hook(item)
+    return obj
+
+
+json_load = functools.partial(json.load, object_hook=jomini_object_hook)
+json_loads = functools.partial(json.loads, object_hook=jomini_object_hook)
+json_dump = functools.partial(json.dump, cls=JominiJSONEncoder, indent=4, ensure_ascii=False)
+json_dumps = functools.partial(json.dumps, cls=JominiJSONEncoder, indent=4, ensure_ascii=False)
+
 # Boolean transformation
 booleans = {"yes": True, "no": False}
 # Tags which must be aggregate as a list in JSON (don't hesitate to add more if needed)
@@ -84,6 +117,8 @@ regex_values = re.compile(r"(([?!<=>]+)\s*\n+)|(\n+\s*([?!<=>]+))")
 regex_line = re.compile(r"\"?(?P<key>[^\s\"]+)\"?\s*(?P<operator>[?!<=>]+)\s*(list\s*)?(?P<value>.*)")
 # Regex to parse independent items in a list
 regex_item = re.compile(r"(\"[^\"]+\"|[\d\.]+|[^\s]+)")
+# Regex to parse dates
+regex_date = re.compile(r"^(?:\d{1,4})\.(?:[1-9]|1[0-2])\.(?:[1-9]|[12]\d|3[01])$")
 # Regex to remove empty lines
 regex_empty = re.compile(r"(\n\s*\n)+", re.MULTILINE)
 # Regex to parse locale files
@@ -151,12 +186,13 @@ def read_file(path, encoding="utf_8_sig"):
         return file.read()
 
 
-def parse_text(text, return_text_on_error=False, comments=False, filename=None, is_global=False):
+def parse_text(text, return_text_on_error=False, comments=False, dates=False, filename=None, is_global=False):
     """
     Parse raw text
     :param text: Text to parse
     :param return_text_on_error: (default false) Return working text document if parsing fails
     :param comments: (default false) Include comments?
+    :param dates: (default false) Include dates?
     :param filename: (default none) Filename (only for debugging)
     :param is_global: (default false) Are variables global?
     :return: Parsed data as dictionary
@@ -251,10 +287,13 @@ def parse_text(text, return_text_on_error=False, comments=False, filename=None,
                     value = val
                 elif value:
                     # Try to convert value to Python value
-                    try:
-                        value = ast.literal_eval(value)
-                    except:  # noqa
-                        pass
+                    if dates and regex_date.fullmatch(value):
+                        value = convert_date(value) or value
+                    else:
+                        try:
+                            value = ast.literal_eval(value)
+                        except:  # noqa
+                            pass
                 # If key is duplicate with direct value
                 if key in node:
                     if node[key] != value:  # Avoid single duplicates
@@ -331,7 +370,7 @@ def parse_text(text, return_text_on_error=False, comments=False, filename=None,
                             node[key] = {"@type": "variable", "@value": value, "@result": result}
                             if result is not None and (key.startswith("@") or len(nodes) == 1):
                                 set_variable(key, result)
-                        elif result := variables.get(value):
+                        elif (result := variables.get(value)) is not None:
                             if not isinstance(result, str):
                                 node[key] = {"@type": "variable", "@value": value, "@result": result}
                                 if result is not None and (key.startswith("@") or len(nodes) == 1):
@@ -426,11 +465,15 @@ def parse_text(text, return_text_on_error=False, comments=False, filename=None,
                                     logger.warning(f"Filename: {filename}")
                                 logger.warning(f'Value for "{value}" cannot be found (line: {line_number})')
                             node.append({"@type": "variable", "@value": item, "@result": result})
+                        elif dates and regex_date.fullmatch(item):
+                            item = convert_date(item) or item
+                            node.append(item)
                         else:
                             try:
-                                node.append(ast.literal_eval(item))
+                                item = ast.literal_eval(item)
                             except:  # noqa
-                                node.append(item)
+                                pass
+                            node.append(item)
         except Exception as error:
             if filename:
                 logger.error(f"Filename: {filename}")
@@ -448,6 +491,7 @@ def parse_file(
     base_dir=None,
     save=False,
     comments=False,
+    dates=False,
     is_global=False,
     patch=None,
 ):
@@ -459,6 +503,7 @@ def parse_file(
     :param base_dir: Base directory (for debug)
     :param save: (default false) Save parsed file in output directory
     :param comments: Include comments?
+    :param dates: Include dates?
     :param is_global: (default false) Are file's variables global?
     :param patch: String replacement patterns
     :return: Parsed data as dictionary or text if parsing fails
@@ -475,7 +520,9 @@ def parse_file(
         text = re.sub(pattern, replacement, text)
     filename = os.path.join(base_dir, os.path.basename(path)).replace(os.sep, "/")
     logger.debug(f"Parsing {filename}")
-    data = parse_text(text, return_text_on_error=True, comments=comments, filename=filename, is_global=is_global)
+    data = parse_text(
+        text, return_text_on_error=True, comments=comments, dates=dates, filename=filename, is_global=is_global
+    )
     if save:
         filename, _ = os.path.splitext(os.path.basename(path))
         directory = os.path.join(output_dir or "output", *base_dir.split("/")).replace(os.sep, "/")
@@ -490,7 +537,7 @@ def parse_file(
         else:
             filename = os.path.join(directory, filename + ".json")
             with open(filename, "w") as file:
-                json.dump(data, file, indent=4)
+                json_dump(data, file)
     total_time = time.monotonic() - start_time
     logger.debug(f"Elapsed time: {total_time:0.3f}s!")
     return data
@@ -503,6 +550,7 @@ def parse_all_files(
     keep_data=False,
     save=False,
     comments=False,
+    dates=False,
     variables_first=True,
     _variables_only=False,
 ):
@@ -514,6 +562,7 @@ def parse_all_files(
     :param keep_data: (default false) Return parsed data of all files in a dictionary
     :param save: (default false) Save every parsed data in output directory
     :param comments: Include comments?
+    :param dates: Include dates?
     :param variables_first: Try to parse variables first
     :return: Dictionary (key: file, value: parsed data if keep_data=True)
     """
@@ -522,12 +571,13 @@ def parse_all_files(
     if variables_first and not _variables_only:
         success.update(
             parse_all_files(
-                path,
-                output_dir,
-                encoding,
-                keep_data,
-                save,
-                comments,
+                path=path,
+                output_dir=output_dir,
+                encoding=encoding,
+                keep_data=keep_data,
+                save=save,
+                comments=comments,
+                dates=dates,
                 variables_first=False,
                 _variables_only=True,
             )
@@ -547,6 +597,7 @@ def parse_all_files(
                 base_dir=path,
                 save=save,
                 comments=comments,
+                dates=dates,
                 is_global=_variables_only,
             )
             if isinstance(data, str):
@@ -561,13 +612,14 @@ def parse_all_files(
     return success
 
 
-def parse_all_locales(path, encoding="utf_8_sig", language="english", save=False):
+def parse_all_locales(path, encoding="utf_8_sig", language="english", save=False, output="_locales.json"):
     """
     Parse all locales strings
     :param path: Path where to find locale files
     :param encoding: Encoding for reading files
     :param language: Target language
     :param save: (default false) save locales in file
+    :param output: locales output file location
     :return: Locales in dictionary
     """
     locales = {}
@@ -595,8 +647,8 @@ def parse_all_locales(path, encoding="utf_8_sig", language="english", save=False
                             key, _, value = match.groups()
                             locales[key] = value
     if save:
-        with open("_locales.json", "w") as file:
-            json.dump(locales, file, indent=4, sort_keys=True)
+        with open(output, "w") as file:
+            json_dump(locales, file, sort_keys=True)
     return locales
 
 
@@ -729,6 +781,8 @@ def revert_value(value, from_key=None, prev_key=None, **kwargs):
             return f'"{value}"'
     elif isinstance(value, dict):
         value = revert(value, from_key=from_key, prev_key=prev_key, depth=0, **kwargs)
+    elif isinstance(value, datetime.date):
+        value = f"{value.year}.{value.month}.{value.day}"
     return value
 
 
@@ -803,7 +857,7 @@ def save_variables(filepath="_variables.json"):
     if not global_variables:
         return
     with open(filepath, "w") as file:
-        json.dump(global_variables, file, indent=4, sort_keys=True)
+        json_dump(global_variables, file, sort_keys=True)
 
 
 def main():
@@ -818,6 +872,7 @@ def main():
     parser.add_argument("--output", type=str, help="output directory for parsing results")
     parser.add_argument("--revert", action="store_true", help="revert JSON files?")
     parser.add_argument("--comments", action="store_true", help="include comments?")
+    parser.add_argument("--dates", action="store_true", help="include dates?")
     parser.add_argument("--debug", action="store_true", help="debug mode?")
     args = parser.parse_args()
 
@@ -847,6 +902,7 @@ def main():
                 encoding=args.encoding,
                 output_dir=args.output,
                 comments=args.comments,
+                dates=args.dates,
                 save=True,
             )
         else:
@@ -855,6 +911,7 @@ def main():
                 encoding=args.encoding,
                 output_dir=args.output,
                 comments=args.comments,
+                dates=args.dates,
                 save=True,
             )
         save_variables()

{ckparser-0.1 → ckparser-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ckparser"
-version = "0.1"
+version = "0.2.1"
 description = "Parse Paradox Jomini data files to Python/JSON and revert them back experimentally."
 readme = "README.md"
 requires-python = ">=3.9"