folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/config_file_load.py

@@ -0,0 +1,66 @@
+import json
+from pathlib import Path
+
+
+def deep_merge(target_dict, source_dict, merge_keys=("name", "fileName", "file_name")):
+    # deep_merge
+    #
+    # Deeply merges nested dictionaries and lists of dictionaries.
+    # **Muatates the target_dict with the changes**, and returns it.
+    #
+    # For lists, attempts to match items based on the first found key in merge_keys in each item.
+    # If no key is found, or no match, the item is appended to the target list.
+    #
+    # Deletes any keys with value Null in the source.
+
+    for k in source_dict:
+        if isinstance(target_dict.get(k, None), dict) and isinstance(source_dict[k], dict):
+            # Recursive, depth-first merge on dictionaries
+            target_dict[k] = deep_merge(target_dict[k], source_dict[k], merge_keys)
+        elif isinstance(target_dict.get(k, None), list) and isinstance(source_dict[k], list):
+            # Merge lists on keys in merge_keys
+            for merging_list_item in source_dict[k]:
+                if not isinstance(merging_list_item, dict):
+                    target_dict[k].append(merging_list_item)
+                    continue
+                merge_key = next((i for i in merge_keys if i in merging_list_item), None)
+                if merge_key is None:
+                    target_dict[k].append(merging_list_item)
+                    continue
+                for index, target in enumerate(target_dict[k]):
+                    if target[merge_key] == merging_list_item[merge_key]:
+                        target_dict[k][index] = deep_merge(
+                            target_dict[k][index],
+                            merging_list_item,
+                            merge_keys,
+                        )
+                        break
+                else:
+                    target_dict[k].append(merging_list_item)
+        else:
+            if source_dict[k] is None:
+                target_dict.pop(k, None)
+            else:
+                target_dict[k] = source_dict[k]
+    return target_dict
+
+
+def merge_load(config_path_str, parsed_config=None):
+    # Recursively load JSON files from a configuration file
+    #
+    # If a configuration file has a "source" key, either a string or list,
+    # the file(s) will be loaded in the order presented, and the config file
+    # will be merged on top.
+    #
+    # To delete a key, set it to `null` in the json source file.
+
+    parsed_config = parsed_config or {}
+    config_path = Path(config_path_str)
+    with open(config_path) as config_file:
+        single_config = json.load(config_file)
+    sources = single_config.get("source", [])
+    if isinstance(sources, str):
+        sources = [sources]
+    for source in sources:
+        parsed_config = merge_load(config_path.parent / source, parsed_config)
+    return deep_merge(parsed_config, single_config)

folio_migration_tools/custom_dict.py

@@ -2,17 +2,19 @@ import csv
 
 
 class InsensitiveDictReader(csv.DictReader):
-    # This class overrides the csv.fieldnames property, which converts all fieldnames without leading and trailing
+    # This class overrides the csv.fieldnames property, which converts all
+    # fieldnames without leading and trailing
     # spaces and to lower case.
     @property
     def fieldnames(self):
-        return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)]
+        return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)] # type: ignore
 
     def next(self):
-        return InsensitiveDict(csv.DictReader.next(self))
+        return InsensitiveDict(csv.DictReader.next(self)) # type: ignore
 
 
 class InsensitiveDict(dict):
-    # This class overrides the __getitem__ method to automatically strip() and lower() the input key
+    # This class overrides the __getitem__ method to automatically strip()
+    # and lower() the input key
     def __getitem__(self, key):
         return dict.__getitem__(self, key.strip().lower())

folio_migration_tools/custom_exceptions.py

@@ -1,24 +1,28 @@
 import logging
+from typing import Union
+import i18n
 
+from folio_migration_tools import StrCoercible
 
-class TransfomationError(Exception):
+
+class TransformationError(Exception):
     pass
 
 
-class TransformationFieldMappingError(TransfomationError):
-    """Raised when the a field mapping fails, but the error is not critical.
+class TransformationFieldMappingError(TransformationError):
+    """Raised when the field mapping fails, but the error is not critical.
     The issue should be logged for the library to act upon it"""
 
-    def __init__(self, index_or_id="", message="", data_value=""):
+    def __init__(self, index_or_id="", message="", data_value: Union[str, StrCoercible]=""):
         self.index_or_id = index_or_id or ""
         self.message = message
-        self.data_value = data_value
+        self.data_value: Union[str, StrCoercible] = data_value
         super().__init__(self.message)
 
     def __str__(self):
         return (
-            f"Data issue. Consider fixing the record. "
-            f"\t{self.index_or_id}\t{self.message}\t{self.data_value}"
+            i18n.t("Data issue. Consider fixing the record. ")
+            + f"\t{self.index_or_id}\t{self.message}\t{self.data_value}"
         )
 
     def log_it(self):
@@ -31,13 +35,13 @@ class TransformationFieldMappingError(TransfomationError):
         )
 
 
-class TransformationRecordFailedError(TransfomationError):
-    """Raised when the a field mapping fails, Error is critical and means tranformation fails"""
+class TransformationRecordFailedError(TransformationError):
+    """Raised when the field mapping fails, Error is critical and means transformation fails"""
 
     def __init__(self, index_or_id, message="", data_value=""):
         self.index_or_id = index_or_id
         self.message = message
-        self.data_value = data_value
+        self.data_value: Union[str, StrCoercible] = data_value
         # logging.log(26, f"RECORD FAILED\t{self.id}\t{self.message}\t{self.data_value}")
         super().__init__(self.message)
 
@@ -48,9 +52,6 @@ class TransformationRecordFailedError(TransfomationError):
         )
 
     def log_it(self):
-        logging.error(
-            "Record failed: %s. See the data issues log for details", self.message
-        )
         logging.log(
             26,
             "RECORD FAILED\t%s\t%s\t%s",
@@ -60,15 +61,16 @@ class TransformationRecordFailedError(TransfomationError):
         )
 
 
-class TransformationProcessError(TransfomationError):
-    """Raised when the mapping fails due to disconnects in ref data.
-    This error should take the process to a halt"""
+class TransformationProcessError(TransformationError):
+    """Raised when the transformation fails due to incorrect configuration,
+    mapping or reference data. This error should take the process to a halt."""
 
     def __init__(
         self,
         index_or_id,
-        message="Critical Process issue. Transformation failed.",
-        data_value="",
+        message="Critical Process issue. Transformation failed."
+        " Check configuration, mapping files and reference data",
+        data_value: Union[str, StrCoercible]="",
     ):
         self.index_or_id = index_or_id
         self.message = message
@@ -77,6 +79,6 @@ class TransformationProcessError(TransfomationError):
 
     def __str__(self):
         return (
-            f"Critical Process issue. Check mapping files and reference data "
+            f"Critical Process issue. Check configuration, mapping files and reference data"
             f"\t{self.index_or_id}\t{self.message}\t{self.data_value}"
         )

folio_migration_tools/extradata_writer.py

@@ -0,0 +1,46 @@
+import json
+import logging
+import os
+from pathlib import Path
+from typing import List
+
+from folio_migration_tools.custom_exceptions import TransformationProcessError
+
+
+class ExtradataWriter:
+    __instance = None
+    __inited = False
+
+    def __new__(cls, path_to_file: Path) -> "ExtradataWriter":
+        if cls.__instance is None:
+            cls.__instance = super().__new__(cls)
+        return cls.__instance
+
+    def __init__(self, path_to_file: Path) -> None:
+        if type(self).__inited:
+            return
+        self.cache: List[str] = []
+        self.path_to_file: Path = path_to_file
+        if self.path_to_file.is_file():
+            os.remove(self.path_to_file)
+        type(self).__inited = True
+
+    def write(self, record_type: str, data_to_write: dict, flush=False):
+        try:
+            if data_to_write:
+                self.cache.append(f"{record_type}\t{json.dumps(data_to_write)}\n")
+            if len(self.cache) > 1000 or flush:
+                with open(self.path_to_file, "a") as extradata_file:
+                    extradata_file.writelines(self.cache)
+                    self.cache = []
+                    logging.debug("Extradata writer flushing the cache")
+        except Exception as ee:
+            error_message = "Something went wrong in extradata Writer"
+            logging.error(error_message)
+            raise TransformationProcessError("", error_message, record_type) from ee
+
+    def flush(self):
+        self.write("", {}, True)
+        if self.path_to_file.is_file() and os.stat(self.path_to_file).st_size == 0:
+            logging.info("Removing extradata file since it is empty")
+            os.remove(self.path_to_file)

folio_migration_tools/folder_structure.py

@@ -1,7 +1,7 @@
 import logging
 import sys
-from pathlib import Path
 import time
+from pathlib import Path
 
 from folio_uuid.folio_namespaces import FOLIONamespaces
 
@@ -15,7 +15,7 @@ class FolderStructure:
         iteration_identifier: str,
         add_time_stamp_to_file_names: bool,
     ):
-        logging.info("Setting up folder structure")
+        logging.info("Validating folder structure")
 
         self.object_type: FOLIONamespaces = object_type
         self.migration_task_name = migration_task_name
@@ -26,25 +26,22 @@ class FolderStructure:
             logging.critical("Base Folder Path is not a folder. Exiting.")
             sys.exit(1)
 
-        self.data_folder = self.base_folder / "data"
-        verify_folder(self.data_folder)
-
-        verify_folder(self.data_folder / str(FOLIONamespaces.instances.name).lower())
-        verify_folder(self.data_folder / str(FOLIONamespaces.holdings.name).lower())
-        verify_folder(self.data_folder / str(FOLIONamespaces.items.name).lower())
-        verify_folder(self.data_folder / str(FOLIONamespaces.users.name).lower())
-        self.archive_folder = self.base_folder / "archive"
-        verify_folder(self.data_folder)
-
-        self.results_folder = self.base_folder / "results"
-        verify_folder(self.results_folder)
-        self.reports_folder = self.base_folder / "reports"
-        verify_folder(self.reports_folder)
-
+        # Basic folders
         self.mapping_files_folder = self.base_folder / "mapping_files"
-        verify_folder(self.mapping_files_folder)
+        self.verify_folder(self.mapping_files_folder)
         gitignore = self.base_folder / ".gitignore"
         verify_git_ignore(gitignore)
+        self.verify_folder(self.base_folder / "iterations")
+
+        # Iteration-specific folders
+        self.iteration_folder = self.base_folder / "iterations" / self.iteration_identifier
+        self.verify_folder(self.iteration_folder)
+        self.data_folder = self.iteration_folder / "source_data"
+        self.verify_folder(self.data_folder)
+        self.results_folder = self.iteration_folder / "results"
+        self.verify_folder(self.results_folder)
+        self.reports_folder = self.iteration_folder / "reports"
+        self.verify_folder(self.reports_folder)
 
     def log_folder_structure(self):
         logging.info("Mapping files folder is %s", self.mapping_files_folder)
@@ -55,104 +52,104 @@ class FolderStructure:
         logging.info("Data folder is %s", self.data_folder)
         logging.info("Source records files folder is %s", self.legacy_records_folder)
         logging.info("Log file will be located at %s", self.transformation_log_path)
-        logging.info(
-            "Extra data will be stored at%s", self.transformation_extra_data_path
-        )
+        logging.info("Extra data will be stored at%s", self.transformation_extra_data_path)
         logging.info("Data issue reports %s", self.data_issue_file_path)
         logging.info("Created objects will be stored at  %s", self.created_objects_path)
-        logging.info(
-            "Migration report file will be saved at %s", self.migration_reports_file
-        )
+        logging.info("Migration report file will be saved at %s", self.migration_reports_file)
 
     def setup_migration_file_structure(self, source_file_type: str = ""):
-        time_stamp = f'_{time.strftime("%Y%m%d-%H%M%S")}'
-        time_str = time_stamp if self.add_time_stamp_to_file_names else ""
-        file_template = (
-            f"{self.iteration_identifier}{time_str}_{self.migration_task_name}"
-        )
+        self.time_stamp = f'_{time.strftime("%Y%m%d-%H%M%S")}'
+        self.time_str = self.time_stamp if self.add_time_stamp_to_file_names else ""
+        self.file_template = f"{self.time_str}_{self.migration_task_name}"
         object_type_string = str(self.object_type.name).lower()
         if source_file_type:
             self.legacy_records_folder = self.data_folder / source_file_type
         elif self.object_type == FOLIONamespaces.other:
             self.legacy_records_folder = self.data_folder
-
         else:
             self.legacy_records_folder = self.data_folder / object_type_string
-        verify_folder(self.legacy_records_folder)
+        self.verify_folder(self.legacy_records_folder)
+
+        # Make sure the items are there if the Holdings processor is run
+        if self.object_type == FOLIONamespaces.holdings:
+            self.verify_folder(self.data_folder / str(FOLIONamespaces.items.name).lower())
 
         self.transformation_log_path = self.reports_folder / (
-            f"log_{object_type_string}_{file_template}.log"
+            f"log_{object_type_string}{self.file_template}.log"
         )
 
         self.failed_recs_path = (
-            self.results_folder / f"failed_records_{file_template}_{time_stamp}.txt"
+            self.results_folder / f"failed_records{self.file_template}{self.time_stamp}.txt"
         )
 
         self.transformation_extra_data_path = (
-            self.results_folder / f"extradata_{file_template}.extradata"
+            self.results_folder / f"extradata{self.file_template}.extradata"
         )
 
         self.data_issue_file_path = (
-            self.reports_folder
-            / f"data_issues_log_{object_type_string}_{file_template}.tsv"
+            self.reports_folder / f"data_issues_log{self.file_template}.tsv"
         )
         self.created_objects_path = (
-            self.results_folder / f"folio_{object_type_string}_{file_template}.json"
+            self.results_folder / f"folio_{object_type_string}{self.file_template}.json"
         )
-
-        self.failed_bibs_file = (
-            self.results_folder
-            / f"failed_bib_records_{self.iteration_identifier}{time_str}.mrc"
-        )
-        self.failed_mfhds_file = (
-            self.results_folder
-            / f"failed_mfhd_records_{self.iteration_identifier}{time_str}.mrc"
+        self.failed_marc_recs_file = (
+            self.results_folder / f"failed_records{self.file_template}.mrc"
         )
 
-        self.migration_reports_file = (
-            self.reports_folder
-            / f"transformation_report_{object_type_string}_{file_template}.md"
-        )
+        self.migration_reports_file = self.reports_folder / f"report{self.file_template}.md"
 
         self.srs_records_path = (
-            self.results_folder / f"folio_srs_{object_type_string}_{file_template}.json"
+            self.results_folder / f"folio_srs_{object_type_string}{self.file_template}.json"
+        )
+        self.data_import_marc_path = (
+            self.results_folder / f"folio_marc_{object_type_string}{self.file_template}.mrc"
+        )
+        self.organizations_id_map_path = (
+            self.results_folder / f"{str(FOLIONamespaces.organizations.name).lower()}_id_map.json"
         )
-
         self.instance_id_map_path = (
-            self.results_folder / f"instance_id_map_{self.iteration_identifier}.json"
+            self.results_folder / f"{str(FOLIONamespaces.instances.name).lower()}_id_map.json"
+        )
+        self.auth_id_map_path = (
+            self.results_folder / f"{str(FOLIONamespaces.authorities.name).lower()}_id_map.json"
         )
 
         self.holdings_id_map_path = (
-            self.results_folder / f"holdings_id_map_{self.iteration_identifier}.json"
+            self.results_folder / f"{str(FOLIONamespaces.holdings.name).lower()}_id_map.json"
+        )
+        self.id_map_path = (
+            self.results_folder / f"{str(self.object_type.name).lower()}_id_map.json"
+        )
+        self.boundwith_relationships_map_path = (
+            self.results_folder / "boundwith_relationships_map.json"
         )
-
         # Mapping files
-        self.temp_locations_map_path = self.mapping_files_folder / "temp_locations.tsv"
         self.material_type_map_path = self.mapping_files_folder / "material_types.tsv"
         self.loan_type_map_path = self.mapping_files_folder / "loan_types.tsv"
         self.temp_loan_type_map_path = self.mapping_files_folder / "temp_loan_types.tsv"
         self.statistical_codes_map_path = self.mapping_files_folder / "statcodes.tsv"
         self.item_statuses_map_path = self.mapping_files_folder / "item_statuses.tsv"
 
+    def verify_folder(self, folder_path: Path):
+        if not folder_path.is_dir():
+            logging.critical("There is no folder located at %s. Exiting.", folder_path)
+            logging.critical("Create a folder by calling\n\tmkdir %s", folder_path)
+            sys.exit(1)
+        else:
+            logging.info("Located %s", folder_path)
+
 
 def verify_git_ignore(gitignore: Path):
     with open(gitignore, "r+") as f:
         contents = f.read()
+        if "reports/" not in contents:
+            f.write("reports/\n")
         if "results/" not in contents:
             f.write("results/\n")
         if "archive/" not in contents:
             f.write("archive/\n")
-        if "data/" not in contents:
-            f.write("data/\n")
+        if "source_data/" not in contents:
+            f.write("source_data/\n")
         if "*.data" not in contents:
             f.write("*.data\n")
     logging.info("Made sure there was a valid .gitignore file at %s", gitignore)
-
-
-def verify_folder(folder_path: Path):
-    if not folder_path.is_dir():
-        logging.critical("There is no folder located at %s. Exiting.", folder_path)
-        logging.critical("Create a folder by calling\n\tmkdir %s", folder_path)
-        sys.exit(1)
-    else:
-        logging.info("Located %s", folder_path)

folio_migration_tools/helper.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import i18n
 
 
 class Helper:
@@ -8,57 +9,64 @@ class Helper:
         report_file, total_records: int, mapped_folio_fields, mapped_legacy_fields
     ):
         details_start = (
-            "<details><summary>Click to expand field report</summary>     \n\n"
+            "<details><summary>" + i18n.t("Click to expand field report") + "</summary>\n\n"
         )
-        details_end = "</details>   \n"
-        report_file.write("\n## Mapped FOLIO fields\n")
+        details_end = "</details>\n"
+        report_file.write("\n## " + i18n.t("Mapped FOLIO fields") + "\n")
         # report_file.write(f"{blurbs[header]}\n")
 
         d_sorted = {k: mapped_folio_fields[k] for k in sorted(mapped_folio_fields)}
         report_file.write(details_start)
-
-        report_file.write("FOLIO Field | Mapped | Unmapped  \n")
-        report_file.write("--- | --- | ---:  \n")
+        columns = [i18n.t("FOLIO Field"), i18n.t("Mapped"), i18n.t("Unmapped")]
+        report_file.write(" | ".join(columns) + "\n")
+        report_file.write("|".join(len(columns) * ["---"]) + "\n")
         for k, v in d_sorted.items():
             unmapped = max(total_records - v[0], 0)
             mapped = v[0]
             mp = mapped / total_records if total_records else 0
             mapped_per = "{:.0%}".format(max(mp, 0))
+            up = unmapped / total_records if total_records else 0
+            unmapped_per = "{:.0%}".format(max(up, 0))
             report_file.write(
-                f"{k} | {max(mapped, 0):,} ({mapped_per}) | {unmapped:,}  \n"
+                f"{k} | {max(mapped, 0):,} ({mapped_per}) | {unmapped:,} ({unmapped_per}) \n"
             )
         report_file.write(details_end)
 
-        report_file.write("\n## Mapped Legacy fields\n")
+        report_file.write("\n## " + i18n.t("Mapped Legacy fields") + "\n")
         # report_file.write(f"{blurbs[header]}\n")
 
         d_sorted = {k: mapped_legacy_fields[k] for k in sorted(mapped_legacy_fields)}
         report_file.write(details_start)
-        report_file.write("Legacy Field | Present | Mapped | Unmapped  \n")
-        report_file.write("--- | --- | --- | ---:  \n")
+        columns = [i18n.t("Legacy Field"), i18n.t("Present"), i18n.t("Mapped"), i18n.t("Unmapped")]
+        report_file.write("|".join(columns) + "\n")
+        report_file.write("|".join(len(columns) * ["---"]) + "\n")
         for k, v in d_sorted.items():
             present = v[0]
-            present_per = "{:.1%}".format(
-                present / total_records if total_records else 0
-            )
+            present_per = "{:.1%}".format(present / total_records if total_records else 0)
             unmapped = present - v[1]
             mapped = v[1]
             mp = mapped / total_records if total_records else 0
             mapped_per = "{:.0%}".format(max(mp, 0))
             report_file.write(
-                f"{k} | {max(present, 0):,} ({present_per}) | {max(mapped, 0):,} ({mapped_per}) | {unmapped:,}  \n"
+                f"{k} | {max(present, 0):,} ({present_per}) | {max(mapped, 0):,} "
+                f"({mapped_per}) | {unmapped:,}  \n"
             )
         report_file.write(details_end)
 
     @staticmethod
     def log_data_issue(index_or_id, message, legacy_value):
         logging.log(26, "DATA ISSUE\t%s\t%s\t%s", index_or_id, message, legacy_value)
+    
+    @staticmethod
+    def log_data_issue_failed(index_or_id, message, legacy_value):
+        logging.log(26, "RECORD FAILED\t%s\t%s\t%s", index_or_id, message, legacy_value)
 
     @staticmethod
-    def write_to_file(file, folio_record, pg_dump=False):
-        """Writes record to file. pg_dump=true for importing directly via the
-        psql copy command"""
-        if pg_dump:
-            file.write("{}\t{}\n".format(folio_record["id"], json.dumps(folio_record)))
-        else:
-            file.write("{}\n".format(json.dumps(folio_record)))
+    def write_to_file(file, folio_record):
+        """Writes record to file.
+
+        Args:
+            file (_type_): _description_
+            folio_record (_type_): _description_
+        """
+        file.write(f"{json.dumps(folio_record)}\n")