folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/mapper_base.py

@@ -1,48 +1,57 @@
+import ast
+import copy
+import json
 import logging
 import sys
-import json
-import requests
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import i18n
+from folio_uuid.folio_namespaces import FOLIONamespaces
+from folio_uuid.folio_uuid import FolioUUID
+from folioclient import FolioClient
+from pymarc import Record
 
 from folio_migration_tools.custom_exceptions import (
+    TransformationFieldMappingError,
     TransformationProcessError,
     TransformationRecordFailedError,
 )
-from folioclient import FolioClient
-from folio_uuid.folio_namespaces import FOLIONamespaces
+from folio_migration_tools.extradata_writer import ExtradataWriter
+from folio_migration_tools.helper import Helper
+from folio_migration_tools.library_configuration import FileDefinition, LibraryConfiguration
 from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
     RefDataMapping,
 )
-from folio_migration_tools.library_configuration import LibraryConfiguration
 from folio_migration_tools.migration_report import MigrationReport
-from folio_migration_tools.report_blurbs import Blurbs
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 
 class MapperBase:
+    legacy_id_template = "Identifier(s) from previous system:"
+    bib_id_template = "Bib id: "
+
     def __init__(
         self,
         library_configuration: LibraryConfiguration,
+        task_configuration: AbstractTaskConfiguration,
         folio_client: FolioClient,
+        parent_id_map: Dict[str, Tuple] = {},
     ):
         logging.info("MapperBase initiating")
-        self.folio_client = folio_client
-        self.hrid_path = "/hrid-settings-storage/hrid-settings"
+        self.parent_id_map: dict[str, tuple] = parent_id_map
+        self.extradata_writer: ExtradataWriter = ExtradataWriter(Path(""))
+        self.start_datetime = datetime.now(timezone.utc)
+        self.folio_client: FolioClient = folio_client
         self.library_configuration: LibraryConfiguration = library_configuration
-        self.hrid_settings = self.folio_client.folio_get_single_object(self.hrid_path)
-        self.instance_hrid_prefix = self.hrid_settings["instances"]["prefix"]
-        self.instance_hrid_counter = self.hrid_settings["instances"]["startNumber"]
-        self.holdings_hrid_prefix = self.hrid_settings["holdings"]["prefix"]
-        self.holdings_hrid_counter = self.hrid_settings["holdings"]["startNumber"]
-        logging.info("Fetched HRID settings.")
-        logging.info("Instance HRID prefix is %s", self.instance_hrid_prefix)
-        logging.info("Instance start number is %s", self.instance_hrid_counter)
-        logging.info("Holdings HRID prefix is %s", self.instance_hrid_prefix)
-        logging.info("Holdings start number is %s", self.holdings_hrid_counter)
-
-        self.mapped_folio_fields = {}
-        self.migration_report = MigrationReport()
+        self.task_configuration: AbstractTaskConfiguration = task_configuration
+        self.mapped_folio_fields: dict = {}
+        self.migration_report: MigrationReport = MigrationReport()
         self.num_criticalerrors = 0
-        self.num_exeptions = 0
-        self.mapped_legacy_fields = {}
+        self.num_exceptions = 0
+        self.mapped_legacy_fields: dict = {}
         self.schema_properties = None
 
     def report_legacy_mapping(self, field_name, present, mapped):
@@ -55,7 +64,7 @@ class MapperBase:
 
     def report_folio_mapping(self, folio_record, schema):
         try:
-            for field_name in flatten(folio_record):
+            for field_name in set(flatten(folio_record)):
                 try:
                     self.mapped_folio_fields[field_name][0] += 1
                 except KeyError:
@@ -73,29 +82,42 @@ class MapperBase:
             logging.error(ee, stack_info=True)
             raise ee from ee
 
+    def report_legacy_mapping_no_schema(self, legacy_object):
+        for field_name, value in legacy_object.items():
+            v = 1 if value else 0
+            if field_name not in self.mapped_legacy_fields:
+                self.mapped_legacy_fields[field_name] = [1, v]
+            else:
+                self.mapped_legacy_fields[field_name][0] += 1
+                self.mapped_legacy_fields[field_name][1] += v
+
+    def report_folio_mapping_no_schema(self, folio_object):
+        for field_name in set(flatten(folio_object)):
+            if field_name not in self.mapped_folio_fields:
+                self.mapped_folio_fields[field_name] = [1, 1]
+            else:
+                self.mapped_folio_fields[field_name][0] += 1
+                self.mapped_folio_fields[field_name][1] += 1
+
     def get_mapped_name(
         self,
         ref_data_mapping: RefDataMapping,
         legacy_object,
         index_or_id,
-        folio_property_name="",
         prevent_default=False,
     ):
         try:
             # Get the values in the fields that will be used for mapping
-            fieldvalues = [
-                legacy_object.get(k) for k in ref_data_mapping.mapped_legacy_keys
-            ]
+            fieldvalues = [legacy_object.get(k) for k in ref_data_mapping.mapped_legacy_keys]
 
             # Gets the first line in the map satisfying all legacy mapping values.
             # Case insensitive, strips away whitespace
-            # TODO: add option for Wild card matching in individual columns
             right_mapping = ref_data_mapping.get_ref_data_mapping(legacy_object)
 
             if not right_mapping:
                 raise StopIteration()
             self.migration_report.add(
-                ref_data_mapping.blurb,
+                ref_data_mapping.blurb_id,
                 (
                     f'{" - ".join(fieldvalues)} '
                     f'-> {right_mapping[f"folio_{ref_data_mapping.key_type}"]}'
@@ -106,15 +128,12 @@ class MapperBase:
         except StopIteration:
             if prevent_default:
                 self.migration_report.add(
-                    ref_data_mapping.blurb,
-                    (
-                        f"Not to be mapped. "
-                        f'(No default) -- {" - ".join(fieldvalues)} -> ""'
-                    ),
+                    ref_data_mapping.blurb_id,
+                    (f"Not to be mapped. " f'(No default) -- {" - ".join(fieldvalues)} -> ""'),
                 )
                 return ""
             self.migration_report.add(
-                ref_data_mapping.blurb,
+                ref_data_mapping.blurb_id,
                 (
                     f"Unmapped (Default value was set) -- "
                     f'{" - ".join(fieldvalues)} -> {ref_data_mapping.default_name}'
@@ -130,8 +149,17 @@ class MapperBase:
                     "a recognized field in the legacy data."
                 ),
             ) from exception
+        except KeyError as exception:
+            raise TransformationProcessError(
+                index_or_id,
+                (
+                    f"{ref_data_mapping.name} mapping - folio_{ref_data_mapping.key_type} "
+                    f"({ref_data_mapping.mapped_legacy_keys})  is not "
+                    f"a recognized field in the legacy data. KeyError: {exception}"
+                ),
+            ) from exception
         except Exception as exception:
-            raise TransformationRecordFailedError(
+            raise TransformationProcessError(
                 index_or_id,
                 (
                     f"{ref_data_mapping.name} - folio_{ref_data_mapping.key_type} "
@@ -139,7 +167,7 @@ class MapperBase:
                 ),
             ) from exception
 
-    def get_mapped_value(
+    def get_mapped_ref_data_value(
         self,
         ref_data_mapping: RefDataMapping,
         legacy_object,
@@ -147,17 +175,13 @@ class MapperBase:
         folio_property_name="",
         prevent_default=False,
     ):
-
         # Gets mapped value from mapping file, translated to the right FOLIO UUID
         try:
             # Get the values in the fields that will be used for mapping
-            fieldvalues = [
-                legacy_object.get(k) for k in ref_data_mapping.mapped_legacy_keys
-            ]
+            fieldvalues = [legacy_object.get(k) for k in ref_data_mapping.mapped_legacy_keys]
 
             # Gets the first line in the map satisfying all legacy mapping values.
             # Case insensitive, strips away whitespace
-            # TODO: add option for Wild card matching in individual columns
             right_mapping = ref_data_mapping.get_ref_data_mapping(legacy_object)
             if not right_mapping:
                 # Not all fields matched. Could it be a hybrid wildcard map?
@@ -166,7 +190,7 @@ class MapperBase:
             if not right_mapping:
                 raise StopIteration()
             self.migration_report.add(
-                ref_data_mapping.blurb,
+                ref_data_mapping.blurb_id,
                 (
                     f'{" - ".join(fieldvalues)} '
                     f'-> {right_mapping[f"folio_{ref_data_mapping.key_type}"]}'
@@ -176,15 +200,12 @@ class MapperBase:
         except StopIteration:
             if prevent_default:
                 self.migration_report.add(
-                    ref_data_mapping.blurb,
-                    (
-                        f"Not to be mapped. "
-                        f'(No default) -- {" - ".join(fieldvalues)} -> ""'
-                    ),
+                    ref_data_mapping.blurb_id,
+                    (f"Not to be mapped. " f'(No default) -- {" - ".join(fieldvalues)} -> ""'),
                 )
                 return ""
             self.migration_report.add(
-                ref_data_mapping.blurb,
+                ref_data_mapping.blurb_id,
                 (
                     f"Unmapped (Default value was set) -- "
                     f'{" - ".join(fieldvalues)} -> {ref_data_mapping.default_name}'
@@ -210,15 +231,13 @@ class MapperBase:
             ) from exception
 
     def handle_transformation_field_mapping_error(self, index_or_id, error):
-        self.migration_report.add(Blurbs.FieldMappingErrors, error)
+        self.migration_report.add("FieldMappingErrors", error)
         error.id = error.id or index_or_id
         error.log_it()
-        self.migration_report.add_general_statistics("Field Mapping Errors found")
+        self.migration_report.add_general_statistics(i18n.t("Field Mapping Errors found"))
 
-    def handle_transformation_process_error(
-        self, idx, error: TransformationProcessError
-    ):
-        self.migration_report.add_general_statistics("Transformation process error")
+    def handle_transformation_process_error(self, idx, error: TransformationProcessError):
+        self.migration_report.add_general_statistics(i18n.t("Transformation process error"))
         logging.critical("%s\t%s", idx, error)
         print(f"\n{error.message}: {error.data_value}")
         sys.exit(1)
@@ -227,16 +246,15 @@ class MapperBase:
         self, records_processed: int, error: TransformationRecordFailedError
     ):
         self.migration_report.add(
-            Blurbs.GeneralStatistics, "Records failed due to an error"
+            "GeneralStatistics", i18n.t("FAILED Records failed due to an error")
         )
-        error.id = error.index_or_id or records_processed
+        error.index_or_id = error.index_or_id or records_processed
         error.log_it()
         self.num_criticalerrors += 1
         if (
             self.num_criticalerrors / (records_processed + 1)
             > (self.library_configuration.failed_percentage_threshold / 100)
-            and self.num_criticalerrors
-            > self.library_configuration.failed_records_threshold
+            and self.num_criticalerrors > self.library_configuration.failed_records_threshold
         ):
             logging.fatal(
                 (
@@ -252,78 +270,76 @@ class MapperBase:
             )
             sys.exit(1)
 
-    @staticmethod
-    def get_id_map_dict(legacy_id, folio_record):
-        return {"legacy_id": legacy_id, "folio_id": folio_record["id"]}
+    def get_id_map_tuple(self, legacy_id: str, folio_record: dict, object_type: FOLIONamespaces):
+        if all(
+            [
+                object_type == FOLIONamespaces.instances,
+                (not getattr(self.task_configuration, "data_import_marc", False)),
+                getattr(self, "create_source_records", True),
+            ]
+        ):
+            return (legacy_id, folio_record["id"], folio_record["hrid"])
+        return (legacy_id, folio_record["id"])
 
-    def handle_generic_exception(self, idx, excepion: Exception):
-        self.num_exeptions += 1
+    def handle_generic_exception(self, idx, exception: Exception):
+        self.num_exceptions += 1
         print("\n=======ERROR===========")
         print(
-            f"Row {idx:,} failed with the following unhandled Exception: {excepion}  "
-            f"of type {type(excepion).__name__}"
+            f"Row {idx:,} failed with the following unhandled Exception: {exception}  "
+            f"of type {type(exception).__name__}"
         )
-        logging.error(excepion, exc_info=True)
-        if self.num_exeptions > 50:
+        logging.error(exception, exc_info=True)
+        if self.num_exceptions > self.library_configuration.generic_exception_threshold:
             logging.fatal(
                 "Stopping. More than %s unhandled exceptions. Code needs fixing",
-                self.num_exeptions,
+                self.num_exceptions,
             )
             sys.exit(1)
 
-    @staticmethod
-    def save_id_map_file(path, legacy_map: dict):
+    def save_id_map_file(self, path, legacy_map: dict):
         with open(path, "w") as legacy_map_file:
             for id_string in legacy_map.values():
                 legacy_map_file.write(f"{json.dumps(id_string)}\n")
-            logging.info("Wrote %s id:s to legacy map", len(legacy_map))
-
-    def store_hrid_settings(self):
-        logging.info("Setting HRID counter to current +1")
-        try:
-            self.hrid_settings["instances"]["startNumber"] = (
-                self.instance_hrid_counter + 1
-            )
-            self.hrid_settings["holdings"]["startNumber"] = (
-                self.holdings_hrid_counter + 1
-            )
-            url = self.folio_client.okapi_url + self.hrid_path
-            resp = requests.put(
-                url,
-                data=json.dumps(self.hrid_settings),
-                headers=self.folio_client.okapi_headers,
-            )
-            resp.raise_for_status()
-            logging.info("%s Successfully set HRID settings.", resp.status_code)
-            a = self.folio_client.folio_get_single_object(self.hrid_path)
-            logging.info("Current hrid settings: %s", json.dumps(a, indent=4))
-        except Exception:
-            logging.exception(
-                f"Something went wrong when setting the HRID settings. "
-                f"Update them manually. {json.dumps(self.hrid_settings)}"
-            )
+                self.migration_report.add(
+                    "GeneralStatistics", i18n.t("Unique ID:s written to legacy map")
+                )
+        logging.info("Wrote legacy id map to %s", path)
 
     @staticmethod
     def validate_required_properties(
         legacy_id, folio_object: dict, schema: dict, object_type: FOLIONamespaces
     ):
         cleaned_folio_object = MapperBase.clean_none_props(folio_object)
-        required = schema["required"]
+        required = []
         missing = []
-        for required_prop in required:
-            if required_prop not in cleaned_folio_object:
-                missing.append(f"Missing: {required_prop}")
-            elif not cleaned_folio_object[required_prop]:
-                missing.append(f"Empty: {required_prop}")
+        if object_type != FOLIONamespaces.note:
+            required = schema.get("required", [])
+            missing = list(MapperBase.list_missing(required, cleaned_folio_object))
+        else:
+            required = (
+                schema.get("properties", {}).get("notes", {}).get("items", {}).get("required", [])
+            )
+            for note in cleaned_folio_object.get("notes", []):
+                missing.extend(MapperBase.list_missing(required, note))
+
         if any(missing):
             raise TransformationRecordFailedError(
                 legacy_id,
                 "One or many required properties empty",
                 f"{json.dumps(missing)}",
             )
-        cleaned_folio_object.pop("type", None)
+        if object_type not in [FOLIONamespaces.users]:
+            cleaned_folio_object.pop("type", None)
         return cleaned_folio_object
 
+    @staticmethod
+    def list_missing(required: list, cleaned_folio_object: dict):
+        for required_prop in required:
+            if required_prop not in cleaned_folio_object:
+                yield f"Missing: {required_prop}"
+            elif not cleaned_folio_object[required_prop]:
+                yield f"Empty: {required_prop}"
+
     @staticmethod
     def clean_none_props(d: dict):
         clean = {}
@@ -338,15 +354,210 @@ class MapperBase:
                 clean[k] = v
         return clean
 
+    def add_legacy_id_to_admin_note(self, folio_record: dict, legacy_id: str):
+        if not legacy_id:
+            raise TransformationFieldMappingError(
+                legacy_id, i18n.t("Legacy id is empty"), legacy_id
+            )
+        if "administrativeNotes" not in folio_record:
+            folio_record["administrativeNotes"] = []
+        if id_string := next(
+            (f for f in folio_record["administrativeNotes"] if MapperBase.legacy_id_template in f),
+            None,
+        ):
+            if legacy_id not in id_string:
+                folio_record["administrativeNotes"] = [
+                    f
+                    for f in folio_record["administrativeNotes"]
+                    if MapperBase.legacy_id_template not in f
+                ]
+
+                folio_record["administrativeNotes"].append(f"{id_string}, {legacy_id}")
+        else:
+            folio_record["administrativeNotes"].append(
+                f"{MapperBase.legacy_id_template} {legacy_id}"
+            )
+
+    def create_and_write_boundwith_part(self, legacy_item_id: str, bound_with_holding_uuid: dict):
+        part = {
+            "id": str(uuid.uuid4()),
+            "holdingsRecordId": bound_with_holding_uuid,
+            "itemId": str(
+                FolioUUID(
+                    self.base_string_for_folio_uuid,
+                    FOLIONamespaces.items,
+                    legacy_item_id,
+                )
+            ),
+        }
+        self.extradata_writer.write("boundwithPart", part)
+
+    def create_bound_with_holdings(
+        self,
+        folio_holding: dict,
+        instance_ids: list,
+        bound_with_holdings_type_id: str,
+    ):
+        if not bound_with_holdings_type_id:
+            raise TransformationProcessError(
+                "Missing task setting holdingsTypeUuidForBoundwiths. Add a "
+                "holdingstype specifically for boundwith holdings and reference "
+                "the UUID in this parameter."
+            )
+        for bwidx, instance_uuid in enumerate(instance_ids):
+            if not instance_uuid:
+                raise ValueError(f"No Instance ID for record {folio_holding}")
+            bound_with_holding = copy.deepcopy(folio_holding)
+            bound_with_holding["instanceId"] = instance_uuid
+
+            if call_number := folio_holding.get("callNumber", None):
+                if "[" in call_number:
+                    try:
+                        call_numbers: List = ast.literal_eval(str(folio_holding["callNumber"]))
+                        bound_with_holding["callNumber"] = call_numbers[bwidx]
+                    except IndexError:
+                        if call_numbers:
+                            bound_with_holding["callNumber"] = call_numbers[0]
+                    except (SyntaxError, ValueError):
+                        bound_with_holding["callNumber"] = call_number
+                else:
+                    bound_with_holding["callNumber"] = call_number
+            bound_with_holding["holdingsTypeId"] = bound_with_holdings_type_id
+
+            # The subsequent copies gets different ids, but the original is maintained.
+            if bwidx > 0:
+                bound_with_holding["id"] = self.generate_boundwith_holding_uuid(
+                    folio_holding["id"], instance_uuid
+                )
+                if bound_with_holding.get("hrid", ""):
+                    bound_with_holding["hrid"] = f'{bound_with_holding["hrid"]}_bw_{bwidx}'
+            self.migration_report.add_general_statistics(i18n.t("Bound-with holdings created"))
+            yield bound_with_holding
+
+    def generate_boundwith_holding_uuid(self, holding_uuid, instance_uuid):
+        return str(
+            FolioUUID(
+                self.base_string_for_folio_uuid,
+                FOLIONamespaces.holdings,
+                f"{holding_uuid}-{instance_uuid}",
+            )
+        )
+
+    def map_statistical_codes(self, folio_record: dict, file_def: FileDefinition, legacy_record: Optional[Union[dict, Record]] = None):
+        """Map statistical codes to the folio record.
+
+        This method checks if the file definition contains statistical codes and
+        if so, it splits the codes by the multi-field delimiter and adds them to
+        the folio record's 'statisticalCodeIds' field. If the field does not exist,
+        it initializes it as an empty list before appending the codes.
+
+        Args:
+            folio_record (dict): The FOLIO record to which the statistical codes will be added.
+            file_def (FileDefinition): The file definition containing the statistical codes.
+            legacy_record (Optional[Union[dict, Record]]): The legacy record from which the statistical codes are derived.
+        """
+        if file_def.statistical_code:
+            code_strings = file_def.statistical_code.split(
+                self.library_configuration.multi_field_delimiter
+            )
+            folio_record["statisticalCodeIds"] = folio_record.get("statisticalCodeIds", []) + code_strings
+
+    def setup_statistical_codes_map(self, statistical_codes_map):
+        if statistical_codes_map:
+            self.statistical_codes_mapping = RefDataMapping(
+                self.folio_client,
+                "/statistical-codes",
+                "statisticalCodes",
+                statistical_codes_map,
+                "code",
+                "StatisticalCodeMapping",
+            )
+            logging.info(f"Statistical codes mapping set up {self.statistical_codes_mapping.mapped_legacy_keys}")
+        else:
+            self.statistical_codes_mapping = None
+            logging.info("Statistical codes map is not set up")
+
+    def get_statistical_code(self, legacy_item: dict, folio_prop_name: str, index_or_id):
+        if self.statistical_codes_mapping:
+            return self.get_mapped_ref_data_value(
+                self.statistical_codes_mapping,
+                legacy_item,
+                index_or_id,
+                folio_prop_name,
+                True,
+            )
+        self.migration_report.add(
+            "StatisticalCodeMapping",
+            i18n.t("Mapping not set up"),
+        )
+        return ""
+
+    def map_statistical_code_ids(
+        self, legacy_ids, folio_record: dict
+    ):
+        if stat_codes := {x: None for x in folio_record.pop("statisticalCodeIds", [])}:
+            folio_code_ids = set()
+            for stat_code in stat_codes:
+                if stat_code_id := self.get_statistical_code({"legacy_stat_code": stat_code}, "statisticalCodeId", legacy_ids):
+                    folio_code_ids.add(stat_code_id)
+                else:
+                    Helper.log_data_issue(
+                        legacy_ids,
+                        i18n.t(
+                            "Statistical code '%{code}' not found in FOLIO",
+                            code=stat_code,
+                        ),
+                        stat_code,
+                    )
+            folio_record["statisticalCodeIds"] = list(folio_code_ids)
+
+    @property
+    def base_string_for_folio_uuid(self):
+        if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
+            return str(self.folio_client.gateway_url)
+        elif self.library_configuration.ecs_tenant_id:
+            return str(self.library_configuration.ecs_tenant_id)
+        else:
+            return str(self.library_configuration.tenant_id)
+
+    @staticmethod
+    def validate_location_map(location_map: List[Dict], locations: List[Dict]) -> List[Dict]:
+        mapped_codes = [x['folio_code'] for x in location_map]
+        existing_codes = [x['code'] for x in locations]
+        missing_codes = set(mapped_codes) - set(existing_codes)
+        if missing_codes:
+            raise TransformationProcessError(
+                "",
+                f"Location map contains codes not found in locations: {', '.join(missing_codes)}",
+                "",
+            )
+        return location_map
+
+    @staticmethod
+    def get_object_type() -> FOLIONamespaces:
+        raise NotImplementedError("This method should be overridden in subclasses")
 
 def flatten(my_dict: dict, path=""):
     for k, v in iter(my_dict.items()):
+        if not path:
+            yield k
         if v:
             if isinstance(v, list):
+                if path and check_if_list_with_dict_keys(v):
+                    yield f"{path}.{k}".strip(".")
                 for e in v:
                     if isinstance(e, dict):
                         yield from flatten(dict(e), f"{path}.{k}")
+                    elif isinstance(e, str) and path:
+                        yield f"{path}.{k}".strip(".")
+
             elif isinstance(v, dict):
+                if path:
+                    yield f"{path}.{k}".strip(".")
                 yield from flatten(dict(v), f"{path}.{k}")
-            else:
+            elif path:
                 yield f"{path}.{k}".strip(".")
+
+
+def check_if_list_with_dict_keys(data):
+    return isinstance(data, list) and all(isinstance(x, dict) for x in data)