folio-migration-tools 1.9.0rc5__tar.gz → 1.9.0rc7__tar.gz

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc5
+Version: 1.9.0rc7
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc5"
+version = "1.9.0rc7"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},

folio_migration_tools-1.9.0rc7/src/folio_migration_tools/__init__.py

@@ -0,0 +1,11 @@
+import importlib.metadata
+from typing import Protocol
+
+__version__ = importlib.metadata.version("folio_migration_tools")
+
+class StrCoercible(Protocol):
+    def __repr__(self) -> str:
+        ...
+
+    def __str__(self) -> str:
+        ...

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/custom_dict.py

@@ -7,10 +7,10 @@ class InsensitiveDictReader(csv.DictReader):
     # spaces and to lower case.
     @property
     def fieldnames(self):
-        return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)]
+        return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)] # type: ignore
 
     def next(self):
-        return InsensitiveDict(csv.DictReader.next(self))
+        return InsensitiveDict(csv.DictReader.next(self)) # type: ignore
 
 
 class InsensitiveDict(dict):

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/custom_exceptions.py

@@ -1,19 +1,22 @@
 import logging
+from typing import Union
 import i18n
 
+from folio_migration_tools import StrCoercible
 
-class TransfomationError(Exception):
+
+class TransformationError(Exception):
     pass
 
 
-class TransformationFieldMappingError(TransfomationError):
-    """Raised when the a field mapping fails, but the error is not critical.
+class TransformationFieldMappingError(TransformationError):
+    """Raised when the field mapping fails, but the error is not critical.
     The issue should be logged for the library to act upon it"""
 
-    def __init__(self, index_or_id="", message="", data_value=""):
+    def __init__(self, index_or_id="", message="", data_value: Union[str, StrCoercible]=""):
         self.index_or_id = index_or_id or ""
         self.message = message
-        self.data_value = data_value
+        self.data_value: Union[str, StrCoercible] = data_value
         super().__init__(self.message)
 
     def __str__(self):
@@ -32,13 +35,13 @@ class TransformationFieldMappingError(TransfomationError):
         )
 
 
-class TransformationRecordFailedError(TransfomationError):
-    """Raised when the a field mapping fails, Error is critical and means tranformation fails"""
+class TransformationRecordFailedError(TransformationError):
+    """Raised when the field mapping fails, Error is critical and means transformation fails"""
 
     def __init__(self, index_or_id, message="", data_value=""):
         self.index_or_id = index_or_id
         self.message = message
-        self.data_value = data_value
+        self.data_value: Union[str, StrCoercible] = data_value
         # logging.log(26, f"RECORD FAILED\t{self.id}\t{self.message}\t{self.data_value}")
         super().__init__(self.message)
 
@@ -58,8 +61,8 @@ class TransformationRecordFailedError(TransfomationError):
         )
 
 
-class TransformationProcessError(TransfomationError):
-    """Raised when the transformation fails due to incorrect configuraiton,
+class TransformationProcessError(TransformationError):
+    """Raised when the transformation fails due to incorrect configuration,
     mapping or reference data. This error should take the process to a halt."""
 
     def __init__(
@@ -67,7 +70,7 @@ class TransformationProcessError(TransfomationError):
         index_or_id,
         message="Critical Process issue. Transformation failed."
         " Check configuration, mapping files and reference data",
-        data_value="",
+        data_value: Union[str, StrCoercible]="",
     ):
         self.index_or_id = index_or_id
         self.message = message

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/mapper_base.py

@@ -6,7 +6,7 @@ import sys
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import List
+from typing import Dict, List
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -35,7 +35,7 @@ class MapperBase:
         self,
         library_configuration: LibraryConfiguration,
         folio_client: FolioClient,
-        parent_id_map: dict[str, tuple] = None,
+        parent_id_map: dict[str, tuple] = {},
     ):
         logging.info("MapperBase initiating")
         self.parent_id_map: dict[str, tuple] = parent_id_map
@@ -318,7 +318,14 @@ class MapperBase:
                     entry["MFHD_ID"],
                 )
             )
-            new_map[mfhd_uuid] = new_map.get(mfhd_uuid, []) + [instance_uuid]
+            if entry["BIB_ID"] in self.parent_id_map:
+                new_map[mfhd_uuid] = new_map.get(mfhd_uuid, []) + [instance_uuid]
+            else:
+                raise TransformationRecordFailedError(
+                    entry["MFHD_ID"],
+                    "Boundwith relationship map contains a BIB_ID id not in the instance id map. No boundwith holdings created.",
+                    entry["BIB_ID"],
+                )
 
         return new_map
 
@@ -469,6 +476,19 @@ class MapperBase:
             )
         )
 
+    @staticmethod
+    def validate_location_map(location_map: List[Dict], locations: List[Dict]) -> List[Dict]:
+        mapped_codes = [x['folio_code'] for x in location_map]
+        existing_codes = [x['code'] for x in locations]
+        missing_codes = set(mapped_codes) - set(existing_codes)
+        if missing_codes:
+            raise TransformationProcessError(
+                "",
+                f"Location map contains codes not found in locations: {', '.join(missing_codes)}",
+                "",
+            )
+        return location_map
+
 
 def flatten(my_dict: dict, path=""):
     for k, v in iter(my_dict.items()):

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/mapping_file_transformation/item_mapper.py

@@ -43,7 +43,7 @@ class ItemMapper(MappingFileMapperBase):
         temporary_location_mapping,
         library_configuration: LibraryConfiguration,
         boundwith_relationship_map,
-        task_configuration: AbstractTaskConfiguration
+        task_configuration: AbstractTaskConfiguration,
     ):
         item_schema = folio_client.get_item_schema()
         super().__init__(
@@ -128,17 +128,24 @@ class ItemMapper(MappingFileMapperBase):
         folio_record["discoverySuppress"] = file_def.discovery_suppressed
         self.migration_report.add(
             "Suppression",
-            i18n.t("Suppressed from discovery") + f' = {folio_record["discoverySuppress"]}',
+            i18n.t("Suppressed from discovery")
+            + f" = {folio_record['discoverySuppress']}",
         )
 
     def setup_status_mapping(self, item_statuses_map):
-        statuses = self.item_schema["properties"]["status"]["properties"]["name"]["enum"]
+        statuses = self.item_schema["properties"]["status"]["properties"]["name"][
+            "enum"
+        ]
         for mapping in item_statuses_map:
             if "folio_name" not in mapping:
-                logging.critical("folio_name is not a column in the status mapping file")
+                logging.critical(
+                    "folio_name is not a column in the status mapping file"
+                )
                 sys.exit(1)
             elif "legacy_code" not in mapping:
-                logging.critical("legacy_code is not a column in the status mapping file")
+                logging.critical(
+                    "legacy_code is not a column in the status mapping file"
+                )
                 sys.exit(1)
             elif mapping["folio_name"] not in statuses:
                 logging.critical(
@@ -153,7 +160,9 @@ class ItemMapper(MappingFileMapperBase):
                 )
                 sys.exit(1)
             elif not all(mapping.values()):
-                logging.critical("empty value in mapping %s. Check mapping file", mapping.values())
+                logging.critical(
+                    "empty value in mapping %s. Check mapping file", mapping.values()
+                )
                 sys.exit(1)
             else:
                 self.status_mapping = {
@@ -206,7 +215,9 @@ class ItemMapper(MappingFileMapperBase):
                 index_or_id,
                 True,
             )
-            self.migration_report.add("TemporaryLoanTypeMapping", f"{folio_prop_name} -> {ltid}")
+            self.migration_report.add(
+                "TemporaryLoanTypeMapping", f"{folio_prop_name} -> {ltid}"
+            )
             return ltid
         elif folio_prop_name == "permanentLoanTypeId":
             return self.get_mapped_ref_data_value(
@@ -232,7 +243,9 @@ class ItemMapper(MappingFileMapperBase):
             normalized_barcode = barcode.strip().lower()
             if normalized_barcode and normalized_barcode in self.unique_barcodes:
                 Helper.log_data_issue(index_or_id, "Duplicate barcode", mapped_value)
-                self.migration_report.add_general_statistics(i18n.t("Duplicate barcodes"))
+                self.migration_report.add_general_statistics(
+                    i18n.t("Duplicate barcodes")
+                )
                 return f"{barcode}-{uuid4()}"
             else:
                 if normalized_barcode:
@@ -257,7 +270,9 @@ class ItemMapper(MappingFileMapperBase):
             self.migration_report.add("UnmappedProperties", f"{folio_prop_name}")
             return ""
 
-    def get_item_level_call_number_type_id(self, legacy_item, folio_prop_name: str, index_or_id):
+    def get_item_level_call_number_type_id(
+        self, legacy_item, folio_prop_name: str, index_or_id
+    ):
         if self.call_number_mapping:
             return self.get_mapped_ref_data_value(
                 self.call_number_mapping, legacy_item, index_or_id, folio_prop_name

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py

@@ -755,12 +755,8 @@ class MappingFileMapperBase(MapperBase):
             for k in data
             if k["folio_field"] == folio_prop_name
             and any(
-                [
-                    is_set_or_bool_or_numeric(k.get("value", "")),
-                    is_set_or_bool_or_numeric(k.get("legacy_field", "")),
-                    is_set_or_bool_or_numeric(k.get("fallback_legacy_field", "")),
-                    is_set_or_bool_or_numeric(k.get("fallback_value", "")),
-                ]
+                is_set_or_bool_or_numeric(k.get(key, ""))
+                for key in ("value", "legacy_field", "fallback_legacy_field", "fallback_value")
             )
         )
 
@@ -972,4 +968,4 @@ def in_deep(dictionary, keys):
 
 
 def is_set_or_bool_or_numeric(any_value):
-    return any(isinstance(any_value, t) for t in [int, bool, float, complex]) or any_value.strip()
+    return (isinstance(any_value, str) and (any_value.strip() not in empty_vals)) or isinstance(any_value, (int, float, complex))

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/marc_rules_transformation/conditions.py

@@ -1,8 +1,9 @@
 import logging
 import re
+import traceback
+from typing import Union
 
 import i18n
-import pymarc
 from folioclient import FolioClient
 from pymarc import field
 
@@ -41,8 +42,8 @@ class Conditions:
         self.folio_release: FolioRelease = folio_release
         self.filter_last_chars = r",$"
         self.folio = folio
-        self.default_contributor_type = ""
-        self.mapper = mapper
+        self.default_contributor_type: dict = {}
+        self.mapper: RulesMapperBase = mapper
         self.ref_data_dicts = {}
         if object_type == "bibs":
             self.setup_reference_data_for_all()
@@ -52,15 +53,16 @@ class Conditions:
         else:
             self.setup_reference_data_for_all()
             self.setup_reference_data_for_items_and_holdings(default_call_number_type_name)
+        self.object_type = object_type
         self.condition_cache: dict = {}
 
     def setup_reference_data_for_bibs(self):
         logging.info("Setting up reference data for bib transformation")
-        logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types))
-        logging.info("%s\tcontributor_types", len(self.folio.contributor_types))
-        logging.info("%s\talt_title_types", len(self.folio.alt_title_types))
-        logging.info("%s\tidentifier_types", len(self.folio.identifier_types))
-        logging.info("%s\tsubject_types", len(self.folio.subject_types))
+        logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types)) # type: ignore
+        logging.info("%s\tcontributor_types", len(self.folio.contributor_types)) # type: ignore
+        logging.info("%s\talt_title_types", len(self.folio.alt_title_types)) # type: ignore
+        logging.info("%s\tidentifier_types", len(self.folio.identifier_types)) # type: ignore
+        logging.info("%s\tsubject_types", len(self.folio.subject_types)) # type: ignore
         # Raise for empty settings
         if not self.folio.contributor_types:
             raise TransformationProcessError("", "No contributor_types in FOLIO")
@@ -75,18 +77,18 @@ class Conditions:
 
         # Set defaults
         logging.info("Setting defaults")
-        self.default_contributor_name_type = self.folio.contrib_name_types[0]["id"]
+        self.default_contributor_name_type: str = self.folio.contrib_name_types[0]["id"] # type: ignore
         logging.info("Contributor name type:\t%s", self.default_contributor_name_type)
         self.default_contributor_type = next(
-            ct for ct in self.folio.contributor_types if ct["code"] == "ctb"
+            ct for ct in self.folio.contributor_types if ct["code"] == "ctb" # type: ignore
         )
         logging.info("Contributor type:\t%s", self.default_contributor_type["id"])
 
     def setup_reference_data_for_items_and_holdings(self, default_call_number_type_name):
-        logging.info(f"{len(self.folio.locations)}\tlocations")
+        logging.info(f"{len(self.folio.locations)}\tlocations") # type: ignore
         self.default_call_number_type = {}
-        logging.info("%s\tholding_note_types", len(self.folio.holding_note_types))
-        logging.info("%s\tcall_number_types", len(self.folio.call_number_types))
+        logging.info("%s\tholding_note_types", len(self.folio.holding_note_types)) # type: ignore
+        logging.info("%s\tcall_number_types", len(self.folio.call_number_types)) # type: ignore
         self.setup_and_validate_holdings_types()
         # Raise for empty settings
         if not self.folio.holding_note_types:
@@ -98,10 +100,10 @@ class Conditions:
 
         # Set defaults
         logging.info("Defaults")
-        self.default_call_number_type = next(
+        self.default_call_number_type: dict = next(
             (
                 ct
-                for ct in self.folio.call_number_types
+                for ct in self.folio.call_number_types # type: ignore
                 if ct["name"] == default_call_number_type_name
             ),
             None,
@@ -124,7 +126,7 @@ class Conditions:
         missing_holdings_types = [
             ht
             for ht in self.holdings_type_map.values()
-            if ht not in [ht_ref["name"] for ht_ref in self.holdings_types]
+            if ht not in [ht_ref["name"] for ht_ref in self.holdings_types] # type: ignore
         ]
         if any(missing_holdings_types):
             raise TransformationProcessError(
@@ -132,15 +134,15 @@ class Conditions:
                 "Holdings types are missing from the tenant. Please set them up",
                 missing_holdings_types,
             )
-        logging.info("%s\tholdings types", len(self.holdings_types))
+        logging.info("%s\tholdings types", len(self.holdings_types)) # type: ignore
 
     def setup_reference_data_for_all(self):
-        logging.info(f"{len(self.folio.class_types)}\tclass_types")
+        logging.info(f"{len(self.folio.class_types)}\tclass_types") # type: ignore
         logging.info(
-            f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships"
+            f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships" # type: ignore
         )
         self.statistical_codes = self.folio.statistical_codes
-        logging.info(f"{len(self.statistical_codes)} \tstatistical_codes")
+        logging.info(f"{len(self.statistical_codes)} \tstatistical_codes") # type: ignore
 
         # Raise for empty settings
         if not self.folio.class_types:
@@ -153,13 +155,13 @@ class Conditions:
             )
         )
         logging.info(f"{len(self.authority_note_types)} \tAuthority note types")
-        logging.info(f"{len(self.folio.identifier_types)} \tidentifier types")
+        logging.info(f"{len(self.folio.identifier_types)} \tidentifier types") # type: ignore
 
     def get_condition(
-        self, name, legacy_id, value, parameter=None, marc_field: field.Field = None
+        self, name, legacy_id, value, parameter=None, marc_field: Union[None, field.Field] = None
     ):
         try:
-            return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field)
+            return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field) # type: ignore
         # Exception should only handle the missing condition from the cache.
         # All other exceptions should propagate up
         except Exception:
@@ -375,10 +377,10 @@ class Conditions:
                 "MappedIdentifierTypes", f"{marc_field.tag} -> {t[1]}"
             )
             return t[0]
-        identifier_type = next(
+        identifier_type: dict = next(
             (
                 f
-                for f in self.folio.identifier_types
+                for f in self.folio.identifier_types # type: ignore
                 if (
                     f["name"] in parameter.get("names", "non existant")
                     or f["name"] in parameter.get("name", "non existant")
@@ -617,7 +619,7 @@ class Conditions:
         return self._extracted_from_condition_set_electronic_access_relations_id_2("8", marc_field)
 
     def condition_set_call_number_type_by_indicator(
-        self, legacy_id, value, parameter, marc_field: pymarc.Field
+        self, legacy_id, value, parameter, marc_field: field.Field
     ):
         self.mapper.migration_report.add(
             "Exceptions",
@@ -629,7 +631,7 @@ class Conditions:
         return self.condition_set_call_number_type_id(legacy_id, value, parameter, marc_field)
 
     def condition_set_call_number_type_id(
-        self, legacy_id, value, parameter, marc_field: pymarc.Field
+        self, legacy_id, value, parameter, marc_field: field.Field
     ):
         first_level_map = {
             "0": "Library of Congress classification",
@@ -690,12 +692,13 @@ class Conditions:
     ):
         for subfield in marc_field.get_subfields("4", "e"):
             normalized_subfield = re.sub(r"[^A-Za-z0-9 ]+", "", subfield.strip())
-            for cont_type in self.folio.contributor_types:
+            for cont_type in self.folio.contributor_types: # type: ignore
                 if normalized_subfield in [cont_type["code"], cont_type["name"]]:
                     return cont_type["name"]
         try:
             return value
-        except IndexError as ee:
+        except IndexError:
+            logging.debug("Exception occurred: %s", traceback.format_exc())
             return ""
 
     def condition_set_alternative_title_type_id(self, legacy_id, value, parameter, marc_field):
@@ -728,7 +731,7 @@ class Conditions:
     ):
         if "legacy_locations" not in self.ref_data_dicts:
             try:
-                d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map}
+                d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map} # type: ignore
                 self.ref_data_dicts["legacy_locations"] = d
                 for folio_code in d.values():
                     t = self.get_ref_data_tuple_by_code(
@@ -812,6 +815,27 @@ class Conditions:
     def condition_set_electronic_access_relations_id(
         self, legacy_id, value, parameter, marc_field: field.Field
     ):
+        """
+        This method handles the mapping of electronic access relationship IDs.
+        If the record type being mapped is FOLIO holdings, it provides an (optional) alternative
+        mapping based on a provided name parameter, bypassing the FOLIO MARC-to-Holdings mapping
+        engine behavior. This requires use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition containing the name parameter.
+        """
+        if self.object_type == "holdings" and "name" in parameter:
+            try:
+                t = self.get_ref_data_tuple_by_name(
+                    self.folio.electronic_access_relationships,
+                    "electronic_access_relationships",
+                    parameter["name"],
+                )
+                self.mapper.migration_report.add("MappedElectronicRelationshipTypes", t[1])
+                return t[0]
+            except Exception:
+                raise TransformationProcessError(
+                    legacy_id,
+                    f"Electronic access relationship not found for {parameter['name']} {marc_field}",
+                )
         return self._extracted_from_condition_set_electronic_access_relations_id_2("3", marc_field)
 
     # TODO Rename this here and in `condition_set_url_relationship` and `condition_set_electronic_access_relations_id`

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_base.py

@@ -597,46 +597,48 @@ class RulesMapperBase(MapperBase):
         legacy_ids,
     ):
         entity_mapping = mapping["entity"]
-        e_parent = entity_mapping[0]["target"].split(".")[0]
-        if mapping.get("entityPerRepeatedSubfield", False):
-            for temp_field in self.grouped(marc_field):
-                entity = self.create_entity(entity_mapping, temp_field, e_parent, legacy_ids)
-                if entity and (
-                    (isinstance(entity, dict) and all(entity.values()))
-                    or (isinstance(entity, list) and all(entity))
+        if entity_indicators_match(entity_mapping, marc_field):
+            entity_mapping = [x for x in entity_mapping if "indicators" not in x]
+            e_parent = entity_mapping[0]["target"].split(".")[0]
+            if mapping.get("entityPerRepeatedSubfield", False):
+                for temp_field in self.grouped(marc_field):
+                    entity = self.create_entity(entity_mapping, temp_field, e_parent, legacy_ids)
+                    if entity and (
+                        (isinstance(entity, dict) and all(entity.values()))
+                        or (isinstance(entity, list) and all(entity))
+                    ):
+                        self.add_entity_to_record(entity, e_parent, folio_record, self.schema)
+            else:
+                if mapping.get("ignoreSubsequentSubfields", False):
+                    marc_field = self.remove_repeated_subfields(marc_field)
+                entity = self.create_entity(entity_mapping, marc_field, e_parent, legacy_ids)
+                if e_parent in ["precedingTitles", "succeedingTitles"]:
+                    self.create_preceding_succeeding_titles(
+                        entity, e_parent, folio_record["id"], marc_field
+                    )
+                elif entity and (
+                    all(
+                        v
+                        for k, v in entity.items()
+                        if k not in ["staffOnly", "primary", "isbnValue", "issnValue"]
+                    )
+                    or e_parent in ["electronicAccess", "publication"]
+                    or (
+                        e_parent.startswith("holdingsStatements") and any(v for k, v in entity.items())
+                    )
                 ):
                     self.add_entity_to_record(entity, e_parent, folio_record, self.schema)
-        else:
-            if mapping.get("ignoreSubsequentSubfields", False):
-                marc_field = self.remove_repeated_subfields(marc_field)
-            entity = self.create_entity(entity_mapping, marc_field, e_parent, legacy_ids)
-            if e_parent in ["precedingTitles", "succeedingTitles"]:
-                self.create_preceding_succeeding_titles(
-                    entity, e_parent, folio_record["id"], marc_field
-                )
-            elif entity and (
-                all(
-                    v
-                    for k, v in entity.items()
-                    if k not in ["staffOnly", "primary", "isbnValue", "issnValue"]
-                )
-                or e_parent in ["electronicAccess", "publication"]
-                or (
-                    e_parent.startswith("holdingsStatements") and any(v for k, v in entity.items())
-                )
-            ):
-                self.add_entity_to_record(entity, e_parent, folio_record, self.schema)
-            else:
-                sfs = " - ".join(
-                    f"{f[0]}:{('has_value' if f[1].strip() else 'empty')}" for f in marc_field
-                )
-                pattern = " - ".join(f"{k}:'{bool(v)}'" for k, v in entity.items())
-                self.migration_report.add(
-                    "IncompleteEntityMapping",
-                    f"{marc_field.tag} {sfs} ->>-->> {e_parent} {pattern}  ",
-                )
-                # Experimental
-                # self.add_entity_to_record(entity, e_parent, rec, self.schema)
+                else:
+                    sfs = " - ".join(
+                        f"{f[0]}:{('has_value' if f[1].strip() else 'empty')}" for f in marc_field
+                    )
+                    pattern = " - ".join(f"{k}:'{bool(v)}'" for k, v in entity.items())
+                    self.migration_report.add(
+                        "IncompleteEntityMapping",
+                        f"{marc_field.tag} {sfs} ->>-->> {e_parent} {pattern}  ",
+                    )
+                    # Experimental
+                    # self.add_entity_to_record(entity, e_parent, rec, self.schema)
 
     def handle_suppression(
         self, folio_record, file_def: FileDefinition, only_discovery_suppress: bool = False
@@ -975,3 +977,34 @@ def is_array_of_strings(schema_property):
 def is_array_of_objects(schema_property):
     sc_prop_type = schema_property.get("type", "string")
     return sc_prop_type == "array" and schema_property["items"]["type"] == "object"
+
+def entity_indicators_match(entity_mapping, marc_field):
+    """
+    Check if the indicators of the entity mapping match the indicators of the MARC field.
+    Entity mappings can limit the fields they are applied to by specifying indicator values that
+    must match the provided MARC field's indicators. If the entity mapping does not specify any
+    indicator values, it is assumed to match all MARC fields. Entity indicator values can be a
+    specific value or a wildcard "*", which matches any value.
+
+    This function compares the indicators of the entity mapping with the indicators of the MARC field.
+    If the entity does not specify any indicator values, the function returns True. If the entity does
+    specify indicator values, the function checks if the MARC field's indicators match the specified
+    values or if the specified values are wildcards. If both indicators match, the function returns True;
+    otherwise, it returns False.
+
+    Args:
+        entity_mapping (dict): _description_
+        marc_field (pymarc.Field): _description_
+
+    Returns:
+        bool: True if the indicators match, False otherwise.
+    """
+    if indicator_rule := [x["indicators"] for x in entity_mapping if "indicators" in x]:
+        return all(
+            [
+                (marc_field.indicator1 == indicator_rule[0]['ind1'] or indicator_rule[0]['ind1'] == "*"),
+                (marc_field.indicator2 == indicator_rule[0]['ind2'] or indicator_rule[0]['ind2'] == "*"),
+            ]
+        )
+    else:
+        return True

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -61,7 +61,10 @@ class RulesMapperHoldings(RulesMapperBase):
         self.boundwith_relationship_map = self.setup_boundwith_relationship_map(
             boundwith_relationship_map
         )
-        self.location_map = location_map
+        self.location_map = self.validate_location_map(
+            location_map,
+            self.folio_client.locations,
+        )
         self.holdings_id_map: dict = {}
         self.ref_data_dicts: dict = {}
         self.fallback_holdings_type_id = self.task_configuration.fallback_holdings_type_id

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -132,6 +132,8 @@ class BatchPoster(MigrationTaskBase):
             ),
         ] = False
 
+    task_configuration: TaskConfiguration
+
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
         return FOLIONamespaces.other
@@ -289,35 +291,57 @@ class BatchPoster(MigrationTaskBase):
             for i in range(0, len(batch), fetch_batch_size):
                 batch_slice = batch[i:i + fetch_batch_size]
                 fetch_tasks.append(
-                    client.get(
+                    self.get_with_retry(
+                        client,
                         query_api,
                         params={
                             "query": f"id==({' OR '.join([record['id'] for record in batch_slice if 'id' in record])})",
                             "limit": fetch_batch_size
                         },
-                        headers=self.folio_client.okapi_headers
                     )
                 )
+
             responses = await asyncio.gather(*fetch_tasks)
 
             for response in responses:
-                if response.status_code == 200:
-                    response_json = response.json()
-                    for record in response_json[object_type]:
-                        updates[record["id"]] = {
+                self.update_record_versions(object_type, updates, response)
+        for record in batch:
+            if record["id"] in updates:
+                record.update(updates[record["id"]])
+
+    @staticmethod
+    def update_record_versions(object_type, updates, response):
+        if response.status_code == 200:
+            response_json = response.json()
+            for record in response_json[object_type]:
+                updates[record["id"]] = {
                             "_version": record["_version"],
                         }
-                        if "status" in record:
-                            updates[record["id"]]["status"] = record["status"]
-                else:
-                    logging.error(
+                if "status" in record:
+                    updates[record["id"]]["status"] = record["status"]
+                if "lastCheckIn" in record:
+                    updates[record["id"]]["lastCheckIn"] = record["lastCheckIn"]
+        else:
+            logging.error(
                         "Failed to fetch current records. HTTP %s\t%s",
                         response.status_code,
                         response.text,
                     )
-        for record in batch:
-            if record["id"] in updates:
-                record.update(updates[record["id"]])
+
+    async def get_with_retry(self, client: httpx.AsyncClient, url: str, params: dict = {}):
+        retries = 3
+        for attempt in range(retries):
+            try:
+                response = await client.get(url, params=params, headers=self.folio_client.okapi_headers)
+                response.raise_for_status()
+                return response
+            except httpx.HTTPError as e:
+                if attempt < retries - 1:
+                    logging.warning(f"Retrying due to {e}")
+                    await asyncio.sleep(2 ** attempt)
+                else:
+                    logging.error(f"Failed to connect after {retries} attempts: {e}")
+                    raise
 
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
@@ -449,7 +473,7 @@ class BatchPoster(MigrationTaskBase):
         )
         logging.info(last_row)
         logging.info("=========Stack trace==============")
-        traceback.logging.info_exc()
+        traceback.logging.info_exc() # type: ignore
         logging.info("=======================", flush=True)
 
     def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -368,12 +368,12 @@ class ItemsTransformer(MigrationTaskBase):
                     self.handle_circiulation_notes(folio_rec, self.folio_client.current_user)
                     self.handle_notes(folio_rec)
                     if folio_rec["holdingsRecordId"] in self.mapper.boundwith_relationship_map:
-                        for idx, instance_id in enumerate(
+                        for idx_, instance_id in enumerate(
                             self.mapper.boundwith_relationship_map.get(
                                 folio_rec["holdingsRecordId"]
                             )
                         ):
-                            if idx == 0:
+                            if idx_ == 0:
                                 bw_id = folio_rec["holdingsRecordId"]
                             else:
                                 bw_id = self.mapper.generate_boundwith_holding_uuid(

{folio_migration_tools-1.9.0rc5 → folio_migration_tools-1.9.0rc7}/src/folio_migration_tools/transaction_migration/legacy_loan.py

@@ -3,9 +3,10 @@ from datetime import datetime
 from zoneinfo import ZoneInfo
 
 from dateutil import tz
-from dateutil.parser import parse
+from dateutil.parser import parse, ParserError
 
 from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.custom_exceptions import TransformationProcessError
 
 utc = ZoneInfo("UTC")
 
@@ -42,46 +43,47 @@ class LegacyLoan(object):
 
         self.tenant_timezone = tenant_timezone
         self.errors = []
+        self.row = row
         for prop in correct_headers:
             if prop not in legacy_loan_dict and prop not in optional_headers:
-                self.errors.append(("Missing properties in legacy data", prop))
+                self.errors.append((f"Missing properties in legacy data {row=}", prop))
             if (
                 prop != "next_item_status"
                 and not legacy_loan_dict.get(prop, "").strip()
                 and prop not in optional_headers
             ):
-                self.errors.append(("Empty properties in legacy data", prop))
+                self.errors.append((f"Empty properties in legacy data {row=}", prop))
         try:
             temp_date_due: datetime = parse(legacy_loan_dict["due_date"])
             if temp_date_due.tzinfo != tz.UTC:
                 temp_date_due = temp_date_due.replace(tzinfo=self.tenant_timezone)
                 self.report(
-                    f"Provided due_date is not UTC, "
-                    f"setting tzinfo to tenant timezone ({self.tenant_timezone})"
+                    f"Provided due_date is not UTC in {row=}, "
+                    f"setting tz-info to tenant timezone ({self.tenant_timezone})"
                 )
             if temp_date_due.hour == 0 and temp_date_due.minute == 0:
                 temp_date_due = temp_date_due.replace(hour=23, minute=59)
                 self.report(
-                    "Hour and minute not specified for due date. "
+                    f"Hour and minute not specified for due date in {row=}. "
                     "Assuming end of local calendar day (23:59)..."
                 )
-        except Exception as ee:
+        except (ParserError, OverflowError) as ee:
             logging.error(ee)
-            self.errors.append(("Parse date failure. Setting UTC NOW", "due_date"))
+            self.errors.append((f"Parse date failure in {row=}. Setting UTC NOW", "due_date"))
             temp_date_due = datetime.now(ZoneInfo("UTC"))
         try:
             temp_date_out: datetime = parse(legacy_loan_dict["out_date"])
             if temp_date_out.tzinfo != tz.UTC:
                 temp_date_out = temp_date_out.replace(tzinfo=self.tenant_timezone)
                 self.report(
-                    f"Provided out_date is not UTC, "
-                    f"setting tzinfo to tenant timezone ({self.tenant_timezone})"
+                    f"Provided out_date is not UTC in {row=}, "
+                    f"setting tz-info to tenant timezone ({self.tenant_timezone})"
                 )
-        except Exception:
+        except (ParserError, OverflowError):
             temp_date_out = datetime.now(
                 ZoneInfo("UTC")
             )  # TODO: Consider moving this assignment block above the temp_date_due
-            self.errors.append(("Parse date failure. Setting UTC NOW", "out_date"))
+            self.errors.append((f"Parse date failure in {row=}. Setting UTC NOW", "out_date"))
 
         # good to go, set properties
         self.item_barcode: str = legacy_loan_dict["item_barcode"].strip()
@@ -94,7 +96,7 @@ class LegacyLoan(object):
         self.renewal_count = self.set_renewal_count(legacy_loan_dict)
         self.next_item_status = legacy_loan_dict.get("next_item_status", "").strip()
         if self.next_item_status not in legal_statuses:
-            self.errors.append(("Not an allowed status", self.next_item_status))
+            self.errors.append((f"Not an allowed status {row=}", self.next_item_status))
         self.service_point_id = (
             legacy_loan_dict["service_point_id"]
             if legacy_loan_dict.get("service_point_id", "")
@@ -107,23 +109,19 @@ class LegacyLoan(object):
             try:
                 return int(renewal_count)
             except ValueError:
-                self.report(
-                    f"Unresolvable {renewal_count=} was replaced with 0.")
+                self.report(f"Unresolvable {renewal_count=} was replaced with 0.")
         else:
             self.report(f"Missing renewal count was replaced with 0.")
         return 0
 
     def correct_for_1_day_loans(self):
-        try:
-            if self.due_date.date() <= self.out_date.date():
-                if self.due_date.hour == 0:
-                    self.due_date = self.due_date.replace(hour=23, minute=59)
-                if self.out_date.hour == 0:
-                    self.out_date = self.out_date.replace(hour=0, minute=1)
-            if self.due_date <= self.out_date:
-                raise ValueError("Due date is before out date")
-        except Exception:
-            self.errors.append(("Time alignment issues", "both dates"))
+        if self.due_date.date() <= self.out_date.date():
+            if self.due_date.hour == 0:
+                self.due_date = self.due_date.replace(hour=23, minute=59)
+            if self.out_date.hour == 0:
+                self.out_date = self.out_date.replace(hour=0, minute=1)
+        if self.due_date <= self.out_date:
+            raise TransformationProcessError(self.row, "Due date is before out date")
 
     def to_dict(self):
         return {
@@ -140,8 +138,8 @@ class LegacyLoan(object):
             if self.tenant_timezone != ZoneInfo("UTC"):
                 self.due_date = self.due_date.astimezone(ZoneInfo("UTC"))
                 self.out_date = self.out_date.astimezone(ZoneInfo("UTC"))
-        except Exception:
-            self.errors.append(("UTC correction issues", "both dates"))
+        except TypeError:
+            self.errors.append((f"UTC correction issues {self.row}", "both dates"))
 
     def report(self, what_to_report: str):
         self.migration_report.add("Details", what_to_report)

folio_migration_tools-1.9.0rc5/src/folio_migration_tools/__init__.py

@@ -1,3 +0,0 @@
-import importlib.metadata
-
-__version__ = importlib.metadata.version("folio_migration_tools")