folio-migration-tools 1.9.0rc5__py3-none-any.whl → 1.9.0rc6__py3-none-any.whl

folio_migration_tools/__init__.py

@@ -1,3 +1,11 @@
 import importlib.metadata
+from typing import Protocol
 
 __version__ = importlib.metadata.version("folio_migration_tools")
+
+class StrCoercible(Protocol):
+    def __repr__(self) -> str:
+        ...
+
+    def __str__(self) -> str:
+        ...

folio_migration_tools/custom_dict.py

@@ -7,10 +7,10 @@ class InsensitiveDictReader(csv.DictReader):
     # spaces and to lower case.
     @property
     def fieldnames(self):
-        return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)]
+        return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)] # type: ignore
 
     def next(self):
-        return InsensitiveDict(csv.DictReader.next(self))
+        return InsensitiveDict(csv.DictReader.next(self)) # type: ignore
 
 
 class InsensitiveDict(dict):

folio_migration_tools/custom_exceptions.py

@@ -1,6 +1,9 @@
 import logging
+from typing import Union
 import i18n
 
+from folio_migration_tools import StrCoercible
+
 
 class TransfomationError(Exception):
     pass
@@ -10,10 +13,10 @@ class TransformationFieldMappingError(TransfomationError):
     """Raised when the a field mapping fails, but the error is not critical.
     The issue should be logged for the library to act upon it"""
 
-    def __init__(self, index_or_id="", message="", data_value=""):
+    def __init__(self, index_or_id="", message="", data_value: Union[str, StrCoercible]=""):
         self.index_or_id = index_or_id or ""
         self.message = message
-        self.data_value = data_value
+        self.data_value: Union[str, StrCoercible] = data_value
         super().__init__(self.message)
 
     def __str__(self):
@@ -38,7 +41,7 @@ class TransformationRecordFailedError(TransfomationError):
     def __init__(self, index_or_id, message="", data_value=""):
         self.index_or_id = index_or_id
         self.message = message
-        self.data_value = data_value
+        self.data_value: Union[str, StrCoercible] = data_value
         # logging.log(26, f"RECORD FAILED\t{self.id}\t{self.message}\t{self.data_value}")
         super().__init__(self.message)
 
@@ -67,7 +70,7 @@ class TransformationProcessError(TransfomationError):
         index_or_id,
         message="Critical Process issue. Transformation failed."
         " Check configuration, mapping files and reference data",
-        data_value="",
+        data_value: Union[str, StrCoercible]="",
     ):
         self.index_or_id = index_or_id
         self.message = message

folio_migration_tools/mapper_base.py

@@ -6,7 +6,7 @@ import sys
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import List
+from typing import Dict, List
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -35,7 +35,7 @@ class MapperBase:
         self,
         library_configuration: LibraryConfiguration,
         folio_client: FolioClient,
-        parent_id_map: dict[str, tuple] = None,
+        parent_id_map: dict[str, tuple] = {},
     ):
         logging.info("MapperBase initiating")
         self.parent_id_map: dict[str, tuple] = parent_id_map
@@ -318,7 +318,14 @@ class MapperBase:
                     entry["MFHD_ID"],
                 )
             )
-            new_map[mfhd_uuid] = new_map.get(mfhd_uuid, []) + [instance_uuid]
+            if entry["BIB_ID"] in self.parent_id_map:
+                new_map[mfhd_uuid] = new_map.get(mfhd_uuid, []) + [instance_uuid]
+            else:
+                raise TransformationRecordFailedError(
+                    entry["MFHD_ID"],
+                    "Boundwith relationship map contains a BIB_ID id not in the instance id map. No boundwith holdings created.",
+                    entry["BIB_ID"],
+                )
 
         return new_map
 
@@ -469,6 +476,19 @@ class MapperBase:
             )
         )
 
+    @staticmethod
+    def validate_location_map(location_map: List[Dict], locations: List[Dict]) -> List[Dict]:
+        mapped_codes = [x['folio_code'] for x in location_map]
+        existing_codes = [x['code'] for x in locations]
+        missing_codes = set(mapped_codes) - set(existing_codes)
+        if missing_codes:
+            raise TransformationProcessError(
+                "",
+                f"Location map contains codes not found in locations: {', '.join(missing_codes)}",
+                "",
+            )
+        return location_map
+
 
 def flatten(my_dict: dict, path=""):
     for k, v in iter(my_dict.items()):

folio_migration_tools/mapping_file_transformation/item_mapper.py

@@ -43,7 +43,7 @@ class ItemMapper(MappingFileMapperBase):
         temporary_location_mapping,
         library_configuration: LibraryConfiguration,
         boundwith_relationship_map,
-        task_configuration: AbstractTaskConfiguration
+        task_configuration: AbstractTaskConfiguration,
     ):
         item_schema = folio_client.get_item_schema()
         super().__init__(
@@ -75,7 +75,9 @@ class ItemMapper(MappingFileMapperBase):
                 self.folio_client,
                 "/locations",
                 "locations",
-                temporary_location_mapping,
+                self.validate_location_map(
+                    temporary_location_mapping, self.folio_client.locations
+                ),
                 "code",
                 "TemporaryLocationMapping",
             )
@@ -116,7 +118,7 @@ class ItemMapper(MappingFileMapperBase):
             self.folio_client,
             "/locations",
             "locations",
-            location_map,
+            self.validate_location_map(location_map, self.folio_client.locations),
             "code",
             "LocationMapping",
         )
@@ -128,17 +130,24 @@ class ItemMapper(MappingFileMapperBase):
         folio_record["discoverySuppress"] = file_def.discovery_suppressed
         self.migration_report.add(
             "Suppression",
-            i18n.t("Suppressed from discovery") + f' = {folio_record["discoverySuppress"]}',
+            i18n.t("Suppressed from discovery")
+            + f" = {folio_record['discoverySuppress']}",
         )
 
     def setup_status_mapping(self, item_statuses_map):
-        statuses = self.item_schema["properties"]["status"]["properties"]["name"]["enum"]
+        statuses = self.item_schema["properties"]["status"]["properties"]["name"][
+            "enum"
+        ]
         for mapping in item_statuses_map:
             if "folio_name" not in mapping:
-                logging.critical("folio_name is not a column in the status mapping file")
+                logging.critical(
+                    "folio_name is not a column in the status mapping file"
+                )
                 sys.exit(1)
             elif "legacy_code" not in mapping:
-                logging.critical("legacy_code is not a column in the status mapping file")
+                logging.critical(
+                    "legacy_code is not a column in the status mapping file"
+                )
                 sys.exit(1)
             elif mapping["folio_name"] not in statuses:
                 logging.critical(
@@ -153,7 +162,9 @@ class ItemMapper(MappingFileMapperBase):
                 )
                 sys.exit(1)
             elif not all(mapping.values()):
-                logging.critical("empty value in mapping %s. Check mapping file", mapping.values())
+                logging.critical(
+                    "empty value in mapping %s. Check mapping file", mapping.values()
+                )
                 sys.exit(1)
             else:
                 self.status_mapping = {
@@ -206,7 +217,9 @@ class ItemMapper(MappingFileMapperBase):
                 index_or_id,
                 True,
             )
-            self.migration_report.add("TemporaryLoanTypeMapping", f"{folio_prop_name} -> {ltid}")
+            self.migration_report.add(
+                "TemporaryLoanTypeMapping", f"{folio_prop_name} -> {ltid}"
+            )
             return ltid
         elif folio_prop_name == "permanentLoanTypeId":
             return self.get_mapped_ref_data_value(
@@ -232,7 +245,9 @@ class ItemMapper(MappingFileMapperBase):
             normalized_barcode = barcode.strip().lower()
             if normalized_barcode and normalized_barcode in self.unique_barcodes:
                 Helper.log_data_issue(index_or_id, "Duplicate barcode", mapped_value)
-                self.migration_report.add_general_statistics(i18n.t("Duplicate barcodes"))
+                self.migration_report.add_general_statistics(
+                    i18n.t("Duplicate barcodes")
+                )
                 return f"{barcode}-{uuid4()}"
             else:
                 if normalized_barcode:
@@ -257,7 +272,9 @@ class ItemMapper(MappingFileMapperBase):
             self.migration_report.add("UnmappedProperties", f"{folio_prop_name}")
             return ""
 
-    def get_item_level_call_number_type_id(self, legacy_item, folio_prop_name: str, index_or_id):
+    def get_item_level_call_number_type_id(
+        self, legacy_item, folio_prop_name: str, index_or_id
+    ):
         if self.call_number_mapping:
             return self.get_mapped_ref_data_value(
                 self.call_number_mapping, legacy_item, index_or_id, folio_prop_name

folio_migration_tools/marc_rules_transformation/conditions.py

@@ -1,8 +1,9 @@
 import logging
 import re
+import traceback
+from typing import Union
 
 import i18n
-import pymarc
 from folioclient import FolioClient
 from pymarc import field
 
@@ -41,8 +42,8 @@ class Conditions:
         self.folio_release: FolioRelease = folio_release
         self.filter_last_chars = r",$"
         self.folio = folio
-        self.default_contributor_type = ""
-        self.mapper = mapper
+        self.default_contributor_type: dict = {}
+        self.mapper: RulesMapperBase = mapper
         self.ref_data_dicts = {}
         if object_type == "bibs":
             self.setup_reference_data_for_all()
@@ -52,15 +53,16 @@ class Conditions:
         else:
             self.setup_reference_data_for_all()
             self.setup_reference_data_for_items_and_holdings(default_call_number_type_name)
+        self.object_type = object_type
         self.condition_cache: dict = {}
 
     def setup_reference_data_for_bibs(self):
         logging.info("Setting up reference data for bib transformation")
-        logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types))
-        logging.info("%s\tcontributor_types", len(self.folio.contributor_types))
-        logging.info("%s\talt_title_types", len(self.folio.alt_title_types))
-        logging.info("%s\tidentifier_types", len(self.folio.identifier_types))
-        logging.info("%s\tsubject_types", len(self.folio.subject_types))
+        logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types)) # type: ignore
+        logging.info("%s\tcontributor_types", len(self.folio.contributor_types)) # type: ignore
+        logging.info("%s\talt_title_types", len(self.folio.alt_title_types)) # type: ignore
+        logging.info("%s\tidentifier_types", len(self.folio.identifier_types)) # type: ignore
+        logging.info("%s\tsubject_types", len(self.folio.subject_types)) # type: ignore
         # Raise for empty settings
         if not self.folio.contributor_types:
             raise TransformationProcessError("", "No contributor_types in FOLIO")
@@ -75,18 +77,18 @@ class Conditions:
 
         # Set defaults
         logging.info("Setting defaults")
-        self.default_contributor_name_type = self.folio.contrib_name_types[0]["id"]
+        self.default_contributor_name_type: str = self.folio.contrib_name_types[0]["id"] # type: ignore
         logging.info("Contributor name type:\t%s", self.default_contributor_name_type)
         self.default_contributor_type = next(
-            ct for ct in self.folio.contributor_types if ct["code"] == "ctb"
+            ct for ct in self.folio.contributor_types if ct["code"] == "ctb" # type: ignore
         )
         logging.info("Contributor type:\t%s", self.default_contributor_type["id"])
 
     def setup_reference_data_for_items_and_holdings(self, default_call_number_type_name):
-        logging.info(f"{len(self.folio.locations)}\tlocations")
+        logging.info(f"{len(self.folio.locations)}\tlocations") # type: ignore
         self.default_call_number_type = {}
-        logging.info("%s\tholding_note_types", len(self.folio.holding_note_types))
-        logging.info("%s\tcall_number_types", len(self.folio.call_number_types))
+        logging.info("%s\tholding_note_types", len(self.folio.holding_note_types)) # type: ignore
+        logging.info("%s\tcall_number_types", len(self.folio.call_number_types)) # type: ignore
         self.setup_and_validate_holdings_types()
         # Raise for empty settings
         if not self.folio.holding_note_types:
@@ -98,10 +100,10 @@ class Conditions:
 
         # Set defaults
         logging.info("Defaults")
-        self.default_call_number_type = next(
+        self.default_call_number_type: dict = next(
             (
                 ct
-                for ct in self.folio.call_number_types
+                for ct in self.folio.call_number_types # type: ignore
                 if ct["name"] == default_call_number_type_name
             ),
             None,
@@ -124,7 +126,7 @@ class Conditions:
         missing_holdings_types = [
             ht
             for ht in self.holdings_type_map.values()
-            if ht not in [ht_ref["name"] for ht_ref in self.holdings_types]
+            if ht not in [ht_ref["name"] for ht_ref in self.holdings_types] # type: ignore
         ]
         if any(missing_holdings_types):
             raise TransformationProcessError(
@@ -132,15 +134,15 @@ class Conditions:
                 "Holdings types are missing from the tenant. Please set them up",
                 missing_holdings_types,
             )
-        logging.info("%s\tholdings types", len(self.holdings_types))
+        logging.info("%s\tholdings types", len(self.holdings_types)) # type: ignore
 
     def setup_reference_data_for_all(self):
-        logging.info(f"{len(self.folio.class_types)}\tclass_types")
+        logging.info(f"{len(self.folio.class_types)}\tclass_types") # type: ignore
         logging.info(
-            f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships"
+            f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships" # type: ignore
         )
         self.statistical_codes = self.folio.statistical_codes
-        logging.info(f"{len(self.statistical_codes)} \tstatistical_codes")
+        logging.info(f"{len(self.statistical_codes)} \tstatistical_codes") # type: ignore
 
         # Raise for empty settings
         if not self.folio.class_types:
@@ -153,13 +155,13 @@ class Conditions:
             )
         )
         logging.info(f"{len(self.authority_note_types)} \tAuthority note types")
-        logging.info(f"{len(self.folio.identifier_types)} \tidentifier types")
+        logging.info(f"{len(self.folio.identifier_types)} \tidentifier types") # type: ignore
 
     def get_condition(
-        self, name, legacy_id, value, parameter=None, marc_field: field.Field = None
+        self, name, legacy_id, value, parameter=None, marc_field: Union[None, field.Field] = None
     ):
         try:
-            return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field)
+            return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field) # type: ignore
         # Exception should only handle the missing condition from the cache.
         # All other exceptions should propagate up
         except Exception:
@@ -375,10 +377,10 @@ class Conditions:
                 "MappedIdentifierTypes", f"{marc_field.tag} -> {t[1]}"
             )
             return t[0]
-        identifier_type = next(
+        identifier_type: dict = next(
             (
                 f
-                for f in self.folio.identifier_types
+                for f in self.folio.identifier_types # type: ignore
                 if (
                     f["name"] in parameter.get("names", "non existant")
                     or f["name"] in parameter.get("name", "non existant")
@@ -617,7 +619,7 @@ class Conditions:
         return self._extracted_from_condition_set_electronic_access_relations_id_2("8", marc_field)
 
     def condition_set_call_number_type_by_indicator(
-        self, legacy_id, value, parameter, marc_field: pymarc.Field
+        self, legacy_id, value, parameter, marc_field: field.Field
     ):
         self.mapper.migration_report.add(
             "Exceptions",
@@ -629,7 +631,7 @@ class Conditions:
         return self.condition_set_call_number_type_id(legacy_id, value, parameter, marc_field)
 
     def condition_set_call_number_type_id(
-        self, legacy_id, value, parameter, marc_field: pymarc.Field
+        self, legacy_id, value, parameter, marc_field: field.Field
     ):
         first_level_map = {
             "0": "Library of Congress classification",
@@ -690,12 +692,13 @@ class Conditions:
     ):
         for subfield in marc_field.get_subfields("4", "e"):
             normalized_subfield = re.sub(r"[^A-Za-z0-9 ]+", "", subfield.strip())
-            for cont_type in self.folio.contributor_types:
+            for cont_type in self.folio.contributor_types: # type: ignore
                 if normalized_subfield in [cont_type["code"], cont_type["name"]]:
                     return cont_type["name"]
         try:
             return value
-        except IndexError as ee:
+        except IndexError:
+            logging.debug("Exception occurred: %s", traceback.format_exc())
             return ""
 
     def condition_set_alternative_title_type_id(self, legacy_id, value, parameter, marc_field):
@@ -728,7 +731,7 @@ class Conditions:
     ):
         if "legacy_locations" not in self.ref_data_dicts:
             try:
-                d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map}
+                d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map} # type: ignore
                 self.ref_data_dicts["legacy_locations"] = d
                 for folio_code in d.values():
                     t = self.get_ref_data_tuple_by_code(
@@ -812,6 +815,27 @@ class Conditions:
     def condition_set_electronic_access_relations_id(
         self, legacy_id, value, parameter, marc_field: field.Field
     ):
+        """
+        This method handles the mapping of electronic access relationship IDs.
+        If the record type being mapped is FOLIO holdings, it provides an (optional) alternative
+        mapping baseed on a provided name parameter, bypassing the FOLIO MARC-to-Holdings mapping
+        engine behavior. This requires use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition containing the name parameter.
+        """
+        if self.object_type == "holdings" and "name" in parameter:
+            try:
+                t = self.get_ref_data_tuple_by_name(
+                    self.folio.electronic_access_relationships,
+                    "electronic_access_relationships",
+                    parameter["name"],
+                )
+                self.mapper.migration_report.add("MappedElectronicRelationshipTypes", t[1])
+                return t[0]
+            except Exception:
+                raise TransformationProcessError(
+                    legacy_id,
+                    f"Electronic access relationship not found for {parameter['name']} {marc_field}",
+                )
         return self._extracted_from_condition_set_electronic_access_relations_id_2("3", marc_field)
 
     # TODO Rename this here and in `condition_set_url_relationship` and `condition_set_electronic_access_relations_id`

folio_migration_tools/marc_rules_transformation/rules_mapper_base.py

@@ -597,46 +597,48 @@ class RulesMapperBase(MapperBase):
         legacy_ids,
     ):
         entity_mapping = mapping["entity"]
-        e_parent = entity_mapping[0]["target"].split(".")[0]
-        if mapping.get("entityPerRepeatedSubfield", False):
-            for temp_field in self.grouped(marc_field):
-                entity = self.create_entity(entity_mapping, temp_field, e_parent, legacy_ids)
-                if entity and (
-                    (isinstance(entity, dict) and all(entity.values()))
-                    or (isinstance(entity, list) and all(entity))
+        if entity_indicators_match(entity_mapping, marc_field):
+            entity_mapping = [x for x in entity_mapping if "indicators" not in x]
+            e_parent = entity_mapping[0]["target"].split(".")[0]
+            if mapping.get("entityPerRepeatedSubfield", False):
+                for temp_field in self.grouped(marc_field):
+                    entity = self.create_entity(entity_mapping, temp_field, e_parent, legacy_ids)
+                    if entity and (
+                        (isinstance(entity, dict) and all(entity.values()))
+                        or (isinstance(entity, list) and all(entity))
+                    ):
+                        self.add_entity_to_record(entity, e_parent, folio_record, self.schema)
+            else:
+                if mapping.get("ignoreSubsequentSubfields", False):
+                    marc_field = self.remove_repeated_subfields(marc_field)
+                entity = self.create_entity(entity_mapping, marc_field, e_parent, legacy_ids)
+                if e_parent in ["precedingTitles", "succeedingTitles"]:
+                    self.create_preceding_succeeding_titles(
+                        entity, e_parent, folio_record["id"], marc_field
+                    )
+                elif entity and (
+                    all(
+                        v
+                        for k, v in entity.items()
+                        if k not in ["staffOnly", "primary", "isbnValue", "issnValue"]
+                    )
+                    or e_parent in ["electronicAccess", "publication"]
+                    or (
+                        e_parent.startswith("holdingsStatements") and any(v for k, v in entity.items())
+                    )
                 ):
                     self.add_entity_to_record(entity, e_parent, folio_record, self.schema)
-        else:
-            if mapping.get("ignoreSubsequentSubfields", False):
-                marc_field = self.remove_repeated_subfields(marc_field)
-            entity = self.create_entity(entity_mapping, marc_field, e_parent, legacy_ids)
-            if e_parent in ["precedingTitles", "succeedingTitles"]:
-                self.create_preceding_succeeding_titles(
-                    entity, e_parent, folio_record["id"], marc_field
-                )
-            elif entity and (
-                all(
-                    v
-                    for k, v in entity.items()
-                    if k not in ["staffOnly", "primary", "isbnValue", "issnValue"]
-                )
-                or e_parent in ["electronicAccess", "publication"]
-                or (
-                    e_parent.startswith("holdingsStatements") and any(v for k, v in entity.items())
-                )
-            ):
-                self.add_entity_to_record(entity, e_parent, folio_record, self.schema)
-            else:
-                sfs = " - ".join(
-                    f"{f[0]}:{('has_value' if f[1].strip() else 'empty')}" for f in marc_field
-                )
-                pattern = " - ".join(f"{k}:'{bool(v)}'" for k, v in entity.items())
-                self.migration_report.add(
-                    "IncompleteEntityMapping",
-                    f"{marc_field.tag} {sfs} ->>-->> {e_parent} {pattern}  ",
-                )
-                # Experimental
-                # self.add_entity_to_record(entity, e_parent, rec, self.schema)
+                else:
+                    sfs = " - ".join(
+                        f"{f[0]}:{('has_value' if f[1].strip() else 'empty')}" for f in marc_field
+                    )
+                    pattern = " - ".join(f"{k}:'{bool(v)}'" for k, v in entity.items())
+                    self.migration_report.add(
+                        "IncompleteEntityMapping",
+                        f"{marc_field.tag} {sfs} ->>-->> {e_parent} {pattern}  ",
+                    )
+                    # Experimental
+                    # self.add_entity_to_record(entity, e_parent, rec, self.schema)
 
     def handle_suppression(
         self, folio_record, file_def: FileDefinition, only_discovery_suppress: bool = False
@@ -975,3 +977,34 @@ def is_array_of_strings(schema_property):
 def is_array_of_objects(schema_property):
     sc_prop_type = schema_property.get("type", "string")
     return sc_prop_type == "array" and schema_property["items"]["type"] == "object"
+
+def entity_indicators_match(entity_mapping, marc_field):
+    """
+    Check if the indicators of the entity mapping match the indicators of the MARC field.
+    Entity mappings can limit the fields they are applied to by specifying indicator values that
+    must match the provided MARC field's indicators. If the entity mapping does not specify any
+    indicator values, it is assumed to match all MARC fields. Entity indicator values can be a
+    specific value or a wildcard "*", which matches any value.
+
+    This function compares the indicators of the entity mapping with the indicators of the MARC field.
+    If the entity does not specify any indicator values, the function returns True. If the entity does
+    specify indicator values, the function checks if the MARC field's indicators match the specified
+    values or if the specified values are wildcards. If both indicators match, the function returns True;
+    otherwise, it returns False.
+
+    Args:
+        entity_mapping (dict): _description_
+        marc_field (pymarc.Field): _description_
+
+    Returns:
+        bool: True if the indicators match, False otherwise.
+    """
+    if indicator_rule := [x["indicators"] for x in entity_mapping if "indicators" in x]:
+        return all(
+            [
+                (marc_field.indicator1 == indicator_rule[0]['ind1'] or indicator_rule[0]['ind1'] == "*"),
+                (marc_field.indicator2 == indicator_rule[0]['ind2'] or indicator_rule[0]['ind2'] == "*"),
+            ]
+        )
+    else:
+        return True

folio_migration_tools/migration_tasks/batch_poster.py

@@ -132,6 +132,8 @@ class BatchPoster(MigrationTaskBase):
             ),
         ] = False
 
+    task_configuration: TaskConfiguration
+
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
         return FOLIONamespaces.other
@@ -289,35 +291,57 @@ class BatchPoster(MigrationTaskBase):
             for i in range(0, len(batch), fetch_batch_size):
                 batch_slice = batch[i:i + fetch_batch_size]
                 fetch_tasks.append(
-                    client.get(
+                    self.get_with_retry(
+                        client,
                         query_api,
                         params={
                             "query": f"id==({' OR '.join([record['id'] for record in batch_slice if 'id' in record])})",
                             "limit": fetch_batch_size
                         },
-                        headers=self.folio_client.okapi_headers
                     )
                 )
+
             responses = await asyncio.gather(*fetch_tasks)
 
             for response in responses:
-                if response.status_code == 200:
-                    response_json = response.json()
-                    for record in response_json[object_type]:
-                        updates[record["id"]] = {
+                self.update_record_versions(object_type, updates, response)
+        for record in batch:
+            if record["id"] in updates:
+                record.update(updates[record["id"]])
+
+    @staticmethod
+    def update_record_versions(object_type, updates, response):
+        if response.status_code == 200:
+            response_json = response.json()
+            for record in response_json[object_type]:
+                updates[record["id"]] = {
                             "_version": record["_version"],
                         }
-                        if "status" in record:
-                            updates[record["id"]]["status"] = record["status"]
-                else:
-                    logging.error(
+                if "status" in record:
+                    updates[record["id"]]["status"] = record["status"]
+                if "lastCheckIn" in record:
+                    updates[record["id"]]["lastCheckIn"] = record["lastCheckIn"]
+        else:
+            logging.error(
                         "Failed to fetch current records. HTTP %s\t%s",
                         response.status_code,
                         response.text,
                     )
-        for record in batch:
-            if record["id"] in updates:
-                record.update(updates[record["id"]])
+
+    async def get_with_retry(self, client: httpx.AsyncClient, url: str, params: dict = {}):
+        retries = 3
+        for attempt in range(retries):
+            try:
+                response = await client.get(url, params=params, headers=self.folio_client.okapi_headers)
+                response.raise_for_status()
+                return response
+            except httpx.HTTPError as e:
+                if attempt < retries - 1:
+                    logging.warning(f"Retrying due to {e}")
+                    await asyncio.sleep(2 ** attempt)
+                else:
+                    logging.error(f"Failed to connect after {retries} attempts: {e}")
+                    raise
 
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
@@ -449,7 +473,7 @@ class BatchPoster(MigrationTaskBase):
         )
         logging.info(last_row)
         logging.info("=========Stack trace==============")
-        traceback.logging.info_exc()
+        traceback.logging.info_exc() # type: ignore
         logging.info("=======================", flush=True)
 
     def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):

folio_migration_tools/migration_tasks/items_transformer.py

@@ -368,12 +368,12 @@ class ItemsTransformer(MigrationTaskBase):
                     self.handle_circiulation_notes(folio_rec, self.folio_client.current_user)
                     self.handle_notes(folio_rec)
                     if folio_rec["holdingsRecordId"] in self.mapper.boundwith_relationship_map:
-                        for idx, instance_id in enumerate(
+                        for idx_, instance_id in enumerate(
                             self.mapper.boundwith_relationship_map.get(
                                 folio_rec["holdingsRecordId"]
                             )
                         ):
-                            if idx == 0:
+                            if idx_ == 0:
                                 bw_id = folio_rec["holdingsRecordId"]
                             else:
                                 bw_id = self.mapper.generate_boundwith_holding_uuid(

{folio_migration_tools-1.9.0rc5.dist-info → folio_migration_tools-1.9.0rc6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc5
+Version: 1.9.0rc6
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.0rc5.dist-info → folio_migration_tools-1.9.0rc6.dist-info}/RECORD

@@ -1,21 +1,21 @@
-folio_migration_tools/__init__.py,sha256=yTPImroNb0CPuYmG4nm6aAcu5gpFJ7o1uM2dJS-47ec,93
+folio_migration_tools/__init__.py,sha256=DXvzUKFSpSZjflFWaNm0L8yhFk0u7RVIvQMskwMmKFc,238
 folio_migration_tools/__main__.py,sha256=0rbCmTq4HTxj8M3UjvX3rOEkq6-YqvaSGEMmuCORbho,7282
 folio_migration_tools/circulation_helper.py,sha256=2kAkLM6caPiep0ZtBkMICbRDh53KdfdH21oEX1eMRDI,14193
 folio_migration_tools/colors.py,sha256=GP0wdI_GZ2WD5SjrbPN-S3u8vvN_u6rGQIBBcWv_0ZM,227
 folio_migration_tools/config_file_load.py,sha256=zHHa6NDkN6EJiQE4DgjrFQPVKsd70POsfbGkB8308jg,2822
-folio_migration_tools/custom_dict.py,sha256=l9jHmlgXJkY9Itx1WnzjPzOlD8u62RgYLjkhU_CPAOI,648
-folio_migration_tools/custom_exceptions.py,sha256=6zjwXzFHyv3xhyAO0SCuyLePxluCQe_1yhSVNLDC6Zk,2509
+folio_migration_tools/custom_dict.py,sha256=-FUnhKp90Dg8EHlY6twx-PYQxBUWEO7FgxL2b7pf-xk,678
+folio_migration_tools/custom_exceptions.py,sha256=fRdMt5AwH_XddZ-bts5ByslN5gYthkLdh7o22JroQeE,2686
 folio_migration_tools/extradata_writer.py,sha256=fuchNcMc6BYb9IyfAcvXg7X4J2TfX6YiROfT2hr0JMw,1678
 folio_migration_tools/folder_structure.py,sha256=yyVvbkM9PbczSHNI8vK0Ru7i0x4nbYGzrRriXrnIh38,6715
 folio_migration_tools/helper.py,sha256=KkOkNAGO_fuYqxdLrsbLzCJLQHUrFZG1NzD4RmpQ-KM,2804
 folio_migration_tools/holdings_helper.py,sha256=yJpz6aJrKRBiJ1MtT5bs2vXAc88uJuGh2_KDuCySOKc,7559
 folio_migration_tools/i18n_config.py,sha256=3AH_2b9zTsxE4XTe4isM_zYtPJSlK0ix6eBmV7kAYUM,228
 folio_migration_tools/library_configuration.py,sha256=JE23VSUKDeCxzj_fOeQwkXF4fwd_y9s_hNy91YtEy7A,3514
-folio_migration_tools/mapper_base.py,sha256=4rtZxmWp1qaAIG2awiJ3RElPM9IVA-n8-Qzzs0R31qc,20317
+folio_migration_tools/mapper_base.py,sha256=ftYQJ6UDsP96EvCr5t3yMJemsmSNXsSH5zChDLu4Pp8,21224
 folio_migration_tools/mapping_file_transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 folio_migration_tools/mapping_file_transformation/courses_mapper.py,sha256=mJQxxeTn1bCYb2zwFYyXJ6EGZpJ0DsmwOY3nED7D_gQ,8091
 folio_migration_tools/mapping_file_transformation/holdings_mapper.py,sha256=nJS-xx1LszvbYfw0qdTUHX9xXHlxS7wP5mYmixFMh8A,7221
-folio_migration_tools/mapping_file_transformation/item_mapper.py,sha256=CWKpve0UdppztCd5ex1dz47g78mTqKkgiij1HEsGMC8,10622
+folio_migration_tools/mapping_file_transformation/item_mapper.py,sha256=CkpxQhwn-ZUpZu4nvybAyrQhdRVX2-wtCgT35-8JLh0,11005
 folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py,sha256=nCkqbxaDHKxMuqQHh_afxQp48YrVD-SeCZ0L1iGvnkk,13402
 folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py,sha256=RacwSOP6r6i28EOywaepq5K5FimD8Ld5MlBo89FYO7c,37963
 folio_migration_tools/mapping_file_transformation/notes_mapper.py,sha256=auLQZqa4rSJo_MIV4Lc5-LG8RcBpp2bnKH243qNYq_0,3470
@@ -24,25 +24,25 @@ folio_migration_tools/mapping_file_transformation/organization_mapper.py,sha256=
 folio_migration_tools/mapping_file_transformation/ref_data_mapping.py,sha256=qFsn_LwKZeKFdOudfEQnNA3DEHOdNQVKzTPdZAlDPX0,8864
 folio_migration_tools/mapping_file_transformation/user_mapper.py,sha256=oWuIPRQL0anF_qTVFibHtc1oOaqyKCBH4O1hX5rQAZQ,7806
 folio_migration_tools/marc_rules_transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-folio_migration_tools/marc_rules_transformation/conditions.py,sha256=17AkmOFU2Ha3Ezg2WTE0NeZafKKW6_MDJkia1TvWdKA,37512
+folio_migration_tools/marc_rules_transformation/conditions.py,sha256=am_VQ5P8vx-AFbVNBVeu0WkIbIh4cQJBLROxNTj_yUE,39143
 folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py,sha256=lTb5QWEAgwyFHy5vdSK6oDl1Q5v2GnzuV04xWV3p4rc,12401
 folio_migration_tools/marc_rules_transformation/hrid_handler.py,sha256=Ihdv0_1q7gL_pZ3HWU3GcfV_jjpIfOLithWk9z_uH3Y,9997
 folio_migration_tools/marc_rules_transformation/loc_language_codes.xml,sha256=ztn2_yKws6qySL4oSsZh7sOjxq5bCC1PhAnXJdtgmJ0,382912
 folio_migration_tools/marc_rules_transformation/marc_file_processor.py,sha256=WkOQRDi7f4PZ5qmVH3Q-1_zdGEKYSvOGC6jixDwDp98,12349
 folio_migration_tools/marc_rules_transformation/marc_reader_wrapper.py,sha256=9ATjYMRAjy0QcXtmNZaHVhHLJ5hE1WUgOcF6KMJjbgo,5309
 folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py,sha256=GFw8j9UtCxnUdLShmPzJa1MpCK8a0NkQIN5C3jyouRs,9604
-folio_migration_tools/marc_rules_transformation/rules_mapper_base.py,sha256=WWSJgYvF9LeY8vh-BtQi7Fm3J-cowUJKWa0Wk2Ge7fc,39358
+folio_migration_tools/marc_rules_transformation/rules_mapper_base.py,sha256=-vJDMNZe-7JbARgybVj6lLtZM79PgWU9V_k23330uLM,41195
 folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py,sha256=ckVeysbpW9s19pmHvogdRFOCouzz17Y6oKJD0_QfQAk,28924
 folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py,sha256=gPJaWcvt-CKIJxrEzheRzohnc3mFEnsznuZIXLPhyZM,18954
 folio_migration_tools/migration_report.py,sha256=BkRspM1hwTBnWeqsHamf7yVEofzLj560Q-9G--O00hw,4258
 folio_migration_tools/migration_tasks/__init__.py,sha256=ZkbY_yGyB84Ke8OMlYUzyyBj4cxxNrhMTwQlu_GbdDs,211
 folio_migration_tools/migration_tasks/authority_transformer.py,sha256=AoXg9s-GLO3yEEDCrQV7hc4YVXxwxsdxDdpj1zhHydE,4251
-folio_migration_tools/migration_tasks/batch_poster.py,sha256=8Oksp7uGjr6eAq-aqSpelQDTum14V9VzYGa_ZfLJQCM,35451
+folio_migration_tools/migration_tasks/batch_poster.py,sha256=wI4lCXU5BQDbKErF6pQxT6srq_Wf_nfFAJc4f1sRCoo,36388
 folio_migration_tools/migration_tasks/bibs_transformer.py,sha256=XzlPo-0uuugJA4SM80xOlOj5nDK6OMDXFnAYg80hOBc,7791
 folio_migration_tools/migration_tasks/courses_migrator.py,sha256=CzXnsu-KGP7B4zcINJzLYUqz47D16NuFfzu_DPqRlTQ,7061
 folio_migration_tools/migration_tasks/holdings_csv_transformer.py,sha256=WT-RlDRm2ILr2-2shfG3TZ3nlSfqxEXT3TklZSqtJCM,22311
 folio_migration_tools/migration_tasks/holdings_marc_transformer.py,sha256=8dtrhxyA9hbISISzpvMJGYaMaDbtZ1MOZeoJJF5lk24,11164
-folio_migration_tools/migration_tasks/items_transformer.py,sha256=7J222fmUIWeJ-d3Oz7YQeb0zGSQ1ff4aJ19-rfYyMKI,18777
+folio_migration_tools/migration_tasks/items_transformer.py,sha256=NNG7AdS_iuWurdySnWZ0o8kw3KNWPAvkRaHpW-18nV0,18779
 folio_migration_tools/migration_tasks/loans_migrator.py,sha256=4n7zbwljX_jgj9ltnxZAegjN3e8QJMjr6JJa5XfVueY,34771
 folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py,sha256=CnmlTge7nChUJ10EiUkriQtJlVxWqglgfhjgneh2_yM,7247
 folio_migration_tools/migration_tasks/migration_task_base.py,sha256=8enHPNrgOHZs5sDGsz0yMPXap0HBprz8-1HVr9udvf0,16704
@@ -60,8 +60,8 @@ folio_migration_tools/transaction_migration/legacy_request.py,sha256=1ulyFzPQw_I
 folio_migration_tools/transaction_migration/legacy_reserve.py,sha256=d0qbh2fWpwlVSYRL6wZyZG20__NAYNxh7sPSsB-LAes,1804
 folio_migration_tools/transaction_migration/transaction_result.py,sha256=cTdCN0BnlI9_ZJB2Z3Fdkl9gpymIi-9mGZsRFlQcmDk,656
 folio_migration_tools/translations/en.json,sha256=HOVpkb_T-SN_x0NpDp8gyvV1hMLCui3SsG7ByyIv0OU,38669
-folio_migration_tools-1.9.0rc5.dist-info/LICENSE,sha256=PhIEkitVi3ejgq56tt6sWoJIG_zmv82cjjd_aYPPGdI,1072
-folio_migration_tools-1.9.0rc5.dist-info/METADATA,sha256=6MxKZNM3wUsGNheI_nzS9YpJjR2XqDTigIedpeAcvzA,7415
-folio_migration_tools-1.9.0rc5.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-folio_migration_tools-1.9.0rc5.dist-info/entry_points.txt,sha256=Hbe-HjqMcU8FwVshVIkeWyZd9XwgT1CCMNf06EpHQu8,77
-folio_migration_tools-1.9.0rc5.dist-info/RECORD,,
+folio_migration_tools-1.9.0rc6.dist-info/LICENSE,sha256=PhIEkitVi3ejgq56tt6sWoJIG_zmv82cjjd_aYPPGdI,1072
+folio_migration_tools-1.9.0rc6.dist-info/METADATA,sha256=EL9MObYfZSyJ1otUDkUQEL0T2P8yxU5E77qZ4HhKFrk,7415
+folio_migration_tools-1.9.0rc6.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+folio_migration_tools-1.9.0rc6.dist-info/entry_points.txt,sha256=Hbe-HjqMcU8FwVshVIkeWyZd9XwgT1CCMNf06EpHQu8,77
+folio_migration_tools-1.9.0rc6.dist-info/RECORD,,