folio-migration-tools 1.9.3__py3-none-any.whl → 1.9.5__py3-none-any.whl

folio_migration_tools/circulation_helper.py

@@ -198,6 +198,14 @@ class CirculationHelper:
                         error_message_from_folio,
                         error_message_from_folio,
                     )
+                elif "Item is already checked out" in error_message_from_folio:
+                    return TransactionResult(
+                        False,
+                        True,
+                        None,
+                        error_message_from_folio,
+                        error_message_from_folio,
+                    )
                 logging.error(
                     f"{error_message} "
                     f"Patron barcode: {legacy_loan.patron_barcode} "

folio_migration_tools/helper.py

@@ -56,6 +56,10 @@ class Helper:
     @staticmethod
     def log_data_issue(index_or_id, message, legacy_value):
         logging.log(26, "DATA ISSUE\t%s\t%s\t%s", index_or_id, message, legacy_value)
+    
+    @staticmethod
+    def log_data_issue_failed(index_or_id, message, legacy_value):
+        logging.log(26, "RECORD FAILED\t%s\t%s\t%s", index_or_id, message, legacy_value)
 
     @staticmethod
     def write_to_file(file, folio_record):

folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -174,11 +174,11 @@ class BibsRulesMapper(RulesMapperBase):
             self.process_marc_field(folio_instance, main_entry_fields[0], ignored_subsequent_fields, legacy_ids)
         try:
             self.process_marc_field(folio_instance, marc_record['245'], ignored_subsequent_fields, legacy_ids)
-        except KeyError:
+        except KeyError as ke:
             raise TransformationRecordFailedError(
                 legacy_ids,
                 "No 245 field in MARC record"
-            )
+            ) from ke
 
     def perform_additional_parsing(
         self,

folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -196,9 +196,15 @@ class RulesMapperHoldings(RulesMapperBase):
         if "004" not in marc_record:
             raise TransformationProcessError(
                 "",
-                ("No 004 in record. The tools only support bib-mfhd linking throuh 004"),
+                ("No 004 in record. The tools only support bib-mfhd linking through 004"),
                 legacy_ids,
             )
+        if len(marc_record.get_fields("004")) > 1:
+            Helper.log_data_issue(
+                legacy_ids,
+                "More than one linked bib (004) found in record. Using the first one",
+                [str(x) for x in marc_record.get_fields("004")],
+            )
         legacy_instance_id = marc_record["004"].data.strip()
         folio_holding["formerIds"].append(f"{self.bib_id_template}{legacy_instance_id}")
         if legacy_instance_id in self.parent_id_map:

folio_migration_tools/migration_tasks/batch_poster.py

@@ -131,6 +131,56 @@ class BatchPoster(MigrationTaskBase):
                 ),
             ),
         ] = False
+        preserve_statistical_codes: Annotated[
+            bool,
+            Field(
+                title="Preserve statistical codes",
+                description=(
+                    "Toggles whether or not to preserve statistical codes "
+                    "during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_administrative_notes: Annotated[
+            bool,
+            Field(
+                title="Preserve administrative notes",
+                description=(
+                    "Toggles whether or not to preserve administrative notes "
+                    "during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_temporary_locations: Annotated[
+            bool,
+            Field(
+                title="Preserve temporary locations",
+                description=(
+                    "Toggles whether or not to preserve temporary locations "
+                    "on items during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_temporary_loan_types: Annotated[
+            bool,
+            Field(
+                title="Preserve temporary loan types",
+                description=(
+                    "Toggles whether or not to preserve temporary loan types "
+                    "on items during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_item_status: Annotated[
+            bool,
+            Field(
+                title="Preserve item status",
+                description=(
+                    "Toggles whether or not to preserve item status "
+                    "on items during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = True
 
     task_configuration: TaskConfiguration
 
@@ -292,7 +342,7 @@ class BatchPoster(MigrationTaskBase):
         """
         fetch_batch_size = 90
         fetch_tasks = []
-        updates = {}
+        existing_records = {}
         async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
             for i in range(0, len(batch), fetch_batch_size):
                 batch_slice = batch[i:i + fetch_batch_size]
@@ -313,31 +363,82 @@ class BatchPoster(MigrationTaskBase):
             responses = await asyncio.gather(*fetch_tasks)
 
             for response in responses:
-                self.update_record_versions(object_type, updates, response)
+                self.collect_existing_records_for_upsert(object_type, response, existing_records)
         for record in batch:
-            if record["id"] in updates:
-                record.update(updates[record["id"]])
+            if record["id"] in existing_records:
+                self.prepare_record_for_upsert(record, existing_records[record["id"]])
+
+    def handle_source_marc(self, new_record: dict, existing_record: dict):
+        updates = {}
+        updates.update(existing_record)
+        self.handle_upsert_for_administrative_notes(updates)
+        self.handle_upsert_for_statistical_codes(updates)
+        keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
+        if "instanceStatusId" in new_record:
+            updates["instanceStatusId"] = new_record["instanceStatusId"]
+        for k, v in keep_new.items():
+            updates[k] = list(dict.fromkeys(updates.get(k, []) + v))
+        new_record.update(updates)
 
     @staticmethod
-    def update_record_versions(object_type, updates, response):
+    def collect_existing_records_for_upsert(object_type: str, response: httpx.Response, existing_records: dict):
         if response.status_code == 200:
             response_json = response.json()
             for record in response_json[object_type]:
-                updates[record["id"]] = {
-                            "_version": record["_version"],
-                        }
-                if "hrid" in record:
-                    updates[record["id"]]["hrid"] = record["hrid"]
-                if "status" in record:
-                    updates[record["id"]]["status"] = record["status"]
-                if "lastCheckIn" in record:
-                    updates[record["id"]]["lastCheckIn"] = record["lastCheckIn"]
+                existing_records[record["id"]] = record
         else:
             logging.error(
-                        "Failed to fetch current records. HTTP %s\t%s",
-                        response.status_code,
-                        response.text,
-                    )
+                "Failed to fetch current records. HTTP %s\t%s",
+                response.status_code,
+                response.text,
+            )
+
+    def handle_upsert_for_statistical_codes(self, updates: dict):
+        if not self.task_configuration.preserve_statistical_codes:
+            updates.pop("statisticalCodeIds", None)
+
+    def handle_upsert_for_administrative_notes(self, updates: dict):
+        if not self.task_configuration.preserve_administrative_notes:
+            updates.pop("administrativeNotes", None)
+
+    def handle_upsert_for_temporary_locations(self, updates: dict):
+        if not self.task_configuration.preserve_temporary_locations:
+            updates.pop("temporaryLocationId", None)
+
+    def handle_upsert_for_temporary_loan_types(self, updates: dict):
+        if not self.task_configuration.preserve_temporary_loan_types:
+            updates.pop("temporaryLoanTypeId", None)
+
+    def keep_existing_fields(self, updates: dict, existing_record: dict):
+        keep_existing_fields = ["hrid", "lastCheckIn"]
+        if self.task_configuration.preserve_item_status:
+            keep_existing_fields.append("status")
+        for key in keep_existing_fields:
+            if key in existing_record:
+                updates[key] = existing_record[key]
+
+    def prepare_record_for_upsert(self, new_record: dict, existing_record: dict):
+        if "source" in existing_record and "MARC" in existing_record["source"]:
+            self.handle_source_marc(new_record, existing_record)
+        else:
+            updates = {
+                "_version": existing_record["_version"],
+            }
+            self.keep_existing_fields(updates, existing_record)
+            keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
+            self.handle_upsert_for_statistical_codes(existing_record)
+            self.handle_upsert_for_administrative_notes(existing_record)
+            self.handle_upsert_for_temporary_locations(existing_record)
+            self.handle_upsert_for_temporary_loan_types(existing_record)
+            for k, v in keep_new.items():
+                updates[k] = list(dict.fromkeys(existing_record.get(k, []) + v))
+            for key in [
+                "temporaryLocationId",
+                "temporaryLoanTypeId",
+            ]:
+                if key in existing_record:
+                    updates[key] = existing_record[key]
+            new_record.update(updates)
 
     async def get_with_retry(self, client: httpx.AsyncClient, url: str, params=None):
         if params is None:

folio_migration_tools/migration_tasks/items_transformer.py

@@ -218,7 +218,10 @@ class ItemsTransformer(MigrationTaskBase):
         self.folio_keys = MappingFileMapperBase.get_mapped_folio_properties_from_map(
             self.items_map
         )
-        if any(k for k in self.folio_keys if k.startswith("statisticalCodeIds")):
+        if (
+            any(k for k in self.folio_keys if k.startswith("statisticalCodeIds"))
+            or any(getattr(k, "statistical_code", "") for k in self.task_configuration.files)
+        ):
             statcode_mapping = self.load_ref_data_mapping_file(
                 "statisticalCodeIds",
                 self.folder_structure.mapping_files_folder

folio_migration_tools/migration_tasks/loans_migrator.py

@@ -6,7 +6,7 @@ import sys
 import time
 import traceback
 from datetime import datetime, timedelta
-from typing import Annotated, Optional
+from typing import Annotated, List, Literal, Optional
 from urllib.error import HTTPError
 from zoneinfo import ZoneInfo
 from pydantic import Field
@@ -55,7 +55,7 @@ class LoansMigrator(MigrationTaskBase):
             Optional[list[FileDefinition]],
             Field(
                 title="Open loans files",
-                description="List of files containing open loan data."
+                description="List of files containing open loan data.",
             ),
         ]
         fallback_service_point_id: Annotated[
@@ -69,10 +69,7 @@ class LoansMigrator(MigrationTaskBase):
             Optional[int],
             Field(
                 title="Starting row",
-                description=(
-                    "The starting row for data processing. "
-                    "By default is 1."
-                ),
+                description=("The starting row for data processing. By default is 1."),
             ),
         ] = 1
         item_files: Annotated[
@@ -80,8 +77,7 @@ class LoansMigrator(MigrationTaskBase):
             Field(
                 title="Item files",
                 description=(
-                    "List of files containing item data. "
-                    "By default is empty list."
+                    "List of files containing item data. By default is empty list."
                 ),
             ),
         ] = []
@@ -90,8 +86,7 @@ class LoansMigrator(MigrationTaskBase):
             Field(
                 title="Patron files",
                 description=(
-                    "List of files containing patron data. "
-                    "By default is empty list."
+                    "List of files containing patron data. By default is empty list."
                 ),
             ),
         ] = []
@@ -104,7 +99,7 @@ class LoansMigrator(MigrationTaskBase):
         self,
         task_configuration: TaskConfiguration,
         library_config: LibraryConfiguration,
-        folio_client
+        folio_client,
     ):
         csv.register_dialect("tsv", delimiter="\t")
         self.patron_item_combos: set = set()
@@ -115,7 +110,7 @@ class LoansMigrator(MigrationTaskBase):
         self.failed: dict = {}
         self.failed_and_not_dupe: dict = {}
         self.migration_report = MigrationReport()
-        self.valid_legacy_loans = []
+        self.valid_legacy_loans: List[LegacyLoan] = []
         super().__init__(library_config, task_configuration, folio_client)
         self.circulation_helper = CirculationHelper(
             self.folio_client,
@@ -129,7 +124,9 @@ class LoansMigrator(MigrationTaskBase):
         my_path = "/configurations/entries?query=(module==ORG%20and%20configName==localeSettings)"
         try:
             self.tenant_timezone_str = json.loads(
-                self.folio_client.folio_get_single_object(my_path)["configs"][0]["value"]
+                self.folio_client.folio_get_single_object(my_path)["configs"][0][
+                    "value"
+                ]
             )["timezone"]
             logging.info("Tenant timezone is: %s", self.tenant_timezone_str)
         except Exception:
@@ -138,10 +135,14 @@ class LoansMigrator(MigrationTaskBase):
         self.tenant_timezone = ZoneInfo(self.tenant_timezone_str)
         self.semi_valid_legacy_loans = []
         for file_def in task_configuration.open_loans_files:
-            loans_file_path = self.folder_structure.legacy_records_folder / file_def.file_name
+            loans_file_path = (
+                self.folder_structure.legacy_records_folder / file_def.file_name
+            )
             with open(loans_file_path, "r", encoding="utf-8") as loans_file:
-                total_rows, empty_rows, reader = MappingFileMapperBase._get_delimited_file_reader(
-                    loans_file, loans_file_path
+                total_rows, empty_rows, reader = (
+                    MappingFileMapperBase._get_delimited_file_reader(
+                        loans_file, loans_file_path
+                    )
                 )
                 logging.info("Source data file contains %d rows", total_rows)
                 logging.info("Source data file contains %d empty rows", empty_rows)
@@ -158,7 +159,8 @@ class LoansMigrator(MigrationTaskBase):
                 self.semi_valid_legacy_loans.extend(
                     self.load_and_validate_legacy_loans(
                         reader,
-                        file_def.service_point_id or task_configuration.fallback_service_point_id,
+                        file_def.service_point_id
+                        or task_configuration.fallback_service_point_id,
                     )
                 )
 
@@ -167,8 +169,12 @@ class LoansMigrator(MigrationTaskBase):
                     len(self.semi_valid_legacy_loans),
                     file_def.file_name,
                 )
-        logging.info("Loaded and validated %s loans in total", len(self.semi_valid_legacy_loans))
-        if any(self.task_configuration.item_files) or any(self.task_configuration.patron_files):
+        logging.info(
+            "Loaded and validated %s loans in total", len(self.semi_valid_legacy_loans)
+        )
+        if any(self.task_configuration.item_files) or any(
+            self.task_configuration.patron_files
+        ):
             self.valid_legacy_loans = list(self.check_barcodes())
             logging.info(
                 "Loaded and validated %s loans against barcodes",
@@ -185,9 +191,9 @@ class LoansMigrator(MigrationTaskBase):
 
     def check_smtp_config(self):
         try:
-            smtp_config = self.folio_client.folio_get_single_object("/smtp-configuration")[
-                "smtpConfigurations"
-            ][0]
+            smtp_config = self.folio_client.folio_get_single_object(
+                "/smtp-configuration"
+            )["smtpConfigurations"][0]
             smtp_config_disabled = "disabled" in smtp_config["host"].lower()
         except IndexError:
             smtp_config_disabled = True
@@ -195,7 +201,9 @@ class LoansMigrator(MigrationTaskBase):
         if not smtp_config_disabled:
             logging.warn("SMTP connection not disabled...")
             for i in range(10, 0, -1):
-                sys.stdout.write("Pausing for {:02d} seconds. Press Ctrl+C to exit...\r".format(i))
+                sys.stdout.write(
+                    "Pausing for {:02d} seconds. Press Ctrl+C to exit...\r".format(i)
+                )
                 time.sleep(1)
         else:
             logging.info("SMTP connection is disabled...")
@@ -219,13 +227,21 @@ class LoansMigrator(MigrationTaskBase):
                 )
                 try:
                     self.checkout_single_loan(legacy_loan)
+                except TransformationRecordFailedError as ee:
+                    logging.error(
+                        f"Transformation failed in row {num_loans}  Item barcode: {legacy_loan.item_barcode} "
+                        f"Patron barcode: {legacy_loan.patron_barcode}"
+                    )
+                    ee.log_it()
                 except Exception as ee:
                     logging.exception(
                         f"Error in row {num_loans}  Item barcode: {legacy_loan.item_barcode} "
                         f"Patron barcode: {legacy_loan.patron_barcode} {ee}"
                     )
                 if num_loans % 25 == 0:
-                    logging.info(f"{timings(self.t0, t0_migration, num_loans)} {num_loans}")
+                    logging.info(
+                        f"{timings(self.t0, t0_migration, num_loans)} {num_loans}"
+                    )
 
     def checkout_single_loan(self, legacy_loan: LegacyLoan):
         """Checks a legacy loan out. Retries once if it fails.
@@ -234,17 +250,23 @@ class LoansMigrator(MigrationTaskBase):
             legacy_loan (LegacyLoan): The Legacy loan
         """
         res_checkout = self.circulation_helper.check_out_by_barcode(legacy_loan)
-        
+
         if res_checkout.was_successful:
             self.migration_report.add("Details", i18n.t("Checked out on first try"))
-            self.migration_report.add_general_statistics(i18n.t("Successfully checked out"))
+            self.migration_report.add_general_statistics(
+                i18n.t("Successfully checked out")
+            )
             self.set_renewal_count(legacy_loan, res_checkout)
             self.set_new_status(legacy_loan, res_checkout)
         elif res_checkout.should_be_retried:
             res_checkout2 = self.handle_checkout_failure(legacy_loan, res_checkout)
             if res_checkout2.was_successful and res_checkout2.folio_loan:
-                self.migration_report.add("Details", i18n.t("Checked out on second try"))
-                self.migration_report.add_general_statistics(i18n.t("Successfully checked out"))
+                self.migration_report.add(
+                    "Details", i18n.t("Checked out on second try")
+                )
+                self.migration_report.add_general_statistics(
+                    i18n.t("Successfully checked out")
+                )
                 logging.info("Checked out on second try")
                 self.set_renewal_count(legacy_loan, res_checkout2)
                 self.set_new_status(legacy_loan, res_checkout2)
@@ -252,7 +274,8 @@ class LoansMigrator(MigrationTaskBase):
                 if res_checkout2.error_message == "Aged to lost and checked out":
                     self.migration_report.add(
                         "Details",
-                        i18n.t("Second failure") + f": {res_checkout2.migration_report_message}",
+                        i18n.t("Second failure")
+                        + f": {res_checkout2.migration_report_message}",
                     )
                     logging.error(
                         f"{res_checkout2.error_message}. Item barcode: {legacy_loan.item_barcode}"
@@ -260,16 +283,23 @@ class LoansMigrator(MigrationTaskBase):
                 else:
                     self.failed[legacy_loan.item_barcode] = legacy_loan
                     self.migration_report.add_general_statistics(i18n.t("Failed loans"))
-                    Helper.log_data_issue(
-                        "", "Loans failing during checkout", json.dumps(legacy_loan.to_dict())
+                    logging.error(
+                        "Failed on second try: %s", res_checkout2.error_message
                     )
-                    logging.error("Failed on second try: %s", res_checkout2.error_message)
                     self.migration_report.add(
                         "Details",
-                        i18n.t("Second failure") + f": {res_checkout2.migration_report_message}",
+                        i18n.t("Second failure")
+                        + f": {res_checkout2.migration_report_message}",
+                    )
+                    raise TransformationRecordFailedError(
+                        f"Row {legacy_loan.row}",
+                        i18n.t("Loans failing during checkout, second try"),
+                        json.dumps(legacy_loan.to_dict()),
                     )
         elif not res_checkout.should_be_retried:
-            logging.error("Failed first time. No retries: %s", res_checkout.error_message)
+            logging.error(
+                "Failed first time. No retries: %s", res_checkout.error_message
+            )
             self.migration_report.add_general_statistics(i18n.t("Failed loans"))
             self.migration_report.add(
                 "Details",
@@ -277,8 +307,10 @@ class LoansMigrator(MigrationTaskBase):
                 + f": {res_checkout.migration_report_message}",
             )
             self.failed[legacy_loan.item_barcode] = legacy_loan
-            Helper.log_data_issue(
-                "", "Loans failing during checkout", json.dumps(legacy_loan.to_dict())
+            raise TransformationRecordFailedError(
+                f"Row {legacy_loan.row}",
+                i18n.t("Loans failing during checkout"),
+                json.dumps(legacy_loan.to_dict()),
             )
 
     def set_new_status(self, legacy_loan: LegacyLoan, res_checkout: TransactionResult):
@@ -296,10 +328,14 @@ class LoansMigrator(MigrationTaskBase):
         elif legacy_loan.next_item_status not in ["Available", "", "Checked out"]:
             self.set_item_status(legacy_loan)
 
-    def set_renewal_count(self, legacy_loan: LegacyLoan, res_checkout: TransactionResult):
+    def set_renewal_count(
+        self, legacy_loan: LegacyLoan, res_checkout: TransactionResult
+    ):
         if legacy_loan.renewal_count > 0:
             self.update_open_loan(res_checkout.folio_loan, legacy_loan)
-            self.migration_report.add_general_statistics(i18n.t("Updated renewal count for loan"))
+            self.migration_report.add_general_statistics(
+                i18n.t("Updated renewal count for loan")
+            )
 
     def wrap_up(self):
         for k, v in self.failed.items():
@@ -316,15 +352,19 @@ class LoansMigrator(MigrationTaskBase):
 
     def write_failed_loans_to_file(self):
         csv_columns = [
-            "due_date",
+            "patron_barcode",
+            "proxy_patron_barcode",
             "item_barcode",
-            "next_item_status",
+            "due_date",
             "out_date",
-            "patron_barcode",
+            "next_item_status",
             "renewal_count",
+            "service_point_id",
         ]
         with open(self.folder_structure.failed_recs_path, "w+") as failed_loans_file:
-            writer = csv.DictWriter(failed_loans_file, fieldnames=csv_columns, dialect="tsv")
+            writer = csv.DictWriter(
+                failed_loans_file, fieldnames=csv_columns, dialect="tsv"
+            )
             writer.writeheader()
             for _k, failed_loan in self.failed_and_not_dupe.items():
                 writer.writerow(failed_loan[0])
@@ -339,12 +379,16 @@ class LoansMigrator(MigrationTaskBase):
             user_barcodes, self.task_configuration.patron_files, self.folder_structure
         )
         for loan in self.semi_valid_legacy_loans:
-            has_item_barcode = loan.item_barcode in item_barcodes or not any(item_barcodes)
-            has_patron_barcode = loan.patron_barcode in user_barcodes or not any(user_barcodes)
+            has_item_barcode = loan.item_barcode in item_barcodes or not any(
+                item_barcodes
+            )
+            has_patron_barcode = loan.patron_barcode in user_barcodes or not any(
+                user_barcodes
+            )
             has_proxy_barcode = True
             if loan.proxy_patron_barcode:
-                has_proxy_barcode = loan.proxy_patron_barcode in user_barcodes or not any(
-                    user_barcodes
+                has_proxy_barcode = (
+                    loan.proxy_patron_barcode in user_barcodes or not any(user_barcodes)
                 )
             if has_item_barcode and has_patron_barcode and has_proxy_barcode:
                 self.migration_report.add_general_statistics(
@@ -364,7 +408,7 @@ class LoansMigrator(MigrationTaskBase):
                     + f": {has_proxy_barcode}",
                 )
             if not has_item_barcode:
-                Helper.log_data_issue(
+                Helper.log_data_issue_failed(
                     "", "Loan without matched item barcode", json.dumps(loan.to_dict())
                 )
             if not has_patron_barcode:
@@ -374,11 +418,15 @@ class LoansMigrator(MigrationTaskBase):
                     json.dumps(loan.to_dict()),
                 )
             if not has_proxy_barcode:
-                Helper.log_data_issue(
-                    "", "Loan without matched proxy patron barcode", json.dumps(loan.to_dict())
+                Helper.log_data_issue_failed(
+                    "",
+                    "Loan without matched proxy patron barcode",
+                    json.dumps(loan.to_dict()),
                 )
 
-    def load_and_validate_legacy_loans(self, loans_reader, service_point_id: str) -> list:
+    def load_and_validate_legacy_loans(
+        self, loans_reader, service_point_id: str
+    ) -> list:
         results = []
         num_bad = 0
         logging.info("Validating legacy loans in file...")
@@ -398,7 +446,9 @@ class LoansMigrator(MigrationTaskBase):
                     )
                     self.migration_report.add_general_statistics(i18n.t("Failed loans"))
                     for error in legacy_loan.errors:
-                        self.migration_report.add("DiscardedLoans", f"{error[0]} - {error[1]}")
+                        self.migration_report.add(
+                            "DiscardedLoans", f"{error[0]} - {error[1]}"
+                        )
                     # Add this loan to failed loans for later correction and re-run.
                     self.failed[
                         legacy_loan.item_barcode or f"no_barcode_{legacy_loan_count}"
@@ -416,7 +466,9 @@ class LoansMigrator(MigrationTaskBase):
                 )
                 trfe.log_it()
                 self.failed[
-                    legacy_loan_dict.get("item_barcode", f"no_barcode_{legacy_loan_count}")
+                    legacy_loan_dict.get(
+                        "item_barcode", f"no_barcode_{legacy_loan_count}"
+                    )
                 ] = legacy_loan_dict
             except ValueError as ve:
                 logging.exception(ve)
@@ -457,13 +509,22 @@ class LoansMigrator(MigrationTaskBase):
         elif folio_checkout.error_message.startswith(
             "Cannot check out item that already has an open loan"
         ):
-            return folio_checkout
+            return self.handle_checked_out_item(legacy_loan)
+        elif "Item is already checked out" in folio_checkout.error_message:
+            return self.handle_checked_out_item(legacy_loan)
         elif "Aged to lost" in folio_checkout.error_message:
-            return self.handle_aged_to_lost_item(legacy_loan)
+            return self.handle_lost_item(legacy_loan, "Aged to lost")
         elif folio_checkout.error_message == "Declared lost":
-            return folio_checkout
-        elif folio_checkout.error_message.startswith("Cannot check out to inactive user"):
+            return self.handle_lost_item(legacy_loan, "Declared lost")
+        elif folio_checkout.error_message.startswith(
+            "Cannot check out to inactive user"
+        ):
             return self.checkout_to_inactive_user(legacy_loan)
+        elif (
+            "has the item status Claimed returned and cannot be checked out"
+            in folio_checkout.error_message
+        ):
+            return self.handle_claimed_returned_item(legacy_loan)
         else:
             self.migration_report.add(
                 "Details",
@@ -489,7 +550,9 @@ class LoansMigrator(MigrationTaskBase):
                     f"Duplicate loans (or failed twice) Item barcode: "
                     f"{legacy_loan.item_barcode} Patron barcode: {legacy_loan.patron_barcode}"
                 )
-                self.migration_report.add("Details", i18n.t("Duplicate loans (or failed twice)"))
+                self.migration_report.add(
+                    "Details", i18n.t("Duplicate loans (or failed twice)")
+                )
                 del self.failed[legacy_loan.item_barcode]
             return TransactionResult(False, False, "", "", "")
 
@@ -500,7 +563,9 @@ class LoansMigrator(MigrationTaskBase):
         user["expirationDate"] = datetime.isoformat(datetime.now() + timedelta(days=1))
         self.activate_user(user)
         logging.debug("Successfully Activated user")
-        res = self.circulation_helper.check_out_by_barcode(legacy_loan)  # checkout_and_update
+        res = self.circulation_helper.check_out_by_barcode(
+            legacy_loan
+        )  # checkout_and_update
         if res.should_be_retried:
             res = self.handle_checkout_failure(legacy_loan, res)
         self.migration_report.add("Details", res.migration_report_message)
@@ -509,28 +574,92 @@ class LoansMigrator(MigrationTaskBase):
         self.migration_report.add("Details", i18n.t("Handled inactive users"))
         return res
 
-    def handle_aged_to_lost_item(self, legacy_loan: LegacyLoan) -> TransactionResult:
+    def handle_checked_out_item(self, legacy_loan: LegacyLoan) -> TransactionResult:
         if self.circulation_helper.is_checked_out(legacy_loan):
             return TransactionResult(
                 False,
                 False,
                 legacy_loan,
-                i18n.t("Aged to lost and checked out"),
-                i18n.t("Aged to lost and checked out"),
+                i18n.t(
+                    "Loan already exists for %{item_barcode}",
+                    item_barcode=legacy_loan.item_barcode,
+                ),
+                i18n.t(
+                    "Loan already exists for %{item_barcode}",
+                    item_barcode=legacy_loan.item_barcode,
+                ),
             )
-
         else:
-            logging.debug("Setting Available")
+            logging.debug(
+                i18n.t(
+                    'Setting item %{item_barcode} to status "Available"',
+                    item_barcode=legacy_loan.item_barcode,
+                )
+            )
             legacy_loan.next_item_status = "Available"
             self.set_item_status(legacy_loan)
             res_checkout = self.circulation_helper.check_out_by_barcode(legacy_loan)
-            legacy_loan.next_item_status = "Aged to lost"
+            legacy_loan.next_item_status = "Checked out"
+            return res_checkout
+
+    def handle_lost_item(
+        self,
+        legacy_loan: LegacyLoan,
+        lost_type: Literal["Aged to lost", "Declared lost"],
+    ) -> TransactionResult:
+        if self.circulation_helper.is_checked_out(legacy_loan):
+            return TransactionResult(
+                False,
+                False,
+                legacy_loan,
+                i18n.t("%{lost_type} and checked out", lost_type=lost_type),
+                i18n.t("%{lost_type} and checked out", lost_type=lost_type),
+            )
+
+        else:
+            logging.debug(
+                'Setting item %{item_barcode} to status "Available"',
+                item_barcode=legacy_loan.item_barcode,
+            )
+            legacy_loan.next_item_status = "Available"
             self.set_item_status(legacy_loan)
-            s = "Successfully Checked out Aged to lost item and put the status back"
+            res_checkout = self.circulation_helper.check_out_by_barcode(legacy_loan)
+            legacy_loan.next_item_status = lost_type
+            if lost_type == "Aged to lost":
+                self.set_item_status(legacy_loan)
+                s = i18n.t(
+                    "Successfully Checked out %{lost_type} item and put the status back",
+                    lost_type=lost_type,
+                )
+            else:
+                s = i18n.t(
+                    "Successfully Checked out %{lost_type} item. Item will be declared lost.",
+                    lost_type=lost_type,
+                )
             logging.info(s)
             self.migration_report.add("Details", s)
             return res_checkout
 
+    def handle_claimed_returned_item(self, legacy_loan: LegacyLoan):
+        if self.circulation_helper.is_checked_out(legacy_loan):
+            return TransactionResult(
+                False,
+                False,
+                legacy_loan,
+                i18n.t("Claimed returned and checked out"),
+                i18n.t("Claimed returned and checked out"),
+            )
+        else:
+            logging.debug(
+                'Setting item %{item_barcode} to status "Available"',
+                item_barcode=legacy_loan.item_barcode,
+            )
+            legacy_loan.next_item_status = "Available"
+            self.set_item_status(legacy_loan)
+            res_checkout = self.circulation_helper.check_out_by_barcode(legacy_loan)
+            legacy_loan.next_item_status = "Claimed returned"
+            return res_checkout
+
     def update_open_loan(self, folio_loan: dict, legacy_loan: LegacyLoan):
         due_date = du_parser.isoparse(str(legacy_loan.due_date))
         out_date = du_parser.isoparse(str(legacy_loan.out_date))
@@ -541,7 +670,9 @@ class LoansMigrator(MigrationTaskBase):
             loan_to_put["dueDate"] = due_date.isoformat()
             loan_to_put["loanDate"] = out_date.isoformat()
             loan_to_put["renewalCount"] = renewal_count
-            url = f"{self.folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
+            url = (
+                f"{self.folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
+            )
             req = self.http_client.put(
                 url,
                 headers=self.folio_client.okapi_headers,
@@ -562,7 +693,8 @@ class LoansMigrator(MigrationTaskBase):
             else:
                 self.migration_report.add(
                     "Details",
-                    i18n.t("Update open loan error http status") + f": {req.status_code}",
+                    i18n.t("Update open loan error http status")
+                    + f": {req.status_code}",
                 )
                 req.raise_for_status()
             logging.debug("Updating open loan was successful")
@@ -592,40 +724,59 @@ class LoansMigrator(MigrationTaskBase):
             "servicePointId": str(self.task_configuration.fallback_service_point_id),
         }
         logging.debug(f"Declare lost data: {json.dumps(data, indent=4)}")
-        if self.folio_put_post(declare_lost_url, data, "POST", i18n.t("Declare item as lost")):
-            self.migration_report.add("Details", i18n.t("Successfully declared loan as lost"))
+        if self.folio_put_post(
+            declare_lost_url, data, "POST", i18n.t("Declare item as lost")
+        ):
+            self.migration_report.add(
+                "Details", i18n.t("Successfully declared loan as lost")
+            )
         else:
             logging.error(f"Unsuccessfully declared loan {folio_loan} as lost")
-            self.migration_report.add("Details", i18n.t("Unsuccessfully declared loan as lost"))
+            self.migration_report.add(
+                "Details", i18n.t("Unsuccessfully declared loan as lost")
+            )
 
     def claim_returned(self, folio_loan):
-        claim_returned_url = f"/circulation/loans/{folio_loan['id']}/claim-item-returned"
+        claim_returned_url = (
+            f"/circulation/loans/{folio_loan['id']}/claim-item-returned"
+        )
         logging.debug(f"Claim returned url:{claim_returned_url}")
         due_date = du_parser.isoparse(folio_loan["dueDate"])
         data = {
-            "itemClaimedReturnedDateTime": datetime.isoformat(due_date + timedelta(days=1)),
+            "itemClaimedReturnedDateTime": datetime.isoformat(
+                due_date + timedelta(days=1)
+            ),
             "comment": "Created at migration. Date is due date + 1 day",
         }
         logging.debug(f"Claim returned data:\t{json.dumps(data)}")
-        if self.folio_put_post(claim_returned_url, data, "POST", i18n.t("Declare item as lost")):
+        if self.folio_put_post(
+            claim_returned_url, data, "POST", i18n.t("Claim item returned")
+        ):
             self.migration_report.add(
                 "Details", i18n.t("Successfully declared loan as Claimed returned")
             )
         else:
-            logging.error(f"Unsuccessfully declared loan {folio_loan} as Claimed returned")
+            logging.error(
+                f"Unsuccessfully declared loan {folio_loan} as Claimed returned"
+            )
             self.migration_report.add(
                 "Details",
                 i18n.t(
-                    "Unsuccessfully declared loan %{loan} as Claimed returned", loan=folio_loan
+                    "Unsuccessfully declared loan %{loan} as Claimed returned",
+                    loan=folio_loan,
                 ),
             )
 
     def set_item_status(self, legacy_loan: LegacyLoan):
         try:
             # Get Item by barcode, update status.
-            item_path = f'item-storage/items?query=(barcode=="{legacy_loan.item_barcode}")'
+            item_path = (
+                f'item-storage/items?query=(barcode=="{legacy_loan.item_barcode}")'
+            )
             item_url = f"{self.folio_client.gateway_url}/{item_path}"
-            resp = self.http_client.get(item_url, headers=self.folio_client.okapi_headers)
+            resp = self.http_client.get(
+                item_url, headers=self.folio_client.okapi_headers
+            )
             resp.raise_for_status()
             data = resp.json()
             folio_item = data["items"][0]
@@ -675,11 +826,11 @@ class LoansMigrator(MigrationTaskBase):
         self.migration_report.add("Details", i18n.t("Successfully deactivated user"))
 
     def update_item(self, item):
-        url = f'/item-storage/items/{item["id"]}'
+        url = f"/item-storage/items/{item['id']}"
         return self.folio_put_post(url, item, "PUT", i18n.t("Update item"))
 
     def update_user(self, user):
-        url = f'/users/{user["id"]}'
+        url = f"/users/{user['id']}"
         self.folio_put_post(url, user, "PUT", i18n.t("Update user"))
 
     def get_user_by_barcode(self, barcode):
@@ -746,7 +897,9 @@ class LoansMigrator(MigrationTaskBase):
         try:
             api_path = f"{folio_loan['id']}/change-due-date"
             api_url = f"{self.folio_client.gateway_url}/circulation/loans/{api_path}"
-            body = {"dueDate": du_parser.isoparse(str(legacy_loan.due_date)).isoformat()}
+            body = {
+                "dueDate": du_parser.isoparse(str(legacy_loan.due_date)).isoformat()
+            }
             req = self.http_client.post(
                 api_url, headers=self.folio_client.okapi_headers, json=body
             )
@@ -762,12 +915,14 @@ class LoansMigrator(MigrationTaskBase):
                 return False
             elif req.status_code == 201:
                 self.migration_report.add(
-                    "Details", i18n.t("Successfully changed due date") + f" ({req.status_code})"
+                    "Details",
+                    i18n.t("Successfully changed due date") + f" ({req.status_code})",
                 )
                 return True, json.loads(req.text), None
             elif req.status_code == 204:
                 self.migration_report.add(
-                    "Details", i18n.t("Successfully changed due date") + f" ({req.status_code})"
+                    "Details",
+                    i18n.t("Successfully changed due date") + f" ({req.status_code})",
                 )
                 return True, None, None
             else:

folio_migration_tools/migration_tasks/organization_transformer.py

@@ -148,6 +148,7 @@ class OrganizationTransformer(MigrationTaskBase):
         self.mapper = OrganizationMapper(
             self.folio_client,
             self.library_configuration,
+            self.task_configuration,
             self.organization_map,
             self.load_ref_data_mapping_file(
                 "organizationTypes",

folio_migration_tools/transaction_migration/legacy_loan.py

@@ -7,6 +7,7 @@ from zoneinfo import ZoneInfo
 from dateutil import tz
 from dateutil.parser import parse, ParserError
 
+from folio_migration_tools.helper import Helper
 from folio_migration_tools.migration_report import MigrationReport
 from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
 
@@ -29,11 +30,13 @@ class LegacyLoan(object):
             "patron_barcode",
             "due_date",
             "out_date",
+        ]
+        optional_headers = [
+            "service_point_id",
+            "proxy_patron_barcode",
             "renewal_count",
             "next_item_status",
-            "service_point_id",
         ]
-        optional_headers = ["service_point_id", "proxy_patron_barcode"]
         legal_statuses = [
             "",
             "Aged to lost",
@@ -60,44 +63,67 @@ class LegacyLoan(object):
             temp_date_due: datetime = parse(self.legacy_loan_dict["due_date"])
             if temp_date_due.tzinfo != tz.UTC:
                 temp_date_due = temp_date_due.replace(tzinfo=self.tenant_timezone)
-                self.report(
+                Helper.log_data_issue(
+                    self.row,
                     f"Provided due_date is not UTC in {row=}, "
-                    f"setting tz-info to tenant timezone ({self.tenant_timezone})"
+                    f"setting tz-info to tenant timezone ({self.tenant_timezone})",
+                    json.dumps(self.legacy_loan_dict)
+                )
+                self.report(
+                    f"Provided due_date is not UTC, setting tz-info to tenant timezone ({self.tenant_timezone})"
                 )
             if temp_date_due.hour == 0 and temp_date_due.minute == 0:
                 temp_date_due = temp_date_due.replace(hour=23, minute=59)
-                self.report(
+                Helper.log_data_issue(
+                    self.row,
                     f"Hour and minute not specified for due date in {row=}. "
-                    "Assuming end of local calendar day (23:59)..."
+                    "Assuming end of local calendar day (23:59)...",
+                    json.dumps(self.legacy_loan_dict)
+                )
+                self.report(
+                    "Hour and minute not specified for due date"
                 )
         except (ParserError, OverflowError) as ee:
             logging.error(ee)
-            self.errors.append((f"Parse date failure in {row=}. Setting UTC NOW", "due_date"))
+            self.errors.append(
+                (f"Parse date failure in {row=}. Setting UTC NOW", "due_date")
+            )
             temp_date_due = datetime.now(ZoneInfo("UTC"))
         try:
             temp_date_out: datetime = parse(self.legacy_loan_dict["out_date"])
             if temp_date_out.tzinfo != tz.UTC:
                 temp_date_out = temp_date_out.replace(tzinfo=self.tenant_timezone)
-                self.report(
+                Helper.log_data_issue(
+                    self.row,
                     f"Provided out_date is not UTC in {row=}, "
-                    f"setting tz-info to tenant timezone ({self.tenant_timezone})"
+                    f"setting tz-info to tenant timezone ({self.tenant_timezone})",
+                    json.dumps(self.legacy_loan_dict)
+                )
+                self.report(
+                    f"Provided out_date is not UTC, setting tz-info to tenant timezone ({self.tenant_timezone})"
                 )
         except (ParserError, OverflowError):
             temp_date_out = datetime.now(
                 ZoneInfo("UTC")
             )  # TODO: Consider moving this assignment block above the temp_date_due
-            self.errors.append((f"Parse date failure in {row=}. Setting UTC NOW", "out_date"))
+            self.errors.append(
+                (f"Parse date failure in {row=}. Setting UTC NOW", "out_date")
+            )
 
         # good to go, set properties
         self.item_barcode: str = self.legacy_loan_dict["item_barcode"].strip()
         self.patron_barcode: str = self.legacy_loan_dict["patron_barcode"].strip()
-        self.proxy_patron_barcode: str = self.legacy_loan_dict.get("proxy_patron_barcode", "")
+        self.proxy_patron_barcode: str = self.legacy_loan_dict.get(
+            "proxy_patron_barcode", ""
+        )
         self.due_date: datetime = temp_date_due
         self.out_date: datetime = temp_date_out
         self.correct_for_1_day_loans()
         self.make_utc()
         self.renewal_count = self.set_renewal_count(self.legacy_loan_dict)
-        self.next_item_status = self.legacy_loan_dict.get("next_item_status", "").strip()
+        self.next_item_status = self.legacy_loan_dict.get(
+            "next_item_status", ""
+        ).strip()
         if self.next_item_status not in legal_statuses:
             self.errors.append((f"Not an allowed status {row=}", self.next_item_status))
         self.service_point_id = (
@@ -112,9 +138,17 @@ class LegacyLoan(object):
             try:
                 return int(renewal_count)
             except ValueError:
-                self.report(f"Unresolvable {renewal_count=} was replaced with 0.")
+                Helper.log_data_issue(
+                    self.row,
+                    i18n.t("Unresolvable %{renewal_count=} was replaced with 0."),
+                    json.dumps(loan)
+                )
         else:
-            self.report(f"Missing renewal count was replaced with 0.")
+            Helper.log_data_issue(
+                self.row,
+                i18n.t("Missing renewal count was replaced with 0."),
+                json.dumps(loan)
+            )
         return 0
 
     def correct_for_1_day_loans(self):
@@ -129,17 +163,19 @@ class LegacyLoan(object):
                 i18n.t(
                     "Due date is before out date, or date information is missing from both"
                 ),
-                json.dumps(self.legacy_loan_dict, indent=2)
+                json.dumps(self.legacy_loan_dict, indent=2),
             )
 
     def to_dict(self):
         return {
             "item_barcode": self.item_barcode,
             "patron_barcode": self.patron_barcode,
+            "proxy_patron_barcode": self.proxy_patron_barcode,
             "due_date": self.due_date.isoformat(),
             "out_date": self.out_date.isoformat(),
             "renewal_count": self.renewal_count,
             "next_item_status": self.next_item_status,
+            "service_point_id": self.service_point_id,
         }
 
     def make_utc(self):

folio_migration_tools/translations/en.json

@@ -6,6 +6,7 @@
   "%{field} a, x and z are missing or empty": "%{field} a, x and z are missing or empty",
   "%{field} subfields a, x, and z missing from field": "%{field} subfields a, x, and z missing from field",
   "%{fro} mapped from %{record}": "%{fro} mapped from %{record}",
+  "%{lost_type} and checked out": "%{lost_type} and checked out",
   "%{props} were concatenated": "%{props} were concatenated",
   "%{schema_value} added to %{prop_name}": "%{schema_value} added to %{prop_name}",
   "%{tag} subfield %{subfield} not in field": "%{tag} subfield %{subfield} not in field",
@@ -36,6 +37,8 @@
   "Check mapping file against the schema.": "Check mapping file against the schema.",
   "Checked out on first try": "Checked out on first try",
   "Checked out on second try": "Checked out on second try",
+  "Claim item returned": "Claim item returned",
+  "Claimed returned and checked out": "Claimed returned and checked out",
   "Click to expand all %{count} things": {
     "many": "Click to expand all %{count} things",
     "one": "Click to expand one thing"
@@ -57,6 +60,7 @@
   "DATA ISSUE Users not in FOLIO": "DATA ISSUE Users not in FOLIO",
   "Data issue. Consider fixing the record. ": "Data issue. Consider fixing the record. ",
   "Declare item as lost": "Declare item as lost",
+  "Declared lost and checked out": "Declared lost and checked out",
   "Discarded reserves": "Discarded reserves",
   "Due date is before out date, or date information is missing from both": "Due date is before out date, or date information is missing from both",
   "Duplicate 001. Creating HRID instead.\n Previous 001 will be stored in a new 035 field": "Duplicate 001. Creating HRID instead.\n Previous 001 will be stored in a new 035 field",
@@ -88,6 +92,7 @@
   "Instances HRID starting number": "Instances HRID starting number",
   "Instances linked using instances_id_map": "Instances linked using instances_id_map",
   "Interfaces": "Interfaces",
+  "Invalid specific retention policy in 008/13-15: %{value}": "Invalid specific retention policy in 008/13-15: %{value}",
   "Inventory records written to disk": "Inventory records written to disk",
   "Item lookups performed": "Item lookups performed",
   "Item transformation report": "Item transformation report",
@@ -95,6 +100,7 @@
   "Legacy Field": "Legacy Field",
   "Legacy bib records without 001": "Legacy bib records without 001",
   "Legacy id is empty": "Legacy id is empty",
+  "Loan already exists for %{item_barcode}": "Loan already exists for %{item_barcode}",
   "Loan already in failed.": "Loan already in failed.",
   "Loans discarded. Had migrated item barcode": "Loans discarded. Had migrated item barcode",
   "Loans failed pre-validation": "Loans failed pre-validation",
@@ -176,6 +182,7 @@
   "Reserve discarded. Could not find migrated barcode": "Reserve discarded. Could not find migrated barcode",
   "Reserve verified against migrated item": "Reserve verified against migrated item",
   "Reserves migration report": "Reserves migration report",
+  "Retention policy 6 indicates a limited period. Specific retention period will be mapped from 008/13-15": "Retention policy 6 indicates a limited period. Specific retention period will be mapped from 008/13-15",
   "Rows merged to create Purchase Orders": "Rows merged to create Purchase Orders",
   "SRS records written to disk": "SRS records written to disk",
   "Second failure": "Second failure",
@@ -186,6 +193,7 @@
   "Set leader 11 (Subfield code count) from %{record} to 2": "Set leader 11 (Subfield code count) from %{record} to 2",
   "Set leader 20-23 from %{field} to 4500": "Set leader 20-23 from %{field} to 4500",
   "Set up statistical code id mapping...": "Set up statistical code id mapping...",
+  "Setting item %{item_barcode} to status \"Available\"": "Setting item %{item_barcode} to status \"Available\"",
   "Source digits": "Source digits",
   "Source of heading or term": "Source of heading or term",
   "Staff suppressed": "Staff suppressed",
@@ -201,6 +209,8 @@
   "Successful matching on %{criteria}": "Successful matching on %{criteria}",
   "Successful user transformations": "Successful user transformations",
   "Successfully %{action}": "Successfully %{action}",
+  "Successfully Checked out %{lost_type} item and put the status back": "Successfully Checked out %{lost_type} item and put the status back",
+  "Successfully Checked out %{lost_type} item. Item will be declared lost.": "Successfully Checked out %{lost_type} item. Item will be declared lost.",
   "Successfully activated user": "Successfully activated user",
   "Successfully changed due date": "Successfully changed due date",
   "Successfully checked out": "Successfully checked out",
@@ -280,6 +290,8 @@
   "blurbs.Details.title": "Details",
   "blurbs.DiffsBetweenOrders.description": "This is a technical report that helps you to identify differences in the mapped order fields. ",
   "blurbs.DiffsBetweenOrders.title": "Differences between generated orders with same Legacy Identifier",
+  "blurbs.DigitizationPolicyMapping.description": "Digitization policies mapped from `008[21]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
+  "blurbs.DigitizationPolicyMapping.title": "Digitization policy",
   "blurbs.DiscardedLoans.description": "List of loans discarded for various resons",
   "blurbs.DiscardedLoans.title": "Discarded loans",
   "blurbs.DiscardedRequests.description": "List of requests discarded for various resons",
@@ -320,6 +332,8 @@
   "blurbs.HoldingsTypeMapping.title": "Holdings type mapping",
   "blurbs.HridHandling.description": "There are two ways of handling HRIDs. The default behaviour is to take the current 001 and move that to a new 035. This will also emerge as an Identifier on the Inventory Instances. The 001 and Instance HRID will be generated from the HRID settings in FOLIO. The second option is to maintain the 001s in the records, and also add this as the Instance HRID",
   "blurbs.HridHandling.title": "HRID and 001/035 handling",
+  "blurbs.ILLPolicyMapping.description": "ILL policies mapped from `008[20]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
+  "blurbs.ILLPolicyMapping.title": "ILL policy",
   "blurbs.IncompleteEntityMapping.description": "**NO ACTION REQUIRED** <br/>This is a coding anomaly that FSE will look into.  <br/>Usually, the library does not have to do anything about it.<br/> One thing to look at is if there are many repeated subfields or unexpected patterns of subfields in the table.",
   "blurbs.IncompleteEntityMapping.title": "Incomplete entity mapping adding entity",
   "blurbs.IncompleteSubPropertyRemoved.description": "Add the missing required information to the record in your current ILS to ensure that it can be migrated over.",
@@ -376,6 +390,8 @@
   "blurbs.MatchedModesOfIssuanceCode.title": "Matched Modes of issuance code",
   "blurbs.MaterialTypeMapping.description": "",
   "blurbs.MaterialTypeMapping.title": "Mapped Material Types",
+  "blurbs.MethodOfAcquisitionMapping.description": "Acquisition methods mapped from `008[7]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
+  "blurbs.MethodOfAcquisitionMapping.title": "Method of acquisition",
   "blurbs.MissingInstanceTypeIds.description": "**IC ACTION REQUIRED** These reords should get an instance type ID mapped from 336, or a default of Undefined, or they will not be transformed.",
   "blurbs.MissingInstanceTypeIds.title": "Records without Instance Type Ids",
   "blurbs.MissingRequiredProperties.description": "",
@@ -408,6 +424,8 @@
   "blurbs.RecourceTypeMapping.title": "Resource Type Mapping (336)",
   "blurbs.ReferenceDataMapping.description": "",
   "blurbs.ReferenceDataMapping.title": "Reference Data Mapping",
+  "blurbs.RetentionPolicyMapping.description": "Retention policies mapped from `008[12-15]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
+  "blurbs.RetentionPolicyMapping.title": "Retention policy",
   "blurbs.Section1.description": "This entries below seem to be related to instances",
   "blurbs.Section1.title": "__Section 1: instances",
   "blurbs.Section2.description": "The entries below seem to be related to holdings",
@@ -446,15 +464,6 @@
   "blurbs.ValueSetInMappingFile.title": "Value set in mapping file",
   "blurbs.ValuesMappedFromLegacyFields.description": "A list fo the values and what they were mapped to",
   "blurbs.ValuesMappedFromLegacyFields.title": "Values mapped from legacy fields",
-  "blurbs.MethodOfAcquisitionMapping.title": "Method of acquisition",
-  "blurbs.MethodOfAcquisitionMapping.description": "Acquisition methods mapped from `008[7]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
-  "blurbs.RetentionPolicyMapping.title": "Retention policy",
-  "blurbs.RetentionPolicyMapping.description": "Retention policies mapped from `008[12-15]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
-  "blurbs.ILLPolicyMapping.title": "ILL policy",
-  "blurbs.ILLPolicyMapping.description": "ILL policies mapped from `008[20]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
-  "blurbs.DigitizationPolicyMapping.title": "Digitization policy",
-  "blurbs.DigitizationPolicyMapping.description": "Digitization policies mapped from `008[21]` (LoC documentation)[https://www.loc.gov/marc/holdings/hd008.html]",
-  "Invalid specific retention policy in 008/13-15: %{value}": "Invalid specific retention policy in 008/13-15: %{value}",
   "created": "created",
   "instance type code (%{code}) not found in FOLIO": "instance type code (%{code}) not found in FOLIO",
   "item barcode": "item barcode",
@@ -462,4 +471,4 @@
   "naturalId mapped from %{fro}": "naturalId mapped from %{fro}",
   "no matching identifier_types in %{names}": "no matching identifier_types in %{names}",
   "subfield present in %{linked_value_tag} but not in %{pattern_field}": "subfield present in %{linked_value_tag} but not in %{pattern_field}"
-}
+}

{folio_migration_tools-1.9.3.dist-info → folio_migration_tools-1.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.3
+Version: 1.9.5
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -19,6 +19,7 @@ Requires-Dist: argparse-prompt (>=0.0.5,<0.0.6)
 Requires-Dist: art (>=6.5,<7.0)
 Requires-Dist: deepdiff (>=6.2.3,<7.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
+Requires-Dist: folio-data-import (>=0.3.2,<0.4.0)
 Requires-Dist: folio-uuid (>=0.2.8,<0.3.0)
 Requires-Dist: folioclient (>=0.70.1,<0.71.0)
 Requires-Dist: pyaml (>=21.10.1,<22.0.0)

{folio_migration_tools-1.9.3.dist-info → folio_migration_tools-1.9.5.dist-info}/RECORD

@@ -1,13 +1,13 @@
 folio_migration_tools/__init__.py,sha256=DXvzUKFSpSZjflFWaNm0L8yhFk0u7RVIvQMskwMmKFc,238
 folio_migration_tools/__main__.py,sha256=kfo4iKf3GJD7deh4RvIizKnC4zvIgCpNo-Bs7HBM34s,8453
-folio_migration_tools/circulation_helper.py,sha256=V2VM30i2OigOKb64B4FFKTeHu9NTkhptalaOfziPhTo,14199
+folio_migration_tools/circulation_helper.py,sha256=iib2p0WGzOcAnwaJppDYiONc-1Jdna66dPAtBmAIuCE,14538
 folio_migration_tools/colors.py,sha256=GP0wdI_GZ2WD5SjrbPN-S3u8vvN_u6rGQIBBcWv_0ZM,227
 folio_migration_tools/config_file_load.py,sha256=zHHa6NDkN6EJiQE4DgjrFQPVKsd70POsfbGkB8308jg,2822
 folio_migration_tools/custom_dict.py,sha256=-FUnhKp90Dg8EHlY6twx-PYQxBUWEO7FgxL2b7pf-xk,678
 folio_migration_tools/custom_exceptions.py,sha256=1zgOKy3NBUVGG6i9YxK6w2Hntlea8MHmm7mdnjBtzvQ,2687
 folio_migration_tools/extradata_writer.py,sha256=fuchNcMc6BYb9IyfAcvXg7X4J2TfX6YiROfT2hr0JMw,1678
 folio_migration_tools/folder_structure.py,sha256=bZlmKGtxdytWcqjnM2lE4Vpx4nHyYRk7CNL1tZhLtXY,6917
-folio_migration_tools/helper.py,sha256=KkOkNAGO_fuYqxdLrsbLzCJLQHUrFZG1NzD4RmpQ-KM,2804
+folio_migration_tools/helper.py,sha256=iWQhdcuzbGzVpEAiHQczO4hdhIH_iciLEp1SCGDynMI,2983
 folio_migration_tools/holdings_helper.py,sha256=yJpz6aJrKRBiJ1MtT5bs2vXAc88uJuGh2_KDuCySOKc,7559
 folio_migration_tools/i18n_config.py,sha256=3AH_2b9zTsxE4XTe4isM_zYtPJSlK0ix6eBmV7kAYUM,228
 folio_migration_tools/library_configuration.py,sha256=LzICsZQdOkXwIqdDfh59x0-Cx77Lb18qVnWroNqekS8,7046
@@ -32,22 +32,22 @@ folio_migration_tools/marc_rules_transformation/marc_file_processor.py,sha256=Qh
 folio_migration_tools/marc_rules_transformation/marc_reader_wrapper.py,sha256=9ATjYMRAjy0QcXtmNZaHVhHLJ5hE1WUgOcF6KMJjbgo,5309
 folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py,sha256=PGt2w8h2pj8_8sGjQe3L-odFDlquURtKnoNFRWQB3GI,9621
 folio_migration_tools/marc_rules_transformation/rules_mapper_base.py,sha256=loNZ9gEYaAwjkP2_wLlXGedjWvSdHoGF_oJN9g6gI3s,45928
-folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py,sha256=GYZmVrEKcHkOEH4U3027-vQjS6mfMbk84GJTqiVrD4E,30350
-folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py,sha256=wT9HDodIRYeGbjutVHDHpSBVWrXsuA2LO8e_MmBMmzE,28498
+folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py,sha256=RD46EzS0NQArn5LCGbrxDm9vbbW9PO_6iNUQwJBAbSg,30364
+folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py,sha256=ZgyDxmNE7LwW8Cd55wRIEE-u6iyMKCRRXdq2ZRjm2nc,28779
 folio_migration_tools/migration_report.py,sha256=BkRspM1hwTBnWeqsHamf7yVEofzLj560Q-9G--O00hw,4258
 folio_migration_tools/migration_tasks/__init__.py,sha256=ZkbY_yGyB84Ke8OMlYUzyyBj4cxxNrhMTwQlu_GbdDs,211
 folio_migration_tools/migration_tasks/authority_transformer.py,sha256=AoXg9s-GLO3yEEDCrQV7hc4YVXxwxsdxDdpj1zhHydE,4251
-folio_migration_tools/migration_tasks/batch_poster.py,sha256=xN1BBZNGW2lZHWPznF6nkYV15XGhwwzcZccCzTbPfA4,40868
+folio_migration_tools/migration_tasks/batch_poster.py,sha256=7gH9KSdtTSbPIS3eXK6_JBi0OshUAupV8AEew9QBSoU,45327
 folio_migration_tools/migration_tasks/bibs_transformer.py,sha256=46d44pcDAodFXDYbrTCMRASISbDciXmA0CXYfhP2IaE,6298
 folio_migration_tools/migration_tasks/courses_migrator.py,sha256=CzXnsu-KGP7B4zcINJzLYUqz47D16NuFfzu_DPqRlTQ,7061
 folio_migration_tools/migration_tasks/holdings_csv_transformer.py,sha256=kMhtHE8DJjA4d6kXBcfflueha3R3nwlBQjdec8CaY8c,21926
 folio_migration_tools/migration_tasks/holdings_marc_transformer.py,sha256=c_ruhOgidyJdSnnRwWUs3wwFMiLqbVMPOhhCaYuH_TI,14343
-folio_migration_tools/migration_tasks/items_transformer.py,sha256=HlTzV7K0AiGBHw56VMascupMKXG0Pv8LS65O9EiQ2VU,19637
-folio_migration_tools/migration_tasks/loans_migrator.py,sha256=_7yZH951p5mhLjbyH1r496DG591dD1tg_mmTtHas62o,35316
+folio_migration_tools/migration_tasks/items_transformer.py,sha256=oTbFX2saF7-ZCb1mO3baLvODnBSEbbN5F_GtSth3iG4,19755
+folio_migration_tools/migration_tasks/loans_migrator.py,sha256=PF8DNpGKsppSDr7wX96Ao13UTFu6dl9cz2URLqSsOzE,40052
 folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py,sha256=CnmlTge7nChUJ10EiUkriQtJlVxWqglgfhjgneh2_yM,7247
 folio_migration_tools/migration_tasks/migration_task_base.py,sha256=Q-57h6rmt74bC9LidA9ZoagEcwVd_ytq8IUWelVOm2E,22521
 folio_migration_tools/migration_tasks/orders_transformer.py,sha256=6SnzU_rUTu2B5hQykI2nRA7vI1rg-uxuF9Ncupe0AEY,14302
-folio_migration_tools/migration_tasks/organization_transformer.py,sha256=vcCjhN1sS55c_a0LXi1Yw1eq3zpDn5E4BGbm2zDQ_Z4,16885
+folio_migration_tools/migration_tasks/organization_transformer.py,sha256=Kuxkh1sKyUVBqm5qAK1Jrq-4xcyNz2JPZvvFRqfwI8s,16922
 folio_migration_tools/migration_tasks/requests_migrator.py,sha256=QP9OBezC3FfcKpI78oMmydxcPaUIYAgHyKevyLwC-WQ,14841
 folio_migration_tools/migration_tasks/reserves_migrator.py,sha256=4sSPer6_6yMwiiY1VYJmYZske_Ah1XG4KAM3NDadPhg,9952
 folio_migration_tools/migration_tasks/user_transformer.py,sha256=aylrMC9n47fdStgsNfW4ZbJh2E4FDSPypsaNv52ynKc,12330
@@ -55,13 +55,13 @@ folio_migration_tools/task_configuration.py,sha256=73OWc8TX--fwPRptv3eQVEVv0-XmN
 folio_migration_tools/test_infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 folio_migration_tools/test_infrastructure/mocked_classes.py,sha256=BurU3NGU_Q8as_BGmW98q9O6bujZDkOfFmvKKdVw9t8,15056
 folio_migration_tools/transaction_migration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-folio_migration_tools/transaction_migration/legacy_loan.py,sha256=phd9oO6xd91qC4ilRq3podZ-rKIIwQ01SXe0JxbZAbQ,6339
+folio_migration_tools/transaction_migration/legacy_loan.py,sha256=sLw2fCBao1VWZykOx-BWpMndLHjJNj-HJZFYqbpVV5A,7561
 folio_migration_tools/transaction_migration/legacy_request.py,sha256=1ulyFzPQw_InOjyPzkWpGnNptgXdQ18nmri0J8Nlpkc,6124
 folio_migration_tools/transaction_migration/legacy_reserve.py,sha256=qzw0okg4axAE_ezXopP9gFsQ_e60o0zh7zqRzFBSWHY,1806
 folio_migration_tools/transaction_migration/transaction_result.py,sha256=cTdCN0BnlI9_ZJB2Z3Fdkl9gpymIi-9mGZsRFlQcmDk,656
-folio_migration_tools/translations/en.json,sha256=TPQRTDdvdkZI2iHczP4hKmFEbd7Hyo5BE37uSo54W_4,40691
-folio_migration_tools-1.9.3.dist-info/LICENSE,sha256=PhIEkitVi3ejgq56tt6sWoJIG_zmv82cjjd_aYPPGdI,1072
-folio_migration_tools-1.9.3.dist-info/METADATA,sha256=dPvDnsZ0qw3K6pPfHatlCPfSCov_7d8Ll7L2pJYSta8,7444
-folio_migration_tools-1.9.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-folio_migration_tools-1.9.3.dist-info/entry_points.txt,sha256=Hbe-HjqMcU8FwVshVIkeWyZd9XwgT1CCMNf06EpHQu8,77
-folio_migration_tools-1.9.3.dist-info/RECORD,,
+folio_migration_tools/translations/en.json,sha256=6IpYYNFCtQoXACndPM0d1Oa25GYuaF-G-b4YpzTjQH0,41656
+folio_migration_tools-1.9.5.dist-info/LICENSE,sha256=PhIEkitVi3ejgq56tt6sWoJIG_zmv82cjjd_aYPPGdI,1072
+folio_migration_tools-1.9.5.dist-info/METADATA,sha256=nkA-xhhnIR5-fawBRyzv7cLMqEycu7IqM00H5QqQpKs,7494
+folio_migration_tools-1.9.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+folio_migration_tools-1.9.5.dist-info/entry_points.txt,sha256=Hbe-HjqMcU8FwVshVIkeWyZd9XwgT1CCMNf06EpHQu8,77
+folio_migration_tools-1.9.5.dist-info/RECORD,,