folio-migration-tools 1.9.0rc10__tar.gz → 1.9.0rc12__tar.gz

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc10
+Version: 1.9.0rc12
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -20,7 +20,7 @@ Requires-Dist: art (>=6.5,<7.0)
 Requires-Dist: deepdiff (>=6.2.3,<7.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
 Requires-Dist: folio-uuid (>=0.2.8,<0.3.0)
-Requires-Dist: folioclient (>=0.61.2,<0.62.0)
+Requires-Dist: folioclient (>=0.70.1,<0.71.0)
 Requires-Dist: pyaml (>=21.10.1,<22.0.0)
 Requires-Dist: pydantic (>=1.10.2,<2.0.0)
 Requires-Dist: pyhumps (>=3.7.3,<4.0.0)

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc10"
+version = "1.9.0rc12"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -44,7 +44,7 @@ profile = "black"
 
 [tool.poetry.dependencies]
 python = "^3.9"
-folioclient = "^0.61.2"
+folioclient = "^0.70.1"
 pyhumps = "^3.7.3"
 defusedxml = "^0.7.1"
 python-dateutil = "^2.8.2"

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/__main__.py

@@ -1,3 +1,4 @@
+from importlib import metadata
 import json
 import logging
 import sys
@@ -38,7 +39,7 @@ def parse_args(args):
         default=environ.get("FOLIO_MIGRATION_TOOLS_TASK_NAME"),
     )
     parser.add_argument(
-        "--okapi_password",
+        "--folio_password", "--okapi_password",
         help="password for the tenant in the configuration file",
         prompt="FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD" not in environ,
         default=environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
@@ -60,11 +61,17 @@ def parse_args(args):
         default=environ.get("FOLIO_MIGRATION_TOOLS_REPORT_LANGUAGE", "en"),
         prompt=False,
     )
+    parser.add_argument(
+        "--version", "-V",
+        help="Show the version of the FOLIO Migration Tools",
+        action="store_true",
+        prompt=False,
+    )
     return parser.parse_args(args)
 
 def prep_library_config(args):
         config_file_humped = merge_load(args.configuration_path)
-        config_file_humped["libraryInformation"]["okapiPassword"] = args.okapi_password
+        config_file_humped["libraryInformation"]["okapiPassword"] = args.folio_password
         config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
         config_file = humps.decamelize(config_file_humped)
         library_config = LibraryConfiguration(**config_file["library_information"])
@@ -78,11 +85,22 @@ def prep_library_config(args):
             sys.exit("ECS Central Iteration Identifier Not Found")
         return config_file, library_config
 
+def print_version(args):
+    if "-V" in args or "--version" in args:
+        print(
+            f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}"
+        )
+        sys.exit(0)
+    return None
+
 
 def main():
     try:
         task_classes = list(inheritors(migration_task_base.MigrationTaskBase))
+        # Check if the script is run with the --version or -V flag
+        print_version(sys.argv)
 
+        # Parse command line arguments
         args = parse_args(sys.argv[1:])
         try:
             i18n.load_config(
@@ -124,10 +142,10 @@ def main():
         try:
             logging.getLogger("httpx").setLevel(logging.WARNING) # Exclude info messages from httpx
             with FolioClient(
-                library_config.okapi_url,
+                library_config.gateway_url,
                 library_config.tenant_id,
-                library_config.okapi_username,
-                library_config.okapi_password,
+                library_config.folio_username,
+                library_config.folio_password,
             ) as folio_client:
                 task_config = task_class.TaskConfiguration(**migration_task_config)
                 task_obj = task_class(task_config, library_config, folio_client)
@@ -139,7 +157,6 @@ def main():
             print("Task failure. Halting.")
             sys.exit(1)
         logging.info("Work done. Shutting down")
-        sys.exit(0)
     except json.decoder.JSONDecodeError as json_error:
         logging.critical(json_error)
         print(json_error.doc)
@@ -180,7 +197,7 @@ def main():
         logging.exception("Unhandled exception")
         print(f"\n{ee}")
         sys.exit(ee.__class__.__name__)
-
+    sys.exit(0)
 
 def inheritors(base_class):
     subclasses = set()

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/circulation_helper.py

@@ -138,7 +138,7 @@ class CirculationHelper:
         if legacy_loan.proxy_patron_barcode:
             data.update({"proxyUserBarcode": legacy_loan.proxy_patron_barcode})
         path = "/circulation/check-out-by-barcode"
-        url = f"{self.folio_client.okapi_url}{path}"
+        url = f"{self.folio_client.gateway_url}{path}"
         try:
             if legacy_loan.patron_barcode in self.missing_patron_barcodes:
                 error_message = i18n.t("Patron barcode already detected as missing")
@@ -249,7 +249,7 @@ class CirculationHelper:
     ):
         try:
             path = "/circulation/requests"
-            url = f"{folio_client.okapi_url}{path}"
+            url = f"{folio_client.gateway_url}{path}"
             data = legacy_request.serialize()
             data["requestProcessingParameters"] = {
                 "overrideBlocks": {
@@ -313,7 +313,7 @@ class CirculationHelper:
             del loan_to_put["metadata"]
             loan_to_put["dueDate"] = extension_due_date.isoformat()
             loan_to_put["loanDate"] = extend_out_date.isoformat()
-            url = f"{folio_client.okapi_url}/circulation/loans/{loan_to_put['id']}"
+            url = f"{folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
 
             req = httpx.put(
                 url, headers=folio_client.okapi_headers, json=loan_to_put, timeout=None

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/library_configuration.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Annotated, Optional
+from typing import Annotated
 
 from pydantic import BaseModel, Field
 from pydantic.types import DirectoryPath
@@ -68,8 +68,28 @@ class FolioRelease(str, Enum):
 
 
 class LibraryConfiguration(BaseModel):
-    okapi_url: str
-    tenant_id: str
+    gateway_url: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway URL",
+            description=(
+                "The URL of the FOLIO API gateway instance. "
+                "You can find this in Settings > Software versions > API gateway services."
+            ),
+            alias="okapi_url"
+        ),
+    ]
+    tenant_id: Annotated[
+        str,
+        Field(
+            title="FOLIO tenant ID",
+            description=(
+                "The ID of the FOLIO tenant instance. "
+                "You can find this in Settings > Software versions > API gateway services. "
+                "In an ECS environment, this is the ID of the central tenant, for all configurations."
+            ),
+        ),
+    ]
     ecs_tenant_id: Annotated[
         str,
         Field(
@@ -80,15 +100,43 @@ class LibraryConfiguration(BaseModel):
             ),
         ),
     ] = ""
-    okapi_username: str
-    okapi_password: str
+    folio_username: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway username",
+            description=(
+                "The username for the FOLIO user account performing the migration. "
+                "User should have a full admin permissions/roles in FOLIO. "
+            ),
+            alias="okapi_username"
+        ),
+    ]
+    folio_password: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway password",
+            description=(
+                "The password for the FOLIO user account performing the migration. "
+            ),
+            alias="okapi_password"
+        )
+    ]
     base_folder: DirectoryPath = Field(
         description=(
             "The base folder for migration. "
             "Should ideally be a github clone of the migration_repo_template"
         )
     )
-    multi_field_delimiter: Optional[str] = "<delimiter>"
+    multi_field_delimiter: Annotated[
+        str,
+        Field(
+            title="Multi field delimiter",
+            description=(
+                "The delimiter used to separate multiple values in a single field. "
+                "This is used for delimited text (CSV/TSV) fields with multiple sub-delimited values."
+            ),
+        ),
+    ] = "<delimiter>"
     failed_records_threshold: Annotated[
         int,
         Field(description=("Number of failed records until the process shuts down")),

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/mapper_base.py

@@ -443,7 +443,7 @@ class MapperBase:
     @property
     def base_string_for_folio_uuid(self):
         if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
-            return str(self.folio_client.okapi_url)
+            return str(self.folio_client.gateway_url)
         elif self.library_configuration.ecs_tenant_id:
             return str(self.library_configuration.ecs_tenant_id)
         else:

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/mapping_file_transformation/organization_mapper.py

@@ -79,28 +79,28 @@ class OrganizationMapper(MappingFileMapperBase):
                 False,
             )
 
-        elif re.compile("addresses\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
+        elif re.compile(r"addresses\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
             return self.get_mapped_ref_data_value(
                 self.address_categories_map,
                 *value_tuple,
                 False,
             )
 
-        elif re.compile("emails\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
+        elif re.compile(r"emails\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
             return self.get_mapped_ref_data_value(
                 self.email_categories_map,
                 *value_tuple,
                 False,
             )
 
-        elif re.compile("phoneNumbers\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
+        elif re.compile(r"phoneNumbers\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
             return self.get_mapped_ref_data_value(
                 self.phone_categories_map,
                 *value_tuple,
                 False,
             )
 
-        elif re.compile("interfaces\[(\d+)\]\.interfaceCredential.interfaceId").fullmatch(
+        elif re.compile(r"interfaces\[(\d+)\]\.interfaceCredential.interfaceId").fullmatch(
             folio_prop_name
         ):
             return "replace_with_interface_id"

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py

@@ -165,23 +165,16 @@ class HoldingsStatementsParser:
             TransformationFieldMappingError: _description_
         """
         for f in marc_record.get_fields(field_textual):
-            codes = [sf.code for sf in f.subfields]
-            if "a" not in codes and "z" not in codes and "x" not in codes:
-                raise TransformationFieldMappingError(
-                    legacy_ids,
-                    i18n.t(
-                        "%{field} subfields a, x, and z missing from field", field=field_textual
-                    ),
-                    f,
-                )
-            if not (
-                len(f.get_subfields("a")) == 0
-                or len(f.get_subfields("z")) == 0
-                or len(f.get_subfields("x")) == 0
+            if all(
+                [
+                    len("".join(f.get_subfields("a")).strip()) == 0,
+                    len("".join(f.get_subfields("z")).strip()) == 0,
+                    len("".join(f.get_subfields("x")).strip()) == 0,
+                ]
             ):
                 raise TransformationFieldMappingError(
                     legacy_ids,
-                    i18n.t("%{field} a,x and z are all empty", field=field_textual),
+                    i18n.t("%{field} a, x and z are missing or empty", field=field_textual),
                     f,
                 )
             return_dict["statements"].append(

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/marc_rules_transformation/hrid_handler.py

@@ -160,7 +160,7 @@ class HRIDHandler:
             self.hrid_settings["instances"]["startNumber"] = self.instance_hrid_counter
             self.hrid_settings["holdings"]["startNumber"] = self.holdings_hrid_counter
             self.hrid_settings["items"]["startNumber"] = self.items_hrid_counter
-            url = self.folio_client.okapi_url + self.hrid_path
+            url = self.folio_client.gateway_url + self.hrid_path
             resp = httpx.put(
                 url,
                 json=self.hrid_settings,

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -10,7 +10,6 @@ from pathlib import Path
 from typing import Dict, Generator, List
 
 import i18n
-import pymarc
 from defusedxml.ElementTree import fromstring
 from folio_uuid.folio_namespaces import FOLIONamespaces
 from folio_uuid.folio_uuid import FolioUUID
@@ -144,7 +143,7 @@ class BibsRulesMapper(RulesMapperBase):
         self.report_folio_mapping(clean_folio_instance, self.schema)
         return [clean_folio_instance]
 
-    def simple_bib_map(self, folio_instnace: dict, marc_record: Record, ignored_subsequent_fields: set, legacy_ids: List[str]):
+    def simple_bib_map(self, folio_instance: dict, marc_record: Record, ignored_subsequent_fields: set, legacy_ids: List[str]):
         """
         This method applies a much simplified MARC-to-instance
         mapping to create a minimal FOLIO Instance record to be
@@ -152,13 +151,27 @@ class BibsRulesMapper(RulesMapperBase):
         than creating SRS records during transformation.
 
         Args:
-            folio_instnace (dict): _description_
+            folio_instance (dict): _description_
             marc_record (Record): _description_
             legacy_ids (List[str]): _description_
             file_def (FileDefinition): _description_
         """
+        main_entry_field_tags = ["100", "110", "111", "130"]
+        main_entry_fields = marc_record.get_fields(*main_entry_field_tags)
+        main_entry_fields.sort(key=lambda x: int(x.tag))
+        if len(main_entry_fields) > 1:
+            Helper.log_data_issue(
+                legacy_ids,
+                "Multiple main entry fields in record. Record will fail Data Import. Creating Instance anyway.",
+                main_entry_fields
+            )
+        if not main_entry_fields:
+            main_entry_fields += marc_record.get_fields("700", "710", "711", "730")
+            main_entry_fields.sort(key=lambda x: int(x.tag))
+        if main_entry_fields:
+            self.process_marc_field(folio_instance, main_entry_fields[0], ignored_subsequent_fields, legacy_ids)
         try:
-            self.process_marc_field(folio_instnace, marc_record['245'], ignored_subsequent_fields, legacy_ids)
+            self.process_marc_field(folio_instance, marc_record['245'], ignored_subsequent_fields, legacy_ids)
         except KeyError:
             raise TransformationRecordFailedError(
                 legacy_ids,

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -34,12 +34,12 @@ from folio_migration_tools.marc_rules_transformation.rules_mapper_base import (
 class RulesMapperHoldings(RulesMapperBase):
     def __init__(
         self,
-        folio_client,
+        folio_client: FolioClient,
         location_map,
         task_configuration,
         library_configuration: LibraryConfiguration,
         parent_id_map: dict,
-        boundwith_relationship_map,
+        boundwith_relationship_map_rows: List[Dict],
     ):
         self.task_configuration = task_configuration
         self.conditions = Conditions(
@@ -58,8 +58,8 @@ class RulesMapperHoldings(RulesMapperBase):
             self.conditions,
             parent_id_map,
         )
-        self.boundwith_relationship_map = self.setup_boundwith_relationship_map(
-            boundwith_relationship_map
+        self.boundwith_relationship_map: Dict = self.setup_boundwith_relationship_map(
+            boundwith_relationship_map_rows
         )
         self.location_map = self.validate_location_map(
             location_map,
@@ -278,6 +278,8 @@ class RulesMapperHoldings(RulesMapperBase):
         self.set_default_call_number_type_if_empty(folio_holding)
         self.pick_first_location_if_many(folio_holding, legacy_ids)
         self.parse_coded_holdings_statements(marc_record, folio_holding, legacy_ids)
+        self.add_mfhd_as_mrk_note(marc_record, folio_holding, legacy_ids)
+        self.add_mfhd_as_mrc_note(marc_record, folio_holding, legacy_ids)
         HoldingsHelper.handle_notes(folio_holding)
         if (
             all([file_def.create_source_records, self.create_source_records])
@@ -341,6 +343,117 @@ class RulesMapperHoldings(RulesMapperBase):
             except TransformationFieldMappingError as tfme:
                 Helper.log_data_issue(tfme.index_or_id, tfme.message, tfme.data_value)
                 self.migration_report.add("FieldMappingErrors", tfme.message)
+        self.collect_mrk_statement_notes(marc_record, folio_holding, legacy_ids)
+
+    def collect_mrk_statement_notes(self, marc_record, folio_holding, legacy_ids):
+        """Collects MFHD holdings statements as MARC Maker field strings in a FOLIO holdings note
+        and adds them to the FOLIO holdings record.
+
+        This is done to preserve the information in the MARC record for future reference.
+
+        Args:
+            marc_record (Record): PyMARC record
+            folio_holding (Dict): FOLIO holdings record
+
+        """
+        if self.task_configuration.include_mrk_statements:
+            mrk_statement_notes = []
+            for field in marc_record.get_fields("853", "854", "855", "863", "864", "865", "866", "867", "868"):
+                mrk_statement_notes.append(str(field))
+            if mrk_statement_notes:
+                folio_holding["notes"] = folio_holding.get("notes", []) + self.add_mrk_statements_note(mrk_statement_notes, legacy_ids)
+
+    def add_mrk_statements_note(self, mrk_statement_notes: List[str], legacy_ids) -> List[Dict]:
+        """Creates a note from the MRK statements
+
+        Args:
+            mrk_statement_notes (List[str]): A list of MFHD holdings statements as MRK strings
+
+        Returns:
+            List: A list containing the FOLIO holdings note object (Dict)
+        """
+        holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
+            self.folio.holding_note_types, "holding_note_types", self.task_configuration.mrk_holdings_note_type
+        )
+        try:
+            holdings_note_type_id = holdings_note_type_tuple[0]
+        except Exception as ee:
+            logging.error(ee)
+            raise TransformationRecordFailedError(
+                legacy_ids,
+                f'Holdings note type mapping error.\tNote type name: {self.task_configuration.mrk_holdings_note_type}\t'
+                f"MFHD holdings statement note type not found in FOLIO.",
+                self.task_configuration.mrk_holdings_note_type,
+            ) from ee
+        return [
+            {
+                "note": "\n".join(mrk_statement_notes),
+                "holdingsNoteTypeId": holdings_note_type_id,
+                "staffOnly": True,
+            }
+        ]
+
+    def add_mfhd_as_mrk_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
+        """Adds the MFHD as a note to the holdings record
+
+        This is done to preserve the information in the MARC record for future reference.
+
+        Args:
+            marc_record (Record): PyMARC record
+            folio_holding (Dict): FOLIO holdings record
+        """
+        if self.task_configuration.include_mfhd_mrk_as_note:
+            holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
+                self.folio.holding_note_types, "holding_note_types", self.task_configuration.mfhd_mrk_note_type
+            )
+            try:
+                holdings_note_type_id = holdings_note_type_tuple[0]
+            except Exception as ee:
+                logging.error(ee)
+                raise TransformationRecordFailedError(
+                    legacy_ids,
+                    f'Holdings note type mapping error.\tNote type name: {self.task_configuration.mfhd_mrk_note_type}\t'
+                    f"Note type not found in FOLIO.",
+                    self.task_configuration.mfhd_mrk_note_type,
+                ) from ee
+            folio_holding["notes"] = folio_holding.get("notes", []) + [
+                {
+                    "note": str(marc_record),
+                    "holdingsNoteTypeId": holdings_note_type_id,
+                    "staffOnly": True,
+                }
+            ]
+
+    def add_mfhd_as_mrc_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
+        """Adds the MFHD as a note to the holdings record
+
+        This is done to preserve the information in the MARC record for future reference.
+
+        Args:
+            marc_record (Record): PyMARC record
+            folio_holding (Dict): FOLIO holdings record
+        """
+        if self.task_configuration.include_mfhd_mrc_as_note:
+            holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
+                self.folio.holding_note_types, "holding_note_types", self.task_configuration.mfhd_mrc_note_type
+            )
+            try:
+                holdings_note_type_id = holdings_note_type_tuple[0]
+            except Exception as ee:
+                logging.error(ee)
+                raise TransformationRecordFailedError(
+                    legacy_ids,
+                    f'Holdings note type mapping error.\tNote type name: {self.task_configuration.mfhd_mrc_note_type}\t'
+                    f"Note type not found in FOLIO.",
+                    self.task_configuration.mfhd_mrc_note_type,
+                ) from ee
+            folio_holding["notes"] = folio_holding.get("notes", []) + [
+                {
+                    "note": marc_record.as_marc().decode("utf-8"),
+                    "holdingsNoteTypeId": holdings_note_type_id,
+                    "staffOnly": True,
+                }
+            ]
 
     def wrap_up(self):
         logging.info("Mapper wrapping up")

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -287,7 +287,7 @@ class BatchPoster(MigrationTaskBase):
         fetch_batch_size = 90
         fetch_tasks = []
         updates = {}
-        async with httpx.AsyncClient(base_url=self.folio_client.okapi_url) as client:
+        async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
             for i in range(0, len(batch), fetch_batch_size):
                 batch_slice = batch[i:i + fetch_batch_size]
                 fetch_tasks.append(
@@ -360,7 +360,7 @@ class BatchPoster(MigrationTaskBase):
     def post_extra_data(self, row: str, num_records: int, failed_recs_file):
         (object_name, data) = row.split("\t")
         endpoint = self.get_extradata_endpoint(self.task_configuration, object_name, data)
-        url = f"{self.folio_client.okapi_url}/{endpoint}"
+        url = f"{self.folio_client.gateway_url}/{endpoint}"
         body = data
         response = self.post_objects(url, body)
         if response.status_code == 201:
@@ -415,7 +415,7 @@ class BatchPoster(MigrationTaskBase):
         if self.api_info["is_batch"]:
             raise TypeError("This record type supports batch processing, use post_batch method")
         api_endpoint = self.api_info.get("api_endpoint")
-        url = f"{self.folio_client.okapi_url}{api_endpoint}"
+        url = f"{self.folio_client.gateway_url}{api_endpoint}"
         response = self.post_objects(url, row)
         if response.status_code == 201:
             self.num_posted += 1
@@ -590,7 +590,7 @@ class BatchPoster(MigrationTaskBase):
 
     def do_post(self, batch):
         path = self.api_info["api_endpoint"]
-        url = self.folio_client.okapi_url + path
+        url = self.folio_client.gateway_url + path
         if self.api_info["object_name"] == "users":
             payload = {self.api_info["object_name"]: list(batch), "totalRecords": len(batch)}
         elif self.api_info["total_records"]:
@@ -675,7 +675,7 @@ class BatchPoster(MigrationTaskBase):
             "processingStartedDate": datetime.utcnow().isoformat(timespec="milliseconds"),
         }
         try:
-            url = f"{self.folio_client.okapi_url}/source-storage/snapshots"
+            url = f"{self.folio_client.gateway_url}/source-storage/snapshots"
             if self.http_client and not self.http_client.is_closed:
                 res = self.http_client.post(
                     url, json=snapshot, headers=self.folio_client.okapi_headers
@@ -684,7 +684,7 @@ class BatchPoster(MigrationTaskBase):
                 res = httpx.post(url, headers=self.okapi_headers, json=snapshot, timeout=None)
             res.raise_for_status()
             logging.info("Posted Snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
-            get_url = f"{self.folio_client.okapi_url}/source-storage/snapshots/{self.snapshot_id}"
+            get_url = f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
             getted = False
             while not getted:
                 logging.info("Sleeping while waiting for the snapshot to get created")
@@ -704,7 +704,7 @@ class BatchPoster(MigrationTaskBase):
     def commit_snapshot(self):
         snapshot = {"jobExecutionId": self.snapshot_id, "status": "COMMITTED"}
         try:
-            url = f"{self.folio_client.okapi_url}/source-storage/snapshots/{self.snapshot_id}"
+            url = f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
             if self.http_client and not self.http_client.is_closed:
                 res = self.http_client.put(
                     url, json=snapshot, headers=self.folio_client.okapi_headers

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/migration_tasks/bibs_transformer.py

@@ -26,7 +26,9 @@ class BibsTransformer(MigrationTaskBase):
         ils_flavour: Annotated[
             IlsFlavour,
             Field(
-                title="ILS flavour", description="The type of ILS you are migrating records from."
+                title="ILS flavour",
+                description="The type of ILS you are migrating records from.",
+                alias="ils_flavor"
             ),
         ]
         custom_bib_id_field: Annotated[

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/migration_tasks/holdings_marc_transformer.py

@@ -143,6 +143,65 @@ class HoldingsMarcTransformer(MigrationTaskBase):
                 description="The name of the file in the mapping_files directory containing supplemental MFHD mapping rules",
             ),
         ] = ""
+        include_mrk_statements: Annotated[
+            bool,
+            Field(
+                title="Include MARC statements (MRK-format) as staff-only Holdings notes",
+                description=(
+                    "If set to true, the MARC statements will be included in the output as MARC Maker format fields. "
+                    "If set to false (default), the MARC statements will not be included in the output."
+                ),
+            ),
+        ] = False
+        mrk_holdings_note_type: Annotated[
+            str,
+            Field(
+                title="MARC Holdings Note type",
+                description=(
+                    "The name of the note type to use for MARC (MRK) statements. "
+                ),
+            ),
+        ] = "Original MARC holdings statements"
+        include_mfhd_mrk_as_note: Annotated[
+            bool,
+            Field(
+                title="Include MARC Record (as MARC Maker Representation) as note",
+                description=(
+                    "If set to true, the MARC statements will be included in the output as a "
+                    "(MRK) note. If set to false (default), the MARC statements will not be "
+                    "included in the output."
+                ),
+            ),
+        ] = False
+        mfhd_mrk_note_type: Annotated[
+            str,
+            Field(
+                title="MARC Record (as MARC Maker Representation) note type",
+                description=(
+                    "The name of the note type to use for MFHD (MRK) note. "
+                ),
+            ),
+        ] = "Original MFHD Record"
+        include_mfhd_mrc_as_note: Annotated[
+            bool,
+            Field(
+                title="Include MARC Record (as MARC21 decoded string) as note",
+                description=(
+                    "If set to true, the MARC record will be included in the output as a "
+                    "decoded binary MARC21 record. If set to false (default), the MARC record will not be "
+                    "included in the output."
+                ),
+            ),
+        ] = False
+        mfhd_mrc_note_type: Annotated[
+            str,
+            Field(
+                title="MARC Record (as MARC21 decoded string) note type",
+                description=(
+                    "The name of the note type to use for MFHD (MRC) note. "
+                ),
+            ),
+        ] = "Original MFHD (MARC21)"
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -183,19 +242,26 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         )
 
         # Load Boundwith relationship map
-        self.boundwith_relationship_map = []
+        self.boundwith_relationship_map_rows = []
         if self.task_config.boundwith_relationship_file_path:
-            with open(
-                self.folder_structure.legacy_records_folder
-                / self.task_config.boundwith_relationship_file_path
-            ) as boundwith_relationship_file:
-                self.boundwith_relationship_map = list(
-                    csv.DictReader(boundwith_relationship_file, dialect="tsv")
+            try:
+                with open(
+                    self.folder_structure.legacy_records_folder
+                    / self.task_config.boundwith_relationship_file_path
+                ) as boundwith_relationship_file:
+                    self.boundwith_relationship_map_rows = list(
+                        csv.DictReader(boundwith_relationship_file, dialect="tsv")
+                    )
+                logging.info(
+                    "Rows in Bound with relationship map: %s",
+                    len(self.boundwith_relationship_map_rows),
+                )
+            except FileNotFoundError:
+                raise TransformationProcessError(
+                    "",
+                    i18n.t("Provided boundwith relationship file not found"),
+                    self.task_config.boundwith_relationship_file_path,
                 )
-            logging.info(
-                "Rows in Bound with relationship map: %s",
-                len(self.boundwith_relationship_map),
-            )
 
         location_map_path = (
             self.folder_structure.mapping_files_folder
@@ -215,7 +281,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
             self.task_config,
             self.library_configuration,
             self.instance_id_map,
-            self.boundwith_relationship_map,
+            self.boundwith_relationship_map_rows,
         )
         self.add_supplemental_mfhd_mappings()
         if (

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/migration_tasks/loans_migrator.py

@@ -525,7 +525,7 @@ class LoansMigrator(MigrationTaskBase):
             loan_to_put["dueDate"] = due_date.isoformat()
             loan_to_put["loanDate"] = out_date.isoformat()
             loan_to_put["renewalCount"] = renewal_count
-            url = f"{self.folio_client.okapi_url}/circulation/loans/{loan_to_put['id']}"
+            url = f"{self.folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
             req = self.http_client.put(
                 url,
                 headers=self.folio_client.okapi_headers,
@@ -608,7 +608,7 @@ class LoansMigrator(MigrationTaskBase):
         try:
             # Get Item by barcode, update status.
             item_path = f'item-storage/items?query=(barcode=="{legacy_loan.item_barcode}")'
-            item_url = f"{self.folio_client.okapi_url}/{item_path}"
+            item_url = f"{self.folio_client.gateway_url}/{item_path}"
             resp = self.http_client.get(item_url, headers=self.folio_client.okapi_headers)
             resp.raise_for_status()
             data = resp.json()
@@ -667,14 +667,14 @@ class LoansMigrator(MigrationTaskBase):
         self.folio_put_post(url, user, "PUT", i18n.t("Update user"))
 
     def get_user_by_barcode(self, barcode):
-        url = f'{self.folio_client.okapi_url}/users?query=(barcode=="{barcode}")'
+        url = f'{self.folio_client.gateway_url}/users?query=(barcode=="{barcode}")'
         resp = self.http_client.get(url, headers=self.folio_client.okapi_headers)
         resp.raise_for_status()
         data = resp.json()
         return data["users"][0]
 
     def folio_put_post(self, url, data_dict, verb, action_description=""):
-        full_url = f"{self.folio_client.okapi_url}{url}"
+        full_url = f"{self.folio_client.gateway_url}{url}"
         try:
             if verb == "PUT":
                 resp = self.http_client.put(
@@ -729,7 +729,7 @@ class LoansMigrator(MigrationTaskBase):
     def change_due_date(self, folio_loan, legacy_loan):
         try:
             api_path = f"{folio_loan['id']}/change-due-date"
-            api_url = f"{self.folio_client.okapi_url}/circulation/loans/{api_path}"
+            api_url = f"{self.folio_client.gateway_url}/circulation/loans/{api_path}"
             body = {"dueDate": du_parser.isoparse(str(legacy_loan.due_date)).isoformat()}
             req = self.http_client.post(
                 api_url, headers=self.folio_client.okapi_headers, json=body

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/migration_tasks/reserves_migrator.py

@@ -190,7 +190,7 @@ class ReservesMigrator(MigrationTaskBase):
             sys.exit(1)
 
     def folio_put_post(self, url, data_dict, verb, action_description=""):
-        full_url = f"{self.folio_client.okapi_url}{url}"
+        full_url = f"{self.folio_client.gateway_url}{url}"
         try:
             if verb == "PUT":
                 resp = httpx.put(

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/test_infrastructure/mocked_classes.py

@@ -212,6 +212,100 @@ def folio_get_all_mocked(ref_data_path, array_name, query="", limit=10):
             }
         ]
 
+    elif ref_data_path == "/holdings-note-types":
+        yield from [
+            {
+                "id": "88914775-f677-4759-b57b-1a33b90b24e0",
+                "name": "Electronic bookplate",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.719+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.719+00:00"
+                }
+            },
+            {
+                "id": "c4407cc7-d79f-4609-95bd-1cefb2e2b5c5",
+                "name": "Copy note",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.722+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.722+00:00"
+                }
+            },
+            {
+                "id": "d6510242-5ec3-42ed-b593-3585d2e48fd6",
+                "name": "Action note",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.723+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.723+00:00"
+                }
+            },
+            {
+                "id": "e19eabab-a85c-4aef-a7b2-33bd9acef24e",
+                "name": "Binding",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.724+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.724+00:00"
+                }
+            },
+            {
+                "id": "db9b4787-95f0-4e78-becf-26748ce6bdeb",
+                "name": "Provenance",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.725+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.725+00:00"
+                }
+            },
+            {
+                "id": "6a41b714-8574-4084-8d64-a9373c3fbb59",
+                "name": "Reproduction",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.728+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "b160f13a-ddba-4053-b9c4-60ec5ea45d56",
+                "name": "Note",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.728+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "841d1873-015b-4bfb-a69f-6cbb41d925ba",
+                "name": "Original MARC holdings statements",
+                "source": "local",
+                "metadata": {
+                    "createdDate": "2025-05-02T01:54:20.728+00:00",
+                    "updatedDate": "2025-05-02T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "09c1e5c9-6f11-432e-bcbe-b9e733ccce57",
+                "name": "Original MFHD Record",
+                "source": "local",
+                "metadata": {
+                    "createdDate": "2025-05-02T01:54:20.728+00:00",
+                    "updatedDate": "2025-05-02T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "474120b0-d64e-4a6f-9c9c-e7d3e76f3cf5",
+                "name": "Original MFHD (MARC21)",
+                "source": "local",
+                "metadata": {
+                    "createdDate": "2025-05-02T01:54:20.728+00:00",
+                    "updatedDate": "2025-05-02T01:54:20.728+00:00"
+                }
+            }
+        ]
+
     elif ref_data_path in super_schema:
         yield from super_schema.get(ref_data_path)
     else:

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/transaction_migration/legacy_loan.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import i18n
 from datetime import datetime
 from zoneinfo import ZoneInfo
 
@@ -41,20 +43,21 @@ class LegacyLoan(object):
             "Lost and paid",
         ]
 
+        self.legacy_loan_dict = legacy_loan_dict
         self.tenant_timezone = tenant_timezone
         self.errors = []
         self.row = row
         for prop in correct_headers:
-            if prop not in legacy_loan_dict and prop not in optional_headers:
+            if prop not in self.legacy_loan_dict and prop not in optional_headers:
                 self.errors.append((f"Missing properties in legacy data {row=}", prop))
             if (
                 prop != "next_item_status"
-                and not legacy_loan_dict.get(prop, "").strip()
+                and not self.legacy_loan_dict.get(prop, "").strip()
                 and prop not in optional_headers
             ):
                 self.errors.append((f"Empty properties in legacy data {row=}", prop))
         try:
-            temp_date_due: datetime = parse(legacy_loan_dict["due_date"])
+            temp_date_due: datetime = parse(self.legacy_loan_dict["due_date"])
             if temp_date_due.tzinfo != tz.UTC:
                 temp_date_due = temp_date_due.replace(tzinfo=self.tenant_timezone)
                 self.report(
@@ -72,7 +75,7 @@ class LegacyLoan(object):
             self.errors.append((f"Parse date failure in {row=}. Setting UTC NOW", "due_date"))
             temp_date_due = datetime.now(ZoneInfo("UTC"))
         try:
-            temp_date_out: datetime = parse(legacy_loan_dict["out_date"])
+            temp_date_out: datetime = parse(self.legacy_loan_dict["out_date"])
             if temp_date_out.tzinfo != tz.UTC:
                 temp_date_out = temp_date_out.replace(tzinfo=self.tenant_timezone)
                 self.report(
@@ -86,20 +89,20 @@ class LegacyLoan(object):
             self.errors.append((f"Parse date failure in {row=}. Setting UTC NOW", "out_date"))
 
         # good to go, set properties
-        self.item_barcode: str = legacy_loan_dict["item_barcode"].strip()
-        self.patron_barcode: str = legacy_loan_dict["patron_barcode"].strip()
-        self.proxy_patron_barcode: str = legacy_loan_dict.get("proxy_patron_barcode", "")
+        self.item_barcode: str = self.legacy_loan_dict["item_barcode"].strip()
+        self.patron_barcode: str = self.legacy_loan_dict["patron_barcode"].strip()
+        self.proxy_patron_barcode: str = self.legacy_loan_dict.get("proxy_patron_barcode", "")
         self.due_date: datetime = temp_date_due
         self.out_date: datetime = temp_date_out
         self.correct_for_1_day_loans()
         self.make_utc()
-        self.renewal_count = self.set_renewal_count(legacy_loan_dict)
-        self.next_item_status = legacy_loan_dict.get("next_item_status", "").strip()
+        self.renewal_count = self.set_renewal_count(self.legacy_loan_dict)
+        self.next_item_status = self.legacy_loan_dict.get("next_item_status", "").strip()
         if self.next_item_status not in legal_statuses:
             self.errors.append((f"Not an allowed status {row=}", self.next_item_status))
         self.service_point_id = (
-            legacy_loan_dict["service_point_id"]
-            if legacy_loan_dict.get("service_point_id", "")
+            self.legacy_loan_dict["service_point_id"]
+            if self.legacy_loan_dict.get("service_point_id", "")
             else fallback_service_point_id
         )
 
@@ -121,7 +124,7 @@ class LegacyLoan(object):
             if self.out_date.hour == 0:
                 self.out_date = self.out_date.replace(hour=0, minute=1)
         if self.due_date <= self.out_date:
-            raise TransformationProcessError(self.row, "Due date is before out date")
+            raise TransformationProcessError(self.row, i18n.t("Due date is before out date, or date information is missing from both"), json.dumps(self.legacy_loan_dict, indent=2))
 
     def to_dict(self):
         return {

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/transaction_migration/legacy_reserve.py

@@ -33,7 +33,7 @@ class LegacyReserve(object):
             self.errors.append(("Missing data.", "legacy_identifier"))
         self.course_listing_id: str = str(
             FolioUUID(
-                folio_client.okapi_url,
+                folio_client.gateway_url,
                 FOLIONamespaces.course_listing,
                 legacy_request_dict["legacy_identifier"],
             )

{folio_migration_tools-1.9.0rc10 → folio_migration_tools-1.9.0rc12}/src/folio_migration_tools/translations/en.json

@@ -3,7 +3,7 @@
   "%{action} error. http status: %{status}": "%{action} error. http status: %{status}",
   "%{action} error: %{message}": "%{action} error: %{message}",
   "%{fields_criteria} empty or not set": "%{fields_criteria} empty or not set",
-  "%{field} a,x and z are all empty": "%{field} a,x and z are all empty",
+  "%{field} a, x and z are missing or empty": "%{field} a, x and z are missing or empty",
   "%{field} subfields a, x, and z missing from field": "%{field} subfields a, x, and z missing from field",
   "%{fro} mapped from %{record}": "%{fro} mapped from %{record}",
   "%{props} were concatenated": "%{props} were concatenated",
@@ -441,5 +441,7 @@
   "legacy id from %{fro}": "legacy id from %{fro}",
   "naturalId mapped from %{fro}": "naturalId mapped from %{fro}",
   "no matching identifier_types in %{names}": "no matching identifier_types in %{names}",
-  "subfield present in %{linked_value_tag} but not in %{pattern_field}": "subfield present in %{linked_value_tag} but not in %{pattern_field}"
+  "subfield present in %{linked_value_tag} but not in %{pattern_field}": "subfield present in %{linked_value_tag} but not in %{pattern_field}",
+  "Provided boundwith relationship file not found": "Provided boundwith relationship file not found",
+  "Due date is before out date, or date information is missing from both": "Due date is before out date, or date information is missing from both"
 }