folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -1,26 +1,29 @@
 """The default mapper, responsible for parsing MARC21 records acording to the
 FOLIO community specifications"""
-import json
+
 import logging
 import sys
 import time
 import typing
 import uuid
-import xml.etree.ElementTree as ET
-from typing import Generator, List
+from pathlib import Path
+from typing import Dict, Generator, List
+
+import i18n
+from defusedxml.ElementTree import fromstring
 from folio_uuid.folio_namespaces import FOLIONamespaces
 from folio_uuid.folio_uuid import FolioUUID
-
-import pymarc
-import requests
 from folioclient import FolioClient
+from pymarc.record import Leader, Record
+from pymarc.field import Field
+
 from folio_migration_tools.custom_exceptions import (
     TransformationProcessError,
     TransformationRecordFailedError,
 )
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
-    FolioRelease,
+    FileDefinition,
     HridHandling,
     IlsFlavour,
     LibraryConfiguration,
@@ -29,10 +32,7 @@ from folio_migration_tools.marc_rules_transformation.conditions import Condition
 from folio_migration_tools.marc_rules_transformation.rules_mapper_base import (
     RulesMapperBase,
 )
-
-from folio_migration_tools.report_blurbs import Blurbs
-from pymarc import Field
-from pymarc.record import Record
+from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskConfigurationBase
 
 
 class BibsRulesMapper(RulesMapperBase):
@@ -41,196 +41,201 @@ class BibsRulesMapper(RulesMapperBase):
 
     def __init__(
         self,
-        folio_client,
+        folio_client: FolioClient,
         library_configuration: LibraryConfiguration,
-        task_configuration,
+        task_configuration: MarcTaskConfigurationBase,
+        statistical_codes_map: Dict[str, str] = None,
     ):
         super().__init__(
             folio_client,
             library_configuration,
-            Conditions(folio_client, self, "bibs"),
-        )
-        self.folio = folio_client
-        self.task_configuration = task_configuration
-        self.record_status = {}
-        self.unique_001s = set()
-        self.holdings_map = {}
-        self.id_map = {}
-        self.srs_recs = []
-        self.schema = self.instance_json_schema
-        self.contrib_name_types = {}
-        self.mapped_folio_fields = {}
-        self.unmapped_folio_fields = {}
-        self.alt_title_map = {}
-        logging.info(
-            f"HRID handling is set to: '{self.task_configuration.hrid_handling}'"
+            task_configuration,
+            statistical_codes_map,
+            self.get_instance_schema(folio_client),
+            Conditions(folio_client, self, "bibs", library_configuration.folio_release),
         )
-        self.hrid_handling: HridHandling = self.task_configuration.hrid_handling
         logging.info("Fetching mapping rules from the tenant")
-        rules_endpoint = (
-            "/mapping-rules"
-            if self.library_configuration.folio_release == FolioRelease.juniper
-            else "/mapping-rules/marc-bib"
-        )
-        self.mappings = self.folio.folio_get_single_object(rules_endpoint)
+        rules_endpoint = "/mapping-rules/marc-bib"
+        self.mappings = self.folio_client.folio_get_single_object(rules_endpoint)
         logging.info("Fetching valid language codes...")
         self.language_codes = list(self.fetch_language_codes())
-        self.unmapped_tags = {}
-        self.unmapped_conditions = {}
-        self.instance_relationships = {}
-        self.instance_relationship_types = {}
-        self.other_mode_of_issuance_id = get_unspecified_mode_of_issuance(self.folio)
+        self.instance_relationships: dict = {}
+        self.instance_relationship_types: dict = {}
+        self.other_mode_of_issuance_id = get_unspecified_mode_of_issuance(self.folio_client)
+        self.data_import_marc = self.task_configuration.data_import_marc
+        if self.data_import_marc:
+            self.hrid_handler.deactivate035_from001 = True
         self.start = time.time()
 
-    def perform_initial_preparation(self, marc_record: pymarc.Record, legacy_ids):
-        folio_instance = {
-            "metadata": self.folio.get_metadata_construct(),
-        }
+    def perform_initial_preparation(self, file_def: FileDefinition, marc_record: Record, legacy_ids: List[str]):
+        folio_instance = {}
         folio_instance["id"] = str(
             FolioUUID(
-                str(self.folio_client.okapi_url),
+                self.base_string_for_folio_uuid,
                 FOLIONamespaces.instances,
                 str(legacy_ids[-1]),
             )
         )
-        leader_05 = marc_record.leader[5]
-        self.migration_report.add(Blurbs.RecordStatus, leader_05 or "Empty")
-        self.handle_hrid(folio_instance, marc_record, legacy_ids)
-        if leader_05 == "d":
-            Helper.log_data_issue(
-                legacy_ids, "d in leader. Is this correct?", marc_record.leader
+        if (
+            all([self.create_source_records, file_def.create_source_records])
+            or self.hrid_handler.handling == HridHandling.preserve001
+        ):
+            self.hrid_handler.handle_hrid(
+                FOLIONamespaces.instances,
+                folio_instance,
+                marc_record,
+                legacy_ids,
             )
+        self.handle_leader_05(marc_record, legacy_ids)
+        if self.task_configuration.add_administrative_notes_with_legacy_ids:
+            for legacy_id in legacy_ids:
+                self.add_legacy_id_to_admin_note(folio_instance, legacy_id)
+
         return folio_instance
 
-    def parse_bib(self, legacy_ids, marc_record: pymarc.Record, suppressed: bool):
+    def handle_leader_05(self, marc_record: Record, legacy_ids: List[str]):
+        leader_05 = marc_record.leader[5] or "Empty"
+        self.migration_report.add("RecordStatus", i18n.t("Original value") + f": {leader_05}")
+        if leader_05 not in ["a", "c", "d", "n", "p"]:
+            marc_record.leader = Leader(f"{marc_record.leader[:5]}c{marc_record.leader[6:]}")
+            self.migration_report.add(
+                "RecordStatus", i18n.t("Changed %{a} to %{b}", a=leader_05, b="c")
+            )
+        if leader_05 == "d":
+            Helper.log_data_issue(legacy_ids, "d in leader. Is this correct?", marc_record.leader)
+
+    def parse_record(
+        self, marc_record: Record, file_def: FileDefinition, legacy_ids: List[str]
+    ) -> list[dict]:
         """Parses a bib recod into a FOLIO Inventory instance object
         Community mapping suggestion: https://bit.ly/2S7Gyp3
-         This is the main function"""
+         This is the main function
+
+        Args:
+            marc_record (Record): _description_
+            file_def (FileDefinition): _description_
+            legacy_ids (List[str]): List of legacy ids in record
+
+        Returns:
+            dict: _description_
+        """
         self.print_progress()
-        ignored_subsequent_fields = set()
+        ignored_subsequent_fields: set = set()
         bad_tags = set(self.task_configuration.tags_to_delete)  # "907"
-        folio_instance = self.perform_initial_preparation(marc_record, legacy_ids)
-        for marc_field in marc_record:
-            self.report_marc_stats(
-                marc_field, bad_tags, legacy_ids, ignored_subsequent_fields
-            )
-            if marc_field.tag not in ignored_subsequent_fields:
-                self.process_marc_field(
-                    folio_instance,
-                    marc_field,
-                    ignored_subsequent_fields,
-                    legacy_ids,
-                )
+        folio_instance = self.perform_initial_preparation(file_def, marc_record, legacy_ids)
+        if self.data_import_marc:
+            self.simple_bib_map(folio_instance, marc_record, ignored_subsequent_fields, legacy_ids)
+        else:
+            for marc_field in marc_record:
+                self.report_marc_stats(marc_field, bad_tags, legacy_ids, ignored_subsequent_fields)
+                if marc_field.tag not in ignored_subsequent_fields:
+                    self.process_marc_field(
+                        folio_instance,
+                        marc_field,
+                        ignored_subsequent_fields,
+                        legacy_ids,
+                    )
 
-        self.perform_additional_parsing(
-            folio_instance, marc_record, legacy_ids, suppressed
-        )
+        self.perform_additional_parsing(folio_instance, marc_record, legacy_ids, file_def)
         clean_folio_instance = self.validate_required_properties(
             "-".join(legacy_ids), folio_instance, self.schema, FOLIONamespaces.instances
         )
         self.dedupe_rec(clean_folio_instance)
         marc_record.remove_fields(*list(bad_tags))
-        self.report_folio_mapping(clean_folio_instance, self.instance_json_schema)
-        if clean_folio_instance["discoverySuppress"]:
-            self.migration_report.add_general_statistics("Suppressed from discovery")
-        # TODO: trim away multiple whitespace and newlines..
-        # TODO: createDate and update date and catalogeddate
-        return clean_folio_instance
-
-    def process_marc_field(
-        self,
-        folio_instance,
-        marc_field,
-        ignored_subsequent_fields,
-        legacy_ids,
-    ):
-        if marc_field.tag == "880" and "6" in marc_field:
-            mappings = self.perform_proxy_mapping(marc_field)
-        else:
-            tags_to_ignore = {"880", "001", "008"}
-            mappings = (
-                self.mappings.get(marc_field.tag, {})
-                if marc_field.tag not in tags_to_ignore
-                else []
-            )
-        if mappings:
-            try:
-                self.map_field_according_to_mapping(
-                    marc_field, mappings, folio_instance, legacy_ids
-                )
-                if any(m.get("ignoreSubsequentFields", False) for m in mappings):
-                    ignored_subsequent_fields.add(marc_field.tag)
-            except Exception as ee:
-                logging.error(
-                    f"map_field_according_to_mapping {marc_field.tag} {marc_field.format_field()} {json.dumps(mappings)}"
-                )
-                raise ee
-
-    def report_marc_stats(
-        self, marc_field, bad_tags, legacy_ids, ignored_subsequent_fields
-    ):
-        self.migration_report.add_general_statistics("Total number of Tags processed")
-        self.report_bad_tags(marc_field, bad_tags, legacy_ids)
-        mapped = marc_field.tag in self.mappings
-        if marc_field.tag in ignored_subsequent_fields:
-            mapped = False
-        self.report_legacy_mapping(marc_field.tag, True, mapped)
-
-    def perform_proxy_mapping(self, marc_field):
-        proxy_mapping = next(iter(self.mappings.get("880", [])), [])
-        if proxy_mapping and "fieldReplacementRule" in proxy_mapping:
-            target_field = next(
-                (
-                    r["targetField"]
-                    for r in proxy_mapping["fieldReplacementRule"]
-                    if r["sourceDigits"] == marc_field["6"][:3]
-                ),
-                "",
-            )
-            mappings = self.mappings.get(target_field, {})
-
-            self.migration_report.add(
-                Blurbs.Field880Mappings,
-                f"Source digits: {marc_field['6'][:3]} Target field: {target_field}",
-            )
-        else:
-            raise TransformationProcessError(
-                "", "Mapping rules for 880 is missing. Halting"
+        self.report_folio_mapping(clean_folio_instance, self.schema)
+        return [clean_folio_instance]
+
+    def simple_bib_map(self, folio_instance: dict, marc_record: Record, ignored_subsequent_fields: set, legacy_ids: List[str]):
+        """
+        This method applies a much simplified MARC-to-instance
+        mapping to create a minimal FOLIO Instance record to be
+        used with a Data Import based MARC loading flow, rather
+        than creating SRS records during transformation.
+
+        Args:
+            folio_instance (dict): _description_
+            marc_record (Record): _description_
+            legacy_ids (List[str]): _description_
+            file_def (FileDefinition): _description_
+        """
+        main_entry_field_tags = ["100", "110", "111", "130"]
+        main_entry_fields = marc_record.get_fields(*main_entry_field_tags)
+        main_entry_fields.sort(key=lambda x: int(x.tag))
+        if len(main_entry_fields) > 1:
+            Helper.log_data_issue(
+                legacy_ids,
+                "Multiple main entry fields in record. Record will fail Data Import. Creating Instance anyway.",
+                [str(field) for field in main_entry_fields]
             )
-        return mappings
+        if not main_entry_fields:
+            main_entry_fields += marc_record.get_fields("700", "710", "711", "730")
+            main_entry_fields.sort(key=lambda x: int(x.tag))
+        if main_entry_fields:
+            self.process_marc_field(folio_instance, main_entry_fields[0], ignored_subsequent_fields, legacy_ids)
+        try:
+            self.process_marc_field(folio_instance, marc_record['245'], ignored_subsequent_fields, legacy_ids)
+        except KeyError as ke:
+            raise TransformationRecordFailedError(
+                legacy_ids,
+                "No 245 field in MARC record"
+            ) from ke
 
     def perform_additional_parsing(
         self,
         folio_instance: dict,
         marc_record: Record,
         legacy_ids: List[str],
-        suppressed: bool,
-    ):
-        """Do stuff not easily captured by the mapping rules"""
-        folio_instance["source"] = "MARC"
+        file_def: FileDefinition,
+    ) -> None:
+        """Do stuff not easily captured by the mapping rules
+
+        Args:
+            folio_instance (dict): _description_
+            marc_record (Record): _description_
+            legacy_ids (List[str]): _description_
+            file_def (FileDefinition): _description_
+        """
+        if file_def.create_source_records and self.create_source_records:
+            folio_instance["source"] = "MARC"
+        else:
+            folio_instance["source"] = "FOLIO"
         folio_instance["instanceFormatIds"] = list(
             set(self.get_instance_format_ids(marc_record, legacy_ids))
         )
-
-        folio_instance["instanceTypeId"] = self.get_instance_type_id(
-            marc_record, legacy_ids
-        )
-
-        folio_instance["modeOfIssuanceId"] = self.get_mode_of_issuance_id(
-            marc_record, legacy_ids
-        )
+        folio_instance["instanceTypeId"] = self.get_instance_type_id(marc_record, legacy_ids)
+
+        folio_instance["modeOfIssuanceId"] = self.get_mode_of_issuance_id(marc_record, legacy_ids)
+        self.handle_languages(folio_instance, marc_record, legacy_ids)
+        self.handle_suppression(folio_instance, file_def)
+        # Map statistical codes from MARC and FileDefinition, then map the IDs
+        self.map_statistical_codes(folio_instance, file_def, marc_record)
+        self.map_statistical_code_ids(legacy_ids, folio_instance)
+        self.handle_holdings(marc_record)
+        if prec_titles := folio_instance.get("precedingTitles", []):
+            self.migration_report.add("PrecedingSuccedingTitles", f"{len(prec_titles)}")
+            del folio_instance["precedingTitles"]
+        if succ_titles := folio_instance.get("succeedingTitles", []):
+            del folio_instance["succeedingTitles"]
+            self.migration_report.add("PrecedingSuccedingTitles", f"{len(succ_titles)}")
+
+    def handle_languages(self, folio_instance: Dict, marc_record: Record, legacy_ids: List[str]):
         if "languages" in folio_instance:
-            folio_instance["languages"].extend(
-                self.get_languages(marc_record, legacy_ids)
+            orig_languages = {lang: None for lang in folio_instance["languages"]}
+            orig_languages.update(
+                {lang: None for lang in self.get_languages(marc_record, legacy_ids)}
             )
+            folio_instance["languages"] = list(orig_languages.keys())
         else:
             folio_instance["languages"] = self.get_languages(marc_record, legacy_ids)
         folio_instance["languages"] = list(
             self.filter_langs(folio_instance["languages"], marc_record, legacy_ids)
         )
-        folio_instance["discoverySuppress"] = suppressed
-        folio_instance["staffSuppress"] = False
-        self.handle_holdings(marc_record)
+
+    def get_instance_schema(self, folio_client: FolioClient):
+        logging.info("Fetching Instance schema...")
+        return folio_client.get_from_github(
+            "folio-org", "mod-inventory-storage", "ramls/instance.json"
+        )
 
     def handle_holdings(self, marc_record: Record):
         if "852" in marc_record:
@@ -239,43 +244,38 @@ class BibsRulesMapper(RulesMapperBase):
             )
             f852s = (f for f in holdingsfields if f.tag == "852")
             f86xs = (
-                f
-                for f in holdingsfields
-                if f.tag in ["866", "867", "868", "865", "864", "863"]
+                f for f in holdingsfields if f.tag in ["866", "867", "868", "865", "864", "863"]
             )
             if f852s and not f86xs:
                 self.migration_report.add(
-                    Blurbs.HoldingsGenerationFromBibs,
-                    "Records with 852s but no 86X",
+                    "HoldingsGenerationFromBibs",
+                    i18n.t(
+                        "Records with %{has_many}s but no %{has_no}", has_many="852", has_no="86X"
+                    ),
                 )
             elif any(f852s):
                 self.migration_report.add(
-                    Blurbs.HoldingsGenerationFromBibs,
-                    "Records with both 852s and at least one 86X",
+                    "HoldingsGenerationFromBibs",
+                    i18n.t(
+                        "Records with both %{has_many}s and at least one %{has_one}",
+                        has_one="86X",
+                        has_many="852",
+                    ),
                 )
 
             elif any(f86xs):
                 self.migration_report.add(
-                    Blurbs.HoldingsGenerationFromBibs,
-                    "Records without 852s but with 86X",
+                    "HoldingsGenerationFromBibs",
+                    i18n.t("Records without %{has_no}s but with %{has}", has="86X", has_no="852"),
                 )
 
     def wrap_up(self):
         logging.info("Mapper wrapping up")
-        self.store_hrid_settings()
-
-    def report_bad_tags(self, marc_field, bad_tags, legacy_ids):
-        if (
-            (not marc_field.tag.isnumeric())
-            and marc_field.tag != "LDR"
-            and marc_field.tag not in bad_tags
-        ):
-            self.migration_report.add(Blurbs.NonNumericTagsInRecord, marc_field.tag)
-            message = "Non-numeric tags in records"
-            Helper.log_data_issue(legacy_ids, message, marc_field.tag)
-            bad_tags.add(marc_field.tag)
+        if self.create_source_records:
+            if self.task_configuration.update_hrid_settings:
+                self.hrid_handler.store_hrid_settings()
 
-    def get_instance_type_id(self, marc_record, legacy_id):
+    def get_instance_type_id(self, marc_record: Record, legacy_ids: List[str]) -> str:
         return_id = ""
 
         def get_folio_id_by_name(f336a: str):
@@ -283,40 +283,37 @@ class BibsRulesMapper(RulesMapperBase):
             match = next(
                 (
                     f["id"]
-                    for f in self.folio.instance_types
+                    for f in self.folio_client.instance_types
                     if f["name"].lower().replace(" ", "") == match_template
                 ),
                 "",
             )
             if match:
                 self.migration_report.add(
-                    Blurbs.RecourceTypeMapping,
-                    f"336$a - Successful matching on  {match_template} ({f336a})",
+                    "RecourceTypeMapping",
+                    "336$a - "
+                    + i18n.t("Successful matching on %{criteria}", criteria=match_template)
+                    + f" ({f336a})",
                 )
             else:
                 self.migration_report.add(
-                    Blurbs.RecourceTypeMapping,
-                    f"336$a - Unsuccessful matching on  {match_template} ({f336a})",
+                    "RecourceTypeMapping",
+                    "336$a - "
+                    + i18n.t("Unsuccessful matching on %{criteria}", criteria=match_template)
+                    + f" ({f336a})",
                 )
                 Helper.log_data_issue(
-                    legacy_id,
+                    legacy_ids,
                     "instance type name (336$a) -Unsuccessful matching",
                     f336a,
                 )
             return match
 
-        if not self.folio.instance_types:
+        if not self.folio_client.instance_types:
             raise TransformationProcessError("", "No instance_types setup in tenant")
 
         if "336" in marc_record and "b" not in marc_record["336"]:
-            self.migration_report.add(
-                Blurbs.RecourceTypeMapping, "Subfield b not in 336"
-            )
-            Helper.log_data_issue(
-                legacy_id,
-                "Subfield b not in 336",
-                "",
-            )
+            self.migration_report.add("RecourceTypeMapping", i18n.t("Subfield b not in 336"))
             if "a" in marc_record["336"]:
                 return_id = get_folio_id_by_name(marc_record["336"]["a"])
 
@@ -324,215 +321,172 @@ class BibsRulesMapper(RulesMapperBase):
             f336_b = marc_record["336"]["b"].lower().replace(" ", "")
             f336_b_norm = f336_b.lower().replace(" ", "")
             t = self.conditions.get_ref_data_tuple_by_code(
-                self.folio.instance_types,
+                self.folio_client.instance_types,
                 "instance_types",
                 f336_b_norm,
             )
             if not t:
                 self.migration_report.add(
-                    Blurbs.RecourceTypeMapping,
-                    f"336$b - Code {f336_b_norm} ('{f336_b}') not found in FOLIO ",
+                    "RecourceTypeMapping",
+                    "336$b - "
+                    + i18n.t(
+                        "Code %{code} ('%{code_raw}') not found in FOLIO ",
+                        code=f336_b_norm,
+                        code_raw=f336_b,
+                    ),
                 )
                 Helper.log_data_issue(
-                    legacy_id,
-                    "instance type code (336$b) not found in FOLIO",
+                    legacy_ids,
+                    i18n.t("instance type code (%{code}) not found in FOLIO", code="336$b"),
                     f336_b,
                 )
             else:
                 self.migration_report.add(
-                    Blurbs.RecourceTypeMapping,
-                    f'336$b {t[1]} mapped from {marc_record["336"]["b"]}',
+                    "RecourceTypeMapping",
+                    "336$b "
+                    + i18n.t(
+                        "%{fro} mapped from %{record}", fro=t[1], record=marc_record["336"]["b"]
+                    ),
                 )
                 return_id = t[0]
 
         if not return_id:
             t = self.conditions.get_ref_data_tuple_by_code(
-                self.folio.instance_types, "instance_types", "zzz"
+                self.folio_client.instance_types, "instance_types", "zzz"
             )
             return_id = t[0]
         return return_id
 
-    def get_instance_format_ids(self, marc_record, legacy_id):
-        # Lambdas
-        def get_folio_id(code: str):
-            try:
-                match = next(
-                    f for f in self.folio.instance_formats if f["code"] == code
-                )
-                self.migration_report.add(
-                    Blurbs.InstanceFormat,
-                    f"Successful match  - {code}->{match['name']}",
-                )
-                return match["id"]
-            except Exception:
-                # TODO: Distinguish between generated codes and proper 338bs
-                Helper.log_data_issue(
-                    legacy_id, "Instance format Code not found in FOLIO", code
-                )
-                self.migration_report.add(
-                    Blurbs.InstanceFormat,
-                    f"Code '{code}' not found in FOLIO",
-                )
-                return ""
+    def get_instance_format_id_by_code(self, legacy_ids: List[str], code: str):
+        try:
+            match = next(f for f in self.folio_client.instance_formats if f["code"] == code)
+            self.migration_report.add(
+                "InstanceFormat",
+                i18n.t("Successful match") + f"  - {code}->{match['name']}",
+            )
+            return match["id"]
+        except Exception:
+            # TODO: Distinguish between generated codes and proper 338bs
+            Helper.log_data_issue(legacy_ids, "Instance format Code not found in FOLIO", code)
+            self.migration_report.add(
+                "InstanceFormat",
+                i18n.t("Code '%{code}' not found in FOLIO", code=code),
+            )
+            return ""
 
-        def get_folio_id_by_name(f337a: str, f338a: str, legacy_id: str):
-            f337a = f337a.lower().replace(" ", "")
-            f338a = f338a.lower().replace(" ", "")
-            match_template = f"{f337a} -- {f338a}"
-            try:
-                match = next(
-                    f
-                    for f in self.folio.instance_formats
-                    if f["name"].lower().replace(" ", "") == match_template
-                )
-                self.migration_report.add(
-                    Blurbs.InstanceFormat,
-                    f"Successful matching on 337$a & 338$a - {match_template}->{match['name']}",
-                )
-                return match["id"]
-            except Exception:
-                Helper.log_data_issue(
-                    legacy_id,
-                    "Unsuccessful matching on 337$a and 338$a",
-                    match_template,
+    def get_instance_format_id_by_name(self, f337a: str, f338a: str, legacy_ids: List[str]):
+        f337a = f337a.lower().strip()
+        f338a = f338a.lower().strip()
+        match_template = f"{f337a} -- {f338a}"
+        try:
+            match = next(
+                f
+                for f in self.folio_client.instance_formats
+                if f["name"].lower() == match_template
+            )
+            self.migration_report.add(
+                "InstanceFormat",
+                i18n.t(
+                    "Successful matching on %{criteria_1} and %{criteria_2}",
+                    criteria_1="337$a",
+                    criteria_2="338$a",
                 )
-                self.migration_report.add(
-                    Blurbs.InstanceFormat,
-                    f"Unsuccessful matching on 337$a and 338$a - {match_template}",
+                + f" - {match_template}->{match['name']}",
+            )
+            return match["id"]
+        except Exception:
+            Helper.log_data_issue(
+                legacy_ids,
+                "Unsuccessful matching on 337$a and 338$a",
+                match_template,
+            )
+            self.migration_report.add(
+                "InstanceFormat",
+                i18n.t(
+                    "Unsuccessful matching on %{criteria_1} and %{criteria_2}",
+                    criteria_1="337$a",
+                    criteria_2="338$a",
                 )
-                return ""
+                + f" - {match_template}",
+            )
+            return ""
+
+    def f338_source_is_rda_carrier(self, field: Field):
+        if "2" not in field:
+            self.migration_report.add(
+                "InstanceFormat",
+                ("Instance Format not mapped from field since 338$2 is missing"),
+            )
+            return False
+        elif field["2"].strip().startswith("rdacarrier"):
+            return True
+        self.migration_report.add(
+            "InstanceFormat",
+            ("InstanceFormat not mapped since 338$2 (Source) " f"is set to {field['2']}. "),
+        )
+        return False
 
+    def get_instance_format_ids_from_a(
+        self, field_index: int, f_338: Field, all_337s: List[Field], legacy_id: List[str]
+    ):
+        self.migration_report.add(
+            "InstanceFormat",
+            i18n.t("338$b is missing. Will try parse from 337$a and 338$a"),
+        )
+        for a in f_338.get_subfields("a"):
+            corresponding_337 = all_337s[field_index] if field_index < len(all_337s) else None
+            if corresponding_337 and "a" in corresponding_337:
+                if fmt_id := self.get_instance_format_id_by_name(
+                    corresponding_337["a"], a, legacy_id
+                ):
+                    yield fmt_id
+
+    def get_instance_format_ids(self, marc_record: Record, legacy_id: List[str]):
         all_337s = marc_record.get_fields("337")
         all_338s = marc_record.get_fields("338")
-        for fidx, f in enumerate(all_338s):
-            source = f["2"] if "2" in f else "Not set"
-            if not source.strip().startswith("rdacarrier"):
-                self.migration_report.add(
-                    Blurbs.InstanceFormat,
-                    (
-                        "InstanceFormat not mapped since 338$2 (Source) "
-                        f"is set to {source}. "
-                    ),
-                )
-            else:
-                if "b" not in f and "a" in f:
-                    self.migration_report.add(
-                        Blurbs.InstanceFormat,
-                        "338$b is missing. Will try parse from 337$a and 338$b",
+        for fidx, f_338 in enumerate(all_338s):
+            if self.f338_source_is_rda_carrier(f_338):
+                if "b" not in f_338 and "a" in f_338:
+                    yield from self.get_instance_format_ids_from_a(
+                        fidx, f_338, all_337s, legacy_id
                     )
-                    for sfidx, a in enumerate(f.get_subfields("a")):
-                        corresponding_337 = (
-                            all_337s[fidx] if fidx < len(all_337s) else None
-                        )
-                        if corresponding_337 and "a" in corresponding_337:
-                            fmt_id = get_folio_id_by_name(
-                                corresponding_337["a"], a, legacy_id
-                            )
-                            if fmt_id:
-                                yield fmt_id
 
-                for sfidx, b in enumerate(f.get_subfields("b")):
+                for sfidx, b in enumerate(f_338.get_subfields("b")):
                     b = b.replace(" ", "")
-                    if len(b) == 2:  # Normal 338b. should be able to map this
-                        yield get_folio_id(b)
+                    if len(b) == 2:
+                        # Normal 338b. should be able to map this
+                        yield self.get_instance_format_id_by_code(legacy_id, b)
                     elif len(b) == 1:
-                        corresponding_337 = (
-                            all_337s[fidx] if fidx < len(all_337s) else None
-                        )
-                        if (
-                            not corresponding_337
-                        ):  # No matching 337. No use mapping the 338
-                            s = "No corresponding 337 to 338 even though 338$b was one charachter code"
+                        corresponding_337 = all_337s[fidx] if fidx < len(all_337s) else None
+                        if not corresponding_337:
+                            # No matching 337. No use mapping the 338
+                            s = i18n.t(
+                                "No corresponding 337 to 338 even though 338$b was one character"
+                            )
                             Helper.log_data_issue(legacy_id, s, b)
                             self.migration_report.add(
-                                Blurbs.InstanceFormat,
+                                "InstanceFormat",
                                 s,
                             )
-                        else:  # Corresponding 337. Try to combine the codes.
+                        else:
+                            # Corresponding 337. Try to combine the codes.
                             corresponding_b = (
                                 corresponding_337.get_subfields("b")[sfidx]
                                 if sfidx < len(corresponding_337.get_subfields("b"))
                                 else None
                             )
                             if not corresponding_b:
-                                s = "No corresponding $b in corresponding 338"
+                                s = i18n.t("No corresponding $b in corresponding 338")
                                 Helper.log_data_issue(legacy_id, s, "")
-                                self.migration_report.add(Blurbs.InstanceFormat, s)
+                                self.migration_report.add("InstanceFormat", s)
                             else:
                                 combined_code = (corresponding_b + b).strip()
                                 if len(combined_code) == 2:
-                                    yield get_folio_id(combined_code)
-
-    def handle_hrid(self, folio_instance, marc_record: Record, legacy_ids) -> None:
-        """Create HRID if not mapped. Add hrid as MARC record 001"""
-        if self.hrid_handling == HridHandling.default or "001" not in marc_record:
-            num_part = str(self.instance_hrid_counter).zfill(11)
-            folio_instance["hrid"] = f"{self.instance_hrid_prefix}{num_part}"
-            new_001 = Field(tag="001", data=folio_instance["hrid"])
-            try:
-                f_001 = marc_record["001"].value()
-                f_003 = (
-                    marc_record["003"].value().strip() if "003" in marc_record else ""
-                )
-                self.migration_report.add(
-                    Blurbs.HridHandling, f'Values in 003: {f_003 or "Empty"}'
-                )
-
-                if self.task_configuration.deactivate035_from001:
-                    self.migration_report.add(
-                        Blurbs.HridHandling, "035 generation from 001 turned off"
-                    )
-                else:
-                    str_035 = f"({f_003}){f_001}" if f_003 else f"{f_001}"
-                    new_035 = Field(
-                        tag="035",
-                        indicators=[" ", " "],
-                        subfields=["a", str_035],
-                    )
-                    marc_record.add_ordered_field(new_035)
-                    self.migration_report.add(Blurbs.HridHandling, "Added 035 from 001")
-                marc_record.remove_fields("001")
-
-            except Exception:
-                if "001" in marc_record:
-                    s = "Failed to create 035 from 001"
-                    self.migration_report.add(Blurbs.HridHandling, s)
-                    Helper.log_data_issue(legacy_ids, s, marc_record["001"])
-                else:
-                    self.migration_report.add(
-                        Blurbs.HridHandling, "Legacy bib records without 001"
-                    )
-            marc_record.add_ordered_field(new_001)
-            self.migration_report.add(
-                Blurbs.HridHandling, "Created HRID using default settings"
-            )
-            self.instance_hrid_counter += 1
-        elif self.hrid_handling == HridHandling.preserve001:
-            value = marc_record["001"].value()
-            if value in self.unique_001s:
-                self.migration_report.add(
-                    Blurbs.HridHandling, "Duplicate 001. Creating HRID instead"
-                )
-                Helper.log_data_issue(
-                    legacy_ids,
-                    "Duplicate 001 for record. HRID created for record",
-                    value,
-                )
-                num_part = str(self.instance_hrid_counter).zfill(11)
-                folio_instance["hrid"] = f"{self.instance_hrid_prefix}{num_part}"
-                new_001 = Field(tag="001", data=folio_instance["hrid"])
-                marc_record.add_ordered_field(new_001)
-                self.instance_hrid_counter += 1
-            else:
-                self.unique_001s.add(value)
-                folio_instance["hrid"] = value
-                self.migration_report.add(Blurbs.HridHandling, "Took HRID from 001")
-        else:
-            raise TransformationProcessError(
-                "", f"Unknown HRID handling: {self.hrid_handling}"
-            )
+                                    yield self.get_instance_format_id_by_code(
+                                        legacy_id, combined_code
+                                    )
 
-    def get_mode_of_issuance_id(self, marc_record: Record, legacy_id: str) -> str:
+    def get_mode_of_issuance_id(self, marc_record: Record, legacy_ids: List[str]) -> str:
         level = marc_record.leader[7]
         try:
             name = "unspecified"
@@ -545,30 +499,29 @@ class BibsRulesMapper(RulesMapperBase):
             ret = next(
                 (
                     i["id"]
-                    for i in self.folio.modes_of_issuance
-                    if str(name).lower() == i["name"].lower()
+                    for i in self.folio_client.modes_of_issuance
+                    if name.lower() == i["name"].lower()
                 ),
                 "",
             )
-            self.migration_report.add(
-                Blurbs.MatchedModesOfIssuanceCode, f"{name} -- {ret}"
-            )
+
+            self.migration_report.add("MatchedModesOfIssuanceCode", f"{name} -- {ret}")
+
             if not ret:
                 self.migration_report.add(
-                    Blurbs.MatchedModesOfIssuanceCode,
-                    f"Unmatched level: {level}",
+                    "MatchedModesOfIssuanceCode", i18n.t("Unmatched level") + f": {level}"
                 )
+
                 return self.other_mode_of_issuance_id
             return ret
         except IndexError:
             self.migration_report.add(
-                Blurbs.PossibleCleaningTasks, f"No Leader[7] in {legacy_id}"
+                "PossibleCleaningTasks", i18n.t("No Leader[7] in") + f" {legacy_ids}"
             )
+
             return self.other_mode_of_issuance_id
         except StopIteration as ee:
-            logging.exception(
-                f"{marc_record.leader} {list(self.folio.modes_of_issuance)}"
-            )
+            logging.exception(f"{marc_record.leader} {list(self.folio_client.modes_of_issuance)}")
             raise ee from ee
 
     def get_nature_of_content(self, marc_record: Record) -> List[str]:
@@ -579,63 +532,74 @@ class BibsRulesMapper(RulesMapperBase):
             return "".join(marc_record["008"].data[35:38])
         return ""
 
-    def get_languages_041(self, marc_record, legacy_id):
-        languages = set()
+    def get_languages_041(self, marc_record: Record, legacy_id: List[str]) -> Dict[str, None]:
+        languages = dict()
         lang_fields = marc_record.get_fields("041")
         if not any(lang_fields):
-            return set()
+            return dict()
         subfields = "abdefghjkmn"
         for lang_tag in lang_fields:
             if "2" in lang_tag:
-                self.migration_report.add(Blurbs.LanguageCodeSources, lang_tag["2"])
-                logging.info(
-                    "Field with other Language code\t%s\t%s",
-                    marc_record["001"],
-                    lang_tag.value(),
+                self.migration_report.add("LanguageCodeSources", lang_tag["2"])
+                Helper.log_data_issue(
+                    legacy_id, "Field with other Language code", lang_tag.value()
                 )
             lang_codes = lang_tag.get_subfields(*list(subfields))
             for lang_code in lang_codes:
                 lang_code = str(lang_code).lower().replace(" ", "")
                 langlength = len(lang_code)
                 if langlength == 3:
-                    languages.add(lang_code.replace(" ", ""))
+                    languages[lang_code.replace(" ", "")] = None
                 elif langlength > 3 and langlength % 3 == 0:
                     lc = lang_code.replace(" ", "")
-                    new_codes = (lc[i : i + 3] for i in range(0, len(lc), 3))
+                    new_codes = {lc[i : i + 3]: None for i in range(0, len(lc), 3)}
                     languages.update(new_codes)
-                    languages.discard(lang_code)
-            languages.update()
-        languages = set(
-            self.filter_langs(filter(None, languages), marc_record, legacy_id)
-        )
+        languages = {
+            str(lang): None
+            for lang in self.filter_langs(
+                list(filter(None, languages.keys())), marc_record, legacy_id
+            )
+            if lang
+        }
         return languages
 
-    def get_languages(self, marc_record: Record, legacy_id: str) -> List[str]:
-        """Get languages and tranforms them to correct codes"""
+    def get_languages(self, marc_record: Record, legacy_id: List[str]) -> List[str]:
+        """Get languages and tranforms them to correct codes
+
+        Args:
+            marc_record (Record): A pymarc Record object
+            legacy_id (List[str]): A list of legacy ids from the legacy record
+
+        Returns:
+            List[str]: List of language codes
+        """
         languages = self.get_languages_041(marc_record, legacy_id)
-        languages.add(self.get_languages_008(marc_record))
+        languages[self.get_languages_008(marc_record)] = None
         for lang in languages:
-            self.migration_report.add(Blurbs.LanguagesInRecords, lang)
+            self.migration_report.add("LanguagesInRecords", lang)
         return list(languages)
 
     def fetch_language_codes(self) -> Generator[str, None, None]:
-        """fetches the list of standardized language codes from LoC"""
-        url = "https://www.loc.gov/standards/codelists/languages.xml"
-        tree = ET.fromstring(requests.get(url).content)
+        """Loads the  list of standardized language codes from LoC
+
+        Yields:
+            Generator[str, None, None]: _description_
+        """
+        path = Path(__file__).parent / "loc_language_codes.xml"
+        with open(path) as f:
+            lines = "".join(f.readlines())
+        tree = fromstring(lines)
         name_space = "{info:lc/xmlns/codelist-v1}"
         xpath_expr = "{0}languages/{0}language/{0}code".format(name_space)
         for code in tree.findall(xpath_expr):
             yield code.text
 
     def filter_langs(
-        self, language_values: List[str], marc_record: Record, index_or_legacy_id
+        self, language_values: List[str], marc_record: Record, index_or_legacy_id: List[str]
     ) -> typing.Generator:
         forbidden_values = ["###", "zxx", "n/a", "N/A", "|||"]
         for language_value in language_values:
-            if (
-                language_value in self.language_codes
-                and language_value not in forbidden_values
-            ):
+            if language_value in self.language_codes and language_value not in forbidden_values:
                 yield language_value
             elif language_value == "jap":
                 yield "jpn"
@@ -653,68 +617,51 @@ class BibsRulesMapper(RulesMapperBase):
                 m = "Unrecognized language codes in record"
                 Helper.log_data_issue(index_or_legacy_id, m, language_value)
                 self.migration_report.add(
-                    Blurbs.UnrecognizedLanguageCodes,
+                    "UnrecognizedLanguageCodes",
                     f"{m}: {language_value}",
                 )
 
-    def get_legacy_ids(
-        self, marc_record: Record, ils_flavour: IlsFlavour, index_or_legacy_id: str
-    ) -> List[str]:
+    def get_legacy_ids(self, marc_record: Record, idx: int) -> List[str]:
+        ils_flavour: IlsFlavour = self.task_configuration.ils_flavour
         if ils_flavour in {IlsFlavour.sierra, IlsFlavour.millennium}:
             return get_iii_bib_id(marc_record)
         elif ils_flavour == IlsFlavour.tag907y:
-            try:
-                return list(set(marc_record["907"].get_subfields("a", "y")))
-            except Exception as e:
-                raise TransformationRecordFailedError(
-                    index_or_legacy_id,
-                    (
-                        "907 $y and $a is missing is missing, although they is "
-                        "required for this legacy ILS choice"
-                    ),
-                    marc_record.as_json(),
-                ) from e
+            return RulesMapperBase.get_bib_id_from_907y(marc_record, idx)
         elif ils_flavour == IlsFlavour.tagf990a:
-            res = {f["a"].strip() for f in marc_record.get_fields("990") if "a" in f}
-            if marc_record["001"].format_field().strip():
-                res.add(marc_record["001"].format_field().strip())
-            if any(res):
-                self.migration_report.add_general_statistics("legacy id from 990$a")
-                return list(res)
+            return RulesMapperBase.get_bib_id_from_990a(marc_record, idx)
         elif ils_flavour == IlsFlavour.aleph:
             return self.get_aleph_bib_id(marc_record)
         elif ils_flavour in {IlsFlavour.voyager, "voyager", IlsFlavour.tag001}:
-            try:
-                return [marc_record["001"].format_field().strip()]
-            except Exception as e:
-                raise TransformationRecordFailedError(
-                    index_or_legacy_id,
-                    "001 is missing, although it is required for Voyager migrations",
-                    marc_record.as_json(),
-                ) from e
+            return RulesMapperBase.get_bib_id_from_001(marc_record, idx)
         elif ils_flavour == IlsFlavour.koha:
             try:
                 return [marc_record["999"]["c"]]
             except Exception as e:
                 raise TransformationRecordFailedError(
-                    index_or_legacy_id,
+                    idx,
                     "999 $c is missing, although it is required for this legacy ILS choice",
                     marc_record.as_json(),
                 ) from e
+        elif ils_flavour == IlsFlavour.custom:
+            return get_custom_bib_id(marc_record, self.task_configuration.custom_bib_id_field)
         elif ils_flavour == IlsFlavour.none:
             return [str(uuid.uuid4())]
         else:
             raise TransformationProcessError("", f"ILS {ils_flavour} not configured")
 
-    def get_aleph_bib_id(self, marc_record: Record):
-        res = {f["b"].strip() for f in marc_record.get_fields("998") if "b" in f}
+    def get_aleph_bib_id(self, marc_record: Record) -> List[str]:
+        res = {f["b"].strip(): None for f in marc_record.get_fields("998") if "b" in f}
         if any(res):
-            self.migration_report.add_general_statistics("legacy id from 998$b")
+            self.migration_report.add_general_statistics(
+                i18n.t("legacy id from %{fro}", fro="998$b")
+            )
             return list(res)
         else:
             try:
                 ret = [marc_record["001"].format_field().strip()]
-                self.migration_report.add_general_statistics("legacy id from 001")
+                self.migration_report.add_general_statistics(
+                    i18n.t("legacy id from %{fro}", fro="001")
+                )
                 return ret
             except Exception as e:
                 raise TransformationRecordFailedError(
@@ -724,7 +671,7 @@ class BibsRulesMapper(RulesMapperBase):
                 ) from e
 
 
-def get_unspecified_mode_of_issuance(folio_client: FolioClient):
+def get_unspecified_mode_of_issuance(folio_client: FolioClient) -> str:
     m_o_is = list(folio_client.modes_of_issuance)
     if not any(m_o_is):
         logging.critical("No Modes of issuance set up in tenant. Quitting...")
@@ -738,7 +685,26 @@ def get_unspecified_mode_of_issuance(folio_client: FolioClient):
     return next(i["id"] for i in m_o_is if i["name"].lower() == "unspecified")
 
 
-def get_iii_bib_id(marc_record: Record):
+def get_custom_bib_id(marc_record: Record, field_string: str):
+    if field_keys := field_string.split("$", maxsplit=1):
+        try:
+            if len(field_keys) == 2:
+                return [marc_record[field_keys[0]][field_keys[1]]]
+            else:
+                return [marc_record[field_keys[0]]]
+        except Exception as e:
+            raise TransformationRecordFailedError(
+                "unknown identifier",
+                f"{field_string} is missing from record but is required in all records",
+                marc_record.as_json(),
+            ) from e
+    else:
+        raise TransformationProcessError(
+            "", 'Critical process issue. No "customBibIdField" specified in task configuration.'
+        )
+
+
+def get_iii_bib_id(marc_record: Record) -> List[str]:
     try:
         return [marc_record["907"]["a"]]
     except Exception as e: