folio-migration-tools 1.9.10__py3-none-any.whl → 1.10.0b1__py3-none-any.whl

folio_migration_tools/migration_tasks/migration_task_base.py

@@ -47,15 +47,13 @@ class MigrationTaskBase:
         logging.info("MigrationTaskBase init")
         self.start_datetime = datetime.now(timezone.utc)
         self.task_configuration = task_configuration
-        logging.info(self.task_configuration.json(indent=4))
+        logging.info(self.task_configuration.model_dump_json(indent=4))
         self.folio_client: FolioClient = folio_client
         self.ecs_tenant_id = (
             task_configuration.ecs_tenant_id or library_configuration.ecs_tenant_id
         )
-        self.ecs_tenant_header = (
-            {"x-okapi-tenant": self.ecs_tenant_id} if self.ecs_tenant_id else {}
-        )
-        self.folio_client.okapi_headers.update(self.ecs_tenant_header)
+        self.folio_client.tenant_id = self.ecs_tenant_id
+
         self.central_folder_structure: Optional[FolderStructure] = None
         if library_configuration.is_ecs and library_configuration.ecs_central_iteration_identifier:
             self.central_folder_structure = FolderStructure(
@@ -133,11 +131,7 @@ class MigrationTaskBase:
             TransformationProcessError: _description_
 
         """
-        files = [
-            source_path / f.file_name
-            for f in file_defs
-            if isfile(source_path / f.file_name)
-        ]
+        files = [source_path / f.file_name for f in file_defs if isfile(source_path / f.file_name)]
         ret_str = ", ".join(f.file_name for f in file_defs)
 
         if files and len(files) < len(file_defs):
@@ -162,12 +156,13 @@ class MigrationTaskBase:
         This is in the base class because multiple tasks need it. It exists because instances in an ECS environment
         are transformed for the central and data tenants separately, but the data tenants need to know about
         the central tenant instance ids. This is a bit of a hack, but it works for now.
-        """
+        """  # noqa: E501
         map_files = []
         instance_id_map = {}
         if self.library_configuration.is_ecs and self.central_folder_structure:
             logging.info(
-                "Loading ECS central tenant instance id map from %s", self.central_folder_structure.instance_id_map_path
+                "Loading ECS central tenant instance id map from %s",
+                self.central_folder_structure.instance_id_map_path,
             )
             instance_id_map = self.load_id_map(
                 self.central_folder_structure.instance_id_map_path,
@@ -176,7 +171,7 @@ class MigrationTaskBase:
             map_files.append(str(self.central_folder_structure.instance_id_map_path))
             logging.info(
                 "Loading member tenant isntance id map from %s",
-                self.folder_structure.instance_id_map_path
+                self.folder_structure.instance_id_map_path,
             )
         instance_id_map = self.load_id_map(
             self.folder_structure.instance_id_map_path,
@@ -190,13 +185,11 @@ class MigrationTaskBase:
         return instance_id_map
 
     @staticmethod
-    def load_id_map(map_path, raise_if_empty=False, existing_id_map={}):
+    def load_id_map(map_path, raise_if_empty=False, existing_id_map=None):
         if not isfile(map_path):
-            logging.warning(
-                "No legacy id map found at %s. Will build one from scratch", map_path
-            )
+            logging.warning("No legacy id map found at %s. Will build one from scratch", map_path)
             return {}
-        id_map = existing_id_map
+        id_map = existing_id_map or {}
         loaded_rows = len(id_map)
         with open(map_path) as id_map_file:
             for index, json_string in enumerate(id_map_file, start=1):
@@ -247,9 +240,7 @@ class MigrationTaskBase:
         else:
             logger.setLevel(logging.INFO)
             stream_handler.setLevel(logging.INFO)
-            stream_handler.addFilter(
-                ExcludeLevelFilter(30)
-            )  # Exclude warnings from pymarc
+            stream_handler.addFilter(ExcludeLevelFilter(30))  # Exclude warnings from pymarc
         stream_handler.setFormatter(formatter)
         logger.addHandler(stream_handler)
 
@@ -280,22 +271,16 @@ class MigrationTaskBase:
     def setup_records_map(self, mapping_file_path):
         with open(mapping_file_path) as mapping_file:
             field_map = json.load(mapping_file)
-            logging.info(
-                "%s fields present in record mapping file", len(field_map["data"])
-            )
+            logging.info("%s fields present in record mapping file", len(field_map["data"]))
             mapped_fields = (
                 f
                 for f in field_map["data"]
                 if f["legacy_field"] and f["legacy_field"] != "Not mapped"
             )
-            logging.info(
-                "%s fields mapped in record mapping file", len(list(mapped_fields))
-            )
+            logging.info("%s fields mapped in record mapping file", len(list(mapped_fields)))
             return field_map
 
-    def log_and_exit_if_too_many_errors(
-        self, error: TransformationRecordFailedError, idx
-    ):
+    def log_and_exit_if_too_many_errors(self, error: TransformationRecordFailedError, idx):
         self.num_exeptions += 1
         error.log_it()
         if self.num_exeptions / (1 + idx) > 0.2 and self.num_exeptions > 5000:
@@ -311,9 +296,7 @@ class MigrationTaskBase:
         if num_processed > 1 and num_processed % 10000 == 0:
             elapsed = num_processed / (time.time() - start_time)
             elapsed_formatted = "{0:.4g}".format(elapsed)
-            logging.info(
-                f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} "
-            )
+            logging.info(f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} ")
 
     def do_work_marc_transformer(
         self,
@@ -322,9 +305,7 @@ class MigrationTaskBase:
         if self.folder_structure.failed_marc_recs_file.is_file():
             os.remove(self.folder_structure.failed_marc_recs_file)
             logging.info("Removed failed marc records file to prevent duplicating data")
-        with open(
-            self.folder_structure.created_objects_path, "w+"
-        ) as created_records_file:
+        with open(self.folder_structure.created_objects_path, "w+") as created_records_file:
             self.processor = MarcFileProcessor(
                 self.mapper, self.folder_structure, created_records_file
             )
@@ -377,7 +358,7 @@ class MigrationTaskBase:
 
         Returns:
             None
-        """
+        """  # noqa: E501
         current_pos = map_file.tell()
         try:
             map_file.seek(0)
@@ -391,13 +372,12 @@ class MigrationTaskBase:
                     "",
                     (
                         f"Mapping file {map_file.name} has rows with different number "
-                        f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} {', '.join(invalid_lines)})"
+                        f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} "
+                        f"{', '.join(invalid_lines)})"
                     ),
                 )
             if not valid_lines:
-                raise TransformationProcessError(
-                    "", f"Map has no rows: {map_file.name}"
-                )
+                raise TransformationProcessError("", f"Map has no rows: {map_file.name}")
         finally:
             map_file.seek(current_pos)
 
@@ -418,15 +398,12 @@ class MigrationTaskBase:
             required (bool): Whether the property is required or not
         """
         if (
-                (
-	            folio_property_name in folio_keys
-	            or required
-	            or folio_property_name.startswith("statisticalCodeIds")
-	            or folio_property_name.startswith("locationMap")
-	            or folio_property_name.startswith("fundsMap")
-                )
-                and map_file_path.is_file()
-        ):
+            folio_property_name in folio_keys
+            or required
+            or folio_property_name.startswith("statisticalCodeIds")
+            or folio_property_name.startswith("locationMap")
+            or folio_property_name.startswith("fundsMap")
+        ) and map_file_path.is_file():
             try:
                 with open(map_file_path) as map_file:
                     # Validate the structure of the mapping file
@@ -486,7 +463,7 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
         deactivate035_from001 (bool):
             Disables the default FOLIO functionality of moving the previous 001 field into a 035 field, prefixed with the value from 003.
             Default is False, meaning the functionality remains active.
-    """
+    """  # noqa: E501
 
     files: Annotated[
         List[library_configuration.FileDefinition],
@@ -500,8 +477,7 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
         Field(
             title="Create source records",
             description=(
-                "Controls whether or not to retain the MARC records in "
-                "Source Record Storage."
+                "Controls whether or not to retain the MARC records in Source Record Storage."
             ),
         ),
     ] = False
@@ -542,12 +518,14 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
             title="Statistical code mapping fields",
             description=(
                 "List of fields + subfields to be used for mapping statistical codes. "
-                "Subfields should be delimited by a \"$\" (eg. 907$a). Single repeating subfields "
-                "will be treated as unique values. Multiple subfields will be concatenated together with a space."
+                'Subfields should be delimited by a "$" (eg. 907$a). Single repeating subfields '
+                "will be treated as unique values. Multiple subfields will be concatenated "
+                "together with a space."
             ),
         ),
     ] = []
 
+
 class ExcludeLevelFilter(logging.Filter):
     def __init__(self, level):
         super().__init__()

folio_migration_tools/migration_tasks/orders_transformer.py

@@ -4,7 +4,6 @@ import json
 import logging
 import sys
 import time
-from os.path import isfile
 from typing import List, Optional, Annotated
 from pydantic import Field
 
@@ -84,50 +83,35 @@ class OrdersTransformer(MigrationTaskBase):
             Optional[str],
             Field(
                 title="Payment Status Map File Name",
-                description=(
-                    "File name for payment status mapping. "
-                    "By default is empty string."
-                ),
+                description=("File name for payment status mapping. By default is empty string."),
             ),
         ] = ""
         receipt_status_map_file_name: Annotated[
             Optional[str],
             Field(
                 title="Receipt Status Map File Name",
-                description=(
-                    "File name for receipt status mapping. "
-                    "By default is empty string."
-                ),
+                description=("File name for receipt status mapping. By default is empty string."),
             ),
         ] = ""
         workflow_status_map_file_name: Annotated[
             Optional[str],
             Field(
                 title="Workflow Status Map File Name",
-                description=(
-                    "File name for workflow status mapping. "
-                    "By default is empty string."
-                ),
+                description=("File name for workflow status mapping. By default is empty string."),
             ),
         ] = ""
         location_map_file_name: Annotated[
             Optional[str],
             Field(
                 title="Location Map File Name",
-                description=(
-                    "File name for location mapping. "
-                    "By default is empty string."
-                ),
+                description=("File name for location mapping. By default is empty string."),
             ),
         ] = ""
         funds_map_file_name: Annotated[
             Optional[str],
             Field(
                 title="Funds Map File Name",
-                description=(
-                    "File name for funds mapping. "
-                    "By default is empty string."
-                ),
+                description=("File name for funds mapping. By default is empty string."),
             ),
         ] = ""
         funds_expense_class_map_file_name: Annotated[
@@ -135,8 +119,7 @@ class OrdersTransformer(MigrationTaskBase):
             Field(
                 title="Funds Expense Class Map File Name",
                 description=(
-                    "File name for funds expense class mapping. "
-                    "By default is empty string."
+                    "File name for funds expense class mapping. By default is empty string."
                 ),
             ),
         ] = ""
@@ -216,8 +199,7 @@ class OrdersTransformer(MigrationTaskBase):
             ),
             self.load_ref_data_mapping_file(  # Required if there was is a fund.
                 "fundsMap",
-                self.folder_structure.mapping_files_folder 
-                / self.task_config.funds_map_file_name,
+                self.folder_structure.mapping_files_folder / self.task_config.funds_map_file_name,
                 self.folio_keys,
                 True,
             ),
@@ -231,26 +213,26 @@ class OrdersTransformer(MigrationTaskBase):
         )
 
     def list_source_files(self):
-        files = [
-            self.folder_structure.data_folder / self.object_type_name / f.file_name
-            for f in self.task_config.files
-            if isfile(self.folder_structure.data_folder / self.object_type_name / f.file_name)
-        ]
-        if not any(files):
-            ret_str = ",".join(f.file_name for f in self.task_config.files)
-            raise TransformationProcessError(
-                f"Files {ret_str} not found in"
-                "{self.folder_structure.data_folder} / {self.object_type_name}"
-            )
+        files = []
+        for f in self.task_config.files:
+            file_path = self.folder_structure.data_folder / self.object_type_name / f.file_name
+
+            if not file_path.is_file():
+                print(f"\n\nERROR: File defined in task not found - {f.file_name}")
+                raise TransformationProcessError(
+                    f"\n\nERROR: File defined in task not found - {f.file_name}"
+                )
+            files.append(file_path)
         logging.info("Files to process:")
         for filename in files:
             logging.info("\t%s", filename)
         return files
 
     def process_single_file(self, filename):
-        with open(filename, encoding="utf-8-sig") as records_file, open(
-            self.folder_structure.created_objects_path, "w+"
-        ) as results_file:
+        with (
+            open(filename, encoding="utf-8-sig") as records_file,
+            open(self.folder_structure.created_objects_path, "w+") as results_file,
+        ):
             self.mapper.migration_report.add_general_statistics(
                 i18n.t("Number of files processed")
             )

folio_migration_tools/migration_tasks/organization_transformer.py

@@ -50,27 +50,21 @@ class OrganizationTransformer(MigrationTaskBase):
             str,
             Field(
                 title="Migration task type",
-                description=(
-                    "The type of migration task you want to perform"
-                ),
+                description=("The type of migration task you want to perform"),
             ),
         ]
         files: Annotated[
             List[FileDefinition],
             Field(
                 title="Source files",
-                description=(
-                    "List of MARC21 files with holdings records"
-                ),
+                description=("List of MARC21 files with holdings records"),
             ),
         ]
         organization_map_path: Annotated[
             str,
             Field(
                 title="Organization map path",
-                description=(
-                    "Path to the organization map file"
-                ),
+                description=("Path to the organization map file"),
             ),
         ]
         organization_types_map_path: Annotated[
@@ -95,18 +89,14 @@ class OrganizationTransformer(MigrationTaskBase):
             Optional[str],
             Field(
                 title="Email categories map path",
-                description=(
-                    "Path to the email categories map file. By default is empty string"
-                ),
+                description=("Path to the email categories map file. By default is empty string"),
             ),
         ] = ""
         phone_categories_map_path: Annotated[
             Optional[str],
             Field(
                 title="Phone categories map path",
-                description=(
-                    "Path to the phone categories map file. By default is empty string"
-                ),
+                description=("Path to the phone categories map file. By default is empty string"),
             ),
         ] = ""
 
@@ -201,9 +191,10 @@ class OrganizationTransformer(MigrationTaskBase):
         return files
 
     def process_single_file(self, filename):
-        with open(filename, encoding="utf-8-sig") as records_file, open(
-            self.folder_structure.created_objects_path, "w+"
-        ) as results_file:
+        with (
+            open(filename, encoding="utf-8-sig") as records_file,
+            open(self.folder_structure.created_objects_path, "w+") as results_file,
+        ):
             self.mapper.migration_report.add_general_statistics(
                 i18n.t("Number of files processed")
             )

folio_migration_tools/migration_tasks/requests_migrator.py

@@ -32,7 +32,7 @@ class RequestsMigrator(MigrationTaskBase):
                 description=(
                     "Name of this migration task. The name is being used to call "
                     "the specific task, and to distinguish tasks of similar types"
-                )
+                ),
             ),
         ]
         migration_task_type: Annotated[
@@ -54,8 +54,7 @@ class RequestsMigrator(MigrationTaskBase):
             Field(
                 title="Starting row",
                 description=(
-                    "Row number to start processing data from. "
-                    "Optional, by default is first row"
+                    "Row number to start processing data from. Optional, by default is first row"
                 ),
             ),
         ] = 1
@@ -64,8 +63,7 @@ class RequestsMigrator(MigrationTaskBase):
             Field(
                 title="Item files",
                 description=(
-                    "List of files containing item data. "
-                    "Optional, by default is empty list"
+                    "List of files containing item data. Optional, by default is empty list"
                 ),
             ),
         ] = []
@@ -74,8 +72,7 @@ class RequestsMigrator(MigrationTaskBase):
             Field(
                 title="Patron files",
                 description=(
-                    "List of files containing patron data. "
-                    "Optional, by default is empty list"
+                    "List of files containing patron data. Optional, by default is empty list"
                 ),
             ),
         ] = []
@@ -88,7 +85,7 @@ class RequestsMigrator(MigrationTaskBase):
         self,
         task_configuration: TaskConfiguration,
         library_config: LibraryConfiguration,
-        folio_client
+        folio_client,
     ):
         csv.register_dialect("tsv", delimiter="\t")
         self.migration_report = MigrationReport()
@@ -135,8 +132,7 @@ class RequestsMigrator(MigrationTaskBase):
             )
         else:
             logging.info(
-                "No item or user files supplied. Not validating against"
-                "previously migrated objects"
+                "No item or user files supplied. Not validating againstpreviously migrated objects"
             )
             self.valid_legacy_requests = self.semi_valid_legacy_requests
 
@@ -188,7 +184,7 @@ class RequestsMigrator(MigrationTaskBase):
         legacy_request.instance_id = holding.get("instanceId")
         if item["status"]["name"] in ["Available"]:
             legacy_request.request_type = "Page"
-            logging.info(f'Setting request to Page, since the status is {item["status"]["name"]}')
+            logging.info(f"Setting request to Page, since the status is {item['status']['name']}")
         self.migration_report.add_general_statistics(
             i18n.t("Valid, prepared requests, ready for posting")
         )
@@ -197,7 +193,7 @@ class RequestsMigrator(MigrationTaskBase):
     def do_work(self):
         logging.info("Starting")
         if self.task_configuration.starting_row > 1:
-            logging.info(f"Skipping {(self.task_configuration.starting_row-1)} records")
+            logging.info(f"Skipping {(self.task_configuration.starting_row - 1)} records")
         for num_requests, legacy_request in enumerate(
             self.valid_legacy_requests[self.task_configuration.starting_row - 1 :],
             start=1,
@@ -281,7 +277,8 @@ class RequestsMigrator(MigrationTaskBase):
                 self.migration_report.add(
                     "DiscardedLoans",
                     i18n.t(
-                        "Requests discarded. Had migrated item barcode: %{item_barcode}.\n Had migrated user barcode: %{patron_barcode}",
+                        "Requests discarded. Had migrated item barcode: %{item_barcode}.\n "
+                        "Had migrated user barcode: %{patron_barcode}",
                         item_barcode=has_item_barcode,
                         patron_barcode=has_patron_barcode,
                     ),
@@ -333,8 +330,7 @@ class RequestsMigrator(MigrationTaskBase):
             except ValueError as ve:
                 logging.exception(ve)
         logging.info(
-            f"Done validating {legacy_reques_count} "
-            f"legacy requests with {num_bad} rotten apples"
+            f"Done validating {legacy_reques_count} legacy requests with {num_bad} rotten apples"
         )
         if num_bad > 0 and (num_bad / legacy_reques_count) > 0.5:
             q = num_bad / legacy_reques_count

folio_migration_tools/migration_tasks/reserves_migrator.py

@@ -59,7 +59,7 @@ class ReservesMigrator(MigrationTaskBase):
         self,
         task_configuration: TaskConfiguration,
         library_config: LibraryConfiguration,
-        folio_client
+        folio_client,
     ):
         csv.register_dialect("tsv", delimiter="\t")
         self.migration_report = MigrationReport()

folio_migration_tools/migration_tasks/user_transformer.py

@@ -49,41 +49,37 @@ class UserTransformer(MigrationTaskBase):
             Field(
                 title="Group map path",
                 description="Define the path for group mapping",
-            )
+            ),
         ]
         departments_map_path: Annotated[
             Optional[str],
             Field(
                 title="Departments map path",
                 description=(
-                    "Define the path for departments mapping. "
-                    "Optional, by dfault is empty string"
+                    "Define the path for departments mapping. Optional, by dfault is empty string"
                 ),
-            )
+            ),
         ] = ""
         use_group_map: Annotated[
             Optional[bool],
             Field(
                 title="Use group map",
-                description=(
-                    "Specify whether to use group mapping. "
-                    "Optional, by default is True"
-                ),
-            )
+                description=("Specify whether to use group mapping. Optional, by default is True"),
+            ),
         ] = True
         user_mapping_file_name: Annotated[
             str,
             Field(
                 title="User mapping file name",
                 description="Specify the user mapping file name",
-            )
+            ),
         ]
         user_file: Annotated[
             FileDefinition,
             Field(
                 title="User file",
                 description="Select the user data file",
-            )
+            ),
         ]
         remove_id_and_request_preferences: Annotated[
             Optional[bool],
@@ -93,7 +89,7 @@ class UserTransformer(MigrationTaskBase):
                     "Specify whether to remove user ID and request preferences. "
                     "Optional, by default is False"
                 ),
-            )
+            ),
         ] = False
         remove_request_preferences: Annotated[
             Optional[bool],
@@ -103,7 +99,7 @@ class UserTransformer(MigrationTaskBase):
                     "Specify whether to remove user request preferences. "
                     "Optional, by default is False"
                 ),
-            )
+            ),
         ] = False
 
     @staticmethod
@@ -306,8 +302,7 @@ def find_primary_addresses(addresses):
         if "primaryAddress" not in address:
             address["primaryAddress"] = False
         elif (
-            isinstance(address["primaryAddress"], bool)
-            and address["primaryAddress"] is True
+            isinstance(address["primaryAddress"], bool) and address["primaryAddress"] is True
         ) or (
             isinstance(address["primaryAddress"], str)
             and address["primaryAddress"].lower() == "true"

folio_migration_tools/task_configuration.py

@@ -1,7 +1,7 @@
 from typing import Annotated
 
 from humps import camelize
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 from pydantic import Field
 
 
@@ -25,9 +25,7 @@ class AbstractTaskConfiguration(BaseModel):
         str,
         Field(
             title="Migration task type",
-            description=(
-                "The type of migration task you want to perform."
-            ),
+            description=("The type of migration task you want to perform."),
         ),
     ]
     ecs_tenant_id: Annotated[
@@ -41,6 +39,7 @@ class AbstractTaskConfiguration(BaseModel):
         ),
     ] = ""
 
-    class Config:
-        alias_generator = to_camel
-        allow_population_by_field_name = True
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+    )