folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/migration_tasks/items_transformer.py

@@ -1,4 +1,3 @@
-'''Main "script."'''
 import csv
 import ctypes
 import json
@@ -7,16 +6,16 @@ import sys
 import time
 import traceback
 import uuid
-from os.path import isfile
-from pathlib import Path
-from typing import List, Optional
+from typing import Annotated, List, Optional
 
+import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
+from pydantic import Field
+
 from folio_migration_tools.custom_exceptions import (
     TransformationProcessError,
     TransformationRecordFailedError,
 )
-from folio_migration_tools.folder_structure import FolderStructure
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
     FileDefinition,
@@ -27,30 +26,171 @@ from folio_migration_tools.mapping_file_transformation.item_mapper import ItemMa
 from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
     MappingFileMapperBase,
 )
-from folio_migration_tools.report_blurbs import Blurbs
-from pydantic.main import BaseModel
-
+from folio_migration_tools.marc_rules_transformation.hrid_handler import HRIDHandler
 from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 
 
 class ItemsTransformer(MigrationTaskBase):
-    class TaskConfiguration(BaseModel):
-        name: str
-        migration_task_type: str
-        hrid_handling: HridHandling
-        files: List[FileDefinition]
-        items_mapping_file_name: str
-        location_map_file_name: str
-        default_call_number_type_name: str
-        temp_location_map_file_name: Optional[str] = ""
-        material_types_map_file_name: str
-        loan_types_map_file_name: str
-        temp_loan_types_map_file_name: Optional[str] = ""
-        statistical_codes_map_file_name: Optional[str] = ""
-        item_statuses_map_file_name: str
-        call_number_type_map_file_name: str
+    class TaskConfiguration(AbstractTaskConfiguration):
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="Name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task.",
+            ),
+        ]
+        hrid_handling: Annotated[
+            HridHandling,
+            Field(
+                title="HRID handling",
+                description=(
+                    "Determining how the HRID generation "
+                    "should be handled."
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of files.",
+            ),
+        ]
+        items_mapping_file_name: Annotated[
+            str,
+            Field(
+                title="Items mapping file name",
+                description="File name for items mapping.",
+            ),
+        ]
+        location_map_file_name: Annotated[
+            str,
+            Field(
+                title="Location map file name",
+                description="File name for location map.",
+            ),
+        ]
+        default_call_number_type_name: Annotated[
+            str,
+            Field(
+                title="Default call number type name",
+                description="Default name for call number type.",
+            ),
+        ]
+        temp_location_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Temporary location map file name",
+                description=(
+                    "Temporary file name for location map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        material_types_map_file_name: Annotated[
+            str,
+            Field(
+                title="Material types map file name",
+                description="File name for material types map.",
+            ),
+        ]
+        loan_types_map_file_name: Annotated[
+            str,
+            Field(
+                title="Loan types map file name",
+                description="File name for loan types map.",
+            ),
+        ]
+        temp_loan_types_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Temporary loan types map file name",
+                description=(
+                    "File name for temporary loan types map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        statistical_codes_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Statistical code map file name",
+                description=(
+                    "Path to the file containing the mapping of statistical codes. "
+                    "The file should be in TSV format with legacy_stat_code "
+                    "and folio_code columns."
+                ),
+            ),
+        ] = ""
+        item_statuses_map_file_name: Annotated[
+            str,
+            Field(
+                title="Item statuses map file name",
+                description="File name for item statuses map.",
+            ),
+        ]
+        call_number_type_map_file_name: Annotated[
+            str,
+            Field(
+                title="Call number type map file name",
+                description="File name for call number type map.",
+            ),
+        ]
+        reset_hrid_settings: Annotated[
+            Optional[bool],
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "At the end of the run "
+                    "reset FOLIO with the HRID settings. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
+        update_hrid_settings: Annotated[
+            bool,
+            Field(
+                title="Update HRID settings",
+                description=(
+                    "At the end of the run "
+                    "update FOLIO with the HRID settings. "
+                    "By default is True."
+                ),
+            ),
+        ] = True
+        boundwith_relationship_file_path: Annotated[
+            str,
+            Field(
+                title="Boundwith relationship file path",
+                description=(
+                    "Path to a file outlining Boundwith relationships, "
+                    "in the style of Voyager. "
+                    "A TSV file with MFHD_ID and BIB_ID headers and values. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        prevent_permanent_location_map_default: Annotated[
+            bool,
+            Field(
+                title="Prevent permanent location map default",
+                description=(
+                    "Prevent the default mapping of permanent location "
+                    "to the default location. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -60,36 +200,28 @@ class ItemsTransformer(MigrationTaskBase):
         self,
         task_config: TaskConfiguration,
         library_config: LibraryConfiguration,
+        folio_client,
         use_logging: bool = True,
     ):
         csv.register_dialect("tsv", delimiter="\t")
-        super().__init__(library_config, task_config, use_logging)
+        super().__init__(library_config, task_config, folio_client, use_logging)
         self.task_config = task_config
-        self.files = [
-            f
-            for f in self.task_config.files
-            if isfile(self.folder_structure.legacy_records_folder / f.file_name)
-        ]
-        if not any(self.files):
-            ret_str = ",".join(f.file_name for f in self.task_config.files)
-            raise TransformationProcessError(
-                "",
-                f"Files {ret_str} not found in {self.folder_structure.data_folder / 'items'}",
-            )
-        logging.info("Files to process:")
-        for filename in self.files:
-            logging.info("\t%s", filename.file_name)
-
+        self.task_configuration = self.task_config
+        self.check_source_files(
+            self.folder_structure.legacy_records_folder, self.task_config.files
+        )
         self.total_records = 0
         self.folio_keys = []
         self.items_map = self.setup_records_map(
-            self.folder_structure.mapping_files_folder
-            / self.task_config.items_mapping_file_name
+            self.folder_structure.mapping_files_folder / self.task_config.items_mapping_file_name
         )
         self.folio_keys = MappingFileMapperBase.get_mapped_folio_properties_from_map(
             self.items_map
         )
-        if any(k for k in self.folio_keys if k.startswith("statisticalCodeIds")):
+        if (
+            any(k for k in self.folio_keys if k.startswith("statisticalCodeIds"))
+            or any(getattr(k, "statistical_code", "") for k in self.task_configuration.files)
+        ):
             statcode_mapping = self.load_ref_data_mapping_file(
                 "statisticalCodeIds",
                 self.folder_structure.mapping_files_folder
@@ -106,7 +238,8 @@ class ItemsTransformer(MigrationTaskBase):
         ).is_file():
             temporary_loan_type_mapping = self.load_ref_data_mapping_file(
                 "temporaryLoanTypeId",
-                self.folder_structure.temp_loan_type_map_path,
+                self.folder_structure.mapping_files_folder
+                / self.task_config.temp_loan_types_map_file_name,
                 self.folio_keys,
             )
         else:
@@ -115,20 +248,24 @@ class ItemsTransformer(MigrationTaskBase):
                 self.folder_structure.temp_loan_type_map_path,
             )
             temporary_loan_type_mapping = None
-
+        # Load Boundwith relationship map
+        self.boundwith_relationship_map = {}
+        if self.task_config.boundwith_relationship_file_path:
+            self.load_boundwith_relationships()
         if (
             self.folder_structure.mapping_files_folder
             / self.task_config.temp_location_map_file_name
         ).is_file():
             temporary_location_mapping = self.load_ref_data_mapping_file(
                 "temporaryLocationId",
-                self.folder_structure.temp_locations_map_path,
+                self.folder_structure.mapping_files_folder
+                / self.task_config.temp_location_map_file_name,
                 self.folio_keys,
             )
         else:
             logging.info(
                 "%s not found. No temporary location mapping will be performed",
-                self.folder_structure.temp_locations_map_path,
+                self.task_config.temp_location_map_file_name,
             )
             temporary_location_mapping = None
         self.mapper = ItemMapper(
@@ -170,42 +307,48 @@ class ItemsTransformer(MigrationTaskBase):
             temporary_loan_type_mapping,
             temporary_location_mapping,
             self.library_configuration,
+            self.task_configuration
         )
+        if (
+            self.task_configuration.reset_hrid_settings
+            and self.task_configuration.update_hrid_settings
+        ):
+            hrid_handler = HRIDHandler(
+                self.folio_client, HridHandling.default, self.mapper.migration_report, True
+            )
+            hrid_handler.reset_item_hrid_counter()
+
         logging.info("Init done")
 
     def do_work(self):
         logging.info("Starting....")
         with open(self.folder_structure.created_objects_path, "w+") as results_file:
-            for file_name in self.files:
+            for file_def in self.task_config.files:
                 try:
-                    self.process_single_file(file_name, results_file)
+                    self.process_single_file(file_def, results_file)
                 except Exception as exception:
-                    error_str = f"\n\nProcessing of {file_name} failed:\n{exception}."
+                    error_str = f"\n\nProcessing of {file_def.file_name} failed:\n{exception}."
                     logging.exception(error_str, stack_info=True)
-                    logging.fatal(
-                        "Check source files for empty lines or missing reference data."
-                    )
+                    logging.fatal("Check source files for empty rows or missing reference data.")
                     self.mapper.migration_report.add(
-                        Blurbs.FailedFiles, f"{file_name} - {exception}"
+                        "FailedFiles", f"{file_def.file_name} - {exception}"
                     )
                     logging.fatal(error_str)
                     sys.exit(1)
         logging.info(
-            f"processed {self.total_records:,} records in {len(self.files)} files"
+            f"processed {self.total_records:,} records in {len(self.task_config.files)} files"
         )
 
-    def process_single_file(self, file_name: FileDefinition, results_file):
-        full_path = self.folder_structure.legacy_records_folder / file_name.file_name
+    def process_single_file(self, file_def: FileDefinition, results_file):
+        full_path = self.folder_structure.legacy_records_folder / file_def.file_name
         logging.info("Processing %s", full_path)
         records_in_file = 0
         with open(full_path, encoding="utf-8-sig") as records_file:
             self.mapper.migration_report.add_general_statistics(
-                "Number of files processed"
+                i18n.t("Number of files processed")
             )
             start = time.time()
-            for idx, record in enumerate(
-                self.mapper.get_objects(records_file, full_path)
-            ):
+            for idx, record in enumerate(self.mapper.get_objects(records_file, full_path)):
                 try:
                     if idx == 0:
                         logging.info("First legacy record:")
@@ -215,44 +358,54 @@ class ItemsTransformer(MigrationTaskBase):
                         record, f"row {idx}", FOLIONamespaces.items
                     )
 
-                    self.handle_circiulation_notes(
-                        folio_rec, self.folio_client.current_user
-                    )
+                    self.mapper.perform_additional_mappings(legacy_id, folio_rec, file_def)
+                    self.handle_circulation_notes(folio_rec, self.folio_client.current_user)
                     self.handle_notes(folio_rec)
+                    if folio_rec["holdingsRecordId"] in self.boundwith_relationship_map:
+                        for idx_, instance_id in enumerate(
+                            self.boundwith_relationship_map.get(
+                                folio_rec["holdingsRecordId"]
+                            )
+                        ):
+                            if idx_ == 0:
+                                bw_id = folio_rec["holdingsRecordId"]
+                            else:
+                                bw_id = self.mapper.generate_boundwith_holding_uuid(
+                                    folio_rec["holdingsRecordId"], instance_id
+                                )
+                            self.mapper.create_and_write_boundwith_part(legacy_id, bw_id)
                     if idx == 0:
                         logging.info("First FOLIO record:")
                         logging.info(json.dumps(folio_rec, indent=4))
-                    # TODO: turn this into a asynchrounous task
+                    # TODO: turn this into a asynchronous task
                     Helper.write_to_file(results_file, folio_rec)
                     self.mapper.migration_report.add_general_statistics(
-                        "Number of records written to disk"
+                        i18n.t("Number of records written to disk")
                     )
                     self.mapper.report_folio_mapping(folio_rec, self.mapper.schema)
                 except TransformationProcessError as process_error:
                     self.mapper.handle_transformation_process_error(idx, process_error)
                 except TransformationRecordFailedError as data_error:
-                    self.mapper.handle_transformation_record_failed_error(
-                        idx, data_error
-                    )
+                    self.mapper.handle_transformation_record_failed_error(idx, data_error)
                 except AttributeError as attribute_error:
                     traceback.print_exc()
                     logging.fatal(attribute_error)
                     logging.info("Quitting...")
                     sys.exit(1)
-                except Exception as excepion:
-                    self.mapper.handle_generic_exception(idx, excepion)
+                except Exception as exception:
+                    self.mapper.handle_generic_exception(idx, exception)
                 self.mapper.migration_report.add(
-                    Blurbs.GeneralStatistics,
-                    f"Number of Legacy items in {file_name}",
+                    "GeneralStatistics",
+                    i18n.t("Number of Legacy items in %{container}", container=file_def),
                 )
                 self.mapper.migration_report.add_general_statistics(
-                    "Number of legacy items in total"
+                    i18n.t("Number of Legacy items in total")
                 )
                 self.print_progress(idx, start)
                 records_in_file = idx + 1
 
             logging.info(
-                f"Done processing {file_name} containing {records_in_file:,} records. "
+                f"Done processing {file_def} containing {records_in_file:,} records. "
                 f"Total records processed: {records_in_file:,}"
             )
         self.total_records += records_in_file
@@ -276,14 +429,14 @@ class ItemsTransformer(MigrationTaskBase):
                 del folio_object["notes"]
 
     @staticmethod
-    def handle_circiulation_notes(folio_rec, current_user_uuid):
+    def handle_circulation_notes(folio_rec, current_user_uuid):
         if not folio_rec.get("circulationNotes", []):
             return
         filtered_notes = []
         for circ_note in folio_rec.get("circulationNotes", []):
             if circ_note.get("noteType", "") not in ["Check in", "Check out"]:
                 raise TransformationProcessError(
-                    "", "Circulation Note types are not mapped correclty"
+                    "", "Circulation Note types are not mapped correctly"
                 )
             if circ_note.get("note", ""):
                 circ_note["id"] = str(uuid.uuid4())
@@ -297,20 +450,47 @@ class ItemsTransformer(MigrationTaskBase):
         else:
             del folio_rec["circulationNotes"]
 
-    def wrap_up(self):
-        logging.info("Work done. Wrapping up...")
-        with open(
-            self.folder_structure.migration_reports_file, "w"
-        ) as migration_report_file:
+    def load_boundwith_relationships(self):
+        try:
+            with open(
+                    self.folder_structure.boundwith_relationships_map_path
+            ) as boundwith_relationship_file:
+                self.boundwith_relationship_map = dict(
+                        json.loads(x) for x in boundwith_relationship_file
+                    )
             logging.info(
-                "Writing migration and mapping report to %s",
-                self.folder_structure.migration_reports_file,
+                "Rows in Bound with relationship map: %s",
+                len(self.boundwith_relationship_map)
+                )
+        except FileNotFoundError:
+            raise TransformationProcessError(
+                "",
+                "Boundwith relationship file specified, but relationships file "
+                "from holdings transformation not found.",
+                self.folder_structure.boundwith_relationships_map_path
+            )
+        except ValueError:
+            raise TransformationProcessError(
+                "",
+                "Boundwith relationship file specified, but relationships file "
+                "from holdings transformation is not a valid line JSON.",
+                self.folder_structure.boundwith_relationships_map_path,
+            )
+
+    def wrap_up(self):
+        logging.info("Done. Transformer wrapping up...")
+        self.extradata_writer.flush()
+        with open(self.folder_structure.migration_reports_file, "w") as migration_report_file:
+            self.mapper.migration_report.write_migration_report(
+                i18n.t("Item transformation report"),
+                migration_report_file,
+                self.mapper.start_datetime,
             )
-            self.mapper.migration_report.write_migration_report(migration_report_file)
             Helper.print_mapping_report(
                 migration_report_file,
                 self.total_records,
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        self.clean_out_empty_logs()
         logging.info("All done!")