folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/migration_tasks/holdings_marc_transformer.py

@@ -1,47 +1,218 @@
-'''Main "script."'''
 import csv
 import json
 import logging
-import sys
-from os import listdir
-from os.path import isfile
-from typing import List, Optional
-from pydantic import BaseModel
+from typing import Annotated, List
 
+import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
-from folio_migration_tools.custom_exceptions import (
-    TransformationProcessError,
-    TransformationRecordFailedError,
-)
+from pydantic import Field
+
+from folio_migration_tools.custom_exceptions import TransformationProcessError
+from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
     FileDefinition,
     HridHandling,
     LibraryConfiguration,
 )
-from folio_migration_tools.marc_rules_transformation.holdings_processor import (
-    HoldingsProcessor,
-)
 from folio_migration_tools.marc_rules_transformation.rules_mapper_holdings import (
     RulesMapperHoldings,
 )
-from pymarc import MARCReader
-
-from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.migration_tasks.migration_task_base import (
+    MarcTaskConfigurationBase,
+    MigrationTaskBase
+)
 
 
 class HoldingsMarcTransformer(MigrationTaskBase):
-    class TaskConfiguration(BaseModel):
-        name: str
-        legacy_id_marc_path: str
-        migration_task_type: str
-        use_tenant_mapping_rules: bool
-        hrid_handling: HridHandling
-        files: List[FileDefinition]
-        mfhd_mapping_file_name: str
-        location_map_file_name: str
-        default_call_number_type_name: str
-        fallback_holdings_type_id: str
-        create_source_records: Optional[bool] = False
+    class TaskConfiguration(MarcTaskConfigurationBase):
+        name: Annotated[
+            str,
+            Field(
+                description=(
+                    "Name of this migration task. The name is being used to call the specific "
+                    "task, and to distinguish tasks of similar types"
+                )
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description=(
+                    "The type of migration task you want to perform"
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Source files",
+                description=(
+                    "List of MARC21 files with holdings records"
+                ),
+            ),
+        ]
+        hrid_handling: Annotated[
+            HridHandling,
+            Field(
+                title="HRID Handling",
+                description=(
+                    "Setting to default will make FOLIO generate HRIDs and move the existing "
+                    "001:s into a 035, concatenated with the 003. Choosing preserve001 means "
+                    "the 001:s will remain in place, and that they will also become the HRIDs"
+                ),
+            ),
+        ] = HridHandling.default
+        holdings_type_uuid_for_boundwiths: Annotated[
+            str,
+            Field(
+                title="Holdings Type for Boundwith Holdings",
+                description=(
+                    "UUID for a Holdings type (set in Settings->Inventory) "
+                    "for Bound-with Holdings)"
+                ),
+            ),
+        ] = ""
+        boundwith_relationship_file_path: Annotated[
+            str,
+            Field(
+                title="Boundwith relationship file path",
+                description=(
+                    "Path to a file outlining Boundwith relationships, in the style of Voyager."
+                    " A TSV file with MFHD_ID and BIB_ID headers and values"
+                ),
+            ),
+        ] = ""
+        update_hrid_settings: Annotated[
+            bool,
+            Field(
+                title="Update HRID settings",
+                description="At the end of the run, update FOLIO with the HRID settings",
+            ),
+        ] = True
+        reset_hrid_settings: Annotated[
+            bool,
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "Setting to true means the task will "
+                    "reset the HRID counters for this particular record type"
+                ),
+            ),
+        ] = False
+        legacy_id_marc_path: Annotated[
+            str,
+            Field(
+                title="Path to legacy id in the records",
+                description=(
+                    "The path to the field where the legacy id is located. "
+                    "Example syntax: '001' or '951$c'"
+                ),
+            ),
+        ]
+        deduplicate_holdings_statements: Annotated[
+            bool,
+            Field(
+                title="Deduplicate holdings statements",
+                description=(
+                    "If set to False, duplicate holding statements within the same record will "
+                    "remain in place"
+                ),
+            ),
+        ] = True
+        location_map_file_name: Annotated[
+            str,
+            Field(
+                title="Path to location map file",
+                description="Must be a TSV file located in the mapping_files folder",
+            ),
+        ]
+        default_call_number_type_name: Annotated[
+            str,
+            Field(
+                title="Default call_number type name",
+                description="The name of the call_number type that will be used as fallback",
+            ),
+        ]
+        fallback_holdings_type_id: Annotated[
+            str,
+            Field(
+                title="Fallback holdings type id",
+                description="The UUID of the Holdings type that will be used for unmapped values",
+            ),
+        ]
+        supplemental_mfhd_mapping_rules_file: Annotated[
+            str,
+            Field(
+                title="Supplemental MFHD mapping rules file",
+                description=(
+                    "The name of the file in the mapping_files directory "
+                    "containing supplemental MFHD mapping rules"
+                ),
+            ),
+        ] = ""
+        include_mrk_statements: Annotated[
+            bool,
+            Field(
+                title="Include MARC statements (MRK-format) as staff-only Holdings notes",
+                description=(
+                    "If set to true, the MARC statements "
+                    "will be included in the output as MARC Maker format fields. "
+                    "If set to false (default), the MARC statements "
+                    "will not be included in the output."
+                ),
+            ),
+        ] = False
+        mrk_holdings_note_type: Annotated[
+            str,
+            Field(
+                title="MARC Holdings Note type",
+                description=(
+                    "The name of the note type to use for MARC (MRK) statements. "
+                ),
+            ),
+        ] = "Original MARC holdings statements"
+        include_mfhd_mrk_as_note: Annotated[
+            bool,
+            Field(
+                title="Include MARC Record (as MARC Maker Representation) as note",
+                description=(
+                    "If set to true, the MARC statements will be included in the output as a "
+                    "(MRK) note. If set to false (default), the MARC statements will not be "
+                    "included in the output."
+                ),
+            ),
+        ] = False
+        mfhd_mrk_note_type: Annotated[
+            str,
+            Field(
+                title="MARC Record (as MARC Maker Representation) note type",
+                description=(
+                    "The name of the note type to use for MFHD (MRK) note. "
+                ),
+            ),
+        ] = "Original MFHD Record"
+        include_mfhd_mrc_as_note: Annotated[
+            bool,
+            Field(
+                title="Include MARC Record (as MARC21 decoded string) as note",
+                description=(
+                    "If set to true, the MARC record will be included in the output as a "
+                    "decoded binary MARC21 record. If set to false (default), "
+                    "the MARC record will not be "
+                    "included in the output."
+                ),
+            ),
+        ] = False
+        mfhd_mrc_note_type: Annotated[
+            str,
+            Field(
+                title="MARC Record (as MARC21 decoded string) note type",
+                description=(
+                    "The name of the note type to use for MFHD (MRC) note. "
+                ),
+            ),
+        ] = "Original MFHD (MARC21)"
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -51,11 +222,21 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         self,
         task_config: TaskConfiguration,
         library_config: LibraryConfiguration,
+        folio_client,
         use_logging: bool = True,
     ):
         csv.register_dialect("tsv", delimiter="\t")
-        super().__init__(library_config, task_config, use_logging)
-        self.task_config = task_config
+        super().__init__(library_config, task_config, folio_client, use_logging)
+        if self.task_configuration.statistical_codes_map_file_name:
+            statcode_mapping = self.load_ref_data_mapping_file(
+                "statisticalCodeIds",
+                self.folder_structure.mapping_files_folder
+                / self.task_configuration.statistical_codes_map_file_name,
+                [],
+                False,
+            )
+        else:
+            statcode_mapping = None
         self.holdings_types = list(
             self.folio_client.folio_get_all("/holdings-types", "holdingsTypes")
         )
@@ -63,111 +244,136 @@ class HoldingsMarcTransformer(MigrationTaskBase):
             (
                 h
                 for h in self.holdings_types
-                if h["id"] == self.task_config.fallback_holdings_type_id
+                if h["id"] == self.task_configuration.fallback_holdings_type_id
             ),
-            "",
+            {"name": ""},
         )
         if not self.default_holdings_type:
             raise TransformationProcessError(
                 "",
                 (
-                    f"Holdings type with ID {self.task_config.fallback_holdings_type_id}"
+                    f"Holdings type with ID {self.task_configuration.fallback_holdings_type_id}"
                     " not found in FOLIO."
                 ),
             )
         logging.info(
             "%s will be used as default holdings type",
-            self.default_holdings_type["name"],
-        )
-        self.instance_id_map = self.load_id_map(
-            self.folder_structure.instance_id_map_path, True
+            self.default_holdings_type.get("name", ""),
         )
-        logging.info("%s Instance ids in map", len(self.instance_id_map))
-        logging.info("Init done")
 
-    def do_work(self):
-        files = self.list_source_files()
-        loc_map_path = (
+        # Load Boundwith relationship map
+        self.boundwith_relationship_map_rows = []
+        if self.task_configuration.boundwith_relationship_file_path:
+            try:
+                with open(
+                    self.folder_structure.legacy_records_folder
+                    / self.task_configuration.boundwith_relationship_file_path
+                ) as boundwith_relationship_file:
+                    self.boundwith_relationship_map_rows = list(
+                        csv.DictReader(boundwith_relationship_file, dialect="tsv")
+                    )
+                logging.info(
+                    "Rows in Bound with relationship map: %s",
+                    len(self.boundwith_relationship_map_rows),
+                )
+            except FileNotFoundError:
+                raise TransformationProcessError(
+                    "",
+                    i18n.t("Provided boundwith relationship file not found"),
+                    self.task_configuration.boundwith_relationship_file_path,
+                )
+
+        location_map_path = (
             self.folder_structure.mapping_files_folder
-            / self.task_config.location_map_file_name
+            / self.task_configuration.location_map_file_name
         )
-        map_path = (
-            self.folder_structure.mapping_files_folder
-            / self.task_config.mfhd_mapping_file_name
+        with open(location_map_path) as location_map_file:
+            self.location_map = list(csv.DictReader(location_map_file, dialect="tsv"))
+            logging.info("Locations in map: %s", len(self.location_map))
+
+        self.check_source_files(
+            self.folder_structure.legacy_records_folder, self.task_configuration.files
         )
-        with open(loc_map_path) as loc_map_f, open(map_path) as map_f:
-            location_map = list(csv.DictReader(loc_map_f, dialect="tsv"))
-            logging.info("Locations in map: %s", len(location_map))
-            rules_file = json.load(map_f)
-            logging.info("Default location code %s", rules_file["defaultLocationCode"])
-            mapper = RulesMapperHoldings(
-                self.folio_client,
-                self.instance_id_map,
-                location_map,
-                self.task_config,
-                self.library_configuration,
-            )
-            mapper.mappings = rules_file["rules"]
-            processor = HoldingsProcessor(mapper, self.folder_structure)
-            for file_def in files:
-                self.process_single_file(file_def, processor)
-            processor.wrap_up()
+        self.instance_id_map = self.load_instance_id_map(True)
+        self.mapper = RulesMapperHoldings(
+            self.folio_client,
+            self.location_map,
+            self.task_configuration,
+            self.library_configuration,
+            self.instance_id_map,
+            self.boundwith_relationship_map_rows,
+            statcode_mapping
+        )
+        self.add_supplemental_mfhd_mappings()
+        if (
+            self.task_configuration.reset_hrid_settings
+            and self.task_configuration.update_hrid_settings
+        ):
+            self.mapper.hrid_handler.reset_holdings_hrid_counter()
+        logging.info("%s Instance ids in map", len(self.instance_id_map))
+        logging.info("Init done")
 
-    def list_source_files(self):
-        files = [
-            f
-            for f in self.task_config.files
-            if isfile(self.folder_structure.legacy_records_folder / f.file_name)
-        ]
-        if not any(files):
-            ret_str = ",".join(f.file_name for f in self.task_config.files)
-            raise TransformationProcessError(
-                "",
-                f"Files {ret_str} not found in {self.folder_structure.data_folder / 'holdings'}",
-            )
+    def add_supplemental_mfhd_mappings(self):
+        if self.task_configuration.supplemental_mfhd_mapping_rules_file:
+            try:
+                with open(
+                    (
+                        self.folder_structure.mapping_files_folder
+                        / self.task_configuration.supplemental_mfhd_mapping_rules_file
+                    ),
+                    "r",
+                ) as new_rules_file:
+                    new_rules = json.load(new_rules_file)
+                    if not isinstance(new_rules, dict):
+                        raise TransformationProcessError(
+                            "",
+                            "Supplemental MFHD mapping rules file must contain a dictionary",
+                            json.dumps(new_rules),
+                        )
+            except FileNotFoundError:
+                raise TransformationProcessError(
+                    "",
+                    "Provided supplemental MFHD mapping rules file not found",
+                    self.task_configuration.supplemental_mfhd_mapping_rules_file,
+                )
+        else:
+            new_rules = {}
+        self.mapper.integrate_supplemental_mfhd_mappings(new_rules)
 
-        return files
+    def do_work(self):
+        self.do_work_marc_transformer()
 
-    def process_single_file(
-        self, file_def: FileDefinition, processor: HoldingsProcessor
-    ):
-        try:
+    def wrap_up(self):
+        logging.info("Done. Transformer Wrapping up...")
+        self.extradata_writer.flush()
+        self.processor.wrap_up()
+        if self.mapper.boundwith_relationship_map:
             with open(
-                self.folder_structure.legacy_records_folder / file_def.file_name,
-                "rb",
-            ) as marc_file:
-                reader = MARCReader(marc_file, to_unicode=True, permissive=True)
-                reader.hide_utf8_warnings = True
-                reader.force_utf8 = True
-                logging.info("Running %s", file_def.file_name)
-                read_records(reader, processor, file_def)
-        except TransformationProcessError as tpe:
-            logging.critical(tpe)
-            sys.exit(1)
-        except Exception:
-            logging.exception(
-                "Failure in Main: %s", file_def.file_name, stack_info=True
-            )
+                self.folder_structure.boundwith_relationships_map_path, "w+"
+            ) as boundwith_relationship_file:
+                logging.info(
+                    "Writing boundwiths relationship map to %s",
+                    boundwith_relationship_file.name,
+                )
+                for key, val in self.mapper.boundwith_relationship_map.items():
+                    boundwith_relationship_file.write(json.dumps((key, val)) + "\n")
 
-    def wrap_up(self):
-        logging.info("wapping up")
+        with open(self.folder_structure.migration_reports_file, "w+") as report_file:
+            self.mapper.migration_report.write_migration_report(
+                i18n.t("Bibliographic records transformation report"),
+                report_file,
+                self.start_datetime,
+            )
+            Helper.print_mapping_report(
+                report_file,
+                self.mapper.parsed_records,
+                self.mapper.mapped_folio_fields,
+                self.mapper.mapped_legacy_fields,
+            )
 
+        logging.info(
+            "Done. Transformation report written to %s",
+            self.folder_structure.migration_reports_file.name,
+        )
 
-def read_records(reader, processor: HoldingsProcessor, file_def: FileDefinition):
-    for idx, record in enumerate(reader):
-        try:
-            if record is None:
-                processor.mapper.migration_report.add_general_statistics(
-                    "Records with encoding errors. See data issues log for details"
-                )
-                raise TransformationRecordFailedError(
-                    f"Index in file:{idx}",
-                    f"MARC parsing error: {reader.current_exception}",
-                    f"{reader.current_chunk}",
-                )
-            else:
-                processor.process_record(record, file_def)
-        except TransformationRecordFailedError as error:
-            error.log_it()
-        except ValueError as error:
-            logging.error(error)
+        self.clean_out_empty_logs()