folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/migration_tasks/bibs_transformer.py

@@ -1,43 +1,106 @@
 import logging
-import sys
-import time
-from datetime import datetime as dt
-from os.path import isfile
-from typing import List, Optional
+from typing import Annotated, List
 
+import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
-from folio_migration_tools.colors import Bcolors
-from folio_migration_tools.custom_exceptions import (
-    TransformationProcessError,
-    TransformationRecordFailedError,
-)
+from pydantic import Field
+
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
-    FileDefinition,
-    HridHandling,
     IlsFlavour,
     LibraryConfiguration,
 )
-from folio_migration_tools.marc_rules_transformation.bibs_processor import BibsProcessor
+from folio_migration_tools.marc_rules_transformation.marc_file_processor import (
+    MarcFileProcessor,
+)
 from folio_migration_tools.marc_rules_transformation.rules_mapper_bibs import (
     BibsRulesMapper,
 )
-from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
-from pydantic import BaseModel
-from pymarc import MARCReader
-from pymarc.record import Record
+from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskConfigurationBase, MigrationTaskBase
 
 
 class BibsTransformer(MigrationTaskBase):
-    class TaskConfiguration(BaseModel):
-        name: str
-        deactivate035_from001: Optional[bool] = False
-        migration_task_type: str
-        use_tenant_mapping_rules: Optional[bool] = True
-        hrid_handling: Optional[HridHandling] = HridHandling.default
-        files: List[FileDefinition]
-        ils_flavour: IlsFlavour
-        tags_to_delete: Optional[List[str]] = []
+    class TaskConfiguration(MarcTaskConfigurationBase):
+        ils_flavour: Annotated[
+            IlsFlavour,
+            Field(
+                title="ILS flavour",
+                description="The type of ILS you are migrating records from.",
+                alias="ils_flavor"
+            ),
+        ]
+        custom_bib_id_field: Annotated[
+            str,
+            Field(
+                title="Custom BIB ID field",
+                description=(
+                    'A string representing a MARC field with optional subfield indicated by a "$" '
+                    '(eg. "991$a") from which to draw legacy Bib ID. Use this in combination '
+                    'with `ilsFlavour: "custom"`. Defaults to "001", and is ignored for all other '
+                    "ilsFlavours."
+                ),
+            ),
+        ] = "001"
+        add_administrative_notes_with_legacy_ids: Annotated[
+            bool,
+            Field(
+                title="Add administrative notes with legacy IDs",
+                description=(
+                    "If set to true, an Administrative note will be added to the records "
+                    "containing the legacy ID. Use this in order to protect the values from "
+                    "getting overwritten by overlays,"
+                ),
+            ),
+        ] = True
+        tags_to_delete: Annotated[
+            List[str],
+            Field(
+                title="Tags to delete from MARC record",
+                description=(
+                    "Tags in the incoming MARC authority that the process should remove "
+                    "before adding them into FOLIO. These tags will be used in the "
+                    "transformation before getting removed."
+                ),
+            ),
+        ] = []
+        data_import_marc: Annotated[
+            bool,
+            Field(
+                title="Generate a MARC file for data import overlay of instances",
+                description=(
+                    "If set to true, the process will generate a file of binary MARC records that can"
+                    "be imported into FOLIO using the Data Import APIs. If set to false, only a file"
+                    "of FOLIO instance records (and optional SRS records) will be generated."
+                ),
+            )
+        ] = True
+        parse_cataloged_date: Annotated[
+            bool,
+            Field(
+                title="Parse cataloged date",
+                description=(
+                    "Parse fields mapped to catalogedDate into a FOLIO accepted date string using "
+                    "dateutil.parser. Verify results carefully when using"
+                ),
+            ),
+        ] = False
+        reset_hrid_settings: Annotated[
+            bool,
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "Setting to true means the task will "
+                    "reset the HRID counters for this particular record type"
+                ),
+            ),
+        ] = False
+        update_hrid_settings: Annotated[
+            bool,
+            Field(
+                title="Update HRID settings",
+                description="At the end of the run, update FOLIO with the HRID settings",
+            ),
+        ] = True
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -47,72 +110,48 @@ class BibsTransformer(MigrationTaskBase):
         self,
         task_config: TaskConfiguration,
         library_config: LibraryConfiguration,
+        folio_client,
         use_logging: bool = True,
     ):
-
-        super().__init__(library_config, task_config, use_logging)
+        super().__init__(library_config, task_config, folio_client, use_logging)
         self.task_config = task_config
-        logging.info(task_config.json(indent=4))
-        self.files = [
-            f
-            for f in self.task_config.files
-            if isfile(self.folder_structure.legacy_records_folder / f.file_name)
-        ]
-        if not any(self.files):
-            ret_str = ",".join(f.file_name for f in self.task_config.files)
-            raise TransformationProcessError(
-                "",
-                f"Files {ret_str} not found in {self.folder_structure.data_folder / 'items'}",
+        self.task_configuration = self.task_config
+        if self.task_config.statistical_codes_map_file_name:
+            statcode_mapping = self.load_ref_data_mapping_file(
+                "statisticalCodeIds",
+                self.folder_structure.mapping_files_folder
+                / self.task_config.statistical_codes_map_file_name,
+                [],
+                False,
             )
-        logging.info(self.files)
-        logging.info("# of files to process: %s", len(self.files))
-        for file_path in self.files:
-            logging.info("\t%s", file_path)
-        self.mapper = BibsRulesMapper(self.folio_client, library_config, task_config)
-        self.processor = None
-        self.bib_ids = set()
+        else:
+            statcode_mapping = None
+        self.processor: MarcFileProcessor
+        self.check_source_files(
+            self.folder_structure.legacy_records_folder, self.task_configuration.files
+        )
+        self.mapper = BibsRulesMapper(self.folio_client, library_config, self.task_configuration, statcode_mapping)
+        self.bib_ids: set = set()
+        if (
+            self.task_configuration.reset_hrid_settings
+            and self.task_configuration.update_hrid_settings
+        ):
+            self.mapper.hrid_handler.reset_instance_hrid_counter()
         logging.info("Init done")
 
     def do_work(self):
-        logging.info("Starting....")
-        with open(
-            self.folder_structure.created_objects_path, "w+"
-        ) as created_records_file:
-            self.processor = BibsProcessor(
-                self.mapper,
-                self.folio_client,
-                created_records_file,
-                self.folder_structure,
-            )
-            for file_obj in self.files:
-                try:
-                    with open(
-                        self.folder_structure.legacy_records_folder
-                        / file_obj.file_name,
-                        "rb",
-                    ) as marc_file:
-                        reader = MARCReader(marc_file, to_unicode=True, permissive=True)
-                        reader.hide_utf8_warnings = True
-                        reader.force_utf8 = False
-                        logging.info("running %s", file_obj.file_name)
-                        self.read_records(reader, file_obj)
-                except TransformationProcessError as tpe:
-                    logging.critical(tpe)
-                    sys.exit(1)
-                except Exception:
-                    logging.exception(file_obj, stack_info=True)
-                    logging.critical(
-                        "File %s failed for unknown reason. Halting", file_obj.file_name
-                    )
-                    sys.exit(1)
+        self.do_work_marc_transformer()
 
     def wrap_up(self):
-        logging.info("Done. Wrapping up...")
+        logging.info("Done. Transformer wrapping up...")
+        self.extradata_writer.flush()
         self.processor.wrap_up()
         with open(self.folder_structure.migration_reports_file, "w+") as report_file:
-            report_file.write("# Bibliographic records transformation results   \n")
-            report_file.write(f"Time Run: {dt.isoformat(dt.utcnow())}   \n")
-            self.mapper.migration_report.write_migration_report(report_file)
+            self.mapper.migration_report.write_migration_report(
+                i18n.t("Bibliographic records transformation report"),
+                report_file,
+                self.start_datetime,
+            )
             Helper.print_mapping_report(
                 report_file,
                 self.mapper.parsed_records,
@@ -124,38 +163,4 @@ class BibsTransformer(MigrationTaskBase):
             "Done. Transformation report written to %s",
             self.folder_structure.migration_reports_file.name,
         )
-
-    def read_records(self, reader, source_file: FileDefinition):
-        with open(self.folder_structure.failed_bibs_file, "wb") as failed_bibs_file:
-            for idx, record in enumerate(reader):
-                self.mapper.migration_report.add_general_statistics(
-                    "Records in file before parsing"
-                )
-                try:
-                    if record is None:
-                        self.mapper.migration_report.add_general_statistics(
-                            "Records with encoding errors - parsing failed",
-                        )
-                        failed_bibs_file.write(reader.current_chunk)
-                        raise TransformationRecordFailedError(
-                            f"Index in {source_file.file_name}:{idx}",
-                            f"MARC parsing error: {reader.current_exception}",
-                            "Failed records stored in results/failed_bib_records.mrc",
-                        )
-
-                    else:
-                        self.set_leader(record)
-                        self.mapper.migration_report.add_general_statistics(
-                            "Records successfully parsed from MARC21",
-                        )
-                        self.processor.process_record(
-                            idx, record, source_file.suppressed
-                        )
-                except TransformationRecordFailedError as error:
-                    error.log_it()
-            logging.info("Done reading %s records from file", idx + 1)
-
-    @staticmethod
-    def set_leader(marc_record: Record):
-        new_leader = marc_record.leader
-        marc_record.leader = f"{new_leader[:9]}a{new_leader[10:]}"
+        self.clean_out_empty_logs()

folio_migration_tools/migration_tasks/courses_migrator.py

@@ -0,0 +1,192 @@
+import csv
+import json
+import logging
+import sys
+import time
+import traceback
+from typing import Optional, Annotated
+from pydantic import Field
+
+import i18n
+from folio_uuid.folio_namespaces import FOLIONamespaces
+
+from folio_migration_tools.custom_exceptions import (
+    TransformationProcessError,
+    TransformationRecordFailedError,
+)
+from folio_migration_tools.library_configuration import (
+    FileDefinition,
+    LibraryConfiguration,
+)
+from folio_migration_tools.mapping_file_transformation.courses_mapper import (
+    CoursesMapper,
+)
+from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
+    MappingFileMapperBase,
+)
+from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
+
+
+class CoursesMigrator(MigrationTaskBase):
+    class TaskConfiguration(AbstractTaskConfiguration):
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task",
+            ),
+        ]
+        composite_course_map_path: Annotated[
+            str,
+            Field(
+                title="Composite course map path",
+                description="Path to the composite course map file",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task",
+            ),
+        ]
+        courses_file: Annotated[
+            FileDefinition,
+            Field(
+                title="Courses file",
+                description="File containing course data",
+            ),
+        ]
+        terms_map_path: Annotated[
+            str,
+            Field(
+                title="Terms map path",
+                description="Path to the terms map file",
+            ),
+        ]
+        departments_map_path: Annotated[
+            str,
+            Field(
+                title="Departments map path",
+                description="Path to the departments map file",
+            ),
+        ]
+        look_up_instructor: Annotated[
+            Optional[bool],
+            Field(
+                title="Look up instructor",
+                description=(
+                    "Flag to indicate whether to look up instructors. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
+
+    @staticmethod
+    def get_object_type() -> FOLIONamespaces:
+        return FOLIONamespaces.course
+
+    def __init__(
+        self,
+        task_configuration: TaskConfiguration,
+        library_config: LibraryConfiguration,
+        folio_client
+    ):
+        csv.register_dialect("tsv", delimiter="\t")
+        self.task_configuration = task_configuration
+        super().__init__(library_config, task_configuration, folio_client)
+        self.t0 = time.time()
+        self.courses_map = self.setup_records_map(
+            self.folder_structure.mapping_files_folder
+            / self.task_configuration.composite_course_map_path
+        )
+        self.folio_keys = MappingFileMapperBase.get_mapped_folio_properties_from_map(
+            self.courses_map
+        )
+        terms_map = self.load_ref_data_mapping_file(
+            "terms",
+            self.folder_structure.mapping_files_folder / self.task_configuration.terms_map_path,
+            self.folio_keys,
+        )
+
+        departments_map = self.load_ref_data_mapping_file(
+            "departments",
+            self.folder_structure.mapping_files_folder
+            / self.task_configuration.departments_map_path,
+            self.folio_keys,
+        )
+        self.mapper: CoursesMapper = CoursesMapper(
+            self.folio_client,
+            self.courses_map,
+            terms_map,
+            departments_map,
+            self.library_configuration,
+            self.task_configuration,
+        )
+        logging.info("Init completed")
+
+    def do_work(self):
+        logging.info("Starting")
+        full_path = (
+            self.folder_structure.legacy_records_folder
+            / self.task_configuration.courses_file.file_name
+        )
+        logging.info("Processing %s", full_path)
+        start = time.time()
+        with open(full_path, encoding="utf-8-sig") as records_file:
+            for idx, record in enumerate(self.mapper.get_objects(records_file, full_path)):
+                try:
+                    if idx == 0:
+                        logging.info("First legacy record:")
+                        logging.info(json.dumps(record, indent=4))
+                        self.mapper.verify_legacy_record(record, idx)
+                    folio_rec, legacy_id = self.mapper.do_map(
+                        record, f"row {idx}", FOLIONamespaces.course
+                    )
+                    self.mapper.perform_additional_mappings((folio_rec, legacy_id))
+                    if idx == 0:
+                        logging.info("First FOLIO record:")
+                        logging.info(json.dumps(folio_rec, indent=4))
+                    self.mapper.store_objects((folio_rec, legacy_id))
+                    self.mapper.notes_mapper.map_notes(
+                        record, legacy_id, folio_rec["course"]["id"], FOLIONamespaces.course
+                    )
+
+                except TransformationProcessError as process_error:
+                    self.mapper.handle_transformation_process_error(idx, process_error)
+                except TransformationRecordFailedError as data_error:
+                    self.mapper.handle_transformation_record_failed_error(idx, data_error)
+                except AttributeError as attribute_error:
+                    traceback.print_exc()
+                    logging.fatal(attribute_error)
+                    logging.info("Quitting...")
+                    sys.exit(1)
+                except Exception as excepion:
+                    self.mapper.handle_generic_exception(idx, excepion)
+                self.mapper.migration_report.add(
+                    "GeneralStatistics",
+                    i18n.t("Number of Legacy items in %{container}", container=full_path),
+                )
+                self.mapper.migration_report.add_general_statistics(
+                    i18n.t("Number of Legacy items in total")
+                )
+                self.print_progress(idx, start)
+
+    def wrap_up(self):
+        self.extradata_writer.flush()
+        with open(self.folder_structure.migration_reports_file, "w+") as report_file:
+            self.mapper.migration_report.write_migration_report(
+                i18n.t("Courses migration report"), report_file, self.mapper.start_datetime
+            )
+        self.clean_out_empty_logs()
+
+
+def timings(t0, t0func, num_objects):
+    avg = (time.time() - t0) / num_objects
+    elapsed = time.time() - t0
+    elapsed_func = time.time() - t0func
+    return (
+        f"Total objects: {num_objects}\tTotal elapsed: {elapsed:.2f}\t"
+        f"Average per object: {avg:.2f}s\tElapsed this time: {elapsed_func:.2f}"
+    )