folio-migration-tools 1.9.0rc2__py3-none-any.whl → 1.9.0rc4__py3-none-any.whl

folio_migration_tools/migration_tasks/items_transformer.py

@@ -1,4 +1,3 @@
-'''Main "script."'''
 import csv
 import ctypes
 import json
@@ -36,26 +35,136 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 
 class ItemsTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        hrid_handling: HridHandling
-        files: List[FileDefinition]
-        items_mapping_file_name: str
-        location_map_file_name: str
-        default_call_number_type_name: str
-        temp_location_map_file_name: Optional[str] = ""
-        material_types_map_file_name: str
-        loan_types_map_file_name: str
-        temp_loan_types_map_file_name: Optional[str] = ""
-        statistical_codes_map_file_name: Optional[str] = ""
-        item_statuses_map_file_name: str
-        call_number_type_map_file_name: str
-        reset_hrid_settings: Optional[bool] = False
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="Name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task.",
+            ),
+        ]
+        hrid_handling: Annotated[
+            HridHandling,
+            Field(
+                title="HRID handling",
+                description=(
+                    "Determining how the HRID generation "
+                    "should be handled."
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of files.",
+            ),
+        ]
+        items_mapping_file_name: Annotated[
+            str,
+            Field(
+                title="Items mapping file name",
+                description="File name for items mapping.",
+            ),
+        ]
+        location_map_file_name: Annotated[
+            str,
+            Field(
+                title="Location map file name",
+                description="File name for location map.",
+            ),
+        ]
+        default_call_number_type_name: Annotated[
+            str,
+            Field(
+                title="Default call number type name",
+                description="Default name for call number type.",
+            ),
+        ]
+        temp_location_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Temporary location map file name",
+                description=(
+                    "Temporary file name for location map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        material_types_map_file_name: Annotated[
+            str,
+            Field(
+                title="Material types map file name",
+                description="File name for material types map.",
+            ),
+        ]
+        loan_types_map_file_name: Annotated[
+            str,
+            Field(
+                title="Loan types map file name",
+                description="File name for loan types map.",
+            ),
+        ]
+        temp_loan_types_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Temporary loan types map file name",
+                description=(
+                    "File name for temporary loan types map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        statistical_codes_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Statistical codes map file name",
+                description=(
+                    "File name for statistical codes map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        item_statuses_map_file_name: Annotated[
+            str,
+            Field(
+                title="Item statuses map file name",
+                description="File name for item statuses map.",
+            ),
+        ]
+        call_number_type_map_file_name: Annotated[
+            str,
+            Field(
+                title="Call number type map file name",
+                description="File name for call number type map.",
+            ),
+        ]
+        reset_hrid_settings: Annotated[
+            Optional[bool],
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "At the end of the run "
+                    "reset FOLIO with the HRID settings. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
         update_hrid_settings: Annotated[
             bool,
             Field(
                 title="Update HRID settings",
-                description="At the end of the run, update FOLIO with the HRID settings",
+                description=(
+                    "At the end of the run "
+                    "update FOLIO with the HRID settings. "
+                    "By default is True."
+                ),
             ),
         ] = True
         boundwith_relationship_file_path: Annotated[
@@ -63,8 +172,10 @@ class ItemsTransformer(MigrationTaskBase):
             Field(
                 title="Boundwith relationship file path",
                 description=(
-                    "Path to a file outlining Boundwith relationships, in the style of Voyager."
-                    " A TSV file with MFHD_ID and BIB_ID headers and values"
+                    "Path to a file outlining Boundwith relationships, "
+                    "in the style of Voyager. "
+                    "A TSV file with MFHD_ID and BIB_ID headers and values. "
+                    "By default is empty string."
                 ),
             ),
         ] = ""
@@ -73,7 +184,9 @@ class ItemsTransformer(MigrationTaskBase):
             Field(
                 title="Prevent permanent location map default",
                 description=(
-                    "Prevent the default mapping of permanent location to the default location."
+                    "Prevent the default mapping of permanent location "
+                    "to the default location. "
+                    "By default is False."
                 ),
             ),
         ] = False

folio_migration_tools/migration_tasks/loans_migrator.py

@@ -6,9 +6,10 @@ import sys
 import time
 import traceback
 from datetime import datetime, timedelta
-from typing import Optional
+from typing import Annotated, Optional
 from urllib.error import HTTPError
 from zoneinfo import ZoneInfo
+from pydantic import Field
 
 import i18n
 from dateutil import parser as du_parser
@@ -35,13 +36,64 @@ from folio_migration_tools.transaction_migration.transaction_result import (
 
 class LoansMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        open_loans_files: list[FileDefinition]
-        fallback_service_point_id: str
-        starting_row: Optional[int] = 1
-        item_files: Optional[list[FileDefinition]] = []
-        patron_files: Optional[list[FileDefinition]] = []
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of the migration task.",
+            ),
+        ]
+        open_loans_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Open loans files",
+                description="List of files containing open loan data."
+            ),
+        ]
+        fallback_service_point_id: Annotated[
+            str,
+            Field(
+                title="Fallback service point ID",
+                description="Identifier of the fallback service point.",
+            ),
+        ]
+        starting_row: Annotated[
+            Optional[int],
+            Field(
+                title="Starting row",
+                description=(
+                    "The starting row for data processing. "
+                    "By default is 1."
+                ),
+            ),
+        ] = 1
+        item_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Item files",
+                description=(
+                    "List of files containing item data. "
+                    "By default is empty list."
+                ),
+            ),
+        ] = []
+        patron_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Patron files",
+                description=(
+                    "List of files containing patron data. "
+                    "By default is empty list."
+                ),
+            ),
+        ] = []
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -729,7 +781,7 @@ def timings(t0, t0func, num_objects):
 
 
 def print_smtp_warning():
-    s = """
+    s = r"""
           _____   __  __   _____   ______   ___
          / ____| |  \/  | |_   _| |  __  | |__ \\
         | (___   | \  / |   | |   | |__|_|    ) |

folio_migration_tools/migration_tasks/migration_task_base.py

@@ -1,4 +1,5 @@
 import csv
+import io
 import json
 import logging
 import os
@@ -119,7 +120,11 @@ class MigrationTaskBase:
             TransformationProcessError: _description_
 
         """
-        files = [source_path / f.file_name for f in file_defs if isfile(source_path / f.file_name)]
+        files = [
+            source_path / f.file_name
+            for f in file_defs
+            if isfile(source_path / f.file_name)
+        ]
         ret_str = ", ".join(f.file_name for f in file_defs)
 
         if files and len(files) < len(file_defs):
@@ -141,7 +146,9 @@ class MigrationTaskBase:
     @staticmethod
     def load_id_map(map_path, raise_if_empty=False):
         if not isfile(map_path):
-            logging.warn("No legacy id map found at %s. Will build one from scratch", map_path)
+            logging.warn(
+                "No legacy id map found at %s. Will build one from scratch", map_path
+            )
             return {}
         id_map = {}
         loaded_rows = 0
@@ -194,7 +201,9 @@ class MigrationTaskBase:
         else:
             logger.setLevel(logging.INFO)
             stream_handler.setLevel(logging.INFO)
-            stream_handler.addFilter(ExcludeLevelFilter(30))  # Exclude warnings from pymarc
+            stream_handler.addFilter(
+                ExcludeLevelFilter(30)
+            )  # Exclude warnings from pymarc
         stream_handler.setFormatter(formatter)
         logger.addHandler(stream_handler)
 
@@ -225,16 +234,22 @@ class MigrationTaskBase:
     def setup_records_map(self, mapping_file_path):
         with open(mapping_file_path) as mapping_file:
             field_map = json.load(mapping_file)
-            logging.info("%s fields present in record mapping file", len(field_map["data"]))
+            logging.info(
+                "%s fields present in record mapping file", len(field_map["data"])
+            )
             mapped_fields = (
                 f
                 for f in field_map["data"]
                 if f["legacy_field"] and f["legacy_field"] != "Not mapped"
             )
-            logging.info("%s fields mapped in record mapping file", len(list(mapped_fields)))
+            logging.info(
+                "%s fields mapped in record mapping file", len(list(mapped_fields))
+            )
             return field_map
 
-    def log_and_exit_if_too_many_errors(self, error: TransformationRecordFailedError, idx):
+    def log_and_exit_if_too_many_errors(
+        self, error: TransformationRecordFailedError, idx
+    ):
         self.num_exeptions += 1
         error.log_it()
         if self.num_exeptions / (1 + idx) > 0.2 and self.num_exeptions > 5000:
@@ -250,7 +265,9 @@ class MigrationTaskBase:
         if num_processed > 1 and num_processed % 10000 == 0:
             elapsed = num_processed / (time.time() - start_time)
             elapsed_formatted = "{0:.4g}".format(elapsed)
-            logging.info(f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} ")
+            logging.info(
+                f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} "
+            )
 
     def do_work_marc_transformer(
         self,
@@ -259,7 +276,9 @@ class MigrationTaskBase:
         if self.folder_structure.failed_marc_recs_file.is_file():
             os.remove(self.folder_structure.failed_marc_recs_file)
             logging.info("Removed failed marc records file to prevent duplicating data")
-        with open(self.folder_structure.created_objects_path, "w+") as created_records_file:
+        with open(
+            self.folder_structure.created_objects_path, "w+"
+        ) as created_records_file:
             self.processor = MarcFileProcessor(
                 self.mapper, self.folder_structure, created_records_file
             )
@@ -271,13 +290,87 @@ class MigrationTaskBase:
                     self.folder_structure,
                 )
 
+    @staticmethod
+    def validate_ref_data_mapping_lines(lines, num_of_columns):
+        """
+        Helper method to validate the structure of individual lines in a mapping file.
+
+        Args:
+            lines (list): List of lines in the mapping file
+            num_of_columns (int): Number of columns expected in each line
+
+        Returns:
+            tuple: A tuple containing a list of invalid lines and a list of valid lines
+        """
+        invalid_lines = []
+        valid_lines = []
+        for idx, row in enumerate(lines, start=2):
+            if not row.strip():
+                if idx == len(lines) + 1:
+                    continue
+                else:
+                    invalid_lines.append(str(idx))
+            else:
+                line_length = len(row.split("\t"))
+                if line_length != num_of_columns:
+                    invalid_lines.append(str(idx))
+                else:
+                    valid_lines.append(str(idx))
+        return invalid_lines, valid_lines
+
+    @staticmethod
+    def verify_ref_data_mapping_file_structure(map_file: io.TextIOBase):
+        """
+        Helper method to validate the structure of a mapping file.
+
+        Args:
+            map_file (io.TextIOBase): The mapping file to validate
+
+        Raises:
+            TransformationProcessError: If the mapping file has rows with different number of columns
+
+        Returns:
+            None
+        """
+        current_pos = map_file.tell()
+        try:
+            map_file.seek(0)
+            num_of_columns = len(map_file.readline().split("\t"))
+            lines = map_file.readlines()
+            invalid_lines, valid_lines = MigrationTaskBase.validate_ref_data_mapping_lines(
+                lines, num_of_columns
+            )
+            if invalid_lines:
+                raise TransformationProcessError(
+                    "",
+                    (
+                        f"Mapping file {map_file.name} has rows with different number "
+                        f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} {', '.join(invalid_lines)})"
+                    ),
+                )
+            if not valid_lines:
+                raise TransformationProcessError(
+                    "", f"Map has no rows: {map_file.name}"
+                )
+        finally:
+            map_file.seek(current_pos)
+
+    @staticmethod
     def load_ref_data_mapping_file(
-        self,
         folio_property_name: str,
         map_file_path: Path,
         folio_keys,
         required: bool = True,
     ):
+        """
+        Helper method to load a reference data mapping file.
+
+        Args:
+            folio_property_name (str): The name of the property in FOLIO
+            map_file_path (Path): The path to the mapping file
+            folio_keys (list): A list of FOLIO keys
+            required (bool): Whether the property is required or not
+        """
         if (
             folio_property_name in folio_keys
             or required
@@ -286,9 +379,9 @@ class MigrationTaskBase:
         ):
             try:
                 with open(map_file_path) as map_file:
+                    # Validate the structure of the mapping file
+                    MigrationTaskBase.verify_ref_data_mapping_file_structure(map_file)
                     ref_data_map = list(csv.DictReader(map_file, dialect="tsv"))
-                    if not ref_data_map:
-                        raise TransformationProcessError("", f"Map has no rows: {map_file_path}")
                     logging.info(
                         "Found %s rows in %s map",
                         len(ref_data_map),

folio_migration_tools/migration_tasks/orders_transformer.py

@@ -5,7 +5,8 @@ import logging
 import sys
 import time
 from os.path import isfile
-from typing import List, Optional
+from typing import List, Optional, Annotated
+from pydantic import Field
 
 import i18n
 from deepdiff import DeepDiff
@@ -26,7 +27,9 @@ from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base
 from folio_migration_tools.mapping_file_transformation.order_mapper import (
     CompositeOrderMapper,
 )
-from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.migration_tasks.migration_task_base import (
+    MigrationTaskBase,
+)
 from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
@@ -35,18 +38,108 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 # Read files and do some work
 class OrdersTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        files: List[FileDefinition]
-        orders_mapping_file_name: str
-        organizations_code_map_file_name: str
-        acquisition_method_map_file_name: str
-        payment_status_map_file_name: Optional[str] = ""
-        receipt_status_map_file_name: Optional[str] = ""
-        workflow_status_map_file_name: Optional[str] = ""
-        location_map_file_name: Optional[str] = ""
-        funds_map_file_name: Optional[str] = ""
-        funds_expense_class_map_file_name: Optional[str] = ""
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of the migration task.",
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of the files.",
+            ),
+        ]
+        orders_mapping_file_name: Annotated[
+            str,
+            Field(
+                title="Orders Mapping File Name",
+                description="File name for orders mapping.",
+            ),
+        ]
+        organizations_code_map_file_name: Annotated[
+            str,
+            Field(
+                title="Organizations Code Map File Name",
+                description="File name for organizations code mapping.",
+            ),
+        ]
+        acquisition_method_map_file_name: Annotated[
+            str,
+            Field(
+                title="Acquisition Method Map File Name",
+                description="File name for acquisition method mapping.",
+            ),
+        ]
+        payment_status_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Payment Status Map File Name",
+                description=(
+                    "File name for payment status mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        receipt_status_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Receipt Status Map File Name",
+                description=(
+                    "File name for receipt status mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        workflow_status_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Workflow Status Map File Name",
+                description=(
+                    "File name for workflow status mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        location_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Location Map File Name",
+                description=(
+                    "File name for location mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        funds_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Funds Map File Name",
+                description=(
+                    "File name for funds mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        funds_expense_class_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Funds Expense Class Map File Name",
+                description=(
+                    "File name for funds expense class mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces: