folio-migration-tools 1.9.0rc3__py3-none-any.whl → 1.9.0rc5__py3-none-any.whl

folio_migration_tools/migration_tasks/loans_migrator.py

@@ -6,9 +6,10 @@ import sys
 import time
 import traceback
 from datetime import datetime, timedelta
-from typing import Optional
+from typing import Annotated, Optional
 from urllib.error import HTTPError
 from zoneinfo import ZoneInfo
+from pydantic import Field
 
 import i18n
 from dateutil import parser as du_parser
@@ -35,13 +36,64 @@ from folio_migration_tools.transaction_migration.transaction_result import (
 
 class LoansMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        open_loans_files: list[FileDefinition]
-        fallback_service_point_id: str
-        starting_row: Optional[int] = 1
-        item_files: Optional[list[FileDefinition]] = []
-        patron_files: Optional[list[FileDefinition]] = []
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of the migration task.",
+            ),
+        ]
+        open_loans_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Open loans files",
+                description="List of files containing open loan data."
+            ),
+        ]
+        fallback_service_point_id: Annotated[
+            str,
+            Field(
+                title="Fallback service point ID",
+                description="Identifier of the fallback service point.",
+            ),
+        ]
+        starting_row: Annotated[
+            Optional[int],
+            Field(
+                title="Starting row",
+                description=(
+                    "The starting row for data processing. "
+                    "By default is 1."
+                ),
+            ),
+        ] = 1
+        item_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Item files",
+                description=(
+                    "List of files containing item data. "
+                    "By default is empty list."
+                ),
+            ),
+        ] = []
+        patron_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Patron files",
+                description=(
+                    "List of files containing patron data. "
+                    "By default is empty list."
+                ),
+            ),
+        ] = []
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -729,7 +781,7 @@ def timings(t0, t0func, num_objects):
 
 
 def print_smtp_warning():
-    s = """
+    s = r"""
           _____   __  __   _____   ______   ___
          / ____| |  \/  | |_   _| |  __  | |__ \\
         | (___   | \  / |   | |   | |__|_|    ) |

folio_migration_tools/migration_tasks/migration_task_base.py

@@ -1,4 +1,5 @@
 import csv
+import io
 import json
 import logging
 import os
@@ -119,7 +120,11 @@ class MigrationTaskBase:
             TransformationProcessError: _description_
 
         """
-        files = [source_path / f.file_name for f in file_defs if isfile(source_path / f.file_name)]
+        files = [
+            source_path / f.file_name
+            for f in file_defs
+            if isfile(source_path / f.file_name)
+        ]
         ret_str = ", ".join(f.file_name for f in file_defs)
 
         if files and len(files) < len(file_defs):
@@ -141,7 +146,9 @@ class MigrationTaskBase:
     @staticmethod
     def load_id_map(map_path, raise_if_empty=False):
         if not isfile(map_path):
-            logging.warn("No legacy id map found at %s. Will build one from scratch", map_path)
+            logging.warn(
+                "No legacy id map found at %s. Will build one from scratch", map_path
+            )
             return {}
         id_map = {}
         loaded_rows = 0
@@ -194,7 +201,9 @@ class MigrationTaskBase:
         else:
             logger.setLevel(logging.INFO)
             stream_handler.setLevel(logging.INFO)
-            stream_handler.addFilter(ExcludeLevelFilter(30))  # Exclude warnings from pymarc
+            stream_handler.addFilter(
+                ExcludeLevelFilter(30)
+            )  # Exclude warnings from pymarc
         stream_handler.setFormatter(formatter)
         logger.addHandler(stream_handler)
 
@@ -225,16 +234,22 @@ class MigrationTaskBase:
     def setup_records_map(self, mapping_file_path):
         with open(mapping_file_path) as mapping_file:
             field_map = json.load(mapping_file)
-            logging.info("%s fields present in record mapping file", len(field_map["data"]))
+            logging.info(
+                "%s fields present in record mapping file", len(field_map["data"])
+            )
             mapped_fields = (
                 f
                 for f in field_map["data"]
                 if f["legacy_field"] and f["legacy_field"] != "Not mapped"
             )
-            logging.info("%s fields mapped in record mapping file", len(list(mapped_fields)))
+            logging.info(
+                "%s fields mapped in record mapping file", len(list(mapped_fields))
+            )
             return field_map
 
-    def log_and_exit_if_too_many_errors(self, error: TransformationRecordFailedError, idx):
+    def log_and_exit_if_too_many_errors(
+        self, error: TransformationRecordFailedError, idx
+    ):
         self.num_exeptions += 1
         error.log_it()
         if self.num_exeptions / (1 + idx) > 0.2 and self.num_exeptions > 5000:
@@ -250,7 +265,9 @@ class MigrationTaskBase:
         if num_processed > 1 and num_processed % 10000 == 0:
             elapsed = num_processed / (time.time() - start_time)
             elapsed_formatted = "{0:.4g}".format(elapsed)
-            logging.info(f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} ")
+            logging.info(
+                f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} "
+            )
 
     def do_work_marc_transformer(
         self,
@@ -259,7 +276,9 @@ class MigrationTaskBase:
         if self.folder_structure.failed_marc_recs_file.is_file():
             os.remove(self.folder_structure.failed_marc_recs_file)
             logging.info("Removed failed marc records file to prevent duplicating data")
-        with open(self.folder_structure.created_objects_path, "w+") as created_records_file:
+        with open(
+            self.folder_structure.created_objects_path, "w+"
+        ) as created_records_file:
             self.processor = MarcFileProcessor(
                 self.mapper, self.folder_structure, created_records_file
             )
@@ -271,13 +290,87 @@ class MigrationTaskBase:
                     self.folder_structure,
                 )
 
+    @staticmethod
+    def validate_ref_data_mapping_lines(lines, num_of_columns):
+        """
+        Helper method to validate the structure of individual lines in a mapping file.
+
+        Args:
+            lines (list): List of lines in the mapping file
+            num_of_columns (int): Number of columns expected in each line
+
+        Returns:
+            tuple: A tuple containing a list of invalid lines and a list of valid lines
+        """
+        invalid_lines = []
+        valid_lines = []
+        for idx, row in enumerate(lines, start=2):
+            if not row.strip():
+                if idx == len(lines) + 1:
+                    continue
+                else:
+                    invalid_lines.append(str(idx))
+            else:
+                line_length = len(row.split("\t"))
+                if line_length != num_of_columns:
+                    invalid_lines.append(str(idx))
+                else:
+                    valid_lines.append(str(idx))
+        return invalid_lines, valid_lines
+
+    @staticmethod
+    def verify_ref_data_mapping_file_structure(map_file: io.TextIOBase):
+        """
+        Helper method to validate the structure of a mapping file.
+
+        Args:
+            map_file (io.TextIOBase): The mapping file to validate
+
+        Raises:
+            TransformationProcessError: If the mapping file has rows with different number of columns
+
+        Returns:
+            None
+        """
+        current_pos = map_file.tell()
+        try:
+            map_file.seek(0)
+            num_of_columns = len(map_file.readline().split("\t"))
+            lines = map_file.readlines()
+            invalid_lines, valid_lines = MigrationTaskBase.validate_ref_data_mapping_lines(
+                lines, num_of_columns
+            )
+            if invalid_lines:
+                raise TransformationProcessError(
+                    "",
+                    (
+                        f"Mapping file {map_file.name} has rows with different number "
+                        f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} {', '.join(invalid_lines)})"
+                    ),
+                )
+            if not valid_lines:
+                raise TransformationProcessError(
+                    "", f"Map has no rows: {map_file.name}"
+                )
+        finally:
+            map_file.seek(current_pos)
+
+    @staticmethod
     def load_ref_data_mapping_file(
-        self,
         folio_property_name: str,
         map_file_path: Path,
         folio_keys,
         required: bool = True,
     ):
+        """
+        Helper method to load a reference data mapping file.
+
+        Args:
+            folio_property_name (str): The name of the property in FOLIO
+            map_file_path (Path): The path to the mapping file
+            folio_keys (list): A list of FOLIO keys
+            required (bool): Whether the property is required or not
+        """
         if (
             folio_property_name in folio_keys
             or required
@@ -286,9 +379,9 @@ class MigrationTaskBase:
         ):
             try:
                 with open(map_file_path) as map_file:
+                    # Validate the structure of the mapping file
+                    MigrationTaskBase.verify_ref_data_mapping_file_structure(map_file)
                     ref_data_map = list(csv.DictReader(map_file, dialect="tsv"))
-                    if not ref_data_map:
-                        raise TransformationProcessError("", f"Map has no rows: {map_file_path}")
                     logging.info(
                         "Found %s rows in %s map",
                         len(ref_data_map),

folio_migration_tools/migration_tasks/orders_transformer.py

@@ -5,7 +5,8 @@ import logging
 import sys
 import time
 from os.path import isfile
-from typing import List, Optional
+from typing import List, Optional, Annotated
+from pydantic import Field
 
 import i18n
 from deepdiff import DeepDiff
@@ -26,7 +27,9 @@ from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base
 from folio_migration_tools.mapping_file_transformation.order_mapper import (
     CompositeOrderMapper,
 )
-from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.migration_tasks.migration_task_base import (
+    MigrationTaskBase,
+)
 from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
@@ -35,18 +38,108 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 # Read files and do some work
 class OrdersTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        files: List[FileDefinition]
-        orders_mapping_file_name: str
-        organizations_code_map_file_name: str
-        acquisition_method_map_file_name: str
-        payment_status_map_file_name: Optional[str] = ""
-        receipt_status_map_file_name: Optional[str] = ""
-        workflow_status_map_file_name: Optional[str] = ""
-        location_map_file_name: Optional[str] = ""
-        funds_map_file_name: Optional[str] = ""
-        funds_expense_class_map_file_name: Optional[str] = ""
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of the migration task.",
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of the files.",
+            ),
+        ]
+        orders_mapping_file_name: Annotated[
+            str,
+            Field(
+                title="Orders Mapping File Name",
+                description="File name for orders mapping.",
+            ),
+        ]
+        organizations_code_map_file_name: Annotated[
+            str,
+            Field(
+                title="Organizations Code Map File Name",
+                description="File name for organizations code mapping.",
+            ),
+        ]
+        acquisition_method_map_file_name: Annotated[
+            str,
+            Field(
+                title="Acquisition Method Map File Name",
+                description="File name for acquisition method mapping.",
+            ),
+        ]
+        payment_status_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Payment Status Map File Name",
+                description=(
+                    "File name for payment status mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        receipt_status_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Receipt Status Map File Name",
+                description=(
+                    "File name for receipt status mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        workflow_status_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Workflow Status Map File Name",
+                description=(
+                    "File name for workflow status mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        location_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Location Map File Name",
+                description=(
+                    "File name for location mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        funds_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Funds Map File Name",
+                description=(
+                    "File name for funds mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
+        funds_expense_class_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Funds Expense Class Map File Name",
+                description=(
+                    "File name for funds expense class mapping. "
+                    "By default is empty string."
+                ),
+            ),
+        ] = ""
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:

folio_migration_tools/migration_tasks/organization_transformer.py

@@ -7,7 +7,8 @@ import time
 import uuid
 from hashlib import sha1
 from os.path import isfile
-from typing import List, Optional
+from typing import List, Optional, Annotated
+from pydantic import Field
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -36,14 +37,78 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 # Read files and do some work
 class OrganizationTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        files: List[FileDefinition]
-        organization_map_path: str
-        organization_types_map_path: Optional[str] = ""
-        address_categories_map_path: Optional[str] = ""
-        email_categories_map_path: Optional[str] = ""
-        phone_categories_map_path: Optional[str] = ""
+        name: Annotated[
+            str,
+            Field(
+                description=(
+                    "Name of this migration task. The name is being used to call the specific "
+                    "task, and to distinguish tasks of similar types"
+                ),
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description=(
+                    "The type of migration task you want to perform"
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Source files",
+                description=(
+                    "List of MARC21 files with holdings records"
+                ),
+            ),
+        ]
+        organization_map_path: Annotated[
+            str,
+            Field(
+                title="Organization map path",
+                description=(
+                    "Path to the organization map file"
+                ),
+            ),
+        ]
+        organization_types_map_path: Annotated[
+            Optional[str],
+            Field(
+                title="Organization types map path",
+                description=(
+                    "Path to the organization types map file. By default is empty string"
+                ),
+            ),
+        ] = ""
+        address_categories_map_path: Annotated[
+            Optional[str],
+            Field(
+                title="Address categories map path",
+                description=(
+                    "Path to the address categories map file. By default is empty string"
+                ),
+            ),
+        ] = ""
+        email_categories_map_path: Annotated[
+            Optional[str],
+            Field(
+                title="Email categories map path",
+                description=(
+                    "Path to the email categories map file. By default is empty string"
+                ),
+            ),
+        ] = ""
+        phone_categories_map_path: Annotated[
+            Optional[str],
+            Field(
+                title="Phone categories map path",
+                description=(
+                    "Path to the phone categories map file. By default is empty string"
+                ),
+            ),
+        ] = ""
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -178,8 +243,8 @@ class OrganizationTransformer(MigrationTaskBase):
                     self.mapper.handle_transformation_process_error(idx, process_error)
                 except TransformationRecordFailedError as error:
                     self.mapper.handle_transformation_record_failed_error(idx, error)
-                except Exception as excepion:
-                    self.mapper.handle_generic_exception(idx, excepion)
+                except Exception as exception:
+                    self.mapper.handle_generic_exception(idx, exception)
 
                 self.mapper.migration_report.add_general_statistics(
                     i18n.t("Number of objects in source data file")
@@ -227,7 +292,7 @@ class OrganizationTransformer(MigrationTaskBase):
                 self.folder_structure.migration_reports_file,
             )
             self.mapper.migration_report.write_migration_report(
-                i18n.t("Ogranization transformation report"),
+                i18n.t("Organization transformation report"),
                 migration_report_file,
                 self.start_datetime,
             )
@@ -269,7 +334,7 @@ class OrganizationTransformer(MigrationTaskBase):
             if address.get("isPrimary") is True:
                 primary_address_exists = True
 
-        # If none of the existing addresses is pimrary
+        # If none of the existing addresses is primary
         # Make the first one primary
         if not primary_address_exists:
             addresses[0]["isPrimary"] = True
@@ -364,7 +429,7 @@ class OrganizationTransformer(MigrationTaskBase):
 
         if len(identical_objects) > 0:
             self.mapper.migration_report.add_general_statistics(
-                i18n.t("Number of reoccuring identical %{type}", type=extradata_object_type)
+                i18n.t("Number of reoccurring identical %{type}", type=extradata_object_type)
             )
             Helper.log_data_issue(
                 f"{self.legacy_id}",

folio_migration_tools/migration_tasks/requests_migrator.py

@@ -3,7 +3,8 @@ import json
 import logging
 import sys
 import time
-from typing import Optional
+from typing import Optional, Annotated
+from pydantic import Field
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -24,12 +25,60 @@ from folio_migration_tools.transaction_migration.legacy_request import LegacyReq
 
 class RequestsMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        open_requests_file: FileDefinition
-        starting_row: Optional[int] = 1
-        item_files: Optional[list[FileDefinition]] = []
-        patron_files: Optional[list[FileDefinition]] = []
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description=(
+                    "Name of this migration task. The name is being used to call "
+                    "the specific task, and to distinguish tasks of similar types"
+                )
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task you want to perform",
+            ),
+        ]
+        open_requests_file: Annotated[
+            FileDefinition,
+            Field(
+                title="Open requests file",
+                description="File with list of open requests",
+            ),
+        ]
+        starting_row: Annotated[
+            Optional[int],
+            Field(
+                title="Starting row",
+                description=(
+                    "Row number to start processing data from. "
+                    "Optional, by default is first row"
+                ),
+            ),
+        ] = 1
+        item_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Item files",
+                description=(
+                    "List of files containing item data. "
+                    "Optional, by default is empty list"
+                ),
+            ),
+        ] = []
+        patron_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Patron files",
+                description=(
+                    "List of files containing patron data. "
+                    "Optional, by default is empty list"
+                ),
+            ),
+        ] = []
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:

folio_migration_tools/migration_tasks/reserves_migrator.py

@@ -4,8 +4,9 @@ import logging
 import sys
 import time
 import traceback
-from typing import Dict
+from typing import Dict, Annotated
 from urllib.error import HTTPError
+from pydantic import Field
 
 import httpx
 import i18n
@@ -25,9 +26,30 @@ from folio_migration_tools.transaction_migration.legacy_reserve import LegacyRes
 
 class ReservesMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        course_reserve_file_path: FileDefinition
+        name: Annotated[
+            str,
+            Field(
+                title="Migration task name",
+                description=(
+                    "Name of this migration task. The name is being used to call the specific "
+                    "task, and to distinguish tasks of similar types"
+                ),
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task you want to perform",
+            ),
+        ]
+        course_reserve_file_path: Annotated[
+            FileDefinition,
+            Field(
+                title="Course reserve file path",
+                description="Path to the file with course reserves",
+            ),
+        ]
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces: