folio-migration-tools 1.10.1__py3-none-any.whl → 1.10.3__py3-none-any.whl

folio_migration_tools/migration_tasks/marc_import.py

@@ -0,0 +1,407 @@
+"""MARCImportTask module for FOLIO MARC data import operations.
+
+This module provides an adapter that wraps folio_data_import.MARCImportJob
+to conform to the folio_migration_tools MigrationTaskBase interface.
+It supports importing MARC records directly into FOLIO using the Data Import
+APIs (change-manager), bypassing the need for SRS record creation during MARC transformation.
+
+This provides an alternative workflow for MARC record loading, using FOLIO's
+native Data Import capabilities.
+"""
+
+import asyncio
+import json
+import logging
+from pathlib import Path
+from typing import Annotated, Dict, List
+
+from folio_data_import._progress import RichProgressReporter
+from folio_data_import.MARCDataImport import MARCImportJob as FDIMARCImportJob
+from folio_uuid.folio_namespaces import FOLIONamespaces
+from pydantic import Field
+
+from folio_migration_tools.library_configuration import (
+    FileDefinition,
+    LibraryConfiguration,
+)
+from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
+
+
+class MARCImportTask(MigrationTaskBase):
+    """MARCImportTask.
+
+    An adapter that wraps folio_data_import.MARCImportJob to provide MARC import
+    functionality via FOLIO's Data Import APIs while conforming to the
+    MigrationTaskBase interface.
+
+    This implementation:
+    - Imports MARC records using the change-manager APIs
+    - Uses configurable Data Import job profiles
+    - Supports MARC record preprocessing
+    - Handles large files with optional splitting
+    - Tracks job IDs for monitoring in FOLIO
+
+    Parents:
+        MigrationTaskBase: Base class for all migration tasks
+
+    Raises:
+        TransformationProcessError: When a critical error occurs during processing
+        FileNotFoundError: When input files are not found
+    """
+
+    class TaskConfiguration(AbstractTaskConfiguration):
+        """Task configuration for MARCImportTask."""
+
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task",
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="List of files",
+                description=(
+                    "List of MARC files to be imported. Files should be in binary MARC "
+                    "format (.mrc). Located in the source_data folder."
+                ),
+            ),
+        ]
+        import_profile_name: Annotated[
+            str,
+            Field(
+                title="Import profile name",
+                description=(
+                    "The name of the Data Import job profile to use in FOLIO. "
+                    "This profile determines how MARC records are processed and "
+                    "what FOLIO records are created/updated."
+                ),
+            ),
+        ]
+        batch_size: Annotated[
+            int,
+            Field(
+                title="Batch size",
+                description="Number of MARC records to include in each batch sent to FOLIO",
+                ge=1,
+                le=1000,
+            ),
+        ] = 10
+        batch_delay: Annotated[
+            float,
+            Field(
+                title="Batch delay",
+                description=(
+                    "Number of seconds to wait between record batches. "
+                    "Use this to throttle requests if needed."
+                ),
+                ge=0.0,
+            ),
+        ] = 0.0
+        marc_record_preprocessors: Annotated[
+            List[str],
+            Field(
+                title="MARC record preprocessors",
+                description=(
+                    "List of preprocessor names to apply to each record before import. "
+                    "Preprocessors can modify MARC records before they are sent to FOLIO."
+                ),
+            ),
+        ] = []
+        preprocessors_args: Annotated[
+            Dict[str, Dict] | str,
+            Field(
+                title="Preprocessor arguments",
+                description=(
+                    "Dictionary of arguments to pass to the MARC record preprocessors. "
+                    "Keys are preprocessor names, values are dicts of arguments."
+                ),
+            ),
+        ] = {}
+        split_files: Annotated[
+            bool,
+            Field(
+                title="Split files",
+                description=(
+                    "Split each file into smaller jobs of size split_size. "
+                    "Useful for very large files that may timeout or be difficult to monitor."
+                ),
+            ),
+        ] = False
+        split_size: Annotated[
+            int,
+            Field(
+                title="Split size",
+                description="Number of records to include in each split file",
+                ge=1,
+            ),
+        ] = 1000
+        split_offset: Annotated[
+            int,
+            Field(
+                title="Split offset",
+                description=(
+                    "Number of split files to skip before starting processing. "
+                    "Useful for resuming a partially completed import."
+                ),
+                ge=0,
+            ),
+        ] = 0
+        show_file_names_in_data_import_logs: Annotated[
+            bool,
+            Field(
+                title="Show file names in Data Import logs",
+                description=(
+                    "If true, set the file name for each job in the Data Import logs. "
+                    "This makes it easier to identify jobs in the FOLIO UI."
+                ),
+            ),
+        ] = False
+        let_summary_fail: Annotated[
+            bool,
+            Field(
+                title="Let summary fail",
+                description=(
+                    "If true, do not retry or fail the import if the final job summary "
+                    "cannot be retrieved. Useful when FOLIO is under heavy load."
+                ),
+            ),
+        ] = False
+        skip_summary: Annotated[
+            bool,
+            Field(
+                title="Skip summary",
+                description=(
+                    "If true, skip fetching the final job summary after import. "
+                    "The import will complete but detailed statistics won't be available."
+                ),
+            ),
+        ] = False
+        no_progress: Annotated[
+            bool,
+            Field(
+                title="No progress",
+                description=(
+                    "Disable progress reporting in the console output. "
+                    "Set to true for non-interactive/CI environments."
+                ),
+            ),
+        ] = False
+
+    task_configuration: TaskConfiguration
+
+    @staticmethod
+    def get_object_type() -> FOLIONamespaces:
+        return FOLIONamespaces.instances  # MARC imports primarily create instances
+
+    def __init__(
+        self,
+        task_config: TaskConfiguration,
+        library_config: LibraryConfiguration,
+        folio_client,
+        use_logging: bool = True,
+    ):
+        """Initialize MarcImport for MARC record import via Data Import APIs.
+
+        Args:
+            task_config (TaskConfiguration): MARC import configuration.
+            library_config (LibraryConfiguration): Library configuration.
+            folio_client: FOLIO API client.
+            use_logging (bool): Whether to set up task logging.
+        """
+        super().__init__(library_config, task_config, folio_client, use_logging)
+        self.migration_report = MigrationReport()
+        self.total_records_sent = 0
+        self.job_ids: List[str] = []
+        self.files_processed: List[str] = []
+
+        logging.info("MARCImportTask initialized")
+        logging.info("Import profile: %s", self.task_configuration.import_profile_name)
+        logging.info("Batch size: %s", self.task_configuration.batch_size)
+        logging.info("Results folder: %s", self.folder_structure.results_folder)
+
+    def _create_fdi_config(self, file_paths: List[Path]) -> FDIMARCImportJob.Config:
+        """Create a folio_data_import.MARCImportJob.Config from our TaskConfiguration.
+
+        Args:
+            file_paths: List of file paths to process
+
+        Returns:
+            FDIMARCImportJob.Config: Configuration for the underlying MARCImportJob
+
+        Note:
+            The folio_data_import MARCImportJob places error files (bad_marc_records_*.mrc,
+            failed_batches_*.mrc) in the parent directory of the first MARC file. Since
+            we're reading from results_folder, error files will also be created there.
+        """
+        # Convert preprocessor list to comma-separated string for folio_data_import
+        # folio_data_import expects List[Callable], str (comma-separated), or None
+        preprocessors_str = (
+            ",".join(self.task_configuration.marc_record_preprocessors)
+            if self.task_configuration.marc_record_preprocessors
+            else None
+        )
+        if isinstance(self.task_configuration.preprocessors_args, str):
+            with open(
+                self.folder_structure.mapping_files_folder
+                / self.task_configuration.preprocessors_args,
+                "r",
+            ) as f:
+                preprocessors_args = json.load(f)
+        else:
+            preprocessors_args = self.task_configuration.preprocessors_args
+
+        return FDIMARCImportJob.Config(
+            marc_files=file_paths,
+            import_profile_name=self.task_configuration.import_profile_name,
+            batch_size=self.task_configuration.batch_size,
+            batch_delay=self.task_configuration.batch_delay,
+            marc_record_preprocessors=preprocessors_str,
+            preprocessors_args=preprocessors_args,
+            no_progress=self.task_configuration.no_progress,
+            no_summary=self.task_configuration.skip_summary,
+            let_summary_fail=self.task_configuration.let_summary_fail,
+            split_files=self.task_configuration.split_files,
+            split_size=self.task_configuration.split_size,
+            split_offset=self.task_configuration.split_offset,
+            job_ids_file_path=self.folder_structure.results_folder / "marc_import_job_ids.txt",
+            show_file_names_in_data_import_logs=(
+                self.task_configuration.show_file_names_in_data_import_logs
+            ),
+        )
+
+    async def _do_work_async(self) -> None:
+        """Async implementation of the work logic."""
+        file_paths: List[Path] = []
+        for file_def in self.task_configuration.files:
+            path = self.folder_structure.results_folder / file_def.file_name
+            if not path.exists():
+                logging.error("File not found: %s", path)
+                raise FileNotFoundError(f"File not found: {path}")
+            file_paths.append(path)
+            self.files_processed.append(file_def.file_name)
+            logging.info("Will process file: %s", path)
+
+        # Create the folio_data_import MARCImportJob config
+        fdi_config = self._create_fdi_config(file_paths)
+
+        # Create progress reporter
+        if self.task_configuration.no_progress:
+            from folio_data_import._progress import NoOpProgressReporter
+
+            reporter = NoOpProgressReporter()
+        else:
+            reporter = RichProgressReporter(enabled=True)
+
+        # Create and run the importer
+        # folio_data_import handles its own error files and progress reporting
+        importer = FDIMARCImportJob(
+            folio_client=self.folio_client,
+            config=fdi_config,
+            reporter=reporter,
+        )
+
+        await importer.do_work()
+        await importer.wrap_up()
+
+        # Capture stats and job IDs from the importer
+        self.total_records_sent = importer.total_records_sent
+        self.job_ids = importer.job_ids
+
+        # Note: Detailed stats (created/updated/discarded/error) are retrieved from
+        # the job summary by folio_data_import and logged via log_job_summary().
+        # We don't have direct access to those stats as they're logged, not returned.
+
+    def do_work(self) -> None:
+        """Main work method that processes MARC files and imports them to FOLIO.
+
+        This method reads MARC records from the configured files and imports them
+        to FOLIO using the Data Import APIs via folio_data_import.MARCImportJob.
+        """
+        logging.info("Starting MARCImportTask work...")
+
+        try:
+            # Run the async work in an event loop
+            asyncio.run(self._do_work_async())
+        except FileNotFoundError as e:
+            logging.error("File not found: %s", e)
+            raise
+        except Exception as e:
+            logging.error("Error during MARC import: %s", e)
+            raise
+
+        logging.info("MARCImportTask work complete")
+
+    def _translate_stats_to_migration_report(self) -> None:
+        """Translate MARC import stats to MigrationReport format.
+
+        Note:
+            Detailed stats (created, updated, discarded, error) are retrieved from
+            the FOLIO job summary and logged by folio_data_import's log_job_summary().
+            We report what we can track directly: records sent and job IDs.
+        """
+        # General statistics
+        self.migration_report.set(
+            "GeneralStatistics",
+            "Records sent to Data Import",
+            self.total_records_sent,
+        )
+
+        self.migration_report.set(
+            "GeneralStatistics",
+            "Data Import jobs created",
+            len(self.job_ids),
+        )
+
+        # Add file information
+        for file_name in self.files_processed:
+            self.migration_report.add("FilesProcessed", file_name)
+
+    def wrap_up(self) -> None:
+        """Finalize the migration task and write reports.
+
+        This method translates statistics to the MigrationReport format and writes
+        both markdown and JSON reports. Error files created by folio_data_import
+        (bad_marc_records_*.mrc, failed_batches_*.mrc) are already in results_folder
+        since that's where we read the input files from.
+        """
+        logging.info("Done. Wrapping up MARCImportTask")
+
+        # Translate stats to migration report
+        self._translate_stats_to_migration_report()
+
+        # Log summary
+        logging.info("=" * 60)
+        logging.info("MARCImportTask Summary")
+        logging.info("=" * 60)
+        logging.info("Records sent to Data Import: %d", self.total_records_sent)
+        logging.info("Files processed: %d", len(self.files_processed))
+        logging.info("Data Import jobs created: %d", len(self.job_ids))
+
+        # Write markdown report
+        with open(self.folder_structure.migration_reports_file, "w+") as report_file:
+            self.migration_report.write_migration_report(
+                "MARC Data Import report",
+                report_file,
+                self.start_datetime,
+            )
+
+        # Write raw JSON report
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.migration_report.write_json_report(raw_report_file)
+
+        # Clean up empty log files
+        self.clean_out_empty_logs()
+
+        logging.info("MARCImportTask wrap up complete")

folio_migration_tools/migration_tasks/migration_task_base.py

@@ -1,3 +1,14 @@
+"""Base class and utilities for all migration tasks.
+
+This module provides the abstract MigrationTaskBase class that all migration tasks
+must inherit from. It handles common functionality including:
+- Folder structure setup and management
+- Logging configuration
+- Reference data mapping file loading and validation
+- MARC file processing workflows
+- Error tracking and reporting
+"""
+
 import csv
 import io
 import json
@@ -44,11 +55,18 @@ class MigrationTaskBase:
         folio_client: folioclient.FolioClient,
         use_logging: bool = True,
     ):
-        logging.info("MigrationTaskBase init")
+        """Initialize base migration task with configurations and FOLIO client.
+
+        Args:
+            library_configuration: Library-specific configuration.
+            task_configuration: Task-specific configuration.
+            folio_client: FOLIO API client.
+            use_logging (bool): Whether to set up task logging.
+        """
         self.start_datetime = datetime.now(timezone.utc)
         self.task_configuration = task_configuration
-        logging.info(self.task_configuration.model_dump_json(indent=4))
         self.folio_client: FolioClient = folio_client
+        self.library_configuration = library_configuration
         self.ecs_tenant_id = (
             task_configuration.ecs_tenant_id or library_configuration.ecs_tenant_id
         )
@@ -72,7 +90,6 @@ class MigrationTaskBase:
             library_configuration.add_time_stamp_to_file_names,
         )
 
-        self.library_configuration = library_configuration
         self.object_type = self.get_object_type()
         try:
             self.folder_structure.setup_migration_file_structure()
@@ -89,8 +106,13 @@ class MigrationTaskBase:
         self.extradata_writer = ExtradataWriter(
             self.folder_structure.transformation_extra_data_path
         )
+
+        # Setup logging after folder_structure, library_configuration, and task_configuration
+        # are initialized since setup_logging depends on all three
         if use_logging:
             self.setup_logging()
+        logging.info("MigrationTaskBase init")
+        logging.info(self.task_configuration.model_dump_json(indent=4))
         self.folder_structure.log_folder_structure()
         logging.info("MigrationTaskBase init done")
 
@@ -123,7 +145,7 @@ class MigrationTaskBase:
     def check_source_files(
         source_path: Path, file_defs: list[library_configuration.FileDefinition]
     ) -> None:
-        """Lists the source data files. Special case since we use the Items folder for holdings
+        """Lists the source data files. Special case since we use the Items folder for holdings.
 
         Args:
             source_path (Path): _description_
@@ -153,12 +175,11 @@ class MigrationTaskBase:
             logging.info("\t%s", filename)
 
     def load_instance_id_map(self, raise_if_empty=True) -> dict:
+        """Load instance ID maps for holdings and other transformations.
+
+        Handles ECS environments where instances are transformed for central and
+        data tenants separately, but data tenants need central tenant instance IDs.
         """
-        This method handles loading instance id maps for holdings and other transformations that require it.
-        This is in the base class because multiple tasks need it. It exists because instances in an ECS environment
-        are transformed for the central and data tenants separately, but the data tenants need to know about
-        the central tenant instance ids. This is a bit of a hack, but it works for now.
-        """  # noqa: E501
         map_files = []
         instance_id_map = {}
         if self.library_configuration.is_ecs and self.central_folder_structure:
@@ -321,8 +342,7 @@ class MigrationTaskBase:
 
     @staticmethod
     def validate_ref_data_mapping_lines(lines, num_of_columns):
-        """
-        Helper method to validate the structure of individual lines in a mapping file.
+        """Helper method to validate the structure of individual lines in a mapping file.
 
         Args:
             lines (list): List of lines in the mapping file
@@ -349,8 +369,7 @@ class MigrationTaskBase:
 
     @staticmethod
     def verify_ref_data_mapping_file_structure(map_file: io.TextIOBase):
-        """
-        Helper method to validate the structure of a mapping file.
+        """Helper method to validate the structure of a mapping file.
 
         Args:
             map_file (io.TextIOBase): The mapping file to validate
@@ -390,8 +409,7 @@ class MigrationTaskBase:
         folio_keys,
         required: bool = True,
     ):
-        """
-        Helper method to load a reference data mapping file.
+        """Helper method to load a reference data mapping file.
 
         Args:
             folio_property_name (str): The name of the property in FOLIO
@@ -445,8 +463,7 @@ class MigrationTaskBase:
 
 
 class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
-    """
-    Base class for MARC task configurations.
+    """Base class for MARC task configurations.
 
     Attributes:
         files (List[library_configuration.FileDefinition]):
@@ -530,6 +547,11 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
 
 class ExcludeLevelFilter(logging.Filter):
     def __init__(self, level):
+        """Initialize filter to exclude logs of a specific level.
+
+        Args:
+            level: Logging level to exclude.
+        """
         super().__init__()
         self.level = level
 
@@ -539,6 +561,11 @@ class ExcludeLevelFilter(logging.Filter):
 
 class TaskNameFilter(logging.Filter):
     def __init__(self, task_configuration_name):
+        """Initialize filter to add task name to log records.
+
+        Args:
+            task_configuration_name (str): Name of the task configuration to add to logs.
+        """
         super().__init__()
         self.task_configuration_name = task_configuration_name
 
@@ -549,6 +576,11 @@ class TaskNameFilter(logging.Filter):
 
 class LevelFilter(logging.Filter):
     def __init__(self, level):
+        """Initialize filter to include only logs of a specific level.
+
+        Args:
+            level: Logging level to include.
+        """
         super().__init__()
         self.level = level
 

folio_migration_tools/migration_tasks/orders_transformer.py

@@ -1,3 +1,9 @@
+"""Purchase orders transformation task.
+
+Transforms purchase order data from CSV files to FOLIO Orders. Handles composite
+orders with embedded purchase order lines, acquisition units, and vendor references.
+"""
+
 import csv
 import ctypes
 import json
@@ -38,6 +44,8 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 # Read files and do some work
 class OrdersTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
+        """Task configuration for OrdersTransformer."""
+
         name: Annotated[
             str,
             Field(
@@ -136,6 +144,14 @@ class OrdersTransformer(MigrationTaskBase):
         folio_client,
         use_logging: bool = True,
     ):
+        """Initialize OrdersTransformer for purchase order transformations.
+
+        Args:
+            task_config (TaskConfiguration): Orders transformation configuration.
+            library_config (LibraryConfiguration): Library configuration.
+            folio_client: FOLIO API client.
+            use_logging (bool): Whether to set up task logging.
+        """
         csv.register_dialect("tsv", delimiter="\t")
 
         super().__init__(library_config, task_config, folio_client, use_logging)

folio_migration_tools/migration_tasks/organization_transformer.py

@@ -1,3 +1,9 @@
+"""Organization records transformation task.
+
+Transforms organization/vendor data from CSV files to FOLIO Organizations. Handles
+embedded interfaces, contacts, and categories with extradata object creation.
+"""
+
 import csv
 import ctypes
 import json
@@ -37,6 +43,8 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 # Read files and do some work
 class OrganizationTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
+        """Task configuration for OrganizationTransformer."""
+
         name: Annotated[
             str,
             Field(
@@ -111,6 +119,14 @@ class OrganizationTransformer(MigrationTaskBase):
         folio_client,
         use_logging: bool = True,
     ):
+        """Initialize OrganizationTransformer for organization transformations.
+
+        Args:
+            task_configuration (TaskConfiguration): Organizations transformation config.
+            library_config (LibraryConfiguration): Library configuration.
+            folio_client: FOLIO API client.
+            use_logging (bool): Whether to set up task logging.
+        """
         csv.register_dialect("tsv", delimiter="\t")
 
         super().__init__(library_config, task_configuration, folio_client, use_logging)
@@ -401,8 +417,7 @@ class OrganizationTransformer(MigrationTaskBase):
         return record
 
     def create_referenced_extradata_object(self, embedded_object, extradata_object_type):
-        """Creates an extradata object from an embedded object,
-        and returns the UUID.
+        """Create an extradata object from an embedded object and return its UUID.
 
         Args:
             embedded_object (_type_): _description_

folio_migration_tools/migration_tasks/requests_migrator.py

@@ -1,3 +1,9 @@
+"""Request records migration task.
+
+Migrates patron requests from legacy ILS to FOLIO. Validates patron and item
+barcodes, handles request types and statuses, and maintains request dates.
+"""
+
 import csv
 import json
 import logging
@@ -26,6 +32,8 @@ from folio_migration_tools.transaction_migration.legacy_request import LegacyReq
 
 class RequestsMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
+        """Task configuration for RequestsMigrator."""
+
         name: Annotated[
             str,
             Field(
@@ -88,6 +96,13 @@ class RequestsMigrator(MigrationTaskBase):
         library_config: LibraryConfiguration,
         folio_client,
     ):
+        """Initialize RequestsMigrator for migrating circulation requests.
+
+        Args:
+            task_configuration (TaskConfiguration): Requests migration configuration.
+            library_config (LibraryConfiguration): Library configuration.
+            folio_client: FOLIO API client.
+        """
         csv.register_dialect("tsv", delimiter="\t")
         self.migration_report = MigrationReport()
         self.valid_legacy_requests = []