folio-data-import 0.2.8rc7__py3-none-any.whl → 0.2.8rc9__py3-none-any.whl

folio_data_import/MARCDataImport.py

@@ -1,18 +1,20 @@
 import argparse
 import asyncio
+import datetime
 import glob
 import importlib
 import io
+import logging
 import os
 import sys
-from typing import List
 import uuid
 from contextlib import ExitStack
-import datetime
 from datetime import datetime as dt
+from functools import cached_property
 from getpass import getpass
 from pathlib import Path
 from time import sleep
+from typing import List
 
 import folioclient
 import httpx
@@ -21,8 +23,6 @@ import pymarc
 import tabulate
 from humps import decamelize
 from tqdm import tqdm
-from zmq import has
-
 
 try:
     datetime_utc = datetime.UTC
@@ -37,6 +37,18 @@ REPORT_SUMMARY_ORDERING = {"created": 0, "updated": 1, "discarded": 2, "error":
 RETRY_TIMEOUT_START = 1
 RETRY_TIMEOUT_RETRY_FACTOR = 2
 
+# Custom log level for data issues, set to 26
+DATA_ISSUE_LVL_NUM = 26
+logging.addLevelName(DATA_ISSUE_LVL_NUM, "DATA_ISSUES")
+
+def data_issues(self, msg, *args, **kws):
+    if self.isEnabledFor(DATA_ISSUE_LVL_NUM):
+        self._log(DATA_ISSUE_LVL_NUM, msg, args, **kws)
+
+logging.Logger.data_issues = data_issues
+
+logger = logging.getLogger(__name__)
+
 class MARCImportJob:
     """
     Class to manage importing MARC data (Bib, Authority) into FOLIO using the Change Manager
@@ -57,7 +69,6 @@ class MARCImportJob:
     bad_records_file: io.TextIOWrapper
     failed_batches_file: io.TextIOWrapper
     job_id: str
-    job_import_profile: dict
     pbar_sent: tqdm
     pbar_imported: tqdm
     http_client: httpx.Client
@@ -78,9 +89,11 @@ class MARCImportJob:
         marc_record_preprocessor=None,
         consolidate=False,
         no_progress=False,
+        let_summary_fail=False,
     ) -> None:
         self.consolidate_files = consolidate
         self.no_progress = no_progress
+        self.let_summary_fail = let_summary_fail
         self.folio_client: folioclient.FolioClient = folio_client
         self.import_files = marc_files
         self.import_profile_name = import_profile_name
@@ -88,6 +101,10 @@ class MARCImportJob:
         self.batch_delay = batch_delay
         self.current_retry_timeout = None
         self.marc_record_preprocessor = marc_record_preprocessor
+        self.pbar_sent: tqdm
+        self.pbar_imported: tqdm
+        self._max_summary_retries: int = 2
+        self._summary_retries: int = 0
 
     async def do_work(self) -> None:
         """
@@ -101,21 +118,25 @@ class MARCImportJob:
         Returns:
             None
         """
-        with httpx.Client() as http_client, open(
-            self.import_files[0].parent.joinpath(
-                f"bad_marc_records_{dt.now(tz=datetime_utc).strftime('%Y%m%d%H%M%S')}.mrc"
-            ),
-            "wb+",
-        ) as bad_marc_file, open(
-            self.import_files[0].parent.joinpath(
-                f"failed_batches_{dt.now(tz=datetime_utc).strftime('%Y%m%d%H%M%S')}.mrc"
-            ),
-            "wb+",
-        ) as failed_batches:
+        with (
+            httpx.Client() as http_client,
+            open(
+                self.import_files[0].parent.joinpath(
+                    f"bad_marc_records_{dt.now(tz=datetime_utc).strftime('%Y%m%d%H%M%S')}.mrc"
+                ),
+                "wb+",
+            ) as bad_marc_file,
+            open(
+                self.import_files[0].parent.joinpath(
+                    f"failed_batches_{dt.now(tz=datetime_utc).strftime('%Y%m%d%H%M%S')}.mrc"
+                ),
+                "wb+",
+            ) as failed_batches,
+        ):
             self.bad_records_file = bad_marc_file
-            print(f"Writing bad records to {self.bad_records_file.name}")
+            logger.info(f"Writing bad records to {self.bad_records_file.name}")
             self.failed_batches_file = failed_batches
-            print(f"Writing failed batches to {self.failed_batches_file.name}")
+            logger.info(f"Writing failed batches to {self.failed_batches_file.name}")
             self.http_client = http_client
             if self.consolidate_files:
                 self.current_file = self.import_files
@@ -136,16 +157,16 @@ class MARCImportJob:
         Returns:
             None
         """
-        self.bad_records_file.seek(0)
-        if not self.bad_records_file.read(1):
-            os.remove(self.bad_records_file.name)
-            print("No bad records found. Removing bad records file.")
-        self.failed_batches_file.seek(0)
-        if not self.failed_batches_file.read(1):
-            os.remove(self.failed_batches_file.name)
-            print("No failed batches. Removing failed batches file.")
-        print("Import complete.")
-        print(f"Total records imported: {self.total_records_sent}")
+        with open(self.bad_records_file.name, "rb") as bad_records:
+            if not bad_records.read(1):
+                os.remove(bad_records.name)
+                logger.info("No bad records found. Removing bad records file.")
+        with open(self.failed_batches_file.name, "rb") as failed_batches:
+            if not failed_batches.read(1):
+                os.remove(failed_batches.name)
+                logger.info("No failed batches. Removing failed batches file.")
+        logger.info("Import complete.")
+        logger.info(f"Total records imported: {self.total_records_sent}")
 
     async def get_job_status(self) -> None:
         """
@@ -159,38 +180,69 @@ class MARCImportJob:
         """
         try:
             self.current_retry_timeout = (
-                self.current_retry_timeout * RETRY_TIMEOUT_RETRY_FACTOR
-            ) if self.current_retry_timeout else RETRY_TIMEOUT_START
-            job_status = self.folio_client.folio_get(
-                "/metadata-provider/jobExecutions?statusNot=DISCARDED&uiStatusAny"
-                "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
+                (self.current_retry_timeout * RETRY_TIMEOUT_RETRY_FACTOR)
+                if self.current_retry_timeout
+                else RETRY_TIMEOUT_START
             )
-            self.current_retry_timeout = None
-        except (httpx.ConnectTimeout, httpx.ReadTimeout):
-            sleep(.25)
             with httpx.Client(
                 timeout=self.current_retry_timeout,
-                verify=self.folio_client.ssl_verify
+                verify=self.folio_client.ssl_verify,
             ) as temp_client:
-                self.folio_client.httpx_client = temp_client
-                return await self.get_job_status()
+                job_status = self.folio_client.folio_get(
+                    "/metadata-provider/jobExecutions?statusNot=DISCARDED&uiStatusAny"
+                    "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
+                )
+                self.current_retry_timeout = None
+        except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+            if not hasattr(e, "response") or e.response.status_code in [502, 504]:
+                error_text = e.response.text if hasattr(e, "response") else str(e)
+                logger.warning(f"SERVER ERROR fetching job status: {error_text}. Retrying.")
+                sleep(0.25)
+                with httpx.Client(
+                    timeout=self.current_retry_timeout,
+                    verify=self.folio_client.ssl_verify,
+                ) as temp_client:
+                    self.folio_client.httpx_client = temp_client
+                    return await self.get_job_status()
+            else:
+                raise e
+        except Exception as e:
+            logger.error(f"Error fetching job status. {e}")
+
         try:
             status = [
                 job for job in job_status["jobExecutions"] if job["id"] == self.job_id
             ][0]
             self.pbar_imported.update(status["progress"]["current"] - self.last_current)
             self.last_current = status["progress"]["current"]
-        except IndexError:
-            job_status = self.folio_client.folio_get(
-                "/metadata-provider/jobExecutions?limit=100&sortBy=completed_date%2Cdesc&statusAny"
-                "=COMMITTED&statusAny=ERROR&statusAny=CANCELLED"
-            )
-            status = [
-                job for job in job_status["jobExecutions"] if job["id"] == self.job_id
-            ][0]
-            self.pbar_imported.update(status["progress"]["current"] - self.last_current)
-            self.last_current = status["progress"]["current"]
-            self.finished = True
+        except (IndexError, ValueError, KeyError):
+            logger.debug(f"No active job found with ID {self.job_id}. Checking for finished job.")
+            try:
+                job_status = self.folio_client.folio_get(
+                    "/metadata-provider/jobExecutions?limit=100&sortBy=completed_date%2Cdesc&statusAny"
+                    "=COMMITTED&statusAny=ERROR&statusAny=CANCELLED"
+                )
+                status = [
+                    job for job in job_status["jobExecutions"] if job["id"] == self.job_id
+                ][0]
+                self.pbar_imported.update(status["progress"]["current"] - self.last_current)
+                self.last_current = status["progress"]["current"]
+                self.finished = True
+            except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+                if not hasattr(e, "response") or e.response.status_code in [502, 504]:
+                    error_text = e.response.text if hasattr(e, "response") else str(e)
+                    logger.warning(
+                        f"SERVER ERROR fetching job status: {error_text}. Retrying."
+                    )
+                    sleep(0.25)
+                    with httpx.Client(
+                        timeout=self.current_retry_timeout,
+                        verify=self.folio_client.ssl_verify,
+                    ) as temp_client:
+                        self.folio_client.httpx_client = temp_client
+                        return await self.get_job_status()
+                else:
+                    raise e
 
     async def create_folio_import_job(self) -> None:
         """
@@ -202,26 +254,36 @@ class MARCImportJob:
         Raises:
             HTTPError: If there is an error creating the job.
         """
-        create_job = self.http_client.post(
-            self.folio_client.okapi_url + "/change-manager/jobExecutions",
-            headers=self.folio_client.okapi_headers,
-            json={"sourceType": "ONLINE", "userId": self.folio_client.current_user},
-        )
         try:
-            create_job.raise_for_status()
-        except httpx.HTTPError as e:
-            print(
-                "Error creating job: "
-                + str(e)
-                + "\n"
-                + getattr(getattr(e, "response", ""), "text", "")
+            create_job = self.http_client.post(
+                self.folio_client.okapi_url + "/change-manager/jobExecutions",
+                headers=self.folio_client.okapi_headers,
+                json={"sourceType": "ONLINE", "userId": self.folio_client.current_user},
             )
-            raise e
+            create_job.raise_for_status()
+        except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+            if not hasattr(e, "response") or e.response.status_code in [502, 504]:
+                logger.warning(f"SERVER ERROR creating job: {e}. Retrying.")
+                sleep(0.25)
+                return await self.create_folio_import_job()
+            else:
+                logger.error(
+                    "Error creating job: "
+                    + str(e)
+                    + "\n"
+                    + getattr(getattr(e, "response", ""), "text", "")
+                )
+                raise e
         self.job_id = create_job.json()["parentJobExecutionId"]
+        logger.info("Created job: " + self.job_id)
 
-    async def get_import_profile(self) -> None:
+    @cached_property
+    def import_profile(self) -> dict:
         """
-        Retrieves the import profile with the specified name.
+        Returns the import profile for the current job execution.
+
+        Returns:
+            dict: The import profile for the current job execution.
         """
         import_profiles = self.folio_client.folio_get(
             "/data-import-profiles/jobProfiles",
@@ -233,7 +295,7 @@ class MARCImportJob:
             for profile in import_profiles
             if profile["name"] == self.import_profile_name
         ][0]
-        self.job_import_profile = profile
+        return profile
 
     async def set_job_profile(self) -> None:
         """
@@ -249,15 +311,15 @@ class MARCImportJob:
             + "/jobProfile",
             headers=self.folio_client.okapi_headers,
             json={
-                "id": self.job_import_profile["id"],
-                "name": self.job_import_profile["name"],
+                "id": self.import_profile["id"],
+                "name": self.import_profile["name"],
                 "dataType": "MARC",
             },
         )
         try:
             set_job_profile.raise_for_status()
         except httpx.HTTPError as e:
-            print(
+            logger.error(
                 "Error creating job: "
                 + str(e)
                 + "\n"
@@ -299,8 +361,13 @@ class MARCImportJob:
                 headers=self.folio_client.okapi_headers,
                 json=batch_payload,
             )
-        except httpx.ReadTimeout:
-            sleep(.25)
+            # if batch_payload["recordsMetadata"]["last"]:
+            #     logger.log(
+            #         25,
+            #         f"Sending last batch of {batch_payload['recordsMetadata']['total']} records.",
+            #     )
+        except (httpx.ConnectTimeout, httpx.ReadTimeout):
+            sleep(0.25)
             return await self.process_record_batch(batch_payload)
         try:
             post_batch.raise_for_status()
@@ -308,12 +375,14 @@ class MARCImportJob:
             self.record_batch = []
             self.pbar_sent.update(len(batch_payload["initialRecords"]))
         except Exception as e:
-            if hasattr(e, "response") and e.response.status_code in [500, 422]: # TODO: #26 Check for specific error code once https://folio-org.atlassian.net/browse/MODSOURMAN-1281 is resolved
+            if (
+                hasattr(e, "response") and e.response.status_code in [500, 422]
+            ):  # TODO: #26 Check for specific error code once https://folio-org.atlassian.net/browse/MODSOURMAN-1281 is resolved
                 self.total_records_sent += len(self.record_batch)
                 self.record_batch = []
                 self.pbar_sent.update(len(batch_payload["initialRecords"]))
             else:
-                print("Error posting batch: " + str(e))
+                logger.error("Error posting batch: " + str(e))
                 for record in self.record_batch:
                     self.failed_batches_file.write(record)
                     self.error_records += len(self.record_batch)
@@ -335,14 +404,20 @@ class MARCImportJob:
         """
         counter = 0
         for import_file in files:
+            file_path = Path(import_file.name)
             self.pbar_sent.set_description(
                 f"Sent ({os.path.basename(import_file.name)}): "
             )
             reader = pymarc.MARCReader(import_file, hide_utf8_warnings=True)
-            for record in reader:
+            for idx, record in enumerate(reader, start=1):
                 if len(self.record_batch) == self.batch_size:
                     await self.process_record_batch(
-                        await self.create_batch_payload(counter, total_records, False),
+                        await self.create_batch_payload(
+                            counter,
+                            total_records,
+                            (counter - self.error_records)
+                            == (total_records - self.error_records),
+                        ),
                     )
                     await self.get_job_status()
                     sleep(0.25)
@@ -354,14 +429,35 @@ class MARCImportJob:
                     self.record_batch.append(record.as_marc())
                     counter += 1
                 else:
+                    logger.data_issues(
+                        "RECORD FAILED\t%s\t%s\t%s",
+                        f"{file_path.name}:{idx}",
+                        f"Error reading {idx} record from {file_path}. Skipping. Writing current chunk to {self.bad_records_file.name}.",
+                        "",
+                    )
                     self.bad_records_file.write(reader.current_chunk)
             if self.record_batch:
                 await self.process_record_batch(
-                    await self.create_batch_payload(counter, total_records, True),
+                    await self.create_batch_payload(
+                        counter,
+                        total_records,
+                        (counter - self.error_records)
+                        == (total_records - self.error_records),
+                    ),
                 )
+            import_complete_path = file_path.parent.joinpath("import_complete")
+            if not import_complete_path.exists():
+                logger.debug(f"Creating import_complete directory: {import_complete_path.absolute()}")
+                import_complete_path.mkdir(exist_ok=True)
+            logger.debug(f"Moving {file_path} to {import_complete_path.absolute()}")
+            file_path.rename(
+                file_path.parent.joinpath("import_complete", file_path.name)
+            )
 
     @staticmethod
-    async def apply_marc_record_preprocessing(record: pymarc.Record, func_or_path) -> pymarc.Record:
+    async def apply_marc_record_preprocessing(
+        record: pymarc.Record, func_or_path
+    ) -> pymarc.Record:
         """
         Apply preprocessing to the MARC record before sending it to FOLIO.
 
@@ -373,25 +469,42 @@ class MARCImportJob:
             pymarc.Record: The preprocessed MARC record.
         """
         if isinstance(func_or_path, str):
-            try:
-                path_parts = func_or_path.rsplit('.')
-                module_path, func_name = ".".join(path_parts[:-1]), path_parts[-1]
-                module = importlib.import_module(module_path)
-                func = getattr(module, func_name)
-            except (ImportError, AttributeError) as e:
-                print(f"Error importing preprocessing function {func_or_path}: {e}. Skipping preprocessing.")
-                return record
+            func_paths = func_or_path.split(",")
+            for func_path in func_paths:
+                record = await MARCImportJob._apply_single_marc_record_preprocessing_by_path(
+                    record, func_path
+                )
         elif callable(func_or_path):
-            func = func_or_path
+            record = func_or_path(record)
         else:
-            print(f"Invalid preprocessing function: {func_or_path}. Skipping preprocessing.")
-            return record
+            logger.warning(
+                f"Invalid preprocessing function: {func_or_path}. Skipping preprocessing."
+            )
+        return record
 
+    async def _apply_single_marc_record_preprocessing_by_path(
+        record: pymarc.Record, func_path: str
+    ) -> pymarc.Record:
+        """
+        Apply a single preprocessing function to the MARC record.
+
+        Args:
+            record (pymarc.Record): The MARC record to preprocess.
+            func_path (str): The path to the preprocessing function.
+
+        Returns:
+            pymarc.Record: The preprocessed MARC record.
+        """
         try:
-            return func(record)
+            module_path, func_name = func_path.rsplit(".", 1)
+            module = importlib.import_module(module_path)
+            func = getattr(module, func_name)
+            record = func(record)
         except Exception as e:
-            print(f"Error applying preprocessing function: {e}. Skipping preprocessing.")
-            return record
+            logger.warning(
+                f"Error applying preprocessing function {func_path}: {e}. Skipping."
+            )
+        return record
 
     async def create_batch_payload(self, counter, total_records, is_last) -> dict:
         """
@@ -435,24 +548,26 @@ class MARCImportJob:
             None
         """
         await self.create_folio_import_job()
-        await self.get_import_profile()
         await self.set_job_profile()
         with ExitStack() as stack:
             files = [
                 stack.enter_context(open(file, "rb")) for file in self.current_file
             ]
             total_records = await self.read_total_records(files)
-            with tqdm(
-                desc="Imported: ",
-                total=total_records,
-                position=1,
-                disable=self.no_progress,
-            ) as pbar_imported, tqdm(
-                desc="Sent: ()",
-                total=total_records,
-                position=0,
-                disable=self.no_progress,
-            ) as pbar_sent:
+            with (
+                tqdm(
+                    desc="Imported: ",
+                    total=total_records,
+                    position=1,
+                    disable=self.no_progress,
+                ) as pbar_imported,
+                tqdm(
+                    desc="Sent: ()",
+                    total=total_records,
+                    position=0,
+                    disable=self.no_progress,
+                ) as pbar_sent,
+            ):
                 self.pbar_sent = pbar_sent
                 self.pbar_imported = pbar_imported
                 await self.process_records(files, total_records)
@@ -461,8 +576,8 @@ class MARCImportJob:
                 sleep(1)
             if self.finished:
                 if job_summary := await self.get_job_summary():
-                    job_summary.pop("jobExecutionId")
-                    job_summary.pop("totalErrors")
+                    job_id = job_summary.pop("jobExecutionId", None)
+                    total_errors = job_summary.pop("totalErrors", 0)
                     columns = ["Summary"] + list(job_summary.keys())
                     rows = set()
                     for key in columns[1:]:
@@ -479,17 +594,20 @@ class MARCImportJob:
                     columns = columns[:1] + [
                         " ".join(decamelize(x).split("_")[:-1]) for x in columns[1:]
                     ]
-                    print(
+                    logger.info(
                         f"Results for {'file' if len(self.current_file) == 1 else 'files'}: "
                         f"{', '.join([os.path.basename(x.name) for x in self.current_file])}"
                     )
-                    print(
-                        tabulate.tabulate(
+                    logger.info(
+                        "\n"
+                        + tabulate.tabulate(
                             table_data, headers=columns, tablefmt="fancy_grid"
                         ),
                     )
+                    if total_errors:
+                        logger.info(f"Total errors: {total_errors}. Job ID: {job_id}.")
                 else:
-                    print(f"No job summary available for job {self.job_id}.")
+                    logger.error(f"No job summary available for job {self.job_id}.")
             self.last_current = 0
             self.finished = False
 
@@ -502,11 +620,12 @@ class MARCImportJob:
         """
         try:
             self.current_retry_timeout = (
-                self.current_retry_timeout * RETRY_TIMEOUT_RETRY_FACTOR
-            ) if self.current_retry_timeout else RETRY_TIMEOUT_START
+                (self.current_retry_timeout * RETRY_TIMEOUT_RETRY_FACTOR)
+                if self.current_retry_timeout
+                else RETRY_TIMEOUT_START
+            )
             with httpx.Client(
-                timeout=self.current_retry_timeout,
-                verify=self.folio_client.ssl_verify
+                timeout=self.current_retry_timeout, verify=self.folio_client.ssl_verify
             ) as temp_client:
                 self.folio_client.httpx_client = temp_client
                 job_summary = self.folio_client.folio_get(
@@ -514,21 +633,75 @@ class MARCImportJob:
                 )
             self.current_retry_timeout = None
         except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
-            if not hasattr(e, "response") or e.response.status_code == 502:
-                sleep(.25)
+            error_text = e.response.text if hasattr(e, "response") else str(e)
+            if (self._max_summary_retries > self._summary_retries and not hasattr(e, "response")) or (
+                e.response.status_code in [502, 504] and not self.let_summary_fail
+            ):
+                logger.warning(f"SERVER ERROR fetching job summary: {e}. Retrying.")
+                sleep(0.25)
                 with httpx.Client(
                     timeout=self.current_retry_timeout,
-                    verify=self.folio_client.ssl_verify
+                    verify=self.folio_client.ssl_verify,
                 ) as temp_client:
                     self.folio_client.httpx_client = temp_client
-                    return await self.get_job_status()
-            elif hasattr(e, "response") and e.response.status_code == 504:
+                    self._summary_retries += 1
+                    return await self.get_job_summary()
+            elif (self._summary_retries >= self._max_summary_retries) or (hasattr(e, "response") and (
+                e.response.status_code in [502, 504] and self.let_summary_fail)
+            ):
+                logger.warning(
+                    f"SERVER ERROR fetching job summary: {error_text}. Skipping final summary check."
+                )
                 job_summary = {}
             else:
                 raise e
         return job_summary
 
 
+def set_up_cli_logging():
+    """
+    This function sets up logging for the CLI.
+    """
+    logger.setLevel(logging.INFO)
+    logger.propagate = False
+
+    # Set up file and stream handlers
+    file_handler = logging.FileHandler(
+        "folio_data_import_{}.log".format(dt.now().strftime("%Y%m%d%H%M%S"))
+    )
+    file_handler.setLevel(logging.INFO)
+    file_handler.addFilter(ExcludeLevelFilter(DATA_ISSUE_LVL_NUM))
+    # file_handler.addFilter(IncludeLevelFilter(25))
+    file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    file_handler.setFormatter(file_formatter)
+    logger.addHandler(file_handler)
+
+    if not any(
+        isinstance(h, logging.StreamHandler) and h.stream == sys.stderr
+        for h in logger.handlers
+    ):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        stream_handler.setLevel(logging.INFO)
+        stream_handler.addFilter(ExcludeLevelFilter(DATA_ISSUE_LVL_NUM))
+        # stream_handler.addFilter(ExcludeLevelFilter(25))
+        stream_formatter = logging.Formatter("%(message)s")
+        stream_handler.setFormatter(stream_formatter)
+        logger.addHandler(stream_handler)
+
+    # Set up data issues logging
+    data_issues_handler = logging.FileHandler(
+        "marc_import_data_issues_{}.log".format(dt.now().strftime("%Y%m%d%H%M%S"))
+    )
+    data_issues_handler.setLevel(26)
+    data_issues_handler.addFilter(IncludeLevelFilter(DATA_ISSUE_LVL_NUM))
+    data_issues_formatter = logging.Formatter("%(message)s")
+    data_issues_handler.setFormatter(data_issues_formatter)
+    logger.addHandler(data_issues_handler)
+
+    # Stop httpx from logging info messages to the console
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+
+
 async def main() -> None:
     """
     Main function to run the MARC import job.
@@ -536,6 +709,7 @@ async def main() -> None:
     This function parses command line arguments, initializes the FolioClient,
     and runs the MARCImportJob.
     """
+    set_up_cli_logging()
     parser = argparse.ArgumentParser()
     parser.add_argument("--gateway_url", type=str, help="The FOLIO API Gateway URL")
     parser.add_argument("--tenant_id", type=str, help="The FOLIO tenant ID")
@@ -592,6 +766,11 @@ async def main() -> None:
         action="store_true",
         help="Disable progress bars (eg. for running in a CI environment)",
     )
+    parser.add_argument(
+        "--let-summary-fail",
+        action="store_true",
+        help="Do not retry fetching the final job summary if it fails",
+    )
     args = parser.parse_args()
     if not args.password:
         args.password = getpass("Enter FOLIO password: ")
@@ -611,10 +790,10 @@ async def main() -> None:
     marc_files.sort()
 
     if len(marc_files) == 0:
-        print(f"No files found matching {args.marc_file_path}. Exiting.")
+        logger.critical(f"No files found matching {args.marc_file_path}. Exiting.")
         sys.exit(1)
     else:
-        print(marc_files)
+        logger.info(marc_files)
 
     if not args.import_profile_name:
         import_profiles = folio_client.folio_get(
@@ -646,12 +825,31 @@ async def main() -> None:
             marc_record_preprocessor=args.preprocessor,
             consolidate=bool(args.consolidate),
             no_progress=bool(args.no_progress),
+            let_summary_fail=bool(args.let_summary_fail),
         ).do_work()
     except Exception as e:
-        print("Error importing files: " + str(e))
+        logger.error("Error importing files: " + str(e))
         raise
 
 
+class ExcludeLevelFilter(logging.Filter):
+    def __init__(self, level):
+        super().__init__()
+        self.level = level
+
+    def filter(self, record):
+        return record.levelno != self.level
+
+
+class IncludeLevelFilter(logging.Filter):
+    def __init__(self, level):
+        super().__init__()
+        self.level = level
+
+    def filter(self, record):
+        return record.levelno == self.level
+
+
 def sync_main() -> None:
     """
     Synchronous main function to run the MARC import job.

folio_data_import/marc_preprocessors/__init__.py

@@ -1 +1 @@
-from ._preprocessors import prepend_ppn_prefix_001, strip_999_ff_fields
+from ._preprocessors import *

folio_data_import/marc_preprocessors/_preprocessors.py

@@ -1,4 +1,8 @@
 import pymarc
+import logging
+
+logger = logging.getLogger("folio_data_import.MARCDataImport")
+
 
 def prepend_prefix_001(record: pymarc.Record, prefix: str) -> pymarc.Record:
     """
@@ -11,9 +15,10 @@ def prepend_prefix_001(record: pymarc.Record, prefix: str) -> pymarc.Record:
     Returns:
         pymarc.Record: The preprocessed MARC record.
     """
-    record['001'].data = f'({prefix})' + record['001'].data
+    record["001"].data = f"({prefix})" + record["001"].data
     return record
 
+
 def prepend_ppn_prefix_001(record: pymarc.Record) -> pymarc.Record:
     """
     Prepend the PPN prefix to the record's 001 field. Useful when
@@ -25,7 +30,8 @@ def prepend_ppn_prefix_001(record: pymarc.Record) -> pymarc.Record:
     Returns:
         pymarc.Record: The preprocessed MARC record.
     """
-    return prepend_prefix_001(record, 'PPN')
+    return prepend_prefix_001(record, "PPN")
+
 
 def prepend_abes_prefix_001(record: pymarc.Record) -> pymarc.Record:
     """
@@ -38,7 +44,8 @@ def prepend_abes_prefix_001(record: pymarc.Record) -> pymarc.Record:
     Returns:
         pymarc.Record: The preprocessed MARC record.
     """
-    return prepend_prefix_001(record, 'ABES')
+    return prepend_prefix_001(record, "ABES")
+
 
 def strip_999_ff_fields(record: pymarc.Record) -> pymarc.Record:
     """
@@ -51,11 +58,35 @@ def strip_999_ff_fields(record: pymarc.Record) -> pymarc.Record:
     Returns:
         pymarc.Record: The preprocessed MARC record.
     """
-    for field in record.get_fields('999'):
-        if field.indicators == pymarc.Indicators(*['f', 'f']):
+    for field in record.get_fields("999"):
+        if field.indicators == pymarc.Indicators(*["f", "f"]):
             record.remove_field(field)
     return record
 
+def clean_999_fields(record: pymarc.Record) -> pymarc.Record:
+    """
+    The presence of 999 fields, with or without ff indicators, can cause
+    issues with data import mapping in FOLIO. This function calls strip_999_ff_fields
+    to remove 999 fields with ff indicators and then copies the remaining 999 fields
+    to 945 fields.
+
+    Args:
+        record (pymarc.Record): The MARC record to preprocess.
+
+    Returns:
+        pymarc.Record: The preprocessed MARC record.
+    """
+    record = strip_999_ff_fields(record)
+    for field in record.get_fields("999"):
+        _945 = pymarc.Field(
+            tag="945",
+            indicators=field.indicators,
+            subfields=field.subfields,
+        )
+        record.add_ordered_field(_945)
+        record.remove_field(field)
+    return record
+
 def sudoc_supercede_prep(record: pymarc.Record) -> pymarc.Record:
     """
     Preprocesses a record from the ABES SUDOC catalog to copy 035 fields
@@ -71,14 +102,232 @@ def sudoc_supercede_prep(record: pymarc.Record) -> pymarc.Record:
         pymarc.Record: The preprocessed MARC record.
     """
     record = prepend_abes_prefix_001(record)
-    for field in record.get_fields('035'):
-        if "a" in field and "9" in field and field['9'] == 'sudoc':
+    for field in record.get_fields("035"):
+        if "a" in field and "9" in field and field["9"] == "sudoc":
             _935 = pymarc.Field(
-                tag='935',
-                indicators=['f', 'f'],
-                subfields=[
-                    pymarc.field.Subfield('a', "(ABES)" + field['a'])
-                ]
+                tag="935",
+                indicators=["f", "f"],
+                subfields=[pymarc.field.Subfield("a", "(ABES)" + field["a"])],
             )
             record.add_ordered_field(_935)
     return record
+
+
+def clean_empty_fields(record: pymarc.Record) -> pymarc.Record:
+    """
+    Remove empty fields and subfields from the record. These can cause
+    data import mapping issues in FOLIO. Removals are logged at custom
+    log level 26, which is used by folio_migration_tools to populate the
+    data issues report.
+
+    Args:
+        record (pymarc.Record): The MARC record to preprocess.
+
+    Returns:
+        pymarc.Record: The preprocessed MARC record.
+    """
+    MAPPED_FIELDS = {
+        "010": ["a", "z"],
+        "020": ["a", "y", "z"],
+        "035": ["a", "z"],
+        "040": ["a", "b", "c", "d", "e", "f", "g", "h", "k", "m", "n", "p", "r", "s"],
+        "050": ["a", "b"],
+        "082": ["a", "b"],
+        "100": ["a", "b", "c", "d", "q"],
+        "110": ["a", "b", "c"],
+        "111": ["a", "c", "d"],
+        "130": [
+            "a",
+            "d",
+            "f",
+            "k",
+            "l",
+            "m",
+            "n",
+            "o",
+            "p",
+            "r",
+            "s",
+            "t",
+            "x",
+            "y",
+            "z",
+        ],
+        "180": ["x", "y", "z"],
+        "210": ["a", "c"],
+        "240": ["a", "f", "k", "l", "m", "n", "o", "p", "r", "s", "t", "x", "y", "z"],
+        "245": ["a", "b", "c", "f", "g", "h", "k", "n", "p", "s"],
+        "246": ["a", "f", "g", "n", "p", "s"],
+        "250": ["a", "b"],
+        "260": ["a", "b", "c", "e", "f", "g"],
+        "300": ["a", "b", "c", "e", "f", "g"],
+        "440": ["a", "n", "p", "v", "x", "y", "z"],
+        "490": ["a", "v", "x", "y", "z"],
+        "500": ["a", "c", "d", "n", "p", "v", "x", "y", "z"],
+        "505": ["a", "g", "r", "t", "u"],
+        "520": ["a", "b", "c", "u"],
+        "600": ["a", "b", "c", "d", "q", "t", "v", "x", "y", "z"],
+        "610": ["a", "b", "c", "d", "t", "v", "x", "y", "z"],
+        "611": ["a", "c", "d", "t", "v", "x", "y", "z"],
+        "630": [
+            "a",
+            "d",
+            "f",
+            "k",
+            "l",
+            "m",
+            "n",
+            "o",
+            "p",
+            "r",
+            "s",
+            "t",
+            "x",
+            "y",
+            "z",
+        ],
+        "650": ["a", "d", "v", "x", "y", "z"],
+        "651": ["a", "v", "x", "y", "z"],
+        "655": ["a", "v", "x", "y", "z"],
+        "700": ["a", "b", "c", "d", "q", "t", "v", "x", "y", "z"],
+        "710": ["a", "b", "c", "d", "t", "v", "x", "y", "z"],
+        "711": ["a", "c", "d", "t", "v", "x", "y", "z"],
+        "730": [
+            "a",
+            "d",
+            "f",
+            "k",
+            "l",
+            "m",
+            "n",
+            "o",
+            "p",
+            "r",
+            "s",
+            "t",
+            "x",
+            "y",
+            "z",
+        ],
+        "740": ["a", "n", "p", "v", "x", "y", "z"],
+        "800": ["a", "b", "c", "d", "q", "t", "v", "x", "y", "z"],
+        "810": ["a", "b", "c", "d", "t", "v", "x", "y", "z"],
+        "811": ["a", "c", "d", "t", "v", "x", "y", "z"],
+        "830": [
+            "a",
+            "d",
+            "f",
+            "k",
+            "l",
+            "m",
+            "n",
+            "o",
+            "p",
+            "r",
+            "s",
+            "t",
+            "x",
+            "y",
+            "z",
+        ],
+        "856": ["u", "y", "z"],
+    }
+
+    for field in list(record.get_fields()):
+        len_subs = len(field.subfields)
+        subfield_value = bool(field.subfields[0].value) if len_subs > 0 else False
+        if not int(field.tag) >= 900 and field.tag in MAPPED_FIELDS:
+            if int(field.tag) > 9 and len_subs == 0:
+                logger.log(
+                    26,
+                    "DATA ISSUE\t%s\t%s\t%s",
+                    record["001"].value(),
+                    f"{field.tag} is empty, removing field",
+                    field,
+                )
+                record.remove_field(field)
+            elif len_subs == 1 and not subfield_value:
+                logger.log(
+                    26,
+                    "DATA ISSUE\t%s\t%s\t%s",
+                    record["001"].value(),
+                    f"{field.tag}${field.subfields[0].code} is empty, no other subfields present, removing field",
+                    field,
+                )
+                record.remove_field(field)
+            else:
+                if len_subs > 1 and "a" in field and not field["a"].strip():
+                    logger.log(
+                        26,
+                        "DATA ISSUE\t%s\t%s\t%s",
+                        record["001"].value(),
+                        f"{field.tag}$a is empty, removing subfield",
+                        field,
+                    )
+                    field.delete_subfield("a")
+                for idx, subfield in enumerate(list(field.subfields), start=1):
+                    if (
+                        subfield.code in MAPPED_FIELDS.get(field.tag, [])
+                        and not subfield.value
+                    ):
+                        logger.log(
+                            26,
+                            "DATA ISSUE\t%s\t%s\t%s",
+                            record["001"].value(),
+                            f"{field.tag}${subfield.code} ({ordinal(idx)} subfield) is empty, but other subfields have values, removing subfield",
+                            field,
+                        )
+                        field.delete_subfield(subfield.code)
+                if len(field.subfields) == 0:
+                    logger.log(
+                        26,
+                        "DATA ISSUE\t%s\t%s\t%s",
+                        record["001"].value(),
+                        f"{field.tag} has no non-empty subfields after cleaning, removing field",
+                        field,
+                    )
+                    record.remove_field(field)
+    return record
+
+
+def fix_leader(record: pymarc.Record) -> pymarc.Record:
+    """
+    Fixes the leader of the record by setting the record status to 'c' (modified
+    record) and the type of record to 'a' (language material).
+
+    Args:
+        record (pymarc.Record): The MARC record to preprocess.
+
+    Returns:
+        pymarc.Record: The preprocessed MARC record.
+    """
+    VALID_STATUSES = ["a", "c", "d", "n", "p"]
+    VALID_TYPES = ["a", "c", "d", "e", "f", "g", "i", "j", "k", "m", "o", "p", "r", "t"]
+    if record.leader[5] not in VALID_STATUSES:
+        logger.log(
+            26,
+            "DATA ISSUE\t%s\t%s\t%s",
+            record["001"].value(),
+            f"Invalid record status: {record.leader[5]}, setting to 'c'",
+            record,
+        )
+        record.leader = pymarc.Leader(record.leader[:5] + "c" + record.leader[6:])
+    if record.leader[6] not in VALID_TYPES:
+        logger.log(
+            26,
+            "DATA ISSUE\t%s\t%s\t%s",
+            record["001"].value(),
+            f"Invalid record type: {record.leader[6]}, setting to 'a'",
+            record,
+        )
+        record.leader = pymarc.Leader(record.leader[:6] + "a" + record.leader[7:])
+    return record
+
+
+def ordinal(n):
+    s = ("th", "st", "nd", "rd") + ("th",) * 10
+    v = n % 100
+    if v > 13:
+        return f"{n}{s[v % 10]}"
+    else:
+        return f"{n}{s[v]}"

{folio_data_import-0.2.8rc7.dist-info → folio_data_import-0.2.8rc9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_data_import
-Version: 0.2.8rc7
+Version: 0.2.8rc9
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis
@@ -108,11 +108,11 @@ Unlike mod-user-import, this importer does not require `externalSystemId` as the
 
 #### Preferred Contact Type Mapping
 
-Another point of departure from the behavior of `mod-user-import` is the handling of `preferredContactTypeId`. This importer will accept either the `"001", "002", "003"...` values stored by the FOLIO, or the human-friendly strings used by `mod-user-import` (`"mail", "email", "text", "phone", "mobile"`). It will also __*set a customizable default for all users that do not otherwise have a valid value specified*__ (using `--default_preferred_contact_type`), unless a (valid) value is already present in the user record being updated.
+Another point of departure from the behavior of `mod-user-import` is the handling of `preferredContactTypeId`. This importer will accept either the `"001", "002", "003"...` values stored by FOLIO, or the human-friendly strings used by `mod-user-import` (`"mail", "email", "text", "phone", "mobile"`). It will also __*set a customizable default for all users that do not otherwise have a valid value specified*__ (using `--default_preferred_contact_type`), unless a (valid) value is already present in the user record being updated.
 
 #### Field Protection (*experimental*)
 
-This script offers a rudimentary field protection implementation using custom fields. To enable this functionality, create a text custom field that has the field name `protectedFields`. In this field, you ca specify a comma-separated list of User schema field names, using dot-notation for nested fields. This protection should support all standard fields except addresses within `personal.addresses`. If you include `personal.addresses` in a user record, any existing addresses will be replaced by the new values.
+This script offers a rudimentary field protection implementation using custom fields. To enable this functionality, create a text custom field that has the field name `protectedFields`. In this field, you can specify a comma-separated list of User schema field names, using dot-notation for nested fields. This protection should support all standard fields except addresses within `personal.addresses`. If you include `personal.addresses` in a user record, any existing addresses will be replaced by the new values.
 
 ##### Example
 

folio_data_import-0.2.8rc9.dist-info/RECORD

@@ -0,0 +1,11 @@
+folio_data_import/MARCDataImport.py,sha256=ImbuGw1ADt4nCmq0lLaqugP2wv5kBrgMGAr0jbKSgFc,33135
+folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
+folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
+folio_data_import/marc_preprocessors/__init__.py,sha256=urExfNTQoZsDCtDPcUY9EEC5OFcUihxhYEQkQFVzbMY,30
+folio_data_import/marc_preprocessors/_preprocessors.py,sha256=4i1_lEnptzZDx3DojX9sfvJ_hmehwFJUC3aZsUADcwA,10851
+folio_data_import-0.2.8rc9.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
+folio_data_import-0.2.8rc9.dist-info/METADATA,sha256=Q80K34yk3xcZPfCf50FBtAYY7Hrxb3ukbAAGAv4uCEs,6112
+folio_data_import-0.2.8rc9.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+folio_data_import-0.2.8rc9.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
+folio_data_import-0.2.8rc9.dist-info/RECORD,,

folio_data_import-0.2.8rc7.dist-info/RECORD

@@ -1,11 +0,0 @@
-folio_data_import/MARCDataImport.py,sha256=b4Qdja0S1_mrrxjVkVsEBatm0Kr2ZfI3b5ZRkzD0kRA,24845
-folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
-folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
-folio_data_import/marc_preprocessors/__init__.py,sha256=Wt-TKkMhUyZWFS-WhAmbShKQLPjXmHKPb2vL6kvkqVA,72
-folio_data_import/marc_preprocessors/_preprocessors.py,sha256=CMG4Xq3vR84xmNFUJfRmnU0A5lchAfK33xDzeiC2AWk,2787
-folio_data_import-0.2.8rc7.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
-folio_data_import-0.2.8rc7.dist-info/METADATA,sha256=NtI0ue7pc177DJ7KtCgYekGYbeeDs-PC_-HI0AtZmCI,6115
-folio_data_import-0.2.8rc7.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-folio_data_import-0.2.8rc7.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
-folio_data_import-0.2.8rc7.dist-info/RECORD,,