folio-data-import 0.2.8rc8__py3-none-any.whl → 0.2.8rc10__py3-none-any.whl

folio_data_import/MARCDataImport.py

@@ -1,7 +1,6 @@
 import argparse
 import asyncio
 import datetime
-from email import message
 import glob
 import importlib
 import io
@@ -104,6 +103,8 @@ class MARCImportJob:
         self.marc_record_preprocessor = marc_record_preprocessor
         self.pbar_sent: tqdm
         self.pbar_imported: tqdm
+        self._max_summary_retries: int = 2
+        self._summary_retries: int = 0
 
     async def do_work(self) -> None:
         """
@@ -183,11 +184,15 @@ class MARCImportJob:
                 if self.current_retry_timeout
                 else RETRY_TIMEOUT_START
             )
-            job_status = self.folio_client.folio_get(
-                "/metadata-provider/jobExecutions?statusNot=DISCARDED&uiStatusAny"
-                "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
-            )
-            self.current_retry_timeout = None
+            with httpx.Client(
+                timeout=self.current_retry_timeout,
+                verify=self.folio_client.ssl_verify,
+            ) as temp_client:
+                job_status = self.folio_client.folio_get(
+                    "/metadata-provider/jobExecutions?statusNot=DISCARDED&uiStatusAny"
+                    "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
+                )
+                self.current_retry_timeout = None
         except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
             if not hasattr(e, "response") or e.response.status_code in [502, 504]:
                 error_text = e.response.text if hasattr(e, "response") else str(e)
@@ -201,13 +206,17 @@ class MARCImportJob:
                     return await self.get_job_status()
             else:
                 raise e
+        except Exception as e:
+            logger.error(f"Error fetching job status. {e}")
+
         try:
             status = [
                 job for job in job_status["jobExecutions"] if job["id"] == self.job_id
             ][0]
             self.pbar_imported.update(status["progress"]["current"] - self.last_current)
             self.last_current = status["progress"]["current"]
-        except IndexError:
+        except (IndexError, ValueError, KeyError):
+            logger.debug(f"No active job found with ID {self.job_id}. Checking for finished job.")
             try:
                 job_status = self.folio_client.folio_get(
                     "/metadata-provider/jobExecutions?limit=100&sortBy=completed_date%2Cdesc&statusAny"
@@ -245,21 +254,26 @@ class MARCImportJob:
         Raises:
             HTTPError: If there is an error creating the job.
         """
-        create_job = self.http_client.post(
-            self.folio_client.okapi_url + "/change-manager/jobExecutions",
-            headers=self.folio_client.okapi_headers,
-            json={"sourceType": "ONLINE", "userId": self.folio_client.current_user},
-        )
         try:
-            create_job.raise_for_status()
-        except httpx.HTTPError as e:
-            logger.error(
-                "Error creating job: "
-                + str(e)
-                + "\n"
-                + getattr(getattr(e, "response", ""), "text", "")
+            create_job = self.http_client.post(
+                self.folio_client.okapi_url + "/change-manager/jobExecutions",
+                headers=self.folio_client.okapi_headers,
+                json={"sourceType": "ONLINE", "userId": self.folio_client.current_user},
             )
-            raise e
+            create_job.raise_for_status()
+        except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+            if not hasattr(e, "response") or e.response.status_code in [502, 504]:
+                logger.warning(f"SERVER ERROR creating job: {e}. Retrying.")
+                sleep(0.25)
+                return await self.create_folio_import_job()
+            else:
+                logger.error(
+                    "Error creating job: "
+                    + str(e)
+                    + "\n"
+                    + getattr(getattr(e, "response", ""), "text", "")
+                )
+                raise e
         self.job_id = create_job.json()["parentJobExecutionId"]
         logger.info("Created job: " + self.job_id)
 
@@ -432,7 +446,7 @@ class MARCImportJob:
                     ),
                 )
             import_complete_path = file_path.parent.joinpath("import_complete")
-            if import_complete_path.exists():
+            if not import_complete_path.exists():
                 logger.debug(f"Creating import_complete directory: {import_complete_path.absolute()}")
                 import_complete_path.mkdir(exist_ok=True)
             logger.debug(f"Moving {file_path} to {import_complete_path.absolute()}")
@@ -455,31 +469,42 @@ class MARCImportJob:
             pymarc.Record: The preprocessed MARC record.
         """
         if isinstance(func_or_path, str):
-            try:
-                path_parts = func_or_path.rsplit(".")
-                module_path, func_name = ".".join(path_parts[:-1]), path_parts[-1]
-                module = importlib.import_module(module_path)
-                func = getattr(module, func_name)
-            except (ImportError, AttributeError) as e:
-                logger.error(
-                    f"Error importing preprocessing function {func_or_path}: {e}. Skipping preprocessing."
+            func_paths = func_or_path.split(",")
+            for func_path in func_paths:
+                record = await MARCImportJob._apply_single_marc_record_preprocessing_by_path(
+                    record, func_path
                 )
-                return record
         elif callable(func_or_path):
-            func = func_or_path
+            record = func_or_path(record)
         else:
             logger.warning(
                 f"Invalid preprocessing function: {func_or_path}. Skipping preprocessing."
             )
-            return record
+        return record
+
+    async def _apply_single_marc_record_preprocessing_by_path(
+        record: pymarc.Record, func_path: str
+    ) -> pymarc.Record:
+        """
+        Apply a single preprocessing function to the MARC record.
+
+        Args:
+            record (pymarc.Record): The MARC record to preprocess.
+            func_path (str): The path to the preprocessing function.
 
+        Returns:
+            pymarc.Record: The preprocessed MARC record.
+        """
         try:
-            return func(record)
+            module_path, func_name = func_path.rsplit(".", 1)
+            module = importlib.import_module(module_path)
+            func = getattr(module, func_name)
+            record = func(record)
         except Exception as e:
-            logger.error(
-                f"Error applying preprocessing function: {e}. Skipping preprocessing."
+            logger.warning(
+                f"Error applying preprocessing function {func_path}: {e}. Skipping."
             )
-            return record
+        return record
 
     async def create_batch_payload(self, counter, total_records, is_last) -> dict:
         """
@@ -609,8 +634,8 @@ class MARCImportJob:
             self.current_retry_timeout = None
         except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
             error_text = e.response.text if hasattr(e, "response") else str(e)
-            if not hasattr(e, "response") or (
-                e.response.status_code in [502, 504] and not self.let_summary_fail
+            if (self._max_summary_retries > self._summary_retries) and (not hasattr(e, "response") or (
+                hasattr(e, "response") and e.response.status_code in [502, 504]) and not self.let_summary_fail
             ):
                 logger.warning(f"SERVER ERROR fetching job summary: {e}. Retrying.")
                 sleep(0.25)
@@ -619,9 +644,10 @@ class MARCImportJob:
                     verify=self.folio_client.ssl_verify,
                 ) as temp_client:
                     self.folio_client.httpx_client = temp_client
+                    self._summary_retries += 1
                     return await self.get_job_summary()
-            elif hasattr(e, "response") and (
-                e.response.status_code in [502, 504] and self.let_summary_fail
+            elif (self._summary_retries >= self._max_summary_retries) or (hasattr(e, "response") and (
+                e.response.status_code in [502, 504] and self.let_summary_fail)
             ):
                 logger.warning(
                     f"SERVER ERROR fetching job summary: {error_text}. Skipping final summary check."
@@ -667,6 +693,7 @@ def set_up_cli_logging():
         "marc_import_data_issues_{}.log".format(dt.now().strftime("%Y%m%d%H%M%S"))
     )
     data_issues_handler.setLevel(26)
+    data_issues_handler.addFilter(IncludeLevelFilter(DATA_ISSUE_LVL_NUM))
     data_issues_formatter = logging.Formatter("%(message)s")
     data_issues_handler.setFormatter(data_issues_formatter)
     logger.addHandler(data_issues_handler)

folio_data_import/marc_preprocessors/_preprocessors.py

@@ -63,6 +63,29 @@ def strip_999_ff_fields(record: pymarc.Record) -> pymarc.Record:
             record.remove_field(field)
     return record
 
+def clean_999_fields(record: pymarc.Record) -> pymarc.Record:
+    """
+    The presence of 999 fields, with or without ff indicators, can cause
+    issues with data import mapping in FOLIO. This function calls strip_999_ff_fields
+    to remove 999 fields with ff indicators and then copies the remaining 999 fields
+    to 945 fields.
+
+    Args:
+        record (pymarc.Record): The MARC record to preprocess.
+
+    Returns:
+        pymarc.Record: The preprocessed MARC record.
+    """
+    record = strip_999_ff_fields(record)
+    for field in record.get_fields("999"):
+        _945 = pymarc.Field(
+            tag="945",
+            indicators=field.indicators,
+            subfields=field.subfields,
+        )
+        record.add_ordered_field(_945)
+        record.remove_field(field)
+    return record
 
 def sudoc_supercede_prep(record: pymarc.Record) -> pymarc.Record:
     """
@@ -219,7 +242,7 @@ def clean_empty_fields(record: pymarc.Record) -> pymarc.Record:
                     26,
                     "DATA ISSUE\t%s\t%s\t%s",
                     record["001"].value(),
-                    f"{field.tag} is empty",
+                    f"{field.tag} is empty, removing field",
                     field,
                 )
                 record.remove_field(field)
@@ -228,7 +251,7 @@ def clean_empty_fields(record: pymarc.Record) -> pymarc.Record:
                     26,
                     "DATA ISSUE\t%s\t%s\t%s",
                     record["001"].value(),
-                    f"{field.tag}${field.subfields[0].code} is empty, removing field",
+                    f"{field.tag}${field.subfields[0].code} is empty, no other subfields present, removing field",
                     field,
                 )
                 record.remove_field(field)
@@ -238,12 +261,15 @@ def clean_empty_fields(record: pymarc.Record) -> pymarc.Record:
                         26,
                         "DATA ISSUE\t%s\t%s\t%s",
                         record["001"].value(),
-                        f"{field.tag}$a is empty, removing field",
+                        f"{field.tag}$a is empty, removing subfield",
                         field,
                     )
                     field.delete_subfield("a")
                 for idx, subfield in enumerate(list(field.subfields), start=1):
-                    if subfield.code in MAPPED_FIELDS.get(field.tag, []) and not subfield.value:
+                    if (
+                        subfield.code in MAPPED_FIELDS.get(field.tag, [])
+                        and not subfield.value
+                    ):
                         logger.log(
                             26,
                             "DATA ISSUE\t%s\t%s\t%s",
@@ -264,6 +290,40 @@ def clean_empty_fields(record: pymarc.Record) -> pymarc.Record:
     return record
 
 
+def fix_leader(record: pymarc.Record) -> pymarc.Record:
+    """
+    Fixes the leader of the record by setting the record status to 'c' (modified
+    record) and the type of record to 'a' (language material).
+
+    Args:
+        record (pymarc.Record): The MARC record to preprocess.
+
+    Returns:
+        pymarc.Record: The preprocessed MARC record.
+    """
+    VALID_STATUSES = ["a", "c", "d", "n", "p"]
+    VALID_TYPES = ["a", "c", "d", "e", "f", "g", "i", "j", "k", "m", "o", "p", "r", "t"]
+    if record.leader[5] not in VALID_STATUSES:
+        logger.log(
+            26,
+            "DATA ISSUE\t%s\t%s\t%s",
+            record["001"].value(),
+            f"Invalid record status: {record.leader[5]}, setting to 'c'",
+            record,
+        )
+        record.leader = pymarc.Leader(record.leader[:5] + "c" + record.leader[6:])
+    if record.leader[6] not in VALID_TYPES:
+        logger.log(
+            26,
+            "DATA ISSUE\t%s\t%s\t%s",
+            record["001"].value(),
+            f"Invalid record type: {record.leader[6]}, setting to 'a'",
+            record,
+        )
+        record.leader = pymarc.Leader(record.leader[:6] + "a" + record.leader[7:])
+    return record
+
+
 def ordinal(n):
     s = ("th", "st", "nd", "rd") + ("th",) * 10
     v = n % 100

{folio_data_import-0.2.8rc8.dist-info → folio_data_import-0.2.8rc10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_data_import
-Version: 0.2.8rc8
+Version: 0.2.8rc10
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis

folio_data_import-0.2.8rc10.dist-info/RECORD

@@ -0,0 +1,11 @@
+folio_data_import/MARCDataImport.py,sha256=3MHwnusWMraUYbxaooVoVlKC3eG4D6-zBr4iguk52iA,33164
+folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
+folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
+folio_data_import/marc_preprocessors/__init__.py,sha256=urExfNTQoZsDCtDPcUY9EEC5OFcUihxhYEQkQFVzbMY,30
+folio_data_import/marc_preprocessors/_preprocessors.py,sha256=4i1_lEnptzZDx3DojX9sfvJ_hmehwFJUC3aZsUADcwA,10851
+folio_data_import-0.2.8rc10.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
+folio_data_import-0.2.8rc10.dist-info/METADATA,sha256=f7nKawpgkE1Ez6j2GeljBRMoWu4yhdSOcVlj5nWtYQ4,6113
+folio_data_import-0.2.8rc10.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+folio_data_import-0.2.8rc10.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
+folio_data_import-0.2.8rc10.dist-info/RECORD,,

folio_data_import-0.2.8rc8.dist-info/RECORD

@@ -1,11 +0,0 @@
-folio_data_import/MARCDataImport.py,sha256=wLh8raMOIXCQiNZd_MLyUPwGlxZU2G-qOrnySKR9XU4,31723
-folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
-folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
-folio_data_import/marc_preprocessors/__init__.py,sha256=urExfNTQoZsDCtDPcUY9EEC5OFcUihxhYEQkQFVzbMY,30
-folio_data_import/marc_preprocessors/_preprocessors.py,sha256=LJdmwW_0oPbcaWb85F0PAcXJWbbBp9HHrFyZQZER5gs,8738
-folio_data_import-0.2.8rc8.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
-folio_data_import-0.2.8rc8.dist-info/METADATA,sha256=Nl8YmVukI1et0xX3ObggSQTbcgjNuYbY3E8GrvG_X8M,6112
-folio_data_import-0.2.8rc8.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-folio_data_import-0.2.8rc8.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
-folio_data_import-0.2.8rc8.dist-info/RECORD,,