folio-data-import 0.2.3__tar.gz → 0.2.5__tar.gz

{folio_data_import-0.2.3 → folio_data_import-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: folio_data_import
-Version: 0.2.3
+Version: 0.2.5
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis
@@ -19,7 +19,7 @@ Requires-Dist: flake8-bugbear (>=24.8.19,<25.0.0)
 Requires-Dist: flake8-docstrings (>=1.7.0,<2.0.0)
 Requires-Dist: flake8-isort (>=6.1.1,<7.0.0)
 Requires-Dist: folioclient (>=0.60.5,<0.61.0)
-Requires-Dist: httpx (>=0.23.0,<0.24.0)
+Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: inquirer (>=3.4.0,<4.0.0)
 Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
 Requires-Dist: pymarc (>=5.2.2,<6.0.0)

{folio_data_import-0.2.3 → folio_data_import-0.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "folio_data_import"
-version = "0.2.3"
+version = "0.2.5"
 description = "A python module to interact with the data importing capabilities of the open-source FOLIO ILS"
 authors = ["Brooks Travis <brooks.travis@gmail.com>"]
 license = "MIT"
@@ -15,7 +15,7 @@ folio-user-import = "folio_data_import.UserImport:sync_main"
 [tool.poetry.dependencies]
 python = "^3.9"
 folioclient = "^0.60.5"
-httpx = "^0.23.0"
+httpx = "^0.27.2"
 pymarc = "^5.2.2"
 pyhumps = "^3.8.0"
 inquirer = "^3.4.0"

{folio_data_import-0.2.3 → folio_data_import-0.2.5}/src/folio_data_import/MARCDataImport.py

@@ -3,6 +3,7 @@ import asyncio
 import glob
 import io
 import os
+import sys
 from typing import List
 import uuid
 from contextlib import ExitStack
@@ -30,6 +31,9 @@ except AttributeError:
 # The order in which the report summary should be displayed
 REPORT_SUMMARY_ORDERING = {"created": 0, "updated": 1, "discarded": 2, "error": 3}
 
+# Set default timeout and backoff values for HTTP requests when retrying job status and final summary checks
+RETRY_TIMEOUT_START = 1
+RETRY_TIMEOUT_RETRY_FACTOR = 2
 
 class MARCImportJob:
     """
@@ -79,6 +83,7 @@ class MARCImportJob:
         self.import_profile_name = import_profile_name
         self.batch_size = batch_size
         self.batch_delay = batch_delay
+        self.current_retry_timeout = None
 
     async def do_work(self) -> None:
         """
@@ -148,10 +153,23 @@ class MARCImportJob:
         Raises:
             IndexError: If the job execution with the specified ID is not found.
         """
-        job_status = self.folio_client.folio_get(
-            "/metadata-provider/jobExecutions?statusNot=DISCARDED&uiStatusAny"
-            "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
-        )
+        try:
+            self.current_retry_timeout = (
+                self.current_retry_timeout * RETRY_TIMEOUT_RETRY_FACTOR
+            ) if self.current_retry_timeout else RETRY_TIMEOUT_START
+            job_status = self.folio_client.folio_get(
+                "/metadata-provider/jobExecutions?statusNot=DISCARDED&uiStatusAny"
+                "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
+            )
+            self.current_retry_timeout = None
+        except httpx.ConnectTimeout:
+            sleep(.25)
+            with httpx.Client(
+                timeout=self.current_retry_timeout,
+                verify=self.folio_client.ssl_verify
+            ) as temp_client:
+                self.folio_client.httpx_client = temp_client
+                return await self.get_job_status()
         try:
             status = [
                 job for job in job_status["jobExecutions"] if job["id"] == self.job_id
@@ -392,9 +410,7 @@ class MARCImportJob:
                     await self.get_job_status()
                 sleep(1)
             if self.finished:
-                job_summary = self.folio_client.folio_get(
-                    f"/metadata-provider/jobSummary/{self.job_id}"
-                )
+                job_summary = await self.get_job_summary()
                 job_summary.pop("jobExecutionId")
                 job_summary.pop("totalErrors")
                 columns = ["Summary"] + list(job_summary.keys())
@@ -425,6 +441,31 @@ class MARCImportJob:
             self.last_current = 0
             self.finished = False
 
+    async def get_job_summary(self) -> dict:
+        """
+        Retrieves the job summary for the current job execution.
+
+        Returns:
+            dict: The job summary for the current job execution.
+        """
+        try:
+            self.current_retry_timeout = (
+                self.current_retry_timeout * RETRY_TIMEOUT_RETRY_FACTOR
+            ) if self.current_retry_timeout else RETRY_TIMEOUT_START
+            job_summary = self.folio_client.folio_get(
+                f"/metadata-provider/jobSummary/{self.job_id}"
+            )
+            self.current_retry_timeout = None
+        except httpx.ReadTimeout:  #
+            sleep(.25)
+            with httpx.Client(
+                timeout=self.current_retry_timeout,
+                verify=self.folio_client.ssl_verify
+            ) as temp_client:
+                self.folio_client.httpx_client = temp_client
+                return await self.get_job_summary()
+        return job_summary
+
 
 async def main() -> None:
     """
@@ -491,6 +532,17 @@ async def main() -> None:
     if args.member_tenant_id:
         folio_client.okapi_headers["x-okapi-tenant"] = args.member_tenant_id
 
+    if os.path.isabs(args.marc_file_path):
+        marc_files = [Path(x) for x in glob.glob(args.marc_file_path)]
+    else:
+        marc_files = list(Path("./").glob(args.marc_file_path))
+
+    if len(marc_files) == 0:
+        print(f"No files found matching {args.marc_file_path}. Exiting.")
+        sys.exit(1)
+    else:
+        print(marc_files)
+
     if not args.import_profile_name:
         import_profiles = folio_client.folio_get(
             "/data-import-profiles/jobProfiles",
@@ -511,8 +563,6 @@ async def main() -> None:
         ]
         answers = inquirer.prompt(questions)
         args.import_profile_name = answers["import_profile_name"]
-    marc_files = [Path(x) for x in glob.glob(args.marc_file_path, root_dir="./")]
-    print(marc_files)
     try:
         await MARCImportJob(
             folio_client,

{folio_data_import-0.2.3 → folio_data_import-0.2.5}/src/folio_data_import/UserImport.py

@@ -181,12 +181,15 @@ class UserImporter:  # noqa: R0902
             KeyError: If an address type name in the user object is not found in address_type_map.
 
         """
-        if "personal" in user_obj and "addresses" in user_obj["personal"]:
-            for address in user_obj["personal"]["addresses"]:
+        if "personal" in user_obj:
+            addresses = user_obj["personal"].pop("addresses", [])
+            mapped_addresses = []
+            for address in addresses:
                 try:
                     address["addressTypeId"] = self.address_type_map[
                         address["addressTypeId"]
                     ]
+                    mapped_addresses.append(address)
                 except KeyError:
                     if address["addressTypeId"] not in self.address_type_map.values():
                         print(
@@ -197,9 +200,8 @@ class UserImporter:  # noqa: R0902
                             f"Row {line_number}: Address type {address['addressTypeId']} not found"
                             f", removing address\n"
                         )
-                        del address
-            if len(user_obj["personal"]["addresses"]) == 0:
-                del user_obj["personal"]["addresses"]
+            if mapped_addresses:
+                user_obj["personal"]["addresses"] = mapped_addresses
 
     async def map_patron_groups(self, user_obj, line_number) -> None:
         """