folio-data-import 0.2.8rc3__tar.gz → 0.2.8rc5__tar.gz

{folio_data_import-0.2.8rc3 → folio_data_import-0.2.8rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_data_import
-Version: 0.2.8rc3
+Version: 0.2.8rc5
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis

{folio_data_import-0.2.8rc3 → folio_data_import-0.2.8rc5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "folio_data_import"
-version = "0.2.8rc3"
+version = "0.2.8rc5"
 description = "A python module to interact with the data importing capabilities of the open-source FOLIO ILS"
 authors = ["Brooks Travis <brooks.travis@gmail.com>"]
 license = "MIT"
@@ -33,6 +33,7 @@ flake8-docstrings = "^1.7.0"
 pytest = "^8.3.2"
 flake8 = "^7.1.1"
 black = "^24.8.0"
+ipykernel = "^6.29.5"
 
 [build-system]
 requires = ["poetry-core"]

{folio_data_import-0.2.8rc3 → folio_data_import-0.2.8rc5}/src/folio_data_import/MARCDataImport.py

@@ -165,7 +165,7 @@ class MARCImportJob:
                 "=PREPARING_FOR_PREVIEW&uiStatusAny=READY_FOR_PREVIEW&uiStatusAny=RUNNING&limit=50"
             )
             self.current_retry_timeout = None
-        except httpx.ConnectTimeout:
+        except (httpx.ConnectTimeout, httpx.ReadTimeout):
             sleep(.25)
             with httpx.Client(
                 timeout=self.current_retry_timeout,
@@ -277,7 +277,7 @@ class MARCImportJob:
         total_records = 0
         for import_file in files:
             while True:
-                chunk = import_file.read(1024)
+                chunk = import_file.read(104857600)
                 if not chunk:
                     break
                 total_records += chunk.count(b"\x1d")
@@ -291,24 +291,33 @@ class MARCImportJob:
         Args:
             batch_payload (dict): A records payload containing the current batch of MARC records.
         """
-        post_batch = self.http_client.post(
-            self.folio_client.okapi_url
-            + f"/change-manager/jobExecutions/{self.job_id}/records",
-            headers=self.folio_client.okapi_headers,
-            json=batch_payload,
-        )
+        try:
+            post_batch = self.http_client.post(
+                self.folio_client.okapi_url
+                + f"/change-manager/jobExecutions/{self.job_id}/records",
+                headers=self.folio_client.okapi_headers,
+                json=batch_payload,
+            )
+        except httpx.ReadTimeout:
+            sleep(.25)
+            return await self.process_record_batch(batch_payload)
         try:
             post_batch.raise_for_status()
             self.total_records_sent += len(self.record_batch)
             self.record_batch = []
             self.pbar_sent.update(len(batch_payload["initialRecords"]))
         except Exception as e:
-            print("Error posting batch: " + str(e))
-            for record in self.record_batch:
-                self.failed_batches_file.write(record)
-                self.error_records += len(self.record_batch)
-                self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
-            self.record_batch = []
+            if hasattr(e, "response") and e.response.status_code in [500, 422]: # TODO: #26 Check for specific error code once https://folio-org.atlassian.net/browse/MODSOURMAN-1281 is resolved
+                self.total_records_sent += len(self.record_batch)
+                self.record_batch = []
+                self.pbar_sent.update(len(batch_payload["initialRecords"]))
+            else:
+                print("Error posting batch: " + str(e))
+                for record in self.record_batch:
+                    self.failed_batches_file.write(record)
+                    self.error_records += len(self.record_batch)
+                    self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
+                self.record_batch = []
         sleep(self.batch_delay)
 
     async def process_records(self, files, total_records) -> None:
@@ -496,7 +505,7 @@ class MARCImportJob:
                 f"/metadata-provider/jobSummary/{self.job_id}"
             )
             self.current_retry_timeout = None
-        except httpx.ReadTimeout:  #
+        except (httpx.ConnectTimeout, httpx.ReadTimeout):
             sleep(.25)
             with httpx.Client(
                 timeout=self.current_retry_timeout,