folio-data-import 0.2.8rc4__py3-none-any.whl → 0.2.8rc5__py3-none-any.whl

folio_data_import/MARCDataImport.py

@@ -277,7 +277,7 @@ class MARCImportJob:
         total_records = 0
         for import_file in files:
             while True:
-                chunk = import_file.read(1024)
+                chunk = import_file.read(104857600)
                 if not chunk:
                     break
                 total_records += chunk.count(b"\x1d")
@@ -291,24 +291,33 @@ class MARCImportJob:
         Args:
             batch_payload (dict): A records payload containing the current batch of MARC records.
         """
-        post_batch = self.http_client.post(
-            self.folio_client.okapi_url
-            + f"/change-manager/jobExecutions/{self.job_id}/records",
-            headers=self.folio_client.okapi_headers,
-            json=batch_payload,
-        )
+        try:
+            post_batch = self.http_client.post(
+                self.folio_client.okapi_url
+                + f"/change-manager/jobExecutions/{self.job_id}/records",
+                headers=self.folio_client.okapi_headers,
+                json=batch_payload,
+            )
+        except httpx.ReadTimeout:
+            sleep(.25)
+            return await self.process_record_batch(batch_payload)
         try:
             post_batch.raise_for_status()
             self.total_records_sent += len(self.record_batch)
             self.record_batch = []
             self.pbar_sent.update(len(batch_payload["initialRecords"]))
         except Exception as e:
-            print("Error posting batch: " + str(e))
-            for record in self.record_batch:
-                self.failed_batches_file.write(record)
-                self.error_records += len(self.record_batch)
-                self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
-            self.record_batch = []
+            if hasattr(e, "response") and e.response.status_code in [500, 422]: # TODO: #26 Check for specific error code once https://folio-org.atlassian.net/browse/MODSOURMAN-1281 is resolved
+                self.total_records_sent += len(self.record_batch)
+                self.record_batch = []
+                self.pbar_sent.update(len(batch_payload["initialRecords"]))
+            else:
+                print("Error posting batch: " + str(e))
+                for record in self.record_batch:
+                    self.failed_batches_file.write(record)
+                    self.error_records += len(self.record_batch)
+                    self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
+                self.record_batch = []
         sleep(self.batch_delay)
 
     async def process_records(self, files, total_records) -> None:

{folio_data_import-0.2.8rc4.dist-info → folio_data_import-0.2.8rc5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_data_import
-Version: 0.2.8rc4
+Version: 0.2.8rc5
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis

{folio_data_import-0.2.8rc4.dist-info → folio_data_import-0.2.8rc5.dist-info}/RECORD

@@ -1,11 +1,11 @@
-folio_data_import/MARCDataImport.py,sha256=DFAHFurz5ijTqRwYU1SS3vMVdAVE4OD7ukYnAsRBPyU,23492
+folio_data_import/MARCDataImport.py,sha256=_HaGBq90lN-NEuDm4GWW9W026NnKujU1kjzORjICzf0,24074
 folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
 folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
 folio_data_import/marc_preprocessors/__init__.py,sha256=Wt-TKkMhUyZWFS-WhAmbShKQLPjXmHKPb2vL6kvkqVA,72
 folio_data_import/marc_preprocessors/_preprocessors.py,sha256=Y9t6tIFxQdzBYYniBZ3Lq05Ukk-3orAXd4YQaJuGNRA,2786
-folio_data_import-0.2.8rc4.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
-folio_data_import-0.2.8rc4.dist-info/METADATA,sha256=8-UXb7KtUNZ4dza7XzazE98L4NVnYu5x5IMRBHnNKx8,6115
-folio_data_import-0.2.8rc4.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-folio_data_import-0.2.8rc4.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
-folio_data_import-0.2.8rc4.dist-info/RECORD,,
+folio_data_import-0.2.8rc5.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
+folio_data_import-0.2.8rc5.dist-info/METADATA,sha256=cBes2BOvOrzY_3jNUohXWvChAMB-DScgYVPArV4lnvo,6115
+folio_data_import-0.2.8rc5.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+folio_data_import-0.2.8rc5.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
+folio_data_import-0.2.8rc5.dist-info/RECORD,,