folio-data-import 0.2.8rc4__py3-none-any.whl → 0.2.8rc6__py3-none-any.whl

folio_data_import/MARCDataImport.py

@@ -277,7 +277,7 @@ class MARCImportJob:
         total_records = 0
         for import_file in files:
             while True:
-                chunk = import_file.read(1024)
+                chunk = import_file.read(104857600)
                 if not chunk:
                     break
                 total_records += chunk.count(b"\x1d")
@@ -291,24 +291,33 @@ class MARCImportJob:
         Args:
             batch_payload (dict): A records payload containing the current batch of MARC records.
         """
-        post_batch = self.http_client.post(
-            self.folio_client.okapi_url
-            + f"/change-manager/jobExecutions/{self.job_id}/records",
-            headers=self.folio_client.okapi_headers,
-            json=batch_payload,
-        )
+        try:
+            post_batch = self.http_client.post(
+                self.folio_client.okapi_url
+                + f"/change-manager/jobExecutions/{self.job_id}/records",
+                headers=self.folio_client.okapi_headers,
+                json=batch_payload,
+            )
+        except httpx.ReadTimeout:
+            sleep(.25)
+            return await self.process_record_batch(batch_payload)
         try:
             post_batch.raise_for_status()
             self.total_records_sent += len(self.record_batch)
             self.record_batch = []
             self.pbar_sent.update(len(batch_payload["initialRecords"]))
         except Exception as e:
-            print("Error posting batch: " + str(e))
-            for record in self.record_batch:
-                self.failed_batches_file.write(record)
-                self.error_records += len(self.record_batch)
-                self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
-            self.record_batch = []
+            if hasattr(e, "response") and e.response.status_code in [500, 422]: # TODO: #26 Check for specific error code once https://folio-org.atlassian.net/browse/MODSOURMAN-1281 is resolved
+                self.total_records_sent += len(self.record_batch)
+                self.record_batch = []
+                self.pbar_sent.update(len(batch_payload["initialRecords"]))
+            else:
+                print("Error posting batch: " + str(e))
+                for record in self.record_batch:
+                    self.failed_batches_file.write(record)
+                    self.error_records += len(self.record_batch)
+                    self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
+                self.record_batch = []
         sleep(self.batch_delay)
 
     async def process_records(self, files, total_records) -> None:
@@ -496,14 +505,17 @@ class MARCImportJob:
                 f"/metadata-provider/jobSummary/{self.job_id}"
             )
             self.current_retry_timeout = None
-        except (httpx.ConnectTimeout, httpx.ReadTimeout):
-            sleep(.25)
-            with httpx.Client(
-                timeout=self.current_retry_timeout,
-                verify=self.folio_client.ssl_verify
-            ) as temp_client:
-                self.folio_client.httpx_client = temp_client
-                return await self.get_job_summary()
+        except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+            if not hasattr(e, "response") or e.response.status_code == 502:
+                sleep(.25)
+                with httpx.Client(
+                    timeout=self.current_retry_timeout,
+                    verify=self.folio_client.ssl_verify
+                ) as temp_client:
+                    self.folio_client.httpx_client = temp_client
+                    return await self.get_job_status()
+            else:
+                raise e
         return job_summary
 
 
@@ -586,6 +598,8 @@ async def main() -> None:
     else:
         marc_files = list(Path("./").glob(args.marc_file_path))
 
+    marc_files.sort()
+
     if len(marc_files) == 0:
         print(f"No files found matching {args.marc_file_path}. Exiting.")
         sys.exit(1)

folio_data_import/marc_preprocessors/_preprocessors.py

@@ -81,4 +81,4 @@ def sudoc_supercede_prep(record: pymarc.Record) -> pymarc.Record:
                 ]
             )
             record.add_ordered_field(_935)
-    return record
+    return record

{folio_data_import-0.2.8rc4.dist-info → folio_data_import-0.2.8rc6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_data_import
-Version: 0.2.8rc4
+Version: 0.2.8rc6
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis

folio_data_import-0.2.8rc6.dist-info/RECORD

@@ -0,0 +1,11 @@
+folio_data_import/MARCDataImport.py,sha256=iIsNDZCBCNeVqkjeBIERvRT3Ee7ryPh1P0M4xTnp2Gw,24270
+folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
+folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
+folio_data_import/marc_preprocessors/__init__.py,sha256=Wt-TKkMhUyZWFS-WhAmbShKQLPjXmHKPb2vL6kvkqVA,72
+folio_data_import/marc_preprocessors/_preprocessors.py,sha256=CMG4Xq3vR84xmNFUJfRmnU0A5lchAfK33xDzeiC2AWk,2787
+folio_data_import-0.2.8rc6.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
+folio_data_import-0.2.8rc6.dist-info/METADATA,sha256=3ZFdaSiFBTbtw_mXnkURBA4mHtgpXy6JnGpN8TZdXSM,6115
+folio_data_import-0.2.8rc6.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+folio_data_import-0.2.8rc6.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
+folio_data_import-0.2.8rc6.dist-info/RECORD,,

{folio_data_import-0.2.8rc4.dist-info → folio_data_import-0.2.8rc6.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any

folio_data_import-0.2.8rc4.dist-info/RECORD

@@ -1,11 +0,0 @@
-folio_data_import/MARCDataImport.py,sha256=DFAHFurz5ijTqRwYU1SS3vMVdAVE4OD7ukYnAsRBPyU,23492
-folio_data_import/UserImport.py,sha256=Y9ZjYoUP_vNJVftx_xUcbBqvC5CwWeuzlmCcSVQfzgo,40976
-folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
-folio_data_import/marc_preprocessors/__init__.py,sha256=Wt-TKkMhUyZWFS-WhAmbShKQLPjXmHKPb2vL6kvkqVA,72
-folio_data_import/marc_preprocessors/_preprocessors.py,sha256=Y9t6tIFxQdzBYYniBZ3Lq05Ukk-3orAXd4YQaJuGNRA,2786
-folio_data_import-0.2.8rc4.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
-folio_data_import-0.2.8rc4.dist-info/METADATA,sha256=8-UXb7KtUNZ4dza7XzazE98L4NVnYu5x5IMRBHnNKx8,6115
-folio_data_import-0.2.8rc4.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-folio_data_import-0.2.8rc4.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
-folio_data_import-0.2.8rc4.dist-info/RECORD,,