folio-data-import 0.2.8rc12__tar.gz → 0.3.0__tar.gz

{folio_data_import-0.2.8rc12 → folio_data_import-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_data_import
-Version: 0.2.8rc12
+Version: 0.3.0
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
 Author: Brooks Travis
@@ -19,8 +19,7 @@ Requires-Dist: flake8-black (>=0.3.6,<0.4.0)
 Requires-Dist: flake8-bugbear (>=24.8.19,<25.0.0)
 Requires-Dist: flake8-docstrings (>=1.7.0,<2.0.0)
 Requires-Dist: flake8-isort (>=6.1.1,<7.0.0)
-Requires-Dist: folioclient (>=0.61.0,<0.62.0)
-Requires-Dist: httpx (>=0.27.2,<0.28.0)
+Requires-Dist: folioclient (>=0.70.1,<0.71.0)
 Requires-Dist: inquirer (>=3.4.0,<4.0.0)
 Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
 Requires-Dist: pymarc (>=5.2.2,<6.0.0)

{folio_data_import-0.2.8rc12 → folio_data_import-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "folio_data_import"
-version = "0.2.8rc12"
+version = "0.3.0"
 description = "A python module to interact with the data importing capabilities of the open-source FOLIO ILS"
 authors = ["Brooks Travis <brooks.travis@gmail.com>"]
 license = "MIT"
@@ -14,8 +14,7 @@ folio-user-import = "folio_data_import.UserImport:sync_main"
 
 [tool.poetry.dependencies]
 python = "^3.9"
-folioclient = "^0.61.0"
-httpx = "^0.27.2"
+folioclient = "^0.70.1"
 pymarc = "^5.2.2"
 pyhumps = "^3.8.0"
 inquirer = "^3.4.0"

{folio_data_import-0.2.8rc12 → folio_data_import-0.3.0}/src/folio_data_import/MARCDataImport.py

@@ -2,8 +2,8 @@ import argparse
 import asyncio
 import datetime
 import glob
-import importlib
 import io
+import json
 import logging
 import math
 import os
@@ -15,7 +15,7 @@ from functools import cached_property
 from getpass import getpass
 from pathlib import Path
 from time import sleep
-from typing import List, Union
+from typing import Any, BinaryIO, Callable, Dict, List, Union
 
 import folioclient
 import httpx
@@ -25,6 +25,9 @@ import tabulate
 from humps import decamelize
 from tqdm import tqdm
 
+from folio_data_import.custom_exceptions import FolioDataImportBatchError
+from folio_data_import.marc_preprocessors._preprocessors import MARCPreprocessor
+
 try:
     datetime_utc = datetime.UTC
 except AttributeError:
@@ -63,7 +66,6 @@ class MARCImportJob:
         import_profile_name (str): The name of the data import job profile to use.
         batch_size (int): The number of source records to include in a record batch (default=10).
         batch_delay (float): The number of seconds to wait between record batches (default=0).
-        consolidate (bool): Consolidate files into a single job. Default is one job for each file.
         no_progress (bool): Disable progress bars (eg. for running in a CI environment).
     """
 
@@ -75,7 +77,6 @@ class MARCImportJob:
     http_client: httpx.Client
     current_file: List[Path]
     record_batch: List[dict] = []
-    error_records: int = 0
     last_current: int = 0
     total_records_sent: int = 0
     finished: bool = False
@@ -92,18 +93,17 @@ class MARCImportJob:
         import_profile_name: str,
         batch_size=10,
         batch_delay=0,
-        marc_record_preprocessor=None,
-        consolidate=False,
+        marc_record_preprocessor: Union[List[Callable], str]=[],
+        preprocessor_args: Dict[str,Dict]={},
         no_progress=False,
         let_summary_fail=False,
         split_files=False,
         split_size=1000,
+        split_offset=0,
     ) -> None:
-        self.consolidate_files = consolidate
         self.split_files = split_files
         self.split_size = split_size
-        if self.split_files and self.consolidate_files:
-            raise ValueError("Cannot consolidate and split files at the same time.")
+        self.split_offset = split_offset
         self.no_progress = no_progress
         self.let_summary_fail = let_summary_fail
         self.folio_client: folioclient.FolioClient = folio_client
@@ -112,16 +112,14 @@ class MARCImportJob:
         self.batch_size = batch_size
         self.batch_delay = batch_delay
         self.current_retry_timeout = None
-        self.marc_record_preprocessor = marc_record_preprocessor
+        self.marc_record_preprocessor: MARCPreprocessor = MARCPreprocessor(marc_record_preprocessor, **preprocessor_args)
 
     async def do_work(self) -> None:
         """
         Performs the necessary work for data import.
 
         This method initializes an HTTP client, files to store records that fail to send,
-        and calls `self.import_marc_records` to import MARC files. If `consolidate_files` is True,
-        it imports all the files specified in `import_files` as a single batch. Otherwise,
-        it imports each file as a separate import job.
+        and calls the appropriate method to import MARC files based on the configuration.
 
         Returns:
             None
@@ -146,27 +144,33 @@ class MARCImportJob:
             self.failed_batches_file = failed_batches
             logger.info(f"Writing failed batches to {self.failed_batches_file.name}")
             self.http_client = http_client
-            if self.consolidate_files:
-                self.current_file = self.import_files
-                await self.import_marc_file()
-            elif self.split_files:
-                for file in self.import_files:
-                    with open(file, "rb") as f:
-                        file_length = await self.read_total_records([f])
-                    expected_batches = math.ceil(file_length /self.split_size)
-                    logger.info(f"{file.name} contains {file_length} records. Splitting into {expected_batches} {self.split_size} record batches.")
-                    zero_pad_parts = len(str(expected_batches)) if expected_batches > 1 else 2
-                    for idx, batch in enumerate(self.split_marc_file(file, self.split_size), start=1):
-                        batch.name = f"{file.name}_part{idx:0{zero_pad_parts}}"
-                        self.current_file = [batch]
-                        await self.import_marc_file()
-                    self.move_file_to_complete(file)
+            if self.split_files:
+                await self.process_split_files()
             else:
                 for file in self.import_files:
                     self.current_file = [file]
                     await self.import_marc_file()
             await self.wrap_up()
 
+    async def process_split_files(self):
+        """
+        Process the import of files in smaller batches.
+        This method is called when `split_files` is set to True.
+        It splits each file into smaller chunks and processes them one by one.
+        """
+        for file in self.import_files:
+            with open(file, "rb") as f:
+                file_length = await self.read_total_records([f])
+            expected_batches = math.ceil(file_length /self.split_size)
+            logger.info(f"{file.name} contains {file_length} records. Splitting into {expected_batches} {self.split_size} record batches.")
+            zero_pad_parts = len(str(expected_batches)) if expected_batches > 1 else 2
+            for idx, batch in enumerate(self.split_marc_file(file, self.split_size), start=1):
+                if idx > self.split_offset:
+                    batch.name = f"{file.name} (Part {idx:0{zero_pad_parts}})"
+                    self.current_file = [batch]
+                    await self.import_marc_file()
+            self.move_file_to_complete(file)
+
     async def wrap_up(self) -> None:
         """
         Wraps up the data import process.
@@ -214,7 +218,7 @@ class MARCImportJob:
                 )
                 self.current_retry_timeout = None
         except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
-            if not hasattr(e, "response") or e.response.status_code in [502, 504]:
+            if not hasattr(e, "response") or e.response.status_code in [502, 504, 401]:
                 error_text = e.response.text if hasattr(e, "response") else str(e)
                 logger.warning(f"SERVER ERROR fetching job status: {error_text}. Retrying.")
                 sleep(0.25)
@@ -276,7 +280,7 @@ class MARCImportJob:
         """
         try:
             create_job = self.http_client.post(
-                self.folio_client.okapi_url + "/change-manager/jobExecutions",
+                self.folio_client.gateway_url + "/change-manager/jobExecutions",
                 headers=self.folio_client.okapi_headers,
                 json={"sourceType": "ONLINE", "userId": self.folio_client.current_user},
             )
@@ -325,7 +329,7 @@ class MARCImportJob:
             The response from the HTTP request to set the job profile.
         """
         set_job_profile = self.http_client.put(
-            self.folio_client.okapi_url
+            self.folio_client.gateway_url
             + "/change-manager/jobExecutions/"
             + self.job_id
             + "/jobProfile",
@@ -350,7 +354,7 @@ class MARCImportJob:
             raise e
 
     @staticmethod
-    async def read_total_records(files) -> int:
+    async def read_total_records(files: List[BinaryIO]) -> int:
         """
         Reads the total number of records from the given files.
 
@@ -379,17 +383,15 @@ class MARCImportJob:
         """
         try:
             post_batch = self.http_client.post(
-                self.folio_client.okapi_url
+                self.folio_client.gateway_url
                 + f"/change-manager/jobExecutions/{self.job_id}/records",
                 headers=self.folio_client.okapi_headers,
                 json=batch_payload,
             )
-            # if batch_payload["recordsMetadata"]["last"]:
-            #     logger.log(
-            #         25,
-            #         f"Sending last batch of {batch_payload['recordsMetadata']['total']} records.",
-            #     )
         except (httpx.ConnectTimeout, httpx.ReadTimeout):
+            logger.warning(
+                f"CONNECTION ERROR posting batch {batch_payload['id']}. Retrying..."
+            )
             sleep(0.25)
             return await self.process_record_batch(batch_payload)
         try:
@@ -397,20 +399,21 @@ class MARCImportJob:
             self.total_records_sent += len(self.record_batch)
             self.record_batch = []
             self.pbar_sent.update(len(batch_payload["initialRecords"]))
-        except Exception as e:
+        except httpx.HTTPStatusError as e:
             if (
-                hasattr(e, "response") and e.response.status_code in [500, 422]
-            ):  # TODO: #26 Check for specific error code once https://folio-org.atlassian.net/browse/MODSOURMAN-1281 is resolved
+                e.response.status_code in [500, 400, 422]
+            ):  # TODO: Update once we no longer have to support < Sunflower to just be 400
                 self.total_records_sent += len(self.record_batch)
                 self.record_batch = []
                 self.pbar_sent.update(len(batch_payload["initialRecords"]))
             else:
-                logger.error("Error posting batch: " + str(e))
                 for record in self.record_batch:
                     self.failed_batches_file.write(record)
-                    self.error_records += len(self.record_batch)
-                    self.pbar_sent.total = self.pbar_sent.total - len(self.record_batch)
-                self.record_batch = []
+                raise FolioDataImportBatchError(
+                    batch_payload['id'],
+                    f"{e}\n{e.response.text}",
+                    e
+                )
         await self.get_job_status()
         sleep(self.batch_delay)
 
@@ -439,16 +442,12 @@ class MARCImportJob:
                         await self.create_batch_payload(
                             counter,
                             total_records,
-                            (counter - self.error_records)
-                            == (total_records - self.error_records),
+                            counter == total_records,
                         ),
                     )
                     sleep(0.25)
                 if record:
-                    if self.marc_record_preprocessor:
-                        record = await self.apply_marc_record_preprocessing(
-                            record, self.marc_record_preprocessor
-                        )
+                    record = self.marc_record_preprocessor.do_work(record)
                     self.record_batch.append(record.as_marc())
                     counter += 1
                 else:
@@ -459,19 +458,18 @@ class MARCImportJob:
                         "",
                     )
                     self.bad_records_file.write(reader.current_chunk)
-            if self.record_batch:
-                await self.process_record_batch(
-                    await self.create_batch_payload(
-                        counter,
-                        total_records,
-                        (counter - self.error_records)
-                        == (total_records - self.error_records),
-                    ),
-                )
             if not self.split_files:
                 self.move_file_to_complete(file_path)
+        if self.record_batch or not self.finished:
+            await self.process_record_batch(
+                await self.create_batch_payload(
+                    counter,
+                    total_records,
+                    counter == total_records,
+                ),
+            )
 
-    def move_file_to_complete(self, file_path):
+    def move_file_to_complete(self, file_path: Path):
         import_complete_path = file_path.parent.joinpath("import_complete")
         if not import_complete_path.exists():
             logger.debug(f"Creating import_complete directory: {import_complete_path.absolute()}")
@@ -481,58 +479,6 @@ class MARCImportJob:
                 file_path.parent.joinpath("import_complete", file_path.name)
             )
 
-    @staticmethod
-    async def apply_marc_record_preprocessing(
-        record: pymarc.Record, func_or_path
-    ) -> pymarc.Record:
-        """
-        Apply preprocessing to the MARC record before sending it to FOLIO.
-
-        Args:
-            record (pymarc.Record): The MARC record to preprocess.
-            func_or_path (Union[Callable, str]): The preprocessing function or its import path.
-
-        Returns:
-            pymarc.Record: The preprocessed MARC record.
-        """
-        if isinstance(func_or_path, str):
-            func_paths = func_or_path.split(",")
-            for func_path in func_paths:
-                record = await MARCImportJob._apply_single_marc_record_preprocessing_by_path(
-                    record, func_path
-                )
-        elif callable(func_or_path):
-            record = func_or_path(record)
-        else:
-            logger.warning(
-                f"Invalid preprocessing function: {func_or_path}. Skipping preprocessing."
-            )
-        return record
-
-    async def _apply_single_marc_record_preprocessing_by_path(
-        record: pymarc.Record, func_path: str
-    ) -> pymarc.Record:
-        """
-        Apply a single preprocessing function to the MARC record.
-
-        Args:
-            record (pymarc.Record): The MARC record to preprocess.
-            func_path (str): The path to the preprocessing function.
-
-        Returns:
-            pymarc.Record: The preprocessed MARC record.
-        """
-        try:
-            module_path, func_name = func_path.rsplit(".", 1)
-            module = importlib.import_module(module_path)
-            func = getattr(module, func_name)
-            record = func(record)
-        except Exception as e:
-            logger.warning(
-                f"Error applying preprocessing function {func_path}: {e}. Skipping."
-            )
-        return record
-
     async def create_batch_payload(self, counter, total_records, is_last) -> dict:
         """
         Create a batch payload for data import.
@@ -549,9 +495,9 @@ class MARCImportJob:
             "id": str(uuid.uuid4()),
             "recordsMetadata": {
                 "last": is_last,
-                "counter": counter - self.error_records,
+                "counter": counter,
                 "contentType": "MARC_RAW",
-                "total": total_records - self.error_records,
+                "total": total_records,
             },
             "initialRecords": [{"record": x.decode()} for x in self.record_batch],
         }
@@ -646,17 +592,47 @@ class MARCImportJob:
                     disable=self.no_progress,
                 ) as pbar_sent,
             ):
-                self.pbar_sent = pbar_sent
-                self.pbar_imported = pbar_imported
-                await self.process_records(files, total_records)
-                while not self.finished:
-                    await self.get_job_status()
-                sleep(1)
+                try:
+                    self.pbar_sent = pbar_sent
+                    self.pbar_imported = pbar_imported
+                    await self.process_records(files, total_records)
+                    while not self.finished:
+                        await self.get_job_status()
+                    sleep(1)
+                except FolioDataImportBatchError as e:
+                    logger.error(
+                        f"Unhandled error posting batch {e.batch_id}: {e.message}"
+                    )
+                    await self.cancel_job()
+                    raise e
             if self.finished:
                 await self.log_job_summary()
             self.last_current = 0
             self.finished = False
 
+    async def cancel_job(self) -> None:
+        """
+        Cancels the current job execution.
+
+        This method sends a request to cancel the job execution and logs the result.
+
+        Returns:
+            None
+        """
+        try:
+            cancel = self.http_client.delete(
+                self.folio_client.gateway_url
+                + f"/change-manager/jobExecutions/{self.job_id}/records",
+                headers=self.folio_client.okapi_headers,
+            )
+            cancel.raise_for_status()
+            self.finished = True
+            logger.info(f"Cancelled job: {self.job_id}")
+        except (httpx.ConnectTimeout, httpx.ReadTimeout):
+            logger.warning(f"CONNECTION ERROR cancelling job {self.job_id}. Retrying...")
+            sleep(0.25)
+            await self.cancel_job()
+
     async def log_job_summary(self):
         if job_summary := await self.get_job_summary():
             job_id = job_summary.pop("jobExecutionId", None)
@@ -835,17 +811,8 @@ async def main() -> None:
         ),
         default=None,
     )
-    # Add mutually exclusive group for consolidate and split-files options
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        "--consolidate",
-        action="store_true",
-        help=(
-            "Consolidate records into a single job. "
-            "Default is to create a new job for each MARC file."
-        ),
-    )
-    group.add_argument(
+
+    parser.add_argument(
         "--split-files",
         action="store_true",
         help="Split files into smaller parts before importing.",
@@ -856,6 +823,12 @@ async def main() -> None:
         help="The number of records to include in each split file.",
         default=1000,
     )
+    parser.add_argument(
+        "--split-offset",
+        type=int,
+        help="The number of record batches of <split-size> to skip before starting import.",
+        default=0,
+    )
 
     parser.add_argument(
         "--no-progress",
@@ -867,6 +840,16 @@ async def main() -> None:
         action="store_true",
         help="Do not retry fetching the final job summary if it fails",
     )
+    parser.add_argument(
+        "--preprocessor-config",
+        type=str,
+        help=(
+            "JSON file containing configuration for preprocessor functions. "
+            "This is passed to MARCPreprocessor class as a dict of dicts."
+        ),
+        default=None,
+    )
+
     args = parser.parse_args()
     if not args.password:
         args.password = getpass("Enter FOLIO password: ")
@@ -891,6 +874,12 @@ async def main() -> None:
     else:
         logger.info(marc_files)
 
+    if args.preprocessor_config:
+        with open(args.preprocessor_config, "r") as f:
+            preprocessor_args = json.load(f)
+    else:
+        preprocessor_args = {}
+
     if not args.import_profile_name:
         import_profiles = folio_client.folio_get(
             "/data-import-profiles/jobProfiles",
@@ -919,11 +908,12 @@ async def main() -> None:
             batch_size=args.batch_size,
             batch_delay=args.batch_delay,
             marc_record_preprocessor=args.preprocessor,
-            consolidate=bool(args.consolidate),
+            preprocessor_args=preprocessor_args,
             no_progress=bool(args.no_progress),
             let_summary_fail=bool(args.let_summary_fail),
             split_files=bool(args.split_files),
             split_size=args.split_size,
+            split_offset=args.split_offset,
         ).do_work()
     except Exception as e:
         logger.error("Error importing files: " + str(e))

{folio_data_import-0.2.8rc12 → folio_data_import-0.3.0}/src/folio_data_import/UserImport.py

@@ -137,7 +137,7 @@ class UserImporter:  # noqa: R0902
         match_key = "id" if ("id" in user_obj) else self.match_key
         try:
             existing_user = await self.http_client.get(
-                self.folio_client.okapi_url + "/users",
+                self.folio_client.gateway_url + "/users",
                 headers=self.folio_client.okapi_headers,
                 params={"query": f"{match_key}=={user_obj[match_key]}"},
             )
@@ -161,7 +161,7 @@ class UserImporter:  # noqa: R0902
         """
         try:
             existing_rp = await self.http_client.get(
-                self.folio_client.okapi_url
+                self.folio_client.gateway_url
                 + "/request-preference-storage/request-preference",
                 headers=self.folio_client.okapi_headers,
                 params={
@@ -188,7 +188,7 @@ class UserImporter:  # noqa: R0902
         """
         try:
             existing_pu = await self.http_client.get(
-                self.folio_client.okapi_url + "/perms/users",
+                self.folio_client.gateway_url + "/perms/users",
                 headers=self.folio_client.okapi_headers,
                 params={
                     "query": f"userId=={existing_user.get('id', user_obj.get('id', ''))}"
@@ -369,7 +369,7 @@ class UserImporter:  # noqa: R0902
             else:
                 existing_user[key] = value
         create_update_user = await self.http_client.put(
-            self.folio_client.okapi_url + f"/users/{existing_user['id']}",
+            self.folio_client.gateway_url + f"/users/{existing_user['id']}",
             headers=self.folio_client.okapi_headers,
             json=existing_user,
         )
@@ -389,7 +389,7 @@ class UserImporter:  # noqa: R0902
             HTTPError: If the HTTP request to create the user fails.
         """
         response = await self.http_client.post(
-            self.folio_client.okapi_url + "/users",
+            self.folio_client.gateway_url + "/users",
             headers=self.folio_client.okapi_headers,
             json=user_obj,
         )
@@ -589,7 +589,7 @@ class UserImporter:  # noqa: R0902
         rp_obj["userId"] = new_user_obj["id"]
         # print(rp_obj)
         response = await self.http_client.post(
-            self.folio_client.okapi_url
+            self.folio_client.gateway_url
             + "/request-preference-storage/request-preference",
             headers=self.folio_client.okapi_headers,
             json=rp_obj,
@@ -613,7 +613,7 @@ class UserImporter:  # noqa: R0902
         existing_rp.update(rp_obj)
         # print(existing_rp)
         response = await self.http_client.put(
-            self.folio_client.okapi_url
+            self.folio_client.gateway_url
             + f"/request-preference-storage/request-preference/{existing_rp['id']}",
             headers=self.folio_client.okapi_headers,
             json=existing_rp,
@@ -635,7 +635,7 @@ class UserImporter:  # noqa: R0902
         """
         perms_user_obj = {"userId": new_user_obj["id"], "permissions": []}
         response = await self.http_client.post(
-            self.folio_client.okapi_url + "/perms/users",
+            self.folio_client.gateway_url + "/perms/users",
             headers=self.folio_client.okapi_headers,
             json=perms_user_obj,
         )
@@ -788,7 +788,7 @@ class UserImporter:  # noqa: R0902
         """
         try:
             existing_spu = await self.http_client.get(
-                self.folio_client.okapi_url + "/service-points-users",
+                self.folio_client.gateway_url + "/service-points-users",
                 headers=self.folio_client.okapi_headers,
                 params={"query": f"userId=={existing_user['id']}"},
             )
@@ -812,7 +812,7 @@ class UserImporter:  # noqa: R0902
         """
         spu_obj["userId"] = existing_user["id"]
         response = await self.http_client.post(
-            self.folio_client.okapi_url + "/service-points-users",
+            self.folio_client.gateway_url + "/service-points-users",
             headers=self.folio_client.okapi_headers,
             json=spu_obj,
         )
@@ -831,7 +831,7 @@ class UserImporter:  # noqa: R0902
         """
         existing_spu.update(spu_obj)
         response = await self.http_client.put(
-            self.folio_client.okapi_url + f"/service-points-users/{existing_spu['id']}",
+            self.folio_client.gateway_url + f"/service-points-users/{existing_spu['id']}",
             headers=self.folio_client.okapi_headers,
             json=existing_spu,
         )

folio_data_import-0.3.0/src/folio_data_import/custom_exceptions.py

@@ -0,0 +1,17 @@
+"""Custom exceptions for the Folio Data Import module."""
+
+class FolioDataImportError(Exception):
+    """Base class for all exceptions in the Folio Data Import module."""
+    pass
+
+class FolioDataImportBatchError(FolioDataImportError):
+    """Exception raised for errors in the Folio Data Import batch process.
+    
+    Attributes:
+        batch_id -- ID of the batch that caused the error
+        message -- explanation of the error
+    """
+    def __init__(self, batch_id, message, exception=None):
+        self.batch_id = batch_id
+        self.message = message
+        super().__init__(f"Unhandled error posting batch {batch_id}: {message}")