folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/__init__.py

@@ -0,0 +1,11 @@
+import importlib.metadata
+from typing import Protocol
+
+__version__ = importlib.metadata.version("folio_migration_tools")
+
+class StrCoercible(Protocol):
+    def __repr__(self) -> str:
+        ...
+
+    def __str__(self) -> str:
+        ...

folio_migration_tools/__main__.py

@@ -1,119 +1,203 @@
+from importlib import metadata
 import json
 import logging
 import sys
+from os import environ
+from pathlib import Path
 
+import httpx
 import humps
-import requests.exceptions
+import i18n
 from argparse_prompt import PromptParser
+from folioclient import FolioClient
 from pydantic import ValidationError
 
+from folio_migration_tools.config_file_load import merge_load
+from folio_migration_tools.custom_exceptions import TransformationProcessError
 from folio_migration_tools.library_configuration import LibraryConfiguration
-from folio_migration_tools.migration_tasks import *  # pylint: disable=wildcard-import, unused-wildcard-import
+from folio_migration_tools.migration_tasks import *  # noqa: F403, F401
 from folio_migration_tools.migration_tasks import migration_task_base
 
 
-def parse_args():
-    """Parse CLI Arguments"""
+def parse_args(args):
     task_classes = iter(inheritors(migration_task_base.MigrationTaskBase))
     parser = PromptParser()
-    parser.add_argument("configuration_path", help="Path to configuration file")
+    parser.add_argument(
+        "configuration_path",
+        help="Path to configuration file",
+        nargs="?" if "FOLIO_MIGRATION_TOOLS_CONFIGURATION_PATH" in environ else None,
+        prompt="FOLIO_MIGRATION_TOOLS_CONFIGURATION_PATH" not in environ,
+        default=environ.get("FOLIO_MIGRATION_TOOLS_CONFIGURATION_PATH"),
+    )
+    tasks_string = ", ".join(sorted(tc.__name__ for tc in task_classes))
+
     parser.add_argument(
         "task_name",
-        help=(
-            "Task name. One of one of "
-            f'{", ".join((tc.__name__ for tc in task_classes))}'
-        ),
+        help=("Task name. Use one of: " f"{tasks_string}"),
+        nargs="?" if "FOLIO_MIGRATION_TOOLS_TASK_NAME" in environ else None,
+        prompt="FOLIO_MIGRATION_TOOLS_TASK_NAME" not in environ,
+        default=environ.get("FOLIO_MIGRATION_TOOLS_TASK_NAME"),
     )
     parser.add_argument(
-        "--okapi_password",
-        help="pasword for the tenant in the configuration file",
+        "--folio_password", "--okapi_password",
+        help="password for the tenant in the configuration file",
+        prompt="FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD" not in environ,
+        default=environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
         secure=True,
     )
     parser.add_argument(
         "--base_folder_path",
         help=(
-            "path to the base folder for this library. "
-            " Built on migration_repo_template"
+            "path to the base folder for this library. Built on migration_repo_template"
+        ),
+        prompt="FOLIO_MIGRATION_TOOLS_BASE_FOLDER_PATH" not in environ,
+        default=environ.get("FOLIO_MIGRATION_TOOLS_BASE_FOLDER_PATH"),
+    )
+    parser.add_argument(
+        "--report_language",
+        help=(
+            "Language to write the reports. Defaults english for untranslated languages/strings."
         ),
+        default=environ.get("FOLIO_MIGRATION_TOOLS_REPORT_LANGUAGE", "en"),
+        prompt=False,
     )
-    return parser.parse_args()
+    parser.add_argument(
+        "--version", "-V",
+        help="Show the version of the FOLIO Migration Tools",
+        action="store_true",
+        prompt=False,
+    )
+    return parser.parse_args(args)
+
+def prep_library_config(args):
+        config_file_humped = merge_load(args.configuration_path)
+        config_file_humped["libraryInformation"]["okapiPassword"] = args.folio_password
+        config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
+        config_file = humps.decamelize(config_file_humped)
+        library_config = LibraryConfiguration(**config_file["library_information"])
+        if library_config.ecs_tenant_id:
+            library_config.is_ecs = True
+        if library_config.ecs_tenant_id and not library_config.ecs_central_iteration_identifier:
+            print(
+                "ECS tenant ID is set, but no central iteration identifier is provided. "
+                "Please provide the central iteration identifier in the configuration file."
+            )
+            sys.exit("ECS Central Iteration Identifier Not Found")
+        return config_file, library_config
+
+def print_version(args):
+    if "-V" in args or "--version" in args:
+        print(
+            f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}"
+        )
+        sys.exit(0)
+    return None
 
 
 def main():
     try:
-        task_classes = [t for t in inheritors(migration_task_base.MigrationTaskBase)]
-        args = parse_args()
-        with open(args.configuration_path) as config_file_path:
-            try:
-                config_file_humped = json.load(config_file_path)
-                config_file_humped["libraryInformation"][
-                    "okapiPassword"
-                ] = args.okapi_password
-                config_file_humped["libraryInformation"][
-                    "baseFolder"
-                ] = args.base_folder_path
-                config_file = humps.decamelize(config_file_humped)
-                library_config = LibraryConfiguration(
-                    **config_file["library_information"]
-                )
-                try:
-                    migration_task_config = next(
-                        t
-                        for t in config_file["migration_tasks"]
-                        if t["name"] == args.task_name
-                    )
-                except StopIteration:
-                    task_names = [
-                        t.get("name", "") for t in config_file["migration_tasks"]
-                    ]
-                    print(
-                        f"Referenced task name {args.task_name} not found in the "
-                        f'configuration file. Use one of {", ".join(task_names)}'
-                        "\nHalting..."
-                    )
-                    sys.exit(2)
-                try:
-                    task_class = next(
-                        tc
-                        for tc in task_classes
-                        if tc.__name__ == migration_task_config["migration_task_type"]
-                    )
-                    task_config = task_class.TaskConfiguration(**migration_task_config)
-                    task_obj = task_class(task_config, library_config)
-                    task_obj.do_work()
-                    task_obj.wrap_up()
-                except StopIteration:
-                    print(
-                        f'Referenced task {migration_task_config["migration_task_type"]} '
-                        "is not a valid option. Update your task to incorporate "
-                        f"one of {json.dumps([tc.__name__ for tc in task_classes], indent=4)}"
-                    )
-                    sys.exit(2)
-            except json.decoder.JSONDecodeError as json_error:
-                logging.critical(json_error)
-                print(
-                    f"\nError parsing configuration file {config_file_path.name}. Halting. "
-                )
-                sys.exit(2)
-            except ValidationError as e:
-                print(e.json())
-                print("Validation errors in configuration file:")
-                print("==========================================")
+        task_classes = list(inheritors(migration_task_base.MigrationTaskBase))
+        # Check if the script is run with the --version or -V flag
+        print_version(sys.argv)
 
-                for validation_message in json.loads(e.json()):
-                    print(
-                        f"{validation_message['msg']}\t"
-                        f"{', '.join(humps.camelize(str(x)) for x in validation_message['loc'])}"
-                    )
-                print("Halting")
-
-            # task_obj.do_work()
-            logging.info("Work done, wrapping up")
-            sys.exit(0)
-        # task_obj.wrap_up()
-    except requests.exceptions.SSLError:
-        print("\nSSL error. Are you connected to the Internet and the VPN?")
+        # Parse command line arguments
+        args = parse_args(sys.argv[1:])
+        try:
+            i18n.load_config(
+                Path(
+                    environ.get("FOLIO_MIGRATION_TOOLS_I18N_CONFIG_BASE_PATH")
+                    or args.base_folder_path
+                )
+                / "i18n_config.py"
+            )
+        except i18n.I18nFileLoadError:
+            i18n.load_config(Path(__file__).parent / "i18n_config.py")
+        i18n.set("locale", args.report_language)
+        config_file, library_config = prep_library_config(args)
+        try:
+            migration_task_config = next(
+                t for t in config_file["migration_tasks"] if t["name"] == args.task_name
+            )
+        except StopIteration:
+            task_names = [t.get("name", "") for t in config_file["migration_tasks"]]
+            print(
+                f"Referenced task name {args.task_name} not found in the "
+                f'configuration file. Use one of {", ".join(task_names)}'
+                "\nHalting..."
+            )
+            sys.exit("Task Name Not Found")
+        try:
+            task_class = next(
+                tc
+                for tc in task_classes
+                if tc.__name__ == migration_task_config["migration_task_type"]
+            )
+        except StopIteration:
+            print(
+                f'Referenced task {migration_task_config["migration_task_type"]} '
+                "is not a valid option. Update your task to incorporate "
+                f"one of {json.dumps([tc.__name__ for tc in task_classes], indent=4)}"
+            )
+            sys.exit("Task Type Not Found")
+        try:
+            logging.getLogger("httpx").setLevel(logging.WARNING) # Exclude info messages from httpx
+            with FolioClient(
+                library_config.gateway_url,
+                library_config.tenant_id,
+                library_config.folio_username,
+                library_config.folio_password,
+            ) as folio_client:
+                task_config = task_class.TaskConfiguration(**migration_task_config)
+                task_obj = task_class(task_config, library_config, folio_client)
+                task_obj.do_work()
+                task_obj.wrap_up()
+        except TransformationProcessError as tpe:
+            logging.critical(tpe.message)
+            print(f"\n{tpe.message}: {tpe.data_value}")
+            print("Task failure. Halting.")
+            sys.exit(1)
+        logging.info("Work done. Shutting down")
+    except json.decoder.JSONDecodeError as json_error:
+        logging.critical(json_error)
+        print(json_error.doc)
+        print(
+            f"\n{json_error}"
+            f"\nError parsing the above JSON mapping or configruation file. Halting."
+        )
+        sys.exit("Invalid JSON")
+    except ValidationError as e:
+        print(e.json())
+        print("Validation errors in configuration file:")
+        print("==========================================")
 
+        for validation_message in json.loads(e.json()):
+            print(
+                f"{validation_message['msg']}\t"
+                f"{', '.join(humps.camelize(str(x)) for x in validation_message['loc'])}"
+            )
+        print("Halting")
+        sys.exit("JSON Not Matching Spec")
+    except httpx.HTTPError as connection_error:
+        if hasattr(connection_error, "response"):
+            print(
+                f"\nHTTP Error when connecting to {connection_error.request.url}. "
+                f"Status code: {connection_error.response.status_code}. "
+                f"\nResponse: {connection_error.response.text}"
+            )
+        else:
+            print(
+                f"\nConnection Error when connecting to {connection_error.request.url}. "
+                "Are you connected to the Internet/VPN? Do you need to update DNS settings?"
+            )
+        sys.exit("HTTP Not Connecting")
+    except FileNotFoundError as fnf_error:
+        print(f"\n{fnf_error.strerror}: {fnf_error.filename}")
+        sys.exit("File not found")
+    except Exception as ee:
+        logging.exception("Unhandled exception")
+        print(f"\n{ee}")
+        sys.exit(ee.__class__.__name__)
+    sys.exit(0)
 
 def inheritors(base_class):
     subclasses = set()

folio_migration_tools/circulation_helper.py

@@ -3,18 +3,20 @@ import json
 import logging
 import re
 import time
+from typing import Set
+
+import httpx
+import i18n
+from folioclient import FolioClient
+from httpx import HTTPError
+
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.migration_report import MigrationReport
-from folio_migration_tools.report_blurbs import Blurbs
 from folio_migration_tools.transaction_migration.legacy_loan import LegacyLoan
+from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
 from folio_migration_tools.transaction_migration.transaction_result import (
     TransactionResult,
 )
-from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
-
-import requests
-from folioclient import FolioClient
-from requests import HTTPError
 
 date_time_format = "%Y-%m-%dT%H:%M:%S.%f+0000"
 
@@ -28,12 +30,15 @@ class CirculationHelper:
     ):
         self.folio_client = folio_client
         self.service_point_id = service_point_id
-        self.missing_patron_barcodes = set()
-        self.missing_item_barcodes = set()
+        self.missing_patron_barcodes: Set[str] = set()
+        self.missing_item_barcodes: Set[str] = set()
         self.migration_report: MigrationReport = migration_report
 
     def get_user_by_barcode(self, user_barcode):
         if user_barcode in self.missing_patron_barcodes:
+            self.migration_report.add_general_statistics(
+                i18n.t("Users already detected as missing")
+            )
             logging.info("User is already detected as missing")
             return {}
         user_path = f"/users?query=barcode=={user_barcode}"
@@ -49,6 +54,9 @@ class CirculationHelper:
 
     def get_item_by_barcode(self, item_barcode):
         if item_barcode in self.missing_item_barcodes:
+            self.migration_report.add_general_statistics(
+                i18n.t("Items already detected as missing")
+            )
             logging.info("Item is already detected as missing")
             return {}
         item_path = f"/item-storage/items?query=barcode=={item_barcode}"
@@ -62,21 +70,64 @@ class CirculationHelper:
             logging.error(f"{ee} {item_path}")
             return {}
 
+    def is_checked_out(self, legacy_loan: LegacyLoan) -> bool:
+        """Makes a deeper check to find out if the loan is already processed.
+        Looks up the item id, and then searches Loan Storage for any open loans.
+        If there are open loans, returns True. Else False.
+
+        Args:
+            legacy_loan (LegacyLoan): _description_
+
+        Returns:
+            bool: _description_
+        """
+        if item := self.get_item_by_barcode(legacy_loan.item_barcode):
+            if self.get_active_loan_by_item_id(item["id"]):
+                return True
+        return False
+
+    def get_active_loan_by_item_id(self, item_id: str) -> dict:
+        """Queries FOLIO for the first open loan.
+
+        Args:
+            item_id (str): the item ID. A uuid string.
+
+        Returns:
+            dict: The open loan, if found. Else an empty dictionary
+        """
+        loan_path = f'/loan-storage/loans?query=(itemId=="{item_id}")'
+        try:
+            loans = self.folio_client.folio_get(loan_path, "loans")
+            return next((loan for loan in loans if loan["status"]["name"] == "Open"), {})
+        except Exception as ee:
+            logging.error(f"{ee} {loan_path}")
+            return {}
+
+    def get_holding_by_uuid(self, holdings_uuid):
+        holdings_path = f"/holdings-storage/holdings/{holdings_uuid}"
+        try:
+            return self.folio_client.folio_get_single_object(holdings_path)
+        except Exception as ee:
+            logging.error(f"{ee} {holdings_path}")
+            return {}
+
     def check_out_by_barcode(self, legacy_loan: LegacyLoan) -> TransactionResult:
         """Checks out a legacy loan using the Endpoint /circulation/check-out-by-barcode
         Adds all possible overrides in order to make the transaction go through
+
         Args:
-            legacy_loan (LegacyLoan): Legacy loan to be posted
+            legacy_loan (LegacyLoan): _description_
 
         Returns:
-            TransactionResult: the result of the transaction
+            TransactionResult: _description_
         """
+
         t0_function = time.time()
         data = {
             "itemBarcode": legacy_loan.item_barcode,
             "userBarcode": legacy_loan.patron_barcode,
             "loanDate": legacy_loan.out_date.isoformat(),
-            "servicePointId": self.service_point_id,
+            "servicePointId": legacy_loan.service_point_id,
             "overrideBlocks": {
                 "itemNotLoanableBlock": {"dueDate": legacy_loan.due_date.isoformat()},
                 "patronBlock": {},
@@ -84,21 +135,19 @@ class CirculationHelper:
                 "comment": "Migrated from legacy system",
             },
         }
+        if legacy_loan.proxy_patron_barcode:
+            data.update({"proxyUserBarcode": legacy_loan.proxy_patron_barcode})
         path = "/circulation/check-out-by-barcode"
-        url = f"{self.folio_client.okapi_url}{path}"
+        url = f"{self.folio_client.gateway_url}{path}"
         try:
             if legacy_loan.patron_barcode in self.missing_patron_barcodes:
-                error_message = "Patron barcode already detected as missing"
+                error_message = i18n.t("Patron barcode already detected as missing")
                 logging.error(
                     f"{error_message} Patron barcode: {legacy_loan.patron_barcode} "
                     f"Item Barcode:{legacy_loan.item_barcode}"
                 )
-                return TransactionResult(
-                    False, False, None, error_message, error_message
-                )
-            req = requests.post(
-                url, headers=self.folio_client.okapi_headers, data=json.dumps(data)
-            )
+                return TransactionResult(False, False, "", error_message, error_message)
+            req = httpx.post(url, headers=self.folio_client.okapi_headers, json=data, timeout=None)
             if req.status_code == 422:
                 error_message_from_folio = json.loads(req.text)["errors"][0]["message"]
                 stat_message = error_message_from_folio
@@ -109,8 +158,7 @@ class CirculationHelper:
                         error_message_from_folio,
                     )[0]
                     error_message = (
-                        f"{stat_message} for item with "
-                        f"barcode {legacy_loan.item_barcode}"
+                        f"{stat_message} for item with barcode {legacy_loan.item_barcode}"
                     )
                     return TransactionResult(
                         False,
@@ -120,9 +168,7 @@ class CirculationHelper:
                         stat_message,
                     )
                 elif "No item with barcode" in error_message_from_folio:
-                    error_message = (
-                        f"No item with barcode {legacy_loan.item_barcode} in FOLIO"
-                    )
+                    error_message = f"No item with barcode {legacy_loan.item_barcode} in FOLIO"
                     stat_message = "Item barcode not in FOLIO"
                     self.missing_item_barcodes.add(legacy_loan.item_barcode)
                     return TransactionResult(
@@ -135,10 +181,8 @@ class CirculationHelper:
 
                 elif "find user with matching barcode" in error_message_from_folio:
                     self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
-                    error_message = (
-                        f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
-                    )
-                    stat_message = "Patron barcode not in FOLIO"
+                    error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
+                    stat_message = i18n.t("Patron barcode not in FOLIO")
                     return TransactionResult(
                         False,
                         False,
@@ -146,10 +190,7 @@ class CirculationHelper:
                         error_message_from_folio,
                         stat_message,
                     )
-                elif (
-                    "Cannot check out item that already has an open"
-                    in error_message_from_folio
-                ):
+                elif "Cannot check out item that already has an open" in error_message_from_folio:
                     return TransactionResult(
                         False,
                         False,
@@ -157,12 +198,20 @@ class CirculationHelper:
                         error_message_from_folio,
                         error_message_from_folio,
                     )
+                elif "Item is already checked out" in error_message_from_folio:
+                    return TransactionResult(
+                        False,
+                        True,
+                        None,
+                        error_message_from_folio,
+                        error_message_from_folio,
+                    )
                 logging.error(
                     f"{error_message} "
                     f"Patron barcode: {legacy_loan.patron_barcode} "
                     f"Item Barcode:{legacy_loan.item_barcode}"
                 )
-                self.migration_report.add(Blurbs.Details, stat_message)
+                self.migration_report.add("Details", stat_message)
                 return TransactionResult(
                     False, True, None, error_message, f"Check out error: {stat_message}"
                 )
@@ -174,7 +223,7 @@ class CirculationHelper:
                     legacy_loan.item_barcode,
                     f"{(time.time() - t0_function):.2f}",
                 )
-                return TransactionResult(True, False, json.loads(req.text), None, stats)
+                return TransactionResult(True, False, json.loads(req.text), "", stats)
             elif req.status_code == 204:
                 stats = "Successfully checked out by barcode"
                 logging.debug(
@@ -183,7 +232,7 @@ class CirculationHelper:
                     legacy_loan.item_barcode,
                     req.status_code,
                 )
-                return TransactionResult(True, False, None, None, stats)
+                return TransactionResult(True, False, None, "", stats)
             else:
                 req.raise_for_status()
         except HTTPError:
@@ -199,19 +248,17 @@ class CirculationHelper:
                 False,
                 None,
                 "5XX",
-                f"Failed checkout http status {req.status_code}",
+                i18n.t("Failed checkout http status %{code}", code=req.status_code),
             )
 
     @staticmethod
     def create_request(
-        folio_client: FolioClient,
-        legacy_request: LegacyRequest,
-        migration_report: MigrationReport,
+        folio_client: FolioClient, legacy_request: LegacyRequest, migration_report: MigrationReport
     ):
         try:
             path = "/circulation/requests"
-            url = f"{folio_client.okapi_url}{path}"
-            data = legacy_request.to_dict()
+            url = f"{folio_client.gateway_url}{path}"
+            data = legacy_request.serialize()
             data["requestProcessingParameters"] = {
                 "overrideBlocks": {
                     "itemNotLoanableBlock": {
@@ -222,20 +269,15 @@ class CirculationHelper:
                     "comment": "Migrated from legacy system",
                 }
             }
-            req = requests.post(
-                url, headers=folio_client.okapi_headers, data=json.dumps(data)
-            )
+            req = httpx.post(url, headers=folio_client.okapi_headers, json=data, timeout=None)
             logging.debug(f"POST {req.status_code}\t{url}\t{json.dumps(data)}")
             if str(req.status_code) == "422":
                 message = json.loads(req.text)["errors"][0]["message"]
-                logging.error(f"{message}\t{json.dumps(data)}")
+                logging.error(f"{message}")
                 migration_report.add_general_statistics(message)
                 return False
             else:
                 req.raise_for_status()
-                migration_report.add_general_statistics(
-                    f"Created {legacy_request.request_type}"
-                )
                 logging.info(
                     "%s Successfully created %s",
                     req.status_code,
@@ -244,7 +286,7 @@ class CirculationHelper:
                 return True
         except Exception as exception:
             logging.error(exception, exc_info=True)
-            migration_report.add(Blurbs.Details, exception)
+            migration_report.add("Details", exception)
             Helper.log_data_issue(
                 legacy_request.item_barcode,
                 exception,
@@ -252,9 +294,7 @@ class CirculationHelper:
             )
             return False
 
-    def load_migrated_user_barcodes(
-        self, user_barcodes, patron_files, folder_structure
-    ):
+    def load_migrated_user_barcodes(self, user_barcodes, patron_files, folder_structure):
         if any(patron_files):
             for filedef in patron_files:
                 my_path = folder_structure.results_folder / filedef.file_name
@@ -275,19 +315,16 @@ class CirculationHelper:
             logging.info("Loaded %s barcodes from items", len(item_barcodes))
 
     @staticmethod
-    def extend_open_loan(
-        folio_client: FolioClient, loan, extension_due_date, extend_out_date
-    ):
-        # TODO: add logging instead of print out
+    def extend_open_loan(folio_client: FolioClient, loan, extension_due_date, extend_out_date):
         try:
             loan_to_put = copy.deepcopy(loan)
             del loan_to_put["metadata"]
             loan_to_put["dueDate"] = extension_due_date.isoformat()
             loan_to_put["loanDate"] = extend_out_date.isoformat()
-            url = f"{folio_client.okapi_url}/circulation/loans/{loan_to_put['id']}"
+            url = f"{folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
 
-            req = requests.put(
-                url, headers=folio_client.okapi_headers, data=json.dumps(loan_to_put)
+            req = httpx.put(
+                url, headers=folio_client.okapi_headers, json=loan_to_put, timeout=None
             )
             logging.info(
                 "%s\tPUT Extend loan %s to %s\t %s",