folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/migration_tasks/requests_migrator.py

@@ -1,37 +1,84 @@
 import csv
-from datetime import datetime
 import json
-import sys
 import logging
+import sys
 import time
-from datetime import timezone
-from pydantic import BaseModel
-from folio_migration_tools.helper import Helper
+from typing import Optional, Annotated
+from pydantic import Field
+
+import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
+from zoneinfo import ZoneInfo
+
 from folio_migration_tools.circulation_helper import CirculationHelper
 from folio_migration_tools.custom_dict import InsensitiveDictReader
+from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
     FileDefinition,
     LibraryConfiguration,
 )
 from folio_migration_tools.migration_report import MigrationReport
 from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
-
-from typing import Dict, List, Optional
-from folio_migration_tools.report_blurbs import Blurbs
-
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
 
 
 class RequestsMigrator(MigrationTaskBase):
-    class TaskConfiguration(BaseModel):
-        name: str
-        utc_difference: int
-        migration_task_type: str
-        open_requests_file: FileDefinition
-        starting_row: Optional[int] = 1
-        item_files: Optional[list[FileDefinition]] = []
-        patron_files: Optional[list[FileDefinition]] = []
+    class TaskConfiguration(AbstractTaskConfiguration):
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description=(
+                    "Name of this migration task. The name is being used to call "
+                    "the specific task, and to distinguish tasks of similar types"
+                )
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task you want to perform",
+            ),
+        ]
+        open_requests_file: Annotated[
+            FileDefinition,
+            Field(
+                title="Open requests file",
+                description="File with list of open requests",
+            ),
+        ]
+        starting_row: Annotated[
+            Optional[int],
+            Field(
+                title="Starting row",
+                description=(
+                    "Row number to start processing data from. "
+                    "Optional, by default is first row"
+                ),
+            ),
+        ] = 1
+        item_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Item files",
+                description=(
+                    "List of files containing item data. "
+                    "Optional, by default is empty list"
+                ),
+            ),
+        ] = []
+        patron_files: Annotated[
+            Optional[list[FileDefinition]],
+            Field(
+                title="Patron files",
+                description=(
+                    "List of files containing patron data. "
+                    "Optional, by default is empty list"
+                ),
+            ),
+        ] = []
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -41,16 +88,30 @@ class RequestsMigrator(MigrationTaskBase):
         self,
         task_configuration: TaskConfiguration,
         library_config: LibraryConfiguration,
+        folio_client
     ):
         csv.register_dialect("tsv", delimiter="\t")
         self.migration_report = MigrationReport()
         self.valid_legacy_requests = []
-        super().__init__(library_config, task_configuration)
+        super().__init__(library_config, task_configuration, folio_client)
         self.circulation_helper = CirculationHelper(
             self.folio_client,
             "",
             self.migration_report,
         )
+        try:
+            logging.info("Attempting to retrieve tenant timezone configuration...")
+            my_path = (
+                "/configurations/entries?query=(module==ORG%20and%20configName==localeSettings)"
+            )
+            self.tenant_timezone_str = json.loads(
+                self.folio_client.folio_get_single_object(my_path)["configs"][0]["value"]
+            )["timezone"]
+            logging.info("Tenant timezone is: %s", self.tenant_timezone_str)
+        except Exception:
+            logging.info('Tenant locale settings not available. Using "UTC".')
+            self.tenant_timezone_str = "UTC"
+        self.tenant_timezone = ZoneInfo(self.tenant_timezone_str)
         with open(
             self.folder_structure.legacy_records_folder
             / task_configuration.open_requests_file.file_name,
@@ -66,9 +127,7 @@ class RequestsMigrator(MigrationTaskBase):
                 "Loaded and validated %s requests in file",
                 len(self.semi_valid_legacy_requests),
             )
-        if any(self.task_configuration.item_files) or any(
-            self.task_configuration.patron_files
-        ):
+        if any(self.task_configuration.item_files) or any(self.task_configuration.patron_files):
             self.valid_legacy_requests = list(self.check_barcodes())
             logging.info(
                 "Loaded and validated %s requests against barcodes",
@@ -80,8 +139,10 @@ class RequestsMigrator(MigrationTaskBase):
                 "previously migrated objects"
             )
             self.valid_legacy_requests = self.semi_valid_legacy_requests
+
         self.valid_legacy_requests.sort(key=lambda x: x.request_date)
         logging.info("Sorted the list of requests by request date")
+
         self.t0 = time.time()
         self.skipped_since_already_added = 0
         self.failed_requests = set()
@@ -89,29 +150,30 @@ class RequestsMigrator(MigrationTaskBase):
         logging.info("Init completed")
 
     def prepare_legacy_request(self, legacy_request: LegacyRequest):
-        patron = self.circulation_helper.get_user_by_barcode(
-            legacy_request.patron_barcode
-        )
+        patron = self.circulation_helper.get_user_by_barcode(legacy_request.patron_barcode)
+        self.migration_report.add_general_statistics(i18n.t("Patron lookups performed"))
+
         if not patron:
-            logging.error(
-                f"No user with barcode {legacy_request.patron_barcode} found in FOLIO"
-            )
+            logging.error(f"No user with barcode {legacy_request.patron_barcode} found in FOLIO")
             Helper.log_data_issue(
                 f"{legacy_request.patron_barcode}",
-                "No user with barcode",
+                "No user with barcode.",
                 f"{legacy_request.patron_barcode}",
             )
-            self.migration_report.add_general_statistics("No user with barcode")
+            self.migration_report.add_general_statistics(
+                i18n.t("No user with barcode found in FOLIO")
+            )
             self.failed_requests.add(legacy_request)
             return False, legacy_request
         legacy_request.patron_id = patron.get("id")
 
         item = self.circulation_helper.get_item_by_barcode(legacy_request.item_barcode)
+        self.migration_report.add_general_statistics(i18n.t("Item lookups performed"))
         if not item:
-            logging.error(
-                f"No item with barcode {legacy_request.item_barcode} found in FOLIO"
+            logging.error(f"No item with barcode {legacy_request.item_barcode} found in FOLIO")
+            self.migration_report.add_general_statistics(
+                i18n.t("No item with barcode found in FOLIO")
             )
-            self.migration_report.add_general_statistics("No item with barcode")
             Helper.log_data_issue(
                 f"{legacy_request.item_barcode}",
                 "No item with barcode",
@@ -119,12 +181,17 @@ class RequestsMigrator(MigrationTaskBase):
             )
             self.failed_requests.add(legacy_request)
             return False, legacy_request
+        holding = self.circulation_helper.get_holding_by_uuid(item.get("holdingsRecordId"))
+        self.migration_report.add_general_statistics(i18n.t("Holdings lookups performed"))
         legacy_request.item_id = item.get("id")
-        if item["status"]["name"] in ["Available", "Aged to lost", "Missing"]:
+        legacy_request.holdings_record_id = item.get("holdingsRecordId")
+        legacy_request.instance_id = holding.get("instanceId")
+        if item["status"]["name"] in ["Available"]:
             legacy_request.request_type = "Page"
-            logging.info(
-                f'Setting request to Page, since the status is {item["status"]["name"]}'
-            )
+            logging.info(f'Setting request to Page, since the status is {item["status"]["name"]}')
+        self.migration_report.add_general_statistics(
+            i18n.t("Valid, prepared requests, ready for posting")
+        )
         return True, legacy_request
 
     def do_work(self):
@@ -135,9 +202,7 @@ class RequestsMigrator(MigrationTaskBase):
             self.valid_legacy_requests[self.task_configuration.starting_row - 1 :],
             start=1,
         ):
-
             t0_migration = time.time()
-            self.migration_report.add_general_statistics("Processed requests")
             try:
                 res, legacy_request = self.prepare_legacy_request(legacy_request)
                 if res:
@@ -145,11 +210,11 @@ class RequestsMigrator(MigrationTaskBase):
                         self.folio_client, legacy_request, self.migration_report
                     ):
                         self.migration_report.add_general_statistics(
-                            "Successfully processed requests"
+                            i18n.t("Successfully migrated requests")
                         )
                     else:
                         self.migration_report.add_general_statistics(
-                            "Unsuccessfully processed requests"
+                            i18n.t("Unsuccessfully migrated requests")
                         )
                         self.failed_requests.add(legacy_request)
                 if num_requests == 1:
@@ -163,19 +228,18 @@ class RequestsMigrator(MigrationTaskBase):
                 )
                 sys.exit(1)
             if num_requests % 10 == 0:
-                logging.info(
-                    f"{timings(self.t0, t0_migration, num_requests)} {num_requests}"
-                )
+                logging.info(f"{timings(self.t0, t0_migration, num_requests)} {num_requests}")
+        logging.info(f"{timings(self.t0, t0_migration, num_requests)} {num_requests}")
 
     def wrap_up(self):
+        self.extradata_writer.flush()
         self.write_failed_request_to_file()
 
         with open(self.folder_structure.migration_reports_file, "w+") as report_file:
-            report_file.write("# Requests migration results   \n")
-            report_file.write(
-                f"Time Finished: {datetime.isoformat(datetime.now(timezone.utc))}\n"
+            self.migration_report.write_migration_report(
+                i18n.t("Requests migration report"), report_file, self.start_datetime
             )
-            self.migration_report.write_migration_report(report_file)
+        self.clean_out_empty_logs()
 
     def write_failed_request_to_file(self):
         csv_columns = [
@@ -188,9 +252,7 @@ class RequestsMigrator(MigrationTaskBase):
             "pickup_servicepoint_id",
         ]
         with open(self.folder_structure.failed_recs_path, "w+") as failed_requests_file:
-            writer = csv.DictWriter(
-                failed_requests_file, fieldnames=csv_columns, dialect="tsv"
-            )
+            writer = csv.DictWriter(failed_requests_file, fieldnames=csv_columns, dialect="tsv")
             writer.writeheader()
             failed: LegacyRequest
             for failed in self.failed_requests:
@@ -212,14 +274,20 @@ class RequestsMigrator(MigrationTaskBase):
             has_patron_barcode = request.patron_barcode in user_barcodes
             if has_item_barcode and has_patron_barcode:
                 self.migration_report.add_general_statistics(
-                    "Requests verified against migrated user and item"
+                    i18n.t("Requests successfully verified against migrated users and items")
                 )
                 yield request
             else:
                 self.migration_report.add(
-                    Blurbs.DiscardedLoans,
-                    f"Requests discarded. Had migrated item barcode: {has_item_barcode}. "
-                    f"Had migrated user barcode: {has_patron_barcode}",
+                    "DiscardedLoans",
+                    i18n.t(
+                        "Requests discarded. Had migrated item barcode: %{item_barcode}.\n Had migrated user barcode: %{patron_barcode}",
+                        item_barcode=has_item_barcode,
+                        patron_barcode=has_patron_barcode,
+                    ),
+                )
+                self.migration_report.add_general_statistics(
+                    i18n.t("Requests that failed verification against migrated users and items")
                 )
             if not has_item_barcode:
                 Helper.log_data_issue(
@@ -237,30 +305,30 @@ class RequestsMigrator(MigrationTaskBase):
     def load_and_validate_legacy_requests(self, requests_reader):
         num_bad = 0
         logging.info("Validating legacy requests in file...")
-        for legacy_reques_count, legacy_request_dict in enumerate(
-            requests_reader, start=1
-        ):
+        for legacy_reques_count, legacy_request_dict in enumerate(requests_reader, start=1):
+            self.migration_report.add_general_statistics(i18n.t("Requests in file"))
             try:
                 legacy_request = LegacyRequest(
                     legacy_request_dict,
-                    self.task_configuration.utc_difference,
+                    self.tenant_timezone,
                     legacy_reques_count,
                 )
                 if any(legacy_request.errors):
                     num_bad += 1
                     self.migration_report.add_general_statistics(
-                        "Requests with valueErrors"
+                        i18n.t("Requests with valueErrors")
                     )
                     for error in legacy_request.errors:
-                        self.migration_report.add(
-                            Blurbs.DiscardedRequests, f"{error[0]} - {error[1]}"
-                        )
+                        self.migration_report.add("DiscardedRequests", f"{error[0]} - {error[1]}")
                         Helper.log_data_issue(
                             legacy_request.item_barcode,
                             f"{error[0]} - {error[1]}",
                             json.dumps(legacy_request.to_source_dict()),
                         )
                 else:
+                    self.migration_report.add_general_statistics(
+                        i18n.t("Requests with valid source data")
+                    )
                     yield legacy_request
             except ValueError as ve:
                 logging.exception(ve)

folio_migration_tools/migration_tasks/reserves_migrator.py

@@ -0,0 +1,253 @@
+import csv
+import json
+import logging
+import sys
+import time
+import traceback
+from typing import Dict, Annotated
+from urllib.error import HTTPError
+from pydantic import Field
+
+import httpx
+import i18n
+from folio_uuid.folio_namespaces import FOLIONamespaces
+
+from folio_migration_tools.custom_dict import InsensitiveDictReader
+from folio_migration_tools.custom_exceptions import TransformationProcessError
+from folio_migration_tools.library_configuration import (
+    FileDefinition,
+    LibraryConfiguration,
+)
+from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
+from folio_migration_tools.transaction_migration.legacy_reserve import LegacyReserve
+
+
+class ReservesMigrator(MigrationTaskBase):
+    class TaskConfiguration(AbstractTaskConfiguration):
+        name: Annotated[
+            str,
+            Field(
+                title="Migration task name",
+                description=(
+                    "Name of this migration task. The name is being used to call the specific "
+                    "task, and to distinguish tasks of similar types"
+                ),
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task you want to perform",
+            ),
+        ]
+        course_reserve_file_path: Annotated[
+            FileDefinition,
+            Field(
+                title="Course reserve file path",
+                description="Path to the file with course reserves",
+            ),
+        ]
+
+    @staticmethod
+    def get_object_type() -> FOLIONamespaces:
+        return FOLIONamespaces.reserve
+
+    def __init__(
+        self,
+        task_configuration: TaskConfiguration,
+        library_config: LibraryConfiguration,
+        folio_client
+    ):
+        csv.register_dialect("tsv", delimiter="\t")
+        self.migration_report = MigrationReport()
+        self.valid_reserves = []
+        super().__init__(library_config, task_configuration, folio_client)
+        with open(
+            self.folder_structure.legacy_records_folder
+            / task_configuration.course_reserve_file_path.file_name,
+            "r",
+            encoding="utf-8",
+        ) as reserves_file:
+            self.semi_valid_reserves = list(
+                self.load_and_validate_legacy_reserves(
+                    InsensitiveDictReader(reserves_file, dialect="tsv")
+                )
+            )
+            logging.info(
+                "Loaded and validated %s reserves in file",
+                len(self.semi_valid_reserves),
+            )
+
+            self.valid_reserves = self.semi_valid_reserves
+        self.t0 = time.time()
+        self.failed: Dict = {}
+        logging.info("Init completed")
+
+    def do_work(self):
+        logging.info("Starting")
+        for num_reserves, legacy_reserve in enumerate(self.valid_reserves, start=1):
+            t0_migration = time.time()
+            self.migration_report.add_general_statistics(i18n.t("Processed reserves"))
+            try:
+                self.post_single_reserve(legacy_reserve)
+            except Exception as ee:
+                logging.exception(
+                    f"Error in row {num_reserves}  Reserve: {json.dumps(legacy_reserve)} {ee}"
+                )
+            if num_reserves % 50 == 0:
+                logging.info(f"{timings(self.t0, t0_migration, num_reserves)} {num_reserves}")
+
+    def post_single_reserve(self, legacy_reserve: LegacyReserve):
+        try:
+            path = f"/coursereserves/courselistings/{legacy_reserve.course_listing_id}/reserves"
+            if self.folio_put_post(
+                path, legacy_reserve.to_dict(), "POST", i18n.t("Posted reserves")
+            ):
+                self.migration_report.add_general_statistics(
+                    i18n.t("Successfully posted reserves")
+                )
+            else:
+                self.migration_report.add_general_statistics(i18n.t("Failure to post reserve"))
+        except Exception as ee:
+            logging.error(ee)
+
+    def wrap_up(self):
+        self.extradata_writer.flush()
+        for k, v in self.failed.items():
+            self.failed_and_not_dupe[k] = [v.to_dict()]
+        self.migration_report.set("GeneralStatistics", "Failed loans", len(self.failed))
+        self.write_failed_reserves_to_file()
+
+        with open(self.folder_structure.migration_reports_file, "w+") as report_file:
+            self.migration_report.write_migration_report(
+                i18n.t("Reserves migration report"), report_file, self.start_datetime
+            )
+        self.clean_out_empty_logs()
+
+    def write_failed_reserves_to_file(self):
+        # POST /coursereserves/courselistings/40a085bd-b44b-42b3-b92f-61894a75e3ce/reserves
+        # Match on legacy course number ()
+
+        csv_columns = ["legacy_identifier", "barcode"]
+        with open(self.folder_structure.failed_recs_path, "w+") as failed_reserves_file:
+            writer = csv.DictWriter(failed_reserves_file, fieldnames=csv_columns, dialect="tsv")
+            writer.writeheader()
+            for _k, failed_reserve in self.failed.items():
+                writer.writerow(failed_reserve[0])
+
+    def check_barcodes(self):
+        """Stub for extension.
+
+        Yields:
+            _type_: _description_
+        """
+        item_barcodes = set()
+        self.circulation_helper.load_migrated_item_barcodes(
+            item_barcodes, self.task_configuration.item_files, self.folder_structure
+        )
+        for loan in self.semi_valid_legacy_loans:
+            has_item_barcode = loan.item_barcode in item_barcodes or not any(item_barcodes)
+            if has_item_barcode:
+                self.migration_report.add_general_statistics(
+                    i18n.t("Reserve verified against migrated item")
+                )
+                yield loan
+            else:
+                self.migration_report.add(
+                    "DiscardedLoans", i18n.t("Reserve discarded. Could not find migrated barcode")
+                )
+
+    def load_and_validate_legacy_reserves(self, reserves_reader):
+        num_bad = 0
+        logging.info("Validating legacy loans in file...")
+        for legacy_reserve_count, legacy_reserve_dict in enumerate(reserves_reader):
+            try:
+                legacy_reserve = LegacyReserve(
+                    legacy_reserve_dict,
+                    self.folio_client,
+                    legacy_reserve_count,
+                )
+                if any(legacy_reserve.errors):
+                    num_bad += 1
+                    self.migration_report.add_general_statistics(i18n.t("Discarded reserves"))
+                    for error in legacy_reserve.errors:
+                        self.migration_report.add("DiscardedReserves", f"{error[0]} - {error[1]}")
+                else:
+                    yield legacy_reserve
+            except ValueError as ve:
+                logging.exception(ve)
+        logging.info(
+            f"Done validating {legacy_reserve_count} legacy reserves with {num_bad} rotten apples"
+        )
+        if num_bad / legacy_reserve_count > 0.5:
+            q = num_bad / legacy_reserve_count
+            logging.error("%s percent of reserves failed to validate.", (q * 100))
+            self.migration_report.log_me()
+            logging.critical("Halting...")
+            sys.exit(1)
+
+    def folio_put_post(self, url, data_dict, verb, action_description=""):
+        full_url = f"{self.folio_client.gateway_url}{url}"
+        try:
+            if verb == "PUT":
+                resp = httpx.put(
+                    full_url,
+                    headers=self.folio_client.okapi_headers,
+                    json=data_dict,
+                )
+            elif verb == "POST":
+                resp = httpx.post(
+                    full_url,
+                    headers=self.folio_client.okapi_headers,
+                    json=data_dict,
+                )
+            else:
+                raise TransformationProcessError("Bad verb supplied. This is a code issue.")
+            if resp.status_code == 422:
+                error_message = json.loads(resp.text)["errors"][0]["message"]
+                logging.error(error_message)
+                self.migration_report.add(
+                    "Details",
+                    i18n.t(
+                        "%{action} error: %{message}",
+                        action=action_description,
+                        message=error_message,
+                    ),
+                )
+                resp.raise_for_status()
+            elif resp.status_code in [201, 204]:
+                self.migration_report.add(
+                    "Details",
+                    i18n.t("Successfully %{action}", action=action_description)
+                    + f" ({resp.status_code})",
+                )
+            else:
+                self.migration_report.add(
+                    "Details",
+                    i18n.t(
+                        "%{action} error. http status: %{status}",
+                        action=action_description,
+                        status=resp.status_code,
+                    ),
+                )
+                logging.error(json.dumps(data_dict))
+                resp.raise_for_status()
+            return True
+        except HTTPError as exception:
+            logging.error(f"{resp.status_code}. {verb} FAILED for {url}")
+            traceback.print_exc()
+            logging.info(exception)
+            return False
+
+
+def timings(t0, t0func, num_objects):
+    avg = num_objects / (time.time() - t0)
+    elapsed = time.time() - t0
+    elapsed_func = time.time() - t0func
+    return (
+        f"Total objects: {num_objects}\tTotal elapsed: {elapsed:.2f}\t"
+        f"Average per object: {avg:.2f}\tElapsed this time: {elapsed_func:.2f}"
+    )