folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/test_infrastructure/mocked_classes.py

@@ -0,0 +1,406 @@
+import json
+import logging
+import uuid
+from pathlib import Path
+from unittest.mock import MagicMock
+from unittest.mock import Mock
+
+from folioclient import FolioClient
+
+from folio_migration_tools.extradata_writer import ExtradataWriter
+from folio_migration_tools.mapping_file_transformation.holdings_mapper import (
+    HoldingsMapper,
+)
+from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.library_configuration import (
+    LibraryConfiguration,
+    FolioRelease,
+)
+
+
+def mocked_holdings_mapper() -> Mock:
+    mock_mapper = Mock(spec=HoldingsMapper)
+    mock_mapper.migration_report = MigrationReport()
+    mock_mapper.extradata_writer = ExtradataWriter(Path(""))
+
+    return mock_mapper
+
+
+def mocked_folio_client() -> FolioClient:
+    try:
+        FolioClient.login = MagicMock(name="login", return_value=None)
+        FolioClient.okapi_token = "token"  # noqa:S105
+        mocked_folio = FolioClient("okapi_url", "tenant_id", "username", "password")
+        mocked_folio.folio_get_single_object = folio_get_single_object_mocked
+        mocked_folio.folio_get_all = folio_get_all_mocked
+        mocked_folio.get_from_github = folio_get_from_github
+        mocked_folio.current_user = str(uuid.uuid4())
+        return mocked_folio
+    except Exception as ee:
+        logging.error(ee)
+        raise ee
+
+
+def folio_get_all_mocked(ref_data_path, array_name, query="", limit=10):
+    with open("./static/reference_data.json", "r") as super_schema_file:
+        super_schema = json.load(super_schema_file)
+    if ref_data_path == "/coursereserves/terms":
+        yield from [
+            {"name": "Fall 2022", "id": "42093be3-d1e7-4bb6-b2b9-18e153d109b2"},
+            {"name": "Summer 2022", "id": "415b14a8-c94c-4aa1-a0a8-d397efae343e"},
+        ]
+    elif ref_data_path == "/coursereserves/departments":
+        yield from [
+            {
+                "id": "7532e5ab-9812-496c-ab77-4fbb6a7e5dbf",
+                "name": "Department_t",
+                "description": "Art & Art History",
+            },
+            {
+                "id": "af7ae6be-c0b2-444d-b76f-4061098d17cd",
+                "name": "Department_FALLBACK",
+                "description": "FALLBACK",
+            },
+        ]
+    elif ref_data_path == "/organizations-storage/categories":
+        yield from [
+            {"id": "c78640d5-a1ec-4721-9a1f-c6f876d4c179", "value": "Returns"},
+            {"id": "604c2c9d-ed3a-46cd-bec4-69926c303b22", "value": "Sales"},
+            {"id": "c5b175bd-34a0-4a4d-9bd9-8eddae8e67f8", "value": "General"},
+            {"id": "97dcb23df-1aba-444e-b88d-804d17c715a5", "value": "Technical Support"},
+            {"id": "e193b0d1-4674-4a9e-818b-375f013d963f", "value": "Moral Support"},
+        ]
+
+    elif ref_data_path == "/organizations-storage/organization-types":
+        yield from [
+            {"id": "837d04b6-d81c-4c49-9efd-2f62515999b3", "name": "Consortium"},
+            {"id": "fc54327d-fd60-4f6a-ba37-a4375511b91b", "name": "Unspecified"},
+        ]
+    elif (
+        ref_data_path == "/organizations-storage/organizations"
+        and query == '?query=(code=="EBSCO")'
+    ):
+        yield from [{"id": "some id", "code": "some code", "name": "EBSCO Information Services"}]
+
+    elif (
+        ref_data_path == "/organizations-storage/organizations"
+        and query == '?query=(code=="LisasAwesomeStartup")'
+    ):
+        yield from []
+
+    elif ref_data_path == "/organizations-storage/organizations":
+        yield from [
+            {"id": "837d04b6-d81c-4c49-9efd-2f62515999b3", "code": "GOBI"},
+            {"id": "fc54327d-fd60-4f6a-ba37-a4375511b91b", "code": "EBSCO"},
+        ]
+
+    elif ref_data_path == "/orders/acquisition-methods":
+        yield from [
+            {"id": "837d04b6-d81c-4c49-9efd-2f62515999b3", "value": "Purchase"},
+            {"id": "fc54327d-fd60-4f6a-ba37-a4375511b91b", "value": "Theft"},
+            {"id": "fc54327d-fd60-4f6a-ba37-a437551sarfs91b", "value": "Other"},
+        ]
+
+    elif ref_data_path == "/groups":
+        yield from [
+            {
+                "group": "FOLIO fallback group name",
+                "desc": "Mocked response",
+                "id": "27ab99d3-0e17-41f0-a20a-99e05acc0e6f",
+            },
+            {
+                "group": "FOLIO group name",
+                "desc": "Mocked response",
+                "id": "5fc96cbd-a860-42a7-8d2b-72af30206712",
+            },
+        ]
+    elif ref_data_path == "/departments":
+        yield from [
+            {
+                "id": "12a2ad12-951d-4124-9fb2-58c70f0b7f71",
+                "name": "FOLIO user department name",
+                "code": "fdp",
+            },
+            {
+                "id": "12a2ad12-951d-4124-9fb2-58c70f0b7f72",
+                "name": "FOLIO user department name 2",
+                "code": "fdp2",
+            },
+            {
+                "id": "2f452d21-507d-4b32-a89d-8ea9753cc946",
+                "name": "FOLIO fallback user department name",
+                "code": "fb",
+            },
+        ]
+    elif ref_data_path == "/owners":
+        yield from [
+            {
+                "owner": "The Best Fee Fine Owner",
+                "desc": "She really is!",
+                "servicePointOwner": [
+                    {"value": "a77b55e7-f9f3-40a1-83e0-241bc606a826", "label": "lisatest"}
+                ],
+                "id": "5abfff3f-50eb-432a-9a43-21f8f7a70194",
+            },
+            {
+                "owner": "The Other Fee Fine Owner",
+                "desc": "heeey",
+                "servicePointOwner": [
+                    {"value": "1543c345-dcaf-4367-84a8-853d95837a3b", "label": "lisatest2 :) <3 "}
+                ],
+                "id": "62a0eb54-de96-46ee-b184-5be6c8114a19",
+            },
+        ]
+    elif ref_data_path == "/feefines":
+        yield from [
+            {
+                "automatic": False,
+                "feeFineType": "Coffee spill",
+                "ownerId": "5abfff3f-50eb-432a-9a43-21f8f7a70194",
+                "id": "6e8dc178-f667-45cd-90b5-338c78c3a85c",
+            },
+            {
+                "automatic": False,
+                "feeFineType": "Coffee spill",
+                "ownerId": "62a0eb54-de96-46ee-b184-5be6c8114a19",
+                "id": "031836ec-521a-4493-9f76-0e02c2e7d241",
+            },
+            {
+                "automatic": False,
+                "feeFineType": "Replacement library card",
+                "ownerId": "5abfff3f-50eb-432a-9a43-21f8f7a70194",
+                "id": "8936606d-223b-428e-9a70-4b8105f60cdb",
+            },
+            {
+                "automatic": True,
+                "feeFineType": "Replacement processing fee",
+                "id": "d20df2fb-45fd-4184-b238-0d25747ffdd9",
+            },
+        ]
+
+    elif ref_data_path == "/service-points":
+        yield from [
+            {
+                "id": "finance_office_uuid",
+                "name": "Finance Office",
+                "code": "fo",
+            },
+            {
+                "id": "library_main_desk_uuid",
+                "name": "Library Main Desk",
+                "code": "lmd",
+            },
+        ]
+
+    elif ref_data_path == "/users" and query == '?query=(externalSystemId=="Some external id")':
+        yield from [{"id": "some id", "barcode": "some barcode", "patronGroup": "some group"}]
+    elif ref_data_path == "/users" and query == '?query=(barcode=="u123")':
+        yield from [{"id": "user123", "barcode": "u123", "patronGroup": "some group"}]
+    elif ref_data_path == "/users" and query == '?query=(barcode=="u456")':
+        yield from [{"id": "user456", "barcode": "u456", "patronGroup": "some group"}]
+
+    elif ref_data_path == "/inventory/items" and query == '?query=(barcode=="some barcode")':
+        yield from [
+            {
+                "id": "a FOLIO item uuid",
+                "title": "Döda fallen i Avesta.",
+                "barcode": "some barcode",
+                "callNumber": "QB611 .C44",
+                "materialType": {
+                    "id": "4eea3f27-8910-46fc-9666-e2b44326c2b8",
+                    "name": "sound recording",
+                },
+                "effectiveLocation": {
+                    "id": "2e48e713-17f3-4c13-a9f8-23845bb210a4",
+                    "name": "Reading room",
+                },
+            }
+        ]
+
+    elif ref_data_path == "/holdings-note-types":
+        yield from [
+            {
+                "id": "88914775-f677-4759-b57b-1a33b90b24e0",
+                "name": "Electronic bookplate",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.719+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.719+00:00"
+                }
+            },
+            {
+                "id": "c4407cc7-d79f-4609-95bd-1cefb2e2b5c5",
+                "name": "Copy note",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.722+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.722+00:00"
+                }
+            },
+            {
+                "id": "d6510242-5ec3-42ed-b593-3585d2e48fd6",
+                "name": "Action note",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.723+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.723+00:00"
+                }
+            },
+            {
+                "id": "e19eabab-a85c-4aef-a7b2-33bd9acef24e",
+                "name": "Binding",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.724+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.724+00:00"
+                }
+            },
+            {
+                "id": "db9b4787-95f0-4e78-becf-26748ce6bdeb",
+                "name": "Provenance",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.725+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.725+00:00"
+                }
+            },
+            {
+                "id": "6a41b714-8574-4084-8d64-a9373c3fbb59",
+                "name": "Reproduction",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.728+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "b160f13a-ddba-4053-b9c4-60ec5ea45d56",
+                "name": "Note",
+                "source": "folio",
+                "metadata": {
+                    "createdDate": "2024-09-04T01:54:20.728+00:00",
+                    "updatedDate": "2024-09-04T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "841d1873-015b-4bfb-a69f-6cbb41d925ba",
+                "name": "Original MARC holdings statements",
+                "source": "local",
+                "metadata": {
+                    "createdDate": "2025-05-02T01:54:20.728+00:00",
+                    "updatedDate": "2025-05-02T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "09c1e5c9-6f11-432e-bcbe-b9e733ccce57",
+                "name": "Original MFHD Record",
+                "source": "local",
+                "metadata": {
+                    "createdDate": "2025-05-02T01:54:20.728+00:00",
+                    "updatedDate": "2025-05-02T01:54:20.728+00:00"
+                }
+            },
+            {
+                "id": "474120b0-d64e-4a6f-9c9c-e7d3e76f3cf5",
+                "name": "Original MFHD (MARC21)",
+                "source": "local",
+                "metadata": {
+                    "createdDate": "2025-05-02T01:54:20.728+00:00",
+                    "updatedDate": "2025-05-02T01:54:20.728+00:00"
+                }
+            }
+        ]
+
+    elif ref_data_path in super_schema:
+        yield from super_schema.get(ref_data_path)
+    else:
+        yield {}
+
+
+def folio_get_single_object_mocked(*args, **kwargs):
+    with open("./static/reference_data.json", "r") as super_schema_file:
+        super_schema = json.load(super_schema_file)
+    if args[0] == "/hrid-settings-storage/hrid-settings":
+        return {
+            "instances": {"prefix": "pref", "startNumber": 1},
+            "holdings": {"prefix": "pref", "startNumber": 1},
+            "items": {"prefix": "pref", "startNumber": 1},
+            "commonRetainLeadingZeroes": True,
+        }
+
+    elif (
+        args[0] == "/configurations/entries?query=(module==ORG%20and%20configName==localeSettings)"
+    ):
+        return {
+            "configs": [
+                {
+                    "value": '{"timezone":"America/New_York"}',
+                }
+            ]
+        }
+
+    elif args[0] in super_schema:
+        return super_schema.get(args[0])
+
+
+def folio_get_from_github(owner, repo, file_path):
+    return FolioClient.get_latest_from_github(owner, repo, file_path, "")
+
+OKAPI_URL = "http://localhost:9130"
+LIBRARY_NAME = "Test Library"
+
+def get_mocked_library_config():
+    return LibraryConfiguration(
+        okapi_url=OKAPI_URL,
+        tenant_id="test_tenant",
+        okapi_username="test_user",
+        okapi_password="test_password",
+        base_folder=Path("."),
+        library_name=LIBRARY_NAME,
+        log_level_debug=False,
+        folio_release=FolioRelease.sunflower,
+        iteration_identifier="test_iteration"
+    )
+
+def get_mocked_ecs_central_libarary_config():
+    return LibraryConfiguration(
+        okapi_url=OKAPI_URL,
+        tenant_id="test_tenant",
+        okapi_username="test_user",
+        okapi_password="test_password",
+        base_folder=Path("."),
+        library_name=LIBRARY_NAME,
+        log_level_debug=False,
+        folio_release=FolioRelease.sunflower,
+        iteration_identifier="central_iteration",
+        is_ecs=True,
+    )
+
+def get_mocked_ecs_member_libarary_config():
+    return LibraryConfiguration(
+        okapi_url=OKAPI_URL,
+        tenant_id="test_tenant",
+        ecs_tenant_id="test_ecs_tenant",
+        okapi_username="test_user",
+        okapi_password="test_password",
+        base_folder=Path("."),
+        library_name=LIBRARY_NAME,
+        log_level_debug=False,
+        folio_release=FolioRelease.sunflower,
+        iteration_identifier="member_iteration",
+        ecs_central_iteration_identifier="central_iteration",
+        is_ecs=True,
+    )
+
+def get_mocked_folder_structure():
+    mock_fs = MagicMock()
+    mock_fs.mapping_files = Path("mapping_files")
+    mock_fs.results_folder = Path("results")
+    mock_fs.legacy_records_folder = Path("source_files")
+    mock_fs.logs_folder = Path("logs")
+    mock_fs.migration_reports_file = Path("/dev/null")
+    mock_fs.transformation_extra_data_path = Path("transformation_extra_data")
+    mock_fs.transformation_log_path = Path("/dev/null")
+    mock_fs.data_issue_file_path = Path("/dev/null")
+    mock_fs.failed_marc_recs_file = Path("failed_marc_recs.txt")
+    return mock_fs

folio_migration_tools/transaction_migration/legacy_loan.py

@@ -1,16 +1,39 @@
-from datetime import datetime, timedelta, timezone
+import json
 import logging
-from dateutil.parser import parse
+import i18n
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from dateutil import tz
+from dateutil.parser import parse, ParserError
+
+from folio_migration_tools.helper import Helper
+from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
+
+utc = ZoneInfo("UTC")
 
 
 class LegacyLoan(object):
-    def __init__(self, legacy_loan_dict, utc_difference=0, row=0):
+    def __init__(
+        self,
+        legacy_loan_dict,
+        fallback_service_point_id: str,
+        migration_report: MigrationReport,
+        tenant_timezone=utc,
+        row=0,
+    ):
+        self.migration_report: MigrationReport = migration_report
         # validate
         correct_headers = [
             "item_barcode",
             "patron_barcode",
             "due_date",
             "out_date",
+        ]
+        optional_headers = [
+            "service_point_id",
+            "proxy_patron_barcode",
             "renewal_count",
             "next_item_status",
         ]
@@ -22,55 +45,146 @@ class LegacyLoan(object):
             "Declared lost",
             "Lost and paid",
         ]
-        self.utc_difference = utc_difference
+
+        self.legacy_loan_dict = legacy_loan_dict
+        self.tenant_timezone = tenant_timezone
         self.errors = []
+        self.row = row
         for prop in correct_headers:
-            if prop not in legacy_loan_dict:
-                self.errors.append(("Missing properties in legacy data", prop))
-            if prop != "next_item_status" and not legacy_loan_dict[prop].strip():
-                self.errors.append(("Empty properties in legacy data", prop))
+            if prop not in self.legacy_loan_dict and prop not in optional_headers:
+                self.errors.append((f"Missing properties in legacy data {row=}", prop))
+            if (
+                prop != "next_item_status"
+                and not self.legacy_loan_dict.get(prop, "").strip()
+                and prop not in optional_headers
+            ):
+                self.errors.append((f"Empty properties in legacy data {row=}", prop))
         try:
-            temp_date_due: datetime = parse(legacy_loan_dict["due_date"])
-        except Exception as ee:
+            temp_date_due: datetime = parse(self.legacy_loan_dict["due_date"])
+            if temp_date_due.tzinfo != tz.UTC:
+                temp_date_due = temp_date_due.replace(tzinfo=self.tenant_timezone)
+                Helper.log_data_issue(
+                    self.row,
+                    f"Provided due_date is not UTC in {row=}, "
+                    f"setting tz-info to tenant timezone ({self.tenant_timezone})",
+                    json.dumps(self.legacy_loan_dict)
+                )
+                self.report(
+                    f"Provided due_date is not UTC, setting tz-info to tenant timezone ({self.tenant_timezone})"
+                )
+            if temp_date_due.hour == 0 and temp_date_due.minute == 0:
+                temp_date_due = temp_date_due.replace(hour=23, minute=59)
+                Helper.log_data_issue(
+                    self.row,
+                    f"Hour and minute not specified for due date in {row=}. "
+                    "Assuming end of local calendar day (23:59)...",
+                    json.dumps(self.legacy_loan_dict)
+                )
+                self.report(
+                    "Hour and minute not specified for due date"
+                )
+        except (ParserError, OverflowError) as ee:
             logging.error(ee)
-            self.errors.append(("Parse date failure. Setting UTC NOW", "due_date"))
-            temp_date_due = datetime.now(timezone.utc)
+            self.errors.append(
+                (f"Parse date failure in {row=}. Setting UTC NOW", "due_date")
+            )
+            temp_date_due = datetime.now(ZoneInfo("UTC"))
         try:
-            temp_date_out: datetime = parse(legacy_loan_dict["out_date"])
-        except Exception as ee:
-            temp_date_out = datetime.now(timezone.utc)
-            self.errors.append(("Parse date failure. Setting UTC NOW", "out_date"))
+            temp_date_out: datetime = parse(self.legacy_loan_dict["out_date"])
+            if temp_date_out.tzinfo != tz.UTC:
+                temp_date_out = temp_date_out.replace(tzinfo=self.tenant_timezone)
+                Helper.log_data_issue(
+                    self.row,
+                    f"Provided out_date is not UTC in {row=}, "
+                    f"setting tz-info to tenant timezone ({self.tenant_timezone})",
+                    json.dumps(self.legacy_loan_dict)
+                )
+                self.report(
+                    f"Provided out_date is not UTC, setting tz-info to tenant timezone ({self.tenant_timezone})"
+                )
+        except (ParserError, OverflowError):
+            temp_date_out = datetime.now(
+                ZoneInfo("UTC")
+            )  # TODO: Consider moving this assignment block above the temp_date_due
+            self.errors.append(
+                (f"Parse date failure in {row=}. Setting UTC NOW", "out_date")
+            )
 
         # good to go, set properties
-        self.item_barcode: str = legacy_loan_dict["item_barcode"].strip()
-        self.patron_barcode: str = legacy_loan_dict["patron_barcode"].strip()
+        self.item_barcode: str = self.legacy_loan_dict["item_barcode"].strip()
+        self.patron_barcode: str = self.legacy_loan_dict["patron_barcode"].strip()
+        self.proxy_patron_barcode: str = self.legacy_loan_dict.get(
+            "proxy_patron_barcode", ""
+        )
         self.due_date: datetime = temp_date_due
         self.out_date: datetime = temp_date_out
-        try:
-            self.make_loan_utc()
-            if self.due_date <= self.out_date:
-                if self.due_date.hour == 0:
-                    self.due_date = self.due_date.replace(hour=23, minute=59)
-                if self.out_date.hour == 0:
-                    self.out_date = self.out_date.replace(hour=0, minute=1)
-        except Exception as ee:
-            self.errors.append(("Time alignment issues", "both dates"))
-        self.renewal_count = int(legacy_loan_dict["renewal_count"])
-        self.next_item_status = legacy_loan_dict.get("next_item_status", "").strip()
+        self.correct_for_1_day_loans()
+        self.make_utc()
+        self.renewal_count = self.set_renewal_count(self.legacy_loan_dict)
+        self.next_item_status = self.legacy_loan_dict.get(
+            "next_item_status", ""
+        ).strip()
         if self.next_item_status not in legal_statuses:
-            self.errors.append(("Not an allowed status", self.next_item_status))
+            self.errors.append((f"Not an allowed status {row=}", self.next_item_status))
+        self.service_point_id = (
+            self.legacy_loan_dict["service_point_id"]
+            if self.legacy_loan_dict.get("service_point_id", "")
+            else fallback_service_point_id
+        )
+
+    def set_renewal_count(self, loan: dict) -> int:
+        if "renewal_count" in loan:
+            renewal_count = loan["renewal_count"]
+            try:
+                return int(renewal_count)
+            except ValueError:
+                Helper.log_data_issue(
+                    self.row,
+                    i18n.t("Unresolvable %{renewal_count=} was replaced with 0."),
+                    json.dumps(loan)
+                )
+        else:
+            Helper.log_data_issue(
+                self.row,
+                i18n.t("Missing renewal count was replaced with 0."),
+                json.dumps(loan)
+            )
+        return 0
+
+    def correct_for_1_day_loans(self):
+        if self.due_date.date() <= self.out_date.date():
+            if self.due_date.hour == 0:
+                self.due_date = self.due_date.replace(hour=23, minute=59)
+            if self.out_date.hour == 0:
+                self.out_date = self.out_date.replace(hour=0, minute=1)
+        if self.due_date <= self.out_date:
+            raise TransformationRecordFailedError(
+                self.row,
+                i18n.t(
+                    "Due date is before out date, or date information is missing from both"
+                ),
+                json.dumps(self.legacy_loan_dict, indent=2),
+            )
 
     def to_dict(self):
         return {
             "item_barcode": self.item_barcode,
             "patron_barcode": self.patron_barcode,
+            "proxy_patron_barcode": self.proxy_patron_barcode,
             "due_date": self.due_date.isoformat(),
             "out_date": self.out_date.isoformat(),
             "renewal_count": self.renewal_count,
             "next_item_status": self.next_item_status,
+            "service_point_id": self.service_point_id,
         }
 
-    def make_loan_utc(self):
-        if self.utc_difference != 0:
-            self.due_date = self.due_date + timedelta(hours=self.utc_difference)
-            self.out_date = self.out_date + timedelta(hours=self.utc_difference)
+    def make_utc(self):
+        try:
+            if self.tenant_timezone != ZoneInfo("UTC"):
+                self.due_date = self.due_date.astimezone(ZoneInfo("UTC"))
+                self.out_date = self.out_date.astimezone(ZoneInfo("UTC"))
+        except TypeError:
+            self.errors.append((f"UTC correction issues {self.row}", "both dates"))
+
+    def report(self, what_to_report: str):
+        self.migration_report.add("Details", what_to_report)