folio-migration-tools 1.9.0rc13__tar.gz → 1.9.1__tar.gz

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc13
+Version: 1.9.1
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc13"
+version = "1.9.1"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/src/folio_migration_tools/mapping_file_transformation/item_mapper.py

@@ -2,7 +2,7 @@ import json
 import logging
 import sys
 from datetime import datetime, timezone
-from typing import Set
+from typing import Dict, List, Set, Union
 from uuid import uuid4
 
 import i18n
@@ -117,12 +117,12 @@ class ItemMapper(MappingFileMapperBase):
             "LocationMapping",
         )
 
-    def perform_additional_mappings(self, legacy_ids, folio_rec, file_def):
+    def perform_additional_mappings(self, legacy_ids: Union[str, List[str]], folio_rec: Dict, file_def: FileDefinition):
         self.handle_suppression(folio_rec, file_def)
         self.map_statistical_codes(folio_rec, file_def)
         self.map_statistical_code_ids(legacy_ids, folio_rec)
 
-    def handle_suppression(self, folio_record, file_def: FileDefinition):
+    def handle_suppression(self, folio_record: Dict, file_def: FileDefinition):
         folio_record["discoverySuppress"] = file_def.discovery_suppressed
         self.migration_report.add(
             "Suppression",

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/src/folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py

@@ -531,9 +531,21 @@ class MappingFileMapperBase(MapperBase):
                             isinstance(res, str)
                             and self.library_configuration.multi_field_delimiter in res
                         ):
+                            for delim_value in res.split(
+                                self.library_configuration.multi_field_delimiter
+                            ):
+                                if delim_value not in empty_vals:
+                                    self.validate_enums(
+                                        delim_value,
+                                        sub_prop,
+                                        sub_prop_name,
+                                        index_or_id,
+                                        required,
+                                    )
                             multi_field_props.append(sub_prop_name)
+                        else:
+                            self.validate_enums(res, sub_prop, sub_prop_name, index_or_id, required)
 
-                        self.validate_enums(res, sub_prop, sub_prop_name, index_or_id, required)
                         if res or isinstance(res, bool):
                             temp_object[sub_prop_name] = res
 
@@ -809,7 +821,7 @@ class MappingFileMapperBase(MapperBase):
         ):
             raise TransformationRecordFailedError(
                 index_or_id,
-                f"Allowed values for {mapped_schema_property_name}"
+                f"Allowed values for {mapped_schema_property_name} "
                 f"are {mapped_schema_property['enum']} "
                 f"Forbidden enum value found: ",
                 mapped_value,

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -1,6 +1,7 @@
 import copy
 import json
 import logging
+import re
 from typing import Dict, List, Set
 
 import i18n
@@ -393,12 +394,29 @@ class RulesMapperHoldings(RulesMapperBase):
             ) from ee
         return [
             {
-                "note": "\n".join(mrk_statement_notes),
+                "note": chunk,
                 "holdingsNoteTypeId": holdings_note_type_id,
                 "staffOnly": True,
-            }
+            } for chunk in self.split_mrk_by_max_note_size("\n".join(mrk_statement_notes))
         ]
 
+    @staticmethod
+    def split_mrk_by_max_note_size(s: str, max_chunk_size: int = 32000) -> List[str]:
+        lines = s.splitlines(keepends=True)
+        chunks = []
+        current_chunk = ""
+        for line in lines:
+            # If adding this line would exceed the limit, start a new chunk
+            if len(current_chunk) + len(line) > max_chunk_size:
+                if current_chunk:
+                    chunks.append(current_chunk)
+                current_chunk = line
+            else:
+                current_chunk += line
+        if current_chunk:
+            chunks.append(current_chunk)
+        return chunks
+
     def add_mfhd_as_mrk_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
         """Adds the MFHD as a note to the holdings record
 
@@ -424,12 +442,29 @@ class RulesMapperHoldings(RulesMapperBase):
                 ) from ee
             folio_holding["notes"] = folio_holding.get("notes", []) + [
                 {
-                    "note": str(marc_record),
+                    "note": chunk,
                     "holdingsNoteTypeId": holdings_note_type_id,
                     "staffOnly": True,
-                }
+                } for chunk in self.split_mrk_by_max_note_size(str(marc_record))
             ]
 
+    @staticmethod
+    def split_mrc_by_max_note_size(data: bytes, sep: bytes = b"\x1e", max_chunk_size: int = 32000) -> List[bytes]:
+        # Split data into segments, each ending with the separator (except possibly the last)
+        pattern = re.compile(b'(.*?' + re.escape(sep) + b'|.+?$)', re.DOTALL)
+        parts = [m.group(0) for m in pattern.finditer(data) if m.group(0)]
+        chunks = []
+        current_chunk = b""
+        for part in parts:
+            if len(current_chunk) + len(part) > max_chunk_size and current_chunk:
+                chunks.append(current_chunk)
+                current_chunk = part
+            else:
+                current_chunk += part
+        if current_chunk:
+            chunks.append(current_chunk)
+        return chunks
+
     def add_mfhd_as_mrc_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
         """Adds the MFHD as a note to the holdings record
 
@@ -455,10 +490,10 @@ class RulesMapperHoldings(RulesMapperBase):
                 ) from ee
             folio_holding["notes"] = folio_holding.get("notes", []) + [
                 {
-                    "note": marc_record.as_marc().decode("utf-8"),
+                    "note": chunk.decode("utf-8"),
                     "holdingsNoteTypeId": holdings_note_type_id,
                     "staffOnly": True,
-                }
+                } for chunk in self.split_mrc_by_max_note_size(marc_record.as_marc())
             ]
 
     def wrap_up(self):

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -6,7 +6,7 @@ import sys
 import time
 import traceback
 from datetime import datetime
-from typing import Annotated, List
+from typing import Annotated, List, Optional
 from uuid import uuid4
 
 import httpx
@@ -173,11 +173,13 @@ class BatchPoster(MigrationTaskBase):
         self.num_posted = 0
         self.okapi_headers = self.folio_client.okapi_headers
         self.http_client = None
+        self.starting_record_count_in_folio: Optional[int] = None
 
     def do_work(self):
         with self.folio_client.get_folio_http_client() as httpx_client:
             self.http_client = httpx_client
             with open(self.folder_structure.failed_recs_path, "w", encoding='utf-8') as failed_recs_file:
+                self.get_starting_record_count()
                 try:
                     batch = []
                     if self.task_configuration.object_type == "SRS":
@@ -317,6 +319,8 @@ class BatchPoster(MigrationTaskBase):
                 updates[record["id"]] = {
                             "_version": record["_version"],
                         }
+                if "hrid" in record:
+                    updates[record["id"]]["hrid"] = record["hrid"]
                 if "status" in record:
                     updates[record["id"]]["status"] = record["status"]
                 if "lastCheckIn" in record:
@@ -604,6 +608,42 @@ class BatchPoster(MigrationTaskBase):
         else:
             return httpx.post(url, headers=self.okapi_headers, json=payload, params=self.query_params, timeout=None)
 
+    def get_current_record_count_in_folio(self):
+        if "query_endpoint" in self.api_info:
+            url = f"{self.folio_client.gateway_url}{self.api_info['query_endpoint']}"
+            query_params = {"query": "cql.allRecords=1", "limit": 0}
+            if self.http_client and not self.http_client.is_closed:
+                res = self.http_client.get(url, headers=self.folio_client.okapi_headers, params=query_params)
+            else:
+                res = httpx.get(url, headers=self.okapi_headers, params=query_params, timeout=None)
+            try:
+                res.raise_for_status()
+                return res.json()["totalRecords"]
+            except httpx.HTTPStatusError:
+                logging.error("Failed to get current record count. HTTP %s", res.status_code)
+                return 0
+            except KeyError:
+                logging.error(f"Failed to get current record count. No 'totalRecords' in response: {res.json()}")
+                return 0
+        else:
+            raise ValueError(
+                "No 'query_endpoint' available for %s. Cannot get current record count.", self.task_configuration.object_type
+            )
+
+    def get_starting_record_count(self):
+        if "query_endpoint" in self.api_info and not self.starting_record_count_in_folio:
+            logging.info("Getting starting record count in FOLIO")
+            self.starting_record_count_in_folio = self.get_current_record_count_in_folio()
+        else:
+            logging.info("No query_endpoint available for %s. Cannot get starting record count.", self.task_configuration.object_type)
+
+    def get_finished_record_count(self):
+        if "query_endpoint" in self.api_info:
+            logging.info("Getting finished record count in FOLIO")
+            self.finished_record_count_in_folio = self.get_current_record_count_in_folio()
+        else:
+            logging.info("No query_endpoint available for %s. Cannot get ending record count.", self.task_configuration.object_type)
+
     def wrap_up(self):
         logging.info("Done. Wrapping up")
         self.extradata_writer.flush()
@@ -621,11 +661,34 @@ class BatchPoster(MigrationTaskBase):
             )
         else:
             logging.info("Done posting %s records. %s failed", self.num_posted, self.num_failures)
-
+        if self.starting_record_count_in_folio:
+            self.get_finished_record_count()
+            total_on_server = self.finished_record_count_in_folio - self.starting_record_count_in_folio
+            discrepancy = self.processed - self.num_failures - total_on_server
+            if discrepancy != 0:
+                logging.error(
+                    (
+                        "Discrepancy in record count. "
+                        "Starting record count: %s. Finished record count: %s. "
+                        "Records posted: %s. Discrepancy: %s"
+                    ),
+                    self.starting_record_count_in_folio,
+                    self.finished_record_count_in_folio,
+                    self.num_posted - self.num_failures,
+                    discrepancy,
+                )
+        else:
+            discrepancy = 0
         run = "second time" if self.performing_rerun else "first time"
         self.migration_report.set("GeneralStatistics", f"Records processed {run}", self.processed)
         self.migration_report.set("GeneralStatistics", f"Records posted {run}", self.num_posted)
         self.migration_report.set("GeneralStatistics", f"Failed to post {run}", self.num_failures)
+        if discrepancy:
+            self.migration_report.set(
+                "GeneralStatistics",
+                f"Discrepancy in record count {run}",
+                discrepancy,
+            )
         self.rerun_run()
         with open(self.folder_structure.migration_reports_file, "w+") as report_file:
             self.migration_report.write_migration_report(

{folio_migration_tools-1.9.0rc13 → folio_migration_tools-1.9.1}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -354,7 +354,7 @@ class ItemsTransformer(MigrationTaskBase):
                         record, f"row {idx}", FOLIONamespaces.items
                     )
 
-                    self.mapper.perform_additional_mappings(folio_rec, file_def)
+                    self.mapper.perform_additional_mappings(legacy_id, folio_rec, file_def)
                     self.handle_circiulation_notes(folio_rec, self.folio_client.current_user)
                     self.handle_notes(folio_rec)
                     if folio_rec["holdingsRecordId"] in self.boundwith_relationship_map: