folio-migration-tools 1.9.0rc3__tar.gz → 1.9.0rc4__tar.gz

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc3
+Version: 1.9.0rc4
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc3"
+version = "1.9.0rc4"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -77,6 +77,10 @@ pandas = "^1.5.3"
 types-requests = "^2.28.11.17"
 types-python-dateutil = "^2.8.19.11"
 ipykernel = "^6.29.5"
+pytest-asyncio = "^0.23.0"
 
 [tool.poetry.extras]
 docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml"]
+
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/mapping_file_transformation/order_mapper.py

@@ -25,7 +25,6 @@ from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
 
 
 class CompositeOrderMapper(MappingFileMapperBase):
-    VALID_PO_NUMBER_CHARACTERS = r"[A-Za-z0-9]"
 
     def __init__(
         self,
@@ -270,9 +269,9 @@ class CompositeOrderMapper(MappingFileMapperBase):
                 and "$ref" in object_schema
                 and object_schema["type"] == "object"
             ):
-                submodule_path["properties"] = CompositeOrderMapper.inject_schema_by_ref(
+                object_schema["properties"] = CompositeOrderMapper.inject_schema_by_ref(
                     submodule_path, github_headers, object_schema
-                ).get("properties")
+                ).get("properties", {})#TODO: Investigate new CustomFields schema and figure out how to actually handle it
 
             for property_name_level1, property_level1 in object_schema.get(
                 "properties", {}
@@ -385,7 +384,7 @@ class CompositeOrderMapper(MappingFileMapperBase):
             index_or_id: str,
             po_number: str,
     ):
-        if re.sub(self.VALID_PO_NUMBER_CHARACTERS, "", po_number):
+        if not self.is_valid_po_number(po_number):
             self.migration_report.add(
                 "PurchaseOrderVendorLinking",
                 i18n.t("RECORD FAILED: PO number has invalid character(s)"),
@@ -396,6 +395,11 @@ class CompositeOrderMapper(MappingFileMapperBase):
                 po_number,
             )
 
+    @staticmethod
+    def is_valid_po_number(po_number: str) -> bool:
+        valid_po_number_characters = r"[A-Za-z0-9]"
+        return re.sub(valid_po_number_characters, "", po_number) == ""
+
     def get_matching_record_from_folio(
         self,
         index_or_id,

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -69,7 +69,6 @@ class RulesMapperHoldings(RulesMapperBase):
         logging.info("Fetching mapping rules from the tenant")
         rules_endpoint = "/mapping-rules/marc-holdings"
         self.mappings = self.folio_client.folio_get_single_object(rules_endpoint)
-        self.fix_853_bug_in_rules()
 
     def fix_853_bug_in_rules(self):
         f852_mappings = self.mappings["852"]
@@ -90,9 +89,10 @@ class RulesMapperHoldings(RulesMapperBase):
                         )
         self.mappings["852"] = new_852_mapping
 
-    def integrate_supplemental_mfhd_mappings(self, new_rules):
+    def integrate_supplemental_mfhd_mappings(self, new_rules={}):
         try:
             self.mappings.update(new_rules)
+            self.fix_853_bug_in_rules()
         except Exception as e:
             raise TransformationProcessError(
                 "",
@@ -102,6 +102,7 @@ class RulesMapperHoldings(RulesMapperBase):
 
     def prep_852_notes(self, marc_record: Record):
         for field in marc_record.get_fields("852"):
+            field.subfields.sort(key=lambda x: x[0])
             new_952 = Field(
                 tag="952",
                 indicators=["f", "f"],

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 import json
 import logging
@@ -248,22 +249,84 @@ class BatchPoster(MigrationTaskBase):
                     if self.task_configuration.object_type == "SRS":
                         self.commit_snapshot()
 
+    @staticmethod
+    def set_consortium_source(json_rec):
+        if json_rec['source'] == 'MARC':
+            json_rec['source'] = 'CONSORTIUM-MARC'
+        elif json_rec['source'] == 'FOLIO':
+            json_rec['source'] = 'CONSORTIUM-FOLIO'
+
+    def set_version(self, batch, query_api, object_type) -> None:
+        """
+        Synchronous wrapper for set_version_async
+        """
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            loop.run_until_complete(self.set_version_async(batch, query_api, object_type))
+        else:
+            asyncio.run(self.set_version_async(batch, query_api, object_type))
+
+    async def set_version_async(self, batch, query_api, object_type) -> None:
+        """
+        Fetches the current version of the records in the batch, if the record exists in FOLIO
+
+        Args:
+            batch (list): List of records to fetch versions for
+            query_api (str): The query API endpoint to use
+            object_type (str): The key in the API response that contains the records
+
+        Returns:
+            None
+        """
+        fetch_batch_size = 90
+        fetch_tasks = []
+        updates = {}
+        async with httpx.AsyncClient(base_url=self.folio_client.okapi_url) as client:
+            for i in range(0, len(batch), fetch_batch_size):
+                batch_slice = batch[i:i + fetch_batch_size]
+                fetch_tasks.append(
+                    client.get(
+                        query_api,
+                        params={
+                            "query": f"id==({' OR '.join([record['id'] for record in batch_slice if 'id' in record])})",
+                            "limit": fetch_batch_size
+                        },
+                        headers=self.folio_client.okapi_headers
+                    )
+                )
+            responses = await asyncio.gather(*fetch_tasks)
+
+            for response in responses:
+                if response.status_code == 200:
+                    response_json = await response.json()
+                    for record in response_json[object_type]:
+                        updates[record["id"]] = {
+                            "_version": record["_version"],
+                        }
+                        if "status" in record:
+                            updates[record["id"]]["status"] = record["status"]
+                else:
+                    logging.error(
+                        "Failed to fetch current records. HTTP %s\t%s",
+                        response.status_code,
+                        response.text,
+                    )
+        for record in batch:
+            if record["id"] in updates:
+                record.update(updates[record["id"]])
+
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
-        if (
-            self.task_configuration.object_type in ["Instances", "Holdings", "Items"]
-            and not self.task_configuration.use_safe_inventory_endpoints
-        ):
-            self.migration_report.add_general_statistics(
-                i18n.t("Set _version to -1 to enable upsert")
-            )
-            json_rec["_version"] = -1
+        if self.task_configuration.object_type == "ShadowInstances":
+            self.set_consortium_source(json_rec)
         if self.task_configuration.object_type == "SRS":
             json_rec["snapshotId"] = self.snapshot_id
         if self.processed == 1:
             logging.info(json.dumps(json_rec, indent=True))
         batch.append(json_rec)
         if len(batch) == int(self.batch_size):
+            if self.query_params.get("upsert", False) and self.api_info.get("query_endpoint", ""):
+                self.set_version(batch, self.api_info['query_endpoint'], self.api_info['object_name'])
             self.post_batch(batch, failed_recs_file, self.processed)
             batch = []
         return batch
@@ -648,6 +711,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
                 if use_safe
                 else "/item-storage/batch/synchronous-unsafe"
             ),
+            "query_endpoint": "/item-storage/items",
             "is_batch": True,
             "total_records": False,
             "addSnapshotId": False,
@@ -660,12 +724,26 @@ def get_api_info(object_type: str, use_safe: bool = True):
                 if use_safe
                 else "/holdings-storage/batch/synchronous-unsafe"
             ),
+            "query_endpoint": "/holdings-storage/holdings",
             "is_batch": True,
             "total_records": False,
             "addSnapshotId": False,
             "supports_upsert": True,
         },
         "Instances": {
+            "object_name": "instances",
+            "api_endpoint": (
+                "/instance-storage/batch/synchronous"
+                if use_safe
+                else "/instance-storage/batch/synchronous-unsafe"
+            ),
+            "query_endpoint": "/instance-storage/instances",
+            "is_batch": True,
+            "total_records": False,
+            "addSnapshotId": False,
+            "supports_upsert": True,
+        },
+        "ShadowInstances": {
             "object_name": "instances",
             "api_endpoint": (
                 "/instance-storage/batch/synchronous"

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/courses_migrator.py

@@ -4,7 +4,8 @@ import logging
 import sys
 import time
 import traceback
-from typing import Optional
+from typing import Optional, Annotated
+from pydantic import Field
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -29,13 +30,58 @@ from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 class CoursesMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        composite_course_map_path: str
-        migration_task_type: str
-        courses_file: FileDefinition
-        terms_map_path: str
-        departments_map_path: str
-        look_up_instructor: Optional[bool] = False
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task",
+            ),
+        ]
+        composite_course_map_path: Annotated[
+            str,
+            Field(
+                title="Composite course map path",
+                description="Path to the composite course map file",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task",
+            ),
+        ]
+        courses_file: Annotated[
+            FileDefinition,
+            Field(
+                title="Courses file",
+                description="File containing course data",
+            ),
+        ]
+        terms_map_path: Annotated[
+            str,
+            Field(
+                title="Terms map path",
+                description="Path to the terms map file",
+            ),
+        ]
+        departments_map_path: Annotated[
+            str,
+            Field(
+                title="Departments map path",
+                description="Path to the departments map file",
+            ),
+        ]
+        look_up_instructor: Annotated[
+            Optional[bool],
+            Field(
+                title="Look up instructor",
+                description=(
+                    "Flag to indicate whether to look up instructors. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/holdings_csv_transformer.py

@@ -39,37 +39,125 @@ csv.register_dialect("tsv", delimiter="\t")
 
 class HoldingsCsvTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        hrid_handling: HridHandling
-        files: List[FileDefinition]
-        holdings_map_file_name: str
-        location_map_file_name: str
-        default_call_number_type_name: str
-        previously_generated_holdings_files: Optional[list[str]] = []
-        fallback_holdings_type_id: str
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="Name of the task",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task",
+            ),
+        ]
+        hrid_handling: Annotated[
+            HridHandling,
+            Field(
+                title="HRID handling",
+                description=(
+                    "Determining how the HRID generation "
+                    "should be handled."
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of files",
+            ),
+        ]
+        holdings_map_file_name: Annotated[
+            str,
+            Field(
+                title="Holdings map file name",
+                description="File name for holdings map",
+            ),
+        ]
+        location_map_file_name: Annotated[
+            str,
+            Field(
+                title="Location map file name",
+                description="File name for location map",
+            ),
+        ]
+        default_call_number_type_name: Annotated[
+            str,
+            Field(
+                title="Default call number type name",
+                description="Default name for call number type",
+            ),
+        ]
+        previously_generated_holdings_files: Annotated[
+            Optional[list[str]],
+            Field(
+                title="Previously generated holdings files",
+                description=(
+                    "List of previously generated holdings files. "
+                    "By default is empty list."
+                ),
+            ),
+        ] = []
+        fallback_holdings_type_id: Annotated[
+            str,
+            Field(
+                title="Fallback holdings type ID",
+                description="ID for fallback holdings type",
+            ),
+        ]
         holdings_type_uuid_for_boundwiths: Annotated[
             str,
             Field(
                 title="Holdings Type for Boundwith Holdings",
                 description=(
                     "UUID for a Holdings type (set in Settings->Inventory) "
-                    "for Bound-with Holdings)"
+                    "for Bound-with Holdings. Default is empty string."
                 ),
             ),
         ] = ""
-        call_number_type_map_file_name: Optional[str]
-        holdings_merge_criteria: Optional[list[str]] = [
+        call_number_type_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Call number type map file name",
+                description="File name for call number type map",
+            ),
+        ]
+        holdings_merge_criteria: Annotated[
+            Optional[list[str]],
+            Field(
+                title="Holdings merge criteria",
+                description=(
+                    "List of holdings merge criteria. "
+                    "Default value is "
+                    "['instanceId', 'permanentLocationId', 'callNumber']."
+                ),
+            ),
+        ] = [
             "instanceId",
             "permanentLocationId",
             "callNumber",
         ]
-        reset_hrid_settings: Optional[bool] = False
+        reset_hrid_settings: Annotated[
+            Optional[bool],
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "At the end of the run reset "
+                    "FOLIO with the HRID settings. Default is FALSE."
+                ),
+            ),
+        ] = False
         update_hrid_settings: Annotated[
             bool,
             Field(
                 title="Update HRID settings",
-                description="At the end of the run, update FOLIO with the HRID settings",
+                description=(
+                    "At the end of the run update "
+                    "FOLIO with the HRID settings. Default is TRUE."
+                ),
             ),
         ] = True
 

{folio_migration_tools-1.9.0rc3 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -1,4 +1,3 @@
-'''Main "script."'''
 import csv
 import ctypes
 import json
@@ -36,26 +35,136 @@ csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
 
 class ItemsTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        hrid_handling: HridHandling
-        files: List[FileDefinition]
-        items_mapping_file_name: str
-        location_map_file_name: str
-        default_call_number_type_name: str
-        temp_location_map_file_name: Optional[str] = ""
-        material_types_map_file_name: str
-        loan_types_map_file_name: str
-        temp_loan_types_map_file_name: Optional[str] = ""
-        statistical_codes_map_file_name: Optional[str] = ""
-        item_statuses_map_file_name: str
-        call_number_type_map_file_name: str
-        reset_hrid_settings: Optional[bool] = False
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="Name of the task.",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task.",
+            ),
+        ]
+        hrid_handling: Annotated[
+            HridHandling,
+            Field(
+                title="HRID handling",
+                description=(
+                    "Determining how the HRID generation "
+                    "should be handled."
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of files.",
+            ),
+        ]
+        items_mapping_file_name: Annotated[
+            str,
+            Field(
+                title="Items mapping file name",
+                description="File name for items mapping.",
+            ),
+        ]
+        location_map_file_name: Annotated[
+            str,
+            Field(
+                title="Location map file name",
+                description="File name for location map.",
+            ),
+        ]
+        default_call_number_type_name: Annotated[
+            str,
+            Field(
+                title="Default call number type name",
+                description="Default name for call number type.",
+            ),
+        ]
+        temp_location_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Temporary location map file name",
+                description=(
+                    "Temporary file name for location map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        material_types_map_file_name: Annotated[
+            str,
+            Field(
+                title="Material types map file name",
+                description="File name for material types map.",
+            ),
+        ]
+        loan_types_map_file_name: Annotated[
+            str,
+            Field(
+                title="Loan types map file name",
+                description="File name for loan types map.",
+            ),
+        ]
+        temp_loan_types_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Temporary loan types map file name",
+                description=(
+                    "File name for temporary loan types map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        statistical_codes_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Statistical codes map file name",
+                description=(
+                    "File name for statistical codes map. "
+                    "Empty string by default."
+                ),
+            ),
+        ] = ""
+        item_statuses_map_file_name: Annotated[
+            str,
+            Field(
+                title="Item statuses map file name",
+                description="File name for item statuses map.",
+            ),
+        ]
+        call_number_type_map_file_name: Annotated[
+            str,
+            Field(
+                title="Call number type map file name",
+                description="File name for call number type map.",
+            ),
+        ]
+        reset_hrid_settings: Annotated[
+            Optional[bool],
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "At the end of the run "
+                    "reset FOLIO with the HRID settings. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
         update_hrid_settings: Annotated[
             bool,
             Field(
                 title="Update HRID settings",
-                description="At the end of the run, update FOLIO with the HRID settings",
+                description=(
+                    "At the end of the run "
+                    "update FOLIO with the HRID settings. "
+                    "By default is True."
+                ),
             ),
         ] = True
         boundwith_relationship_file_path: Annotated[
@@ -63,8 +172,10 @@ class ItemsTransformer(MigrationTaskBase):
             Field(
                 title="Boundwith relationship file path",
                 description=(
-                    "Path to a file outlining Boundwith relationships, in the style of Voyager."
-                    " A TSV file with MFHD_ID and BIB_ID headers and values"
+                    "Path to a file outlining Boundwith relationships, "
+                    "in the style of Voyager. "
+                    "A TSV file with MFHD_ID and BIB_ID headers and values. "
+                    "By default is empty string."
                 ),
             ),
         ] = ""
@@ -73,7 +184,9 @@ class ItemsTransformer(MigrationTaskBase):
             Field(
                 title="Prevent permanent location map default",
                 description=(
-                    "Prevent the default mapping of permanent location to the default location."
+                    "Prevent the default mapping of permanent location "
+                    "to the default location. "
+                    "By default is False."
                 ),
             ),
         ] = False