folio-migration-tools 1.9.0rc2__tar.gz → 1.9.0rc4__tar.gz

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc2
+Version: 1.9.0rc4
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc2"
+version = "1.9.0rc4"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -77,6 +77,10 @@ pandas = "^1.5.3"
 types-requests = "^2.28.11.17"
 types-python-dateutil = "^2.8.19.11"
 ipykernel = "^6.29.5"
+pytest-asyncio = "^0.23.0"
 
 [tool.poetry.extras]
 docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml"]
+
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/mapping_file_transformation/holdings_mapper.py

@@ -101,7 +101,7 @@ class HoldingsMapper(MappingFileMapperBase):
         if legacy_value.startswith("[") and len(legacy_value.split(",")) == 1:
             try:
                 legacy_value = ast.literal_eval(str(legacy_value))[0]
-            except SyntaxError:
+            except (SyntaxError, ValueError):
                 return legacy_value
         return legacy_value
 

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/mapping_file_transformation/order_mapper.py

@@ -25,7 +25,6 @@ from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
 
 
 class CompositeOrderMapper(MappingFileMapperBase):
-    VALID_PO_NUMBER_CHARACTERS = r"[A-Za-z0-9]"
 
     def __init__(
         self,
@@ -270,9 +269,9 @@ class CompositeOrderMapper(MappingFileMapperBase):
                 and "$ref" in object_schema
                 and object_schema["type"] == "object"
             ):
-                submodule_path["properties"] = CompositeOrderMapper.inject_schema_by_ref(
+                object_schema["properties"] = CompositeOrderMapper.inject_schema_by_ref(
                     submodule_path, github_headers, object_schema
-                ).get("properties")
+                ).get("properties", {})#TODO: Investigate new CustomFields schema and figure out how to actually handle it
 
             for property_name_level1, property_level1 in object_schema.get(
                 "properties", {}
@@ -385,7 +384,7 @@ class CompositeOrderMapper(MappingFileMapperBase):
             index_or_id: str,
             po_number: str,
     ):
-        if re.sub(self.VALID_PO_NUMBER_CHARACTERS, "", po_number):
+        if not self.is_valid_po_number(po_number):
             self.migration_report.add(
                 "PurchaseOrderVendorLinking",
                 i18n.t("RECORD FAILED: PO number has invalid character(s)"),
@@ -396,6 +395,11 @@ class CompositeOrderMapper(MappingFileMapperBase):
                 po_number,
             )
 
+    @staticmethod
+    def is_valid_po_number(po_number: str) -> bool:
+        valid_po_number_characters = r"[A-Za-z0-9]"
+        return re.sub(valid_po_number_characters, "", po_number) == ""
+
     def get_matching_record_from_folio(
         self,
         index_or_id,

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -69,7 +69,6 @@ class RulesMapperHoldings(RulesMapperBase):
         logging.info("Fetching mapping rules from the tenant")
         rules_endpoint = "/mapping-rules/marc-holdings"
         self.mappings = self.folio_client.folio_get_single_object(rules_endpoint)
-        self.fix_853_bug_in_rules()
 
     def fix_853_bug_in_rules(self):
         f852_mappings = self.mappings["852"]
@@ -90,6 +89,27 @@ class RulesMapperHoldings(RulesMapperBase):
                         )
         self.mappings["852"] = new_852_mapping
 
+    def integrate_supplemental_mfhd_mappings(self, new_rules={}):
+        try:
+            self.mappings.update(new_rules)
+            self.fix_853_bug_in_rules()
+        except Exception as e:
+            raise TransformationProcessError(
+                "",
+                "Failed to integrate supplemental mfhd mappings",
+                str(e),
+            )
+
+    def prep_852_notes(self, marc_record: Record):
+        for field in marc_record.get_fields("852"):
+            field.subfields.sort(key=lambda x: x[0])
+            new_952 = Field(
+                tag="952",
+                indicators=["f", "f"],
+                subfields=field.subfields
+            )
+            marc_record.add_ordered_field(new_952)
+
     def parse_record(
         self, marc_record: Record, file_def: FileDefinition, legacy_ids: List[str]
     ) -> list[dict]:
@@ -111,7 +131,7 @@ class RulesMapperHoldings(RulesMapperBase):
 
         self.print_progress()
         folio_holding = self.perform_initial_preparation(marc_record, legacy_ids)
-
+        self.prep_852_notes(marc_record)
         self.migration_report.add("RecordStatus", marc_record.leader[5])
         ignored_subsequent_fields: set = set()
         num_852s = 0

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 import json
 import logging
@@ -50,46 +51,84 @@ class BatchPoster(MigrationTaskBase):
     """
 
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        object_type: str
-        files: List[FileDefinition]
-        batch_size: int
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task",
+            ),
+        ]
+        object_type: Annotated[
+            str,
+            Field(
+                title="Object type",
+                description=(
+                    "The type of object being migrated"
+                    "Examples of possible values: "
+                    "'Extradata', 'SRS', Instances', 'Holdings', 'Items'"
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="List of files",
+                description="List of files to be processed",
+            ),
+        ]
+        batch_size: Annotated[
+            int,
+            Field(
+                title="Batch size",
+                description="The batch size for processing files",
+            ),
+        ]
         rerun_failed_records: Annotated[
             bool,
             Field(
+                title="Rerun failed records",
                 description=(
-                    "Toggles whether or not BatchPoster should try to rerun failed batches or "
-                    "just leave the failing records on disk."
-                )
+                    "Toggles whether or not BatchPoster should try to rerun "
+                    "failed batches or just leave the failing records on disk."
+                ),
             ),
         ] = True
         use_safe_inventory_endpoints: Annotated[
             bool,
             Field(
+                title="Use safe inventory endpoints",
                 description=(
-                    "Toggles the use of the safe/unsafe Inventory storage endpoints. "
-                    "Unsafe circumvents the Optimistic locking in FOLIO. Defaults to "
-                    "True (using the 'safe' options)"
-                )
+                    "Toggles the use of the safe/unsafe Inventory storage "
+                    "endpoints. Unsafe circumvents the Optimistic locking "
+                    "in FOLIO. Defaults to True (using the 'safe' options)"
+                ),
             ),
         ] = True
         extradata_endpoints: Annotated[
             dict,
             Field(
+                title="Extradata endpoints",
                 description=(
                     "A dictionary of extradata endpoints. "
                     "The key is the object type and the value is the endpoint"
-                )
+                ),
             ),
         ] = {}
         upsert: Annotated[
             bool,
             Field(
+                title="Upsert",
                 description=(
-                    "Toggles whether or not to use the upsert feature of the Inventory storage "
-                    "endpoints. Defaults to False"
-                )
+                    "Toggles whether or not to use the upsert feature "
+                    "of the Inventory storage endpoints. Defaults to False"
+                ),
             ),
         ] = False
 
@@ -188,9 +227,10 @@ class BatchPoster(MigrationTaskBase):
 
                 except Exception as ee:
                     if "idx" in locals() and self.task_configuration.files[idx:]:
-                        for file in self.task_configuration.files[idx:]:
+                        for file_def in self.task_configuration.files[idx:]:
+                            path = self.folder_structure.results_folder / file_def.file_name
                             try:
-                                with open(file, "r") as failed_file:
+                                with open(path, "r") as failed_file:
                                     failed_file.seek(self.processed)
                                     failed_recs_file.write(failed_file.read())
                                     self.processed = 0
@@ -209,22 +249,84 @@ class BatchPoster(MigrationTaskBase):
                     if self.task_configuration.object_type == "SRS":
                         self.commit_snapshot()
 
+    @staticmethod
+    def set_consortium_source(json_rec):
+        if json_rec['source'] == 'MARC':
+            json_rec['source'] = 'CONSORTIUM-MARC'
+        elif json_rec['source'] == 'FOLIO':
+            json_rec['source'] = 'CONSORTIUM-FOLIO'
+
+    def set_version(self, batch, query_api, object_type) -> None:
+        """
+        Synchronous wrapper for set_version_async
+        """
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            loop.run_until_complete(self.set_version_async(batch, query_api, object_type))
+        else:
+            asyncio.run(self.set_version_async(batch, query_api, object_type))
+
+    async def set_version_async(self, batch, query_api, object_type) -> None:
+        """
+        Fetches the current version of the records in the batch, if the record exists in FOLIO
+
+        Args:
+            batch (list): List of records to fetch versions for
+            query_api (str): The query API endpoint to use
+            object_type (str): The key in the API response that contains the records
+
+        Returns:
+            None
+        """
+        fetch_batch_size = 90
+        fetch_tasks = []
+        updates = {}
+        async with httpx.AsyncClient(base_url=self.folio_client.okapi_url) as client:
+            for i in range(0, len(batch), fetch_batch_size):
+                batch_slice = batch[i:i + fetch_batch_size]
+                fetch_tasks.append(
+                    client.get(
+                        query_api,
+                        params={
+                            "query": f"id==({' OR '.join([record['id'] for record in batch_slice if 'id' in record])})",
+                            "limit": fetch_batch_size
+                        },
+                        headers=self.folio_client.okapi_headers
+                    )
+                )
+            responses = await asyncio.gather(*fetch_tasks)
+
+            for response in responses:
+                if response.status_code == 200:
+                    response_json = await response.json()
+                    for record in response_json[object_type]:
+                        updates[record["id"]] = {
+                            "_version": record["_version"],
+                        }
+                        if "status" in record:
+                            updates[record["id"]]["status"] = record["status"]
+                else:
+                    logging.error(
+                        "Failed to fetch current records. HTTP %s\t%s",
+                        response.status_code,
+                        response.text,
+                    )
+        for record in batch:
+            if record["id"] in updates:
+                record.update(updates[record["id"]])
+
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
-        if (
-            self.task_configuration.object_type in ["Instances", "Holdings", "Items"]
-            and not self.task_configuration.use_safe_inventory_endpoints
-        ):
-            self.migration_report.add_general_statistics(
-                i18n.t("Set _version to -1 to enable upsert")
-            )
-            json_rec["_version"] = -1
+        if self.task_configuration.object_type == "ShadowInstances":
+            self.set_consortium_source(json_rec)
         if self.task_configuration.object_type == "SRS":
             json_rec["snapshotId"] = self.snapshot_id
         if self.processed == 1:
             logging.info(json.dumps(json_rec, indent=True))
         batch.append(json_rec)
         if len(batch) == int(self.batch_size):
+            if self.query_params.get("upsert", False) and self.api_info.get("query_endpoint", ""):
+                self.set_version(batch, self.api_info['query_endpoint'], self.api_info['object_name'])
             self.post_batch(batch, failed_recs_file, self.processed)
             batch = []
         return batch
@@ -609,6 +711,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
                 if use_safe
                 else "/item-storage/batch/synchronous-unsafe"
             ),
+            "query_endpoint": "/item-storage/items",
             "is_batch": True,
             "total_records": False,
             "addSnapshotId": False,
@@ -621,12 +724,26 @@ def get_api_info(object_type: str, use_safe: bool = True):
                 if use_safe
                 else "/holdings-storage/batch/synchronous-unsafe"
             ),
+            "query_endpoint": "/holdings-storage/holdings",
             "is_batch": True,
             "total_records": False,
             "addSnapshotId": False,
             "supports_upsert": True,
         },
         "Instances": {
+            "object_name": "instances",
+            "api_endpoint": (
+                "/instance-storage/batch/synchronous"
+                if use_safe
+                else "/instance-storage/batch/synchronous-unsafe"
+            ),
+            "query_endpoint": "/instance-storage/instances",
+            "is_batch": True,
+            "total_records": False,
+            "addSnapshotId": False,
+            "supports_upsert": True,
+        },
+        "ShadowInstances": {
             "object_name": "instances",
             "api_endpoint": (
                 "/instance-storage/batch/synchronous"

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/courses_migrator.py

@@ -4,7 +4,8 @@ import logging
 import sys
 import time
 import traceback
-from typing import Optional
+from typing import Optional, Annotated
+from pydantic import Field
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -29,13 +30,58 @@ from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 class CoursesMigrator(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        composite_course_map_path: str
-        migration_task_type: str
-        courses_file: FileDefinition
-        terms_map_path: str
-        departments_map_path: str
-        look_up_instructor: Optional[bool] = False
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task",
+            ),
+        ]
+        composite_course_map_path: Annotated[
+            str,
+            Field(
+                title="Composite course map path",
+                description="Path to the composite course map file",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task",
+            ),
+        ]
+        courses_file: Annotated[
+            FileDefinition,
+            Field(
+                title="Courses file",
+                description="File containing course data",
+            ),
+        ]
+        terms_map_path: Annotated[
+            str,
+            Field(
+                title="Terms map path",
+                description="Path to the terms map file",
+            ),
+        ]
+        departments_map_path: Annotated[
+            str,
+            Field(
+                title="Departments map path",
+                description="Path to the departments map file",
+            ),
+        ]
+        look_up_instructor: Annotated[
+            Optional[bool],
+            Field(
+                title="Look up instructor",
+                description=(
+                    "Flag to indicate whether to look up instructors. "
+                    "By default is False."
+                ),
+            ),
+        ] = False
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc4}/src/folio_migration_tools/migration_tasks/holdings_csv_transformer.py

@@ -39,37 +39,125 @@ csv.register_dialect("tsv", delimiter="\t")
 
 class HoldingsCsvTransformer(MigrationTaskBase):
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        hrid_handling: HridHandling
-        files: List[FileDefinition]
-        holdings_map_file_name: str
-        location_map_file_name: str
-        default_call_number_type_name: str
-        previously_generated_holdings_files: Optional[list[str]] = []
-        fallback_holdings_type_id: str
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="Name of the task",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="Type of migration task",
+            ),
+        ]
+        hrid_handling: Annotated[
+            HridHandling,
+            Field(
+                title="HRID handling",
+                description=(
+                    "Determining how the HRID generation "
+                    "should be handled."
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Files",
+                description="List of files",
+            ),
+        ]
+        holdings_map_file_name: Annotated[
+            str,
+            Field(
+                title="Holdings map file name",
+                description="File name for holdings map",
+            ),
+        ]
+        location_map_file_name: Annotated[
+            str,
+            Field(
+                title="Location map file name",
+                description="File name for location map",
+            ),
+        ]
+        default_call_number_type_name: Annotated[
+            str,
+            Field(
+                title="Default call number type name",
+                description="Default name for call number type",
+            ),
+        ]
+        previously_generated_holdings_files: Annotated[
+            Optional[list[str]],
+            Field(
+                title="Previously generated holdings files",
+                description=(
+                    "List of previously generated holdings files. "
+                    "By default is empty list."
+                ),
+            ),
+        ] = []
+        fallback_holdings_type_id: Annotated[
+            str,
+            Field(
+                title="Fallback holdings type ID",
+                description="ID for fallback holdings type",
+            ),
+        ]
         holdings_type_uuid_for_boundwiths: Annotated[
             str,
             Field(
                 title="Holdings Type for Boundwith Holdings",
                 description=(
                     "UUID for a Holdings type (set in Settings->Inventory) "
-                    "for Bound-with Holdings)"
+                    "for Bound-with Holdings. Default is empty string."
                 ),
             ),
         ] = ""
-        call_number_type_map_file_name: Optional[str]
-        holdings_merge_criteria: Optional[list[str]] = [
+        call_number_type_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Call number type map file name",
+                description="File name for call number type map",
+            ),
+        ]
+        holdings_merge_criteria: Annotated[
+            Optional[list[str]],
+            Field(
+                title="Holdings merge criteria",
+                description=(
+                    "List of holdings merge criteria. "
+                    "Default value is "
+                    "['instanceId', 'permanentLocationId', 'callNumber']."
+                ),
+            ),
+        ] = [
             "instanceId",
             "permanentLocationId",
             "callNumber",
         ]
-        reset_hrid_settings: Optional[bool] = False
+        reset_hrid_settings: Annotated[
+            Optional[bool],
+            Field(
+                title="Reset HRID settings",
+                description=(
+                    "At the end of the run reset "
+                    "FOLIO with the HRID settings. Default is FALSE."
+                ),
+            ),
+        ] = False
         update_hrid_settings: Annotated[
             bool,
             Field(
                 title="Update HRID settings",
-                description="At the end of the run, update FOLIO with the HRID settings",
+                description=(
+                    "At the end of the run update "
+                    "FOLIO with the HRID settings. Default is TRUE."
+                ),
             ),
         ] = True