folio-migration-tools 1.9.0rc2__tar.gz → 1.9.0rc3__tar.gz

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc2
+Version: 1.9.0rc3
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc2"
+version = "1.9.0rc3"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/src/folio_migration_tools/mapping_file_transformation/holdings_mapper.py

@@ -101,7 +101,7 @@ class HoldingsMapper(MappingFileMapperBase):
         if legacy_value.startswith("[") and len(legacy_value.split(",")) == 1:
             try:
                 legacy_value = ast.literal_eval(str(legacy_value))[0]
-            except SyntaxError:
+            except (SyntaxError, ValueError):
                 return legacy_value
         return legacy_value
 

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -90,6 +90,25 @@ class RulesMapperHoldings(RulesMapperBase):
                         )
         self.mappings["852"] = new_852_mapping
 
+    def integrate_supplemental_mfhd_mappings(self, new_rules):
+        try:
+            self.mappings.update(new_rules)
+        except Exception as e:
+            raise TransformationProcessError(
+                "",
+                "Failed to integrate supplemental mfhd mappings",
+                str(e),
+            )
+
+    def prep_852_notes(self, marc_record: Record):
+        for field in marc_record.get_fields("852"):
+            new_952 = Field(
+                tag="952",
+                indicators=["f", "f"],
+                subfields=field.subfields
+            )
+            marc_record.add_ordered_field(new_952)
+
     def parse_record(
         self, marc_record: Record, file_def: FileDefinition, legacy_ids: List[str]
     ) -> list[dict]:
@@ -111,7 +130,7 @@ class RulesMapperHoldings(RulesMapperBase):
 
         self.print_progress()
         folio_holding = self.perform_initial_preparation(marc_record, legacy_ids)
-
+        self.prep_852_notes(marc_record)
         self.migration_report.add("RecordStatus", marc_record.leader[5])
         ignored_subsequent_fields: set = set()
         num_852s = 0

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -50,46 +50,84 @@ class BatchPoster(MigrationTaskBase):
     """
 
     class TaskConfiguration(AbstractTaskConfiguration):
-        name: str
-        migration_task_type: str
-        object_type: str
-        files: List[FileDefinition]
-        batch_size: int
+        name: Annotated[
+            str,
+            Field(
+                title="Task name",
+                description="The name of the task",
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description="The type of migration task",
+            ),
+        ]
+        object_type: Annotated[
+            str,
+            Field(
+                title="Object type",
+                description=(
+                    "The type of object being migrated"
+                    "Examples of possible values: "
+                    "'Extradata', 'SRS', Instances', 'Holdings', 'Items'"
+                ),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="List of files",
+                description="List of files to be processed",
+            ),
+        ]
+        batch_size: Annotated[
+            int,
+            Field(
+                title="Batch size",
+                description="The batch size for processing files",
+            ),
+        ]
         rerun_failed_records: Annotated[
             bool,
             Field(
+                title="Rerun failed records",
                 description=(
-                    "Toggles whether or not BatchPoster should try to rerun failed batches or "
-                    "just leave the failing records on disk."
-                )
+                    "Toggles whether or not BatchPoster should try to rerun "
+                    "failed batches or just leave the failing records on disk."
+                ),
             ),
         ] = True
         use_safe_inventory_endpoints: Annotated[
             bool,
             Field(
+                title="Use safe inventory endpoints",
                 description=(
-                    "Toggles the use of the safe/unsafe Inventory storage endpoints. "
-                    "Unsafe circumvents the Optimistic locking in FOLIO. Defaults to "
-                    "True (using the 'safe' options)"
-                )
+                    "Toggles the use of the safe/unsafe Inventory storage "
+                    "endpoints. Unsafe circumvents the Optimistic locking "
+                    "in FOLIO. Defaults to True (using the 'safe' options)"
+                ),
             ),
         ] = True
         extradata_endpoints: Annotated[
             dict,
             Field(
+                title="Extradata endpoints",
                 description=(
                     "A dictionary of extradata endpoints. "
                     "The key is the object type and the value is the endpoint"
-                )
+                ),
             ),
         ] = {}
         upsert: Annotated[
             bool,
             Field(
+                title="Upsert",
                 description=(
-                    "Toggles whether or not to use the upsert feature of the Inventory storage "
-                    "endpoints. Defaults to False"
-                )
+                    "Toggles whether or not to use the upsert feature "
+                    "of the Inventory storage endpoints. Defaults to False"
+                ),
             ),
         ] = False
 
@@ -188,9 +226,10 @@ class BatchPoster(MigrationTaskBase):
 
                 except Exception as ee:
                     if "idx" in locals() and self.task_configuration.files[idx:]:
-                        for file in self.task_configuration.files[idx:]:
+                        for file_def in self.task_configuration.files[idx:]:
+                            path = self.folder_structure.results_folder / file_def.file_name
                             try:
-                                with open(file, "r") as failed_file:
+                                with open(path, "r") as failed_file:
                                     failed_file.seek(self.processed)
                                     failed_recs_file.write(failed_file.read())
                                     self.processed = 0

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/src/folio_migration_tools/migration_tasks/holdings_marc_transformer.py

@@ -1,5 +1,7 @@
 '''Main "script."'''
+
 import csv
+import json
 import logging
 from typing import Annotated, List
 
@@ -42,7 +44,8 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         files: Annotated[
             List[FileDefinition],
             Field(
-                title="Source files", description=("List of MARC21 files with holdings records")
+                title="Source files",
+                description=("List of MARC21 files with holdings records"),
             ),
         ]
         hrid_handling: Annotated[
@@ -154,6 +157,13 @@ class HoldingsMarcTransformer(MigrationTaskBase):
                 description="The UUID of the Holdings type that will be used for unmapped values",
             ),
         ]
+        supplemental_mfhd_mapping_rules_file: Annotated[
+            str,
+            Field(
+                title="Supplemental MFHD mapping rules file",
+                description="The name of the file in the mapping_files directory containing supplemental MFHD mapping rules",
+            ),
+        ] = ""
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -204,11 +214,13 @@ class HoldingsMarcTransformer(MigrationTaskBase):
                     csv.DictReader(boundwith_relationship_file, dialect="tsv")
                 )
             logging.info(
-                "Rows in Bound with relationship map: %s", len(self.boundwith_relationship_map)
+                "Rows in Bound with relationship map: %s",
+                len(self.boundwith_relationship_map),
             )
 
         location_map_path = (
-            self.folder_structure.mapping_files_folder / self.task_config.location_map_file_name
+            self.folder_structure.mapping_files_folder
+            / self.task_config.location_map_file_name
         )
         with open(location_map_path) as location_map_file:
             self.location_map = list(csv.DictReader(location_map_file, dialect="tsv"))
@@ -217,7 +229,9 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         self.check_source_files(
             self.folder_structure.legacy_records_folder, self.task_config.files
         )
-        self.instance_id_map = self.load_id_map(self.folder_structure.instance_id_map_path, True)
+        self.instance_id_map = self.load_id_map(
+            self.folder_structure.instance_id_map_path, True
+        )
         self.mapper = RulesMapperHoldings(
             self.folio_client,
             self.location_map,
@@ -226,6 +240,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
             self.instance_id_map,
             self.boundwith_relationship_map,
         )
+        self.add_supplemental_mfhd_mappings()
         if (
             self.task_configuration.reset_hrid_settings
             and self.task_configuration.update_hrid_settings
@@ -234,6 +249,33 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         logging.info("%s Instance ids in map", len(self.instance_id_map))
         logging.info("Init done")
 
+    def add_supplemental_mfhd_mappings(self):
+        if self.task_config.supplemental_mfhd_mapping_rules_file:
+            try:
+                with open(
+                    (
+                        self.folder_structure.mapping_files_folder
+                        / self.task_config.supplemental_mfhd_mapping_rules_file
+                    ),
+                    "r",
+                ) as new_rules_file:
+                    new_rules = json.load(new_rules_file)
+                    if not isinstance(new_rules, dict):
+                        raise TransformationProcessError(
+                            "",
+                            "Supplemental MFHD mapping rules file must contain a dictionary",
+                            json.dumps(new_rules),
+                        )
+            except FileNotFoundError:
+                raise TransformationProcessError(
+                    "",
+                    "Provided supplemental MFHD mapping rules file not found",
+                    self.task_config.supplemental_mfhd_mapping_rules_file,
+                )
+        else:
+            new_rules = {}
+        self.mapper.integrate_supplemental_mfhd_mappings(new_rules)
+
     def do_work(self):
         self.do_work_marc_transformer()
 

{folio_migration_tools-1.9.0rc2 → folio_migration_tools-1.9.0rc3}/src/folio_migration_tools/task_configuration.py

@@ -16,8 +16,8 @@ class AbstractTaskConfiguration(BaseModel):
         Field(
             title="ECS tenant ID",
             description=(
-                "The tenant ID to use when making requests to FOLIO APIs for this task, if ",
-                "different from library configuration",
+                "The tenant ID to use when making requests to FOLIO APIs "
+                "for this task, if different from library configuration",
             ),
         ),
     ] = ""