folio-migration-tools 1.9.3__tar.gz → 1.9.4__tar.gz

{folio_migration_tools-1.9.3 → folio_migration_tools-1.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.3
+Version: 1.9.4
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -19,6 +19,7 @@ Requires-Dist: argparse-prompt (>=0.0.5,<0.0.6)
 Requires-Dist: art (>=6.5,<7.0)
 Requires-Dist: deepdiff (>=6.2.3,<7.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
+Requires-Dist: folio-data-import (>=0.3.2,<0.4.0)
 Requires-Dist: folio-uuid (>=0.2.8,<0.3.0)
 Requires-Dist: folioclient (>=0.70.1,<0.71.0)
 Requires-Dist: pyaml (>=21.10.1,<22.0.0)

{folio_migration_tools-1.9.3 → folio_migration_tools-1.9.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.3"
+version = "1.9.4"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -56,6 +56,7 @@ deepdiff = "^6.2.3"
 pyaml = "^21.10.1"
 python-i18n = "^0.3.9"
 art = "^6.5"
+folio-data-import = "^0.3.2"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.1.3"
@@ -63,24 +64,31 @@ lxml = ">4.9"
 coverage = {extras = ["toml"], version = "^6.5.0"}
 pytest-cov = "^4.0.0"
 black = "^22.10.0"
-flake8 = "^5.0.4"
+flake8 = "^7.3.0"
 mypy = "^0.982"
-flake8-black = "^0.3.3"
-flake8-bugbear = "^22.9.23"
+flake8-black = "^0.3.6"
+flake8-bugbear = "^24.12.12"
 flake8-bandit = "^4.1.1"
-flake8-isort = "^5.0.0"
+flake8-isort = "^6.1.2"
 flake8-docstrings = "^1.6.0"
 darglint = "^1.8.1"
-sphinx = "^5.3.0"
-sphinx-autodoc-typehints = "^1.19.4"
-myst-parser = "^0.18.1"
 types-requests = "^2.28.11.17"
 types-python-dateutil = "^2.8.19.11"
 ipykernel = "^6.29.5"
 pytest-asyncio = "^0.23.0"
+sphinx-autobuild = "^2024.10.3"
+
+[tool.poetry.group.docs.dependencies]
+m2r = "^0.2.1"
+myst-parser = "<4.0.0"
+sphinx = ">=6.0.0, <8.0.0"
+sphinx-autodoc-typehints = "^1.17.0"
+sphinx-rtd-theme = "^1.0.0"
+toml = "^0.10.2"
+sphinx-book-theme = "^1.1.4"
 
 [tool.poetry.extras]
-docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml"]
+docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml",]
 
 [tool.poetry.requires-plugins]
 poetry-plugin-export = ">=1.8"

{folio_migration_tools-1.9.3 → folio_migration_tools-1.9.4}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -174,11 +174,11 @@ class BibsRulesMapper(RulesMapperBase):
             self.process_marc_field(folio_instance, main_entry_fields[0], ignored_subsequent_fields, legacy_ids)
         try:
             self.process_marc_field(folio_instance, marc_record['245'], ignored_subsequent_fields, legacy_ids)
-        except KeyError:
+        except KeyError as ke:
             raise TransformationRecordFailedError(
                 legacy_ids,
                 "No 245 field in MARC record"
-            )
+            ) from ke
 
     def perform_additional_parsing(
         self,

{folio_migration_tools-1.9.3 → folio_migration_tools-1.9.4}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -196,9 +196,15 @@ class RulesMapperHoldings(RulesMapperBase):
         if "004" not in marc_record:
             raise TransformationProcessError(
                 "",
-                ("No 004 in record. The tools only support bib-mfhd linking throuh 004"),
+                ("No 004 in record. The tools only support bib-mfhd linking through 004"),
                 legacy_ids,
             )
+        if len(marc_record.get_fields("004")) > 1:
+            Helper.log_data_issue(
+                legacy_ids,
+                "More than one linked bib (004) found in record. Using the first one",
+                [str(x) for x in marc_record.get_fields("004")],
+            )
         legacy_instance_id = marc_record["004"].data.strip()
         folio_holding["formerIds"].append(f"{self.bib_id_template}{legacy_instance_id}")
         if legacy_instance_id in self.parent_id_map:

{folio_migration_tools-1.9.3 → folio_migration_tools-1.9.4}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -131,6 +131,56 @@ class BatchPoster(MigrationTaskBase):
                 ),
             ),
         ] = False
+        preserve_statistical_codes: Annotated[
+            bool,
+            Field(
+                title="Preserve statistical codes",
+                description=(
+                    "Toggles whether or not to preserve statistical codes "
+                    "during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_administrative_notes: Annotated[
+            bool,
+            Field(
+                title="Preserve administrative notes",
+                description=(
+                    "Toggles whether or not to preserve administrative notes "
+                    "during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_temporary_locations: Annotated[
+            bool,
+            Field(
+                title="Preserve temporary locations",
+                description=(
+                    "Toggles whether or not to preserve temporary locations "
+                    "on items during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_temporary_loan_types: Annotated[
+            bool,
+            Field(
+                title="Preserve temporary loan types",
+                description=(
+                    "Toggles whether or not to preserve temporary loan types "
+                    "on items during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = False
+        preserve_item_status: Annotated[
+            bool,
+            Field(
+                title="Preserve item status",
+                description=(
+                    "Toggles whether or not to preserve item status "
+                    "on items during the upsert process. Defaults to False"
+                ),
+            ),
+        ] = True
 
     task_configuration: TaskConfiguration
 
@@ -292,7 +342,7 @@ class BatchPoster(MigrationTaskBase):
         """
         fetch_batch_size = 90
         fetch_tasks = []
-        updates = {}
+        existing_records = {}
         async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
             for i in range(0, len(batch), fetch_batch_size):
                 batch_slice = batch[i:i + fetch_batch_size]
@@ -313,31 +363,82 @@ class BatchPoster(MigrationTaskBase):
             responses = await asyncio.gather(*fetch_tasks)
 
             for response in responses:
-                self.update_record_versions(object_type, updates, response)
+                self.collect_existing_records_for_upsert(object_type, response, existing_records)
         for record in batch:
-            if record["id"] in updates:
-                record.update(updates[record["id"]])
+            if record["id"] in existing_records:
+                self.prepare_record_for_upsert(record, existing_records[record["id"]])
+
+    def handle_source_marc(self, new_record: dict, existing_record: dict):
+        updates = {}
+        updates.update(existing_record)
+        self.handle_upsert_for_administrative_notes(updates)
+        self.handle_upsert_for_statistical_codes(updates)
+        keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
+        if "instanceStatusId" in new_record:
+            updates["instanceStatusId"] = new_record["instanceStatusId"]
+        for k, v in keep_new.items():
+            updates[k] = list(dict.fromkeys(updates.get(k, []) + v))
+        new_record.update(updates)
 
     @staticmethod
-    def update_record_versions(object_type, updates, response):
+    def collect_existing_records_for_upsert(object_type: str, response: httpx.Response, existing_records: dict):
         if response.status_code == 200:
             response_json = response.json()
             for record in response_json[object_type]:
-                updates[record["id"]] = {
-                            "_version": record["_version"],
-                        }
-                if "hrid" in record:
-                    updates[record["id"]]["hrid"] = record["hrid"]
-                if "status" in record:
-                    updates[record["id"]]["status"] = record["status"]
-                if "lastCheckIn" in record:
-                    updates[record["id"]]["lastCheckIn"] = record["lastCheckIn"]
+                existing_records[record["id"]] = record
         else:
             logging.error(
-                        "Failed to fetch current records. HTTP %s\t%s",
-                        response.status_code,
-                        response.text,
-                    )
+                "Failed to fetch current records. HTTP %s\t%s",
+                response.status_code,
+                response.text,
+            )
+
+    def handle_upsert_for_statistical_codes(self, updates: dict):
+        if not self.task_configuration.preserve_statistical_codes:
+            updates.pop("statisticalCodeIds", None)
+
+    def handle_upsert_for_administrative_notes(self, updates: dict):
+        if not self.task_configuration.preserve_administrative_notes:
+            updates.pop("administrativeNotes", None)
+
+    def handle_upsert_for_temporary_locations(self, updates: dict):
+        if not self.task_configuration.preserve_temporary_locations:
+            updates.pop("temporaryLocationId", None)
+
+    def handle_upsert_for_temporary_loan_types(self, updates: dict):
+        if not self.task_configuration.preserve_temporary_loan_types:
+            updates.pop("temporaryLoanTypeId", None)
+
+    def keep_existing_fields(self, updates: dict, existing_record: dict):
+        keep_existing_fields = ["hrid", "lastCheckIn"]
+        if self.task_configuration.preserve_item_status:
+            keep_existing_fields.append("status")
+        for key in keep_existing_fields:
+            if key in existing_record:
+                updates[key] = existing_record[key]
+
+    def prepare_record_for_upsert(self, new_record: dict, existing_record: dict):
+        if "source" in existing_record and "MARC" in existing_record["source"]:
+            self.handle_source_marc(new_record, existing_record)
+        else:
+            updates = {
+                "_version": existing_record["_version"],
+            }
+            self.keep_existing_fields(updates, existing_record)
+            keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
+            self.handle_upsert_for_statistical_codes(existing_record)
+            self.handle_upsert_for_administrative_notes(existing_record)
+            self.handle_upsert_for_temporary_locations(existing_record)
+            self.handle_upsert_for_temporary_loan_types(existing_record)
+            for k, v in keep_new.items():
+                updates[k] = list(dict.fromkeys(existing_record.get(k, []) + v))
+            for key in [
+                "temporaryLocationId",
+                "temporaryLoanTypeId",
+            ]:
+                if key in existing_record:
+                    updates[key] = existing_record[key]
+            new_record.update(updates)
 
     async def get_with_retry(self, client: httpx.AsyncClient, url: str, params=None):
         if params is None:

{folio_migration_tools-1.9.3 → folio_migration_tools-1.9.4}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -218,7 +218,10 @@ class ItemsTransformer(MigrationTaskBase):
         self.folio_keys = MappingFileMapperBase.get_mapped_folio_properties_from_map(
             self.items_map
         )
-        if any(k for k in self.folio_keys if k.startswith("statisticalCodeIds")):
+        if (
+            any(k for k in self.folio_keys if k.startswith("statisticalCodeIds"))
+            or any(getattr(k, "statistical_code", "") for k in self.task_configuration.files)
+        ):
             statcode_mapping = self.load_ref_data_mapping_file(
                 "statisticalCodeIds",
                 self.folder_structure.mapping_files_folder