folio-migration-tools 1.9.1__tar.gz → 1.9.3__tar.gz

{folio_migration_tools-1.9.1 → folio_migration_tools-1.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.1
+Version: 1.9.3
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data

{folio_migration_tools-1.9.1 → folio_migration_tools-1.9.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.1"
+version = "1.9.3"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},

{folio_migration_tools-1.9.1 → folio_migration_tools-1.9.3}/src/folio_migration_tools/mapping_file_transformation/holdings_mapper.py

@@ -1,10 +1,12 @@
 import ast
+import json
+import logging
 
 import i18n
 from folio_uuid.folio_uuid import FOLIONamespaces
 from folioclient import FolioClient
 
-from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
+from folio_migration_tools.custom_exceptions import TransformationProcessError, TransformationRecordFailedError
 from folio_migration_tools.library_configuration import (
     FileDefinition,
     LibraryConfiguration,
@@ -59,6 +61,21 @@ class HoldingsMapper(MappingFileMapperBase):
                 "name",
                 "CallNumberTypeMapping",
             )
+        self.holdings_sources = self.get_holdings_sources()
+
+    def get_holdings_sources(self):
+        res = {}
+        holdings_sources = list(
+            self.folio_client.folio_get_all("/holdings-sources", "holdingsRecordsSources")
+        )
+        logging.info("Fetched %s holdingsRecordsSources from tenant", len(holdings_sources))
+        res = {n["name"].upper(): n["id"] for n in holdings_sources}
+        if "FOLIO" not in res:
+            raise TransformationProcessError("", "No holdings source with name FOLIO in tenant")
+        if "MARC" not in res:
+            raise TransformationProcessError("", "No holdings source with name MARC in tenant")
+        logging.info(json.dumps(res, indent=4))
+        return res
 
     def perform_additional_mappings(self, legacy_ids, folio_rec, file_def):
         self.handle_suppression(folio_rec, file_def)

{folio_migration_tools-1.9.1 → folio_migration_tools-1.9.3}/src/folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py

@@ -171,14 +171,15 @@ class MappingFileMapperBase(MapperBase):
         object_type: FOLIONamespaces,
         accept_duplicate_ids: bool = False,
     ):
+        folio_object = {}
         if self.ignore_legacy_identifier:
-            return (
+            folio_object.update(
                 {
                     "id": str(uuid.uuid4()),
                     "type": "object",
-                },
-                index_or_id,
+                }
             )
+            return folio_object, index_or_id
 
         if not (
             legacy_id := " ".join(
@@ -205,13 +206,21 @@ class MappingFileMapperBase(MapperBase):
             )
         else:
             self.unique_record_ids.add(generated_id)
-        return (
+        folio_object.update(
             {
                 "id": generated_id,
                 "type": "object",
-            },
-            legacy_id,
+            }
         )
+        if object_type == FOLIONamespaces.holdings and hasattr(self, "holdings_sources"):
+            folio_object['sourceId'] = self.holdings_sources.get("FOLIO")
+        elif object_type == FOLIONamespaces.holdings and not hasattr(self, "holdings_sources"):
+            raise TransformationProcessError(
+                index_or_id,
+                "Holdings source not set in the mapper",
+                None
+            )
+        return folio_object, legacy_id
 
     def get_statistical_code(self, legacy_item: dict, folio_prop_name: str, index_or_id):
         if self.statistical_codes_mapping:

{folio_migration_tools-1.9.1 → folio_migration_tools-1.9.3}/src/folio_migration_tools/mapping_file_transformation/user_mapper.py

@@ -59,8 +59,8 @@ class UserMapper(MappingFileMapperBase):
                 True,
             )
             self.notes_mapper.migration_report = self.migration_report
-            self.setup_departments_mapping(departments_mapping)
-            self.setup_groups_mapping(groups_map)
+            self.departments_mapping = self.setup_departments_mapping(departments_mapping)
+            self.groups_mapping = self.setup_groups_mapping(groups_map)
 
             for m in self.record_map["data"]:
                 if m["folio_field"].startswith("customFields"):
@@ -120,7 +120,8 @@ class UserMapper(MappingFileMapperBase):
 
         return clean_folio_object
 
-    def get_users(self, source_file, file_format: str):
+    @staticmethod
+    def get_users(source_file, file_format: str):
         csv.register_dialect("tsv", delimiter="\t")
         if file_format == "tsv":
             reader = csv.DictReader(source_file, dialect="tsv")
@@ -156,12 +157,25 @@ class UserMapper(MappingFileMapperBase):
                     "No Departments mapping set up. Set up a departments mapping file "
                     " or remove the mapping of the Departments field",
                 )
-            return self.get_mapped_name(
-                self.departments_mapping,
-                legacy_user,
-                index_or_id,
-                True,
-            )
+            if len(self.departments_mapping.mapped_legacy_keys) == 1 and self.library_configuration.multi_field_delimiter in legacy_user.get(self.departments_mapping.mapped_legacy_keys[0], ""):
+                split_departments = legacy_user.get(self.departments_mapping.mapped_legacy_keys[0], "").split(
+                    self.library_configuration.multi_field_delimiter
+                )
+                return self.library_configuration.multi_field_delimiter.join([
+                    self.get_mapped_name(
+                        self.departments_mapping,
+                        {self.departments_mapping.mapped_legacy_keys[0]: dept},
+                        index_or_id,
+                        True,
+                    ) for dept in split_departments
+                ])
+            else:
+                return self.get_mapped_name(
+                    self.departments_mapping,
+                    legacy_user,
+                    index_or_id,
+                    True,
+                )
         elif folio_prop_name in ["expirationDate", "enrollmentDate", "personal.dateOfBirth"]:
             return self.get_parsed_date(mapped_value, folio_prop_name)
         return mapped_value
@@ -184,27 +198,21 @@ class UserMapper(MappingFileMapperBase):
             return ""
 
     def setup_groups_mapping(self, groups_map):
-        if groups_map:
-            self.groups_mapping = RefDataMapping(
-                self.folio_client,
-                "/groups",
-                "usergroups",
-                groups_map,
-                "group",
-                "UserGroupMapping",
-            )
-        else:
-            self.groups_mapping = None
+        return RefDataMapping(
+            self.folio_client,
+            "/groups",
+            "usergroups",
+            groups_map,
+            "group",
+            "UserGroupMapping",
+        ) if groups_map else None
 
     def setup_departments_mapping(self, departments_mapping):
-        if departments_mapping:
-            self.departments_mapping = RefDataMapping(
-                self.folio_client,
-                "/departments",
-                "departments",
-                departments_mapping,
-                "name",
-                "DepartmentsMapping",
-            )
-        else:
-            self.departments_mapping = None
+        return RefDataMapping(
+            self.folio_client,
+            "/departments",
+            "departments",
+            departments_mapping,
+            "name",
+            "DepartmentsMapping",
+        ) if departments_mapping else None

{folio_migration_tools-1.9.1 → folio_migration_tools-1.9.3}/src/folio_migration_tools/marc_rules_transformation/conditions.py

@@ -90,6 +90,7 @@ class Conditions:
         logging.info("%s\tholding_note_types", len(self.folio.holding_note_types)) # type: ignore
         logging.info("%s\tcall_number_types", len(self.folio.call_number_types)) # type: ignore
         self.setup_and_validate_holdings_types()
+        self.ill_policies = self.folio.folio_get_all("/ill-policies", "illPolicies")
         # Raise for empty settings
         if not self.folio.holding_note_types:
             raise TransformationProcessError("", "No holding_note_types in FOLIO")
@@ -466,36 +467,6 @@ class Conditions:
     def condition_char_select(self, legacy_id, value, parameter, marc_field: field.Field):
         return value[parameter["from"] : parameter["to"]]
 
-    def condition_set_receipt_status(self, legacy_id, value, parameter, marc_field: field.Field):
-        if len(value) < 7:
-            self.mapper.migration_report.add(
-                "ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
-            )
-            return ""
-        try:
-            status_map = {
-                "0": "Unknown",
-                "1": "Other receipt or acquisition status",
-                "2": "Received and complete or ceased",
-                "3": "On order",
-                "4": "Currently received",
-                "5": "Not currently received",
-                "6": "External access",
-            }
-            mapped_value = status_map[value[6]]
-            self.mapper.migration_report.add(
-                "ReceiptStatusMapping",
-                i18n.t(
-                    "%{value} mapped to %{mapped_value}", value=value[6], mapped_value=mapped_value
-                ),
-            )
-
-            return
-        except Exception:
-            self.mapper.migration_report.add(
-                "ReceiptStatusMapping", i18n.t("%{value} not found in map.", value=value)
-            )
-            return "Unknown"
 
     def condition_set_identifier_type_id_by_name(
         self, legacy_id, value, parameter, marc_field: field.Field
@@ -915,3 +886,241 @@ class Conditions:
                 legacy_id,
                 f"Subject source not found for {value} {marc_field}",
             )
+
+    def condition_set_receipt_status(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the receipt status based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and 
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 7:
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        
+        status_map = {
+            "0": "Unknown",
+            "1": "Other receipt or acquisition status",
+            "2": "Received and complete or ceased",
+            "3": "On order",
+            "4": "Currently received",
+            "5": "Not currently received",
+            "6": "External access",
+        }
+
+        try:
+            mapped_value = status_map[value[6]]
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}",
+                    value=value[6],
+                    mapped_value=mapped_value,
+                ),
+            )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping", i18n.t("%{value} not found in map.", value=value[6])
+            )
+            return ""
+
+    def condition_set_acquisition_method(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the acquisition method based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and 
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 8:
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+
+        try:
+            acq_methods = {
+                "c": "Cooperative or consortial purchase",
+                "d": "Deposit",
+                "e": "Exchange",
+                "f": "Free",
+                "g": "Gift",
+                "l": "Legal deposit",
+                "m": "Membership",
+                "n": "Non-library purchase",
+                "p": "Purchase",
+                "q": "Lease",
+                "u": "Unknown",
+                "z": "Other method of acquisition",
+            }
+            mapped_value = acq_methods[value[7]]
+            self.mapper.migration_report.add(
+                "MethodOfAcquisitionMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}", value=value[7], mapped_value=mapped_value
+                ),
+            )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "MethodOfAcquisitionMapping", i18n.t("%{value} not found in map.", value=value[8])
+            )
+            return ""
+
+    def condition_set_retention_policy(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the retention policy based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 13:
+            self.mapper.migration_report.add(
+                "RetentionPolicyMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        value = value.replace("|", " ").replace("#", " ") # Replace pipe with space for mapping consistency
+        try:
+            retention_policies = {
+                "0": "Unknown",
+                "1": "Other general retention policy",
+                "2": "Retained except as replaced by updates",
+                "3": "Sample issue retained",
+                "4": "Retained until replaced by microform",
+                "5": "Retained until replaced by cumulation, replacement volume, or revision",
+                "6": "Retained for a limited period",
+                "7": "Not retained",
+                "8": "Permanently retained",
+            }
+            mapped_value = retention_policies[value[12]]
+            self.mapper.migration_report.add(
+                "RetentionPolicyMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}",
+                    value=value[12],
+                    mapped_value=mapped_value,
+                ),
+            )
+            if value[12] == "6":
+                policy_types = {
+                    "l": "Latest",
+                    "p": "Previous",
+                }
+                unit_types = {
+                    "m": "Day",
+                    "w": "Month",
+                    "y": "Year",
+                    "e": "Edition",
+                    "i": "Issue",
+                    "s": "Supplement"
+                }
+                try:
+                    specific_retention_policy = ""
+                    if value[13].strip() or value[15].strip():
+                        if value[14].strip() and int(value[14]) > 1:
+                            specific_retention_policy = f"{policy_types.get(value[13], '')} {value[14]} {unit_types.get(value[15], '')}s retained".strip()
+                        else:
+                            specific_retention_policy = f"{policy_types.get(value[13], '')} {unit_types.get(value[15], '')} retained".strip()
+                    if specific_retention_policy:
+                        self.mapper.migration_report.add(
+                            "RetentionPolicyMapping",
+                            i18n.t(
+                                "Retention policy 6 indicates a limited period. Specific retention period will be mapped from 008/13-15",
+                            )
+                        )
+                        return specific_retention_policy
+                    else:
+                        raise ValueError(
+                            "Specific retention policy is empty or invalid in 008/13-15"
+                        )
+                except ValueError:
+                    self.mapper.migration_report.add(
+                        "RetentionPolicyMapping",
+                        i18n.t("Invalid specific retention policy in 008/13-15: %{value}", value=value[13:16]),
+                    )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "RetentionPolicyMapping", i18n.t("%{value} not found in map.", value=value[12])
+            )
+            return ""
+
+    def condition_set_ill_policy(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the ILL policy based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition."""
+        if len(value) < 21:
+            self.mapper.migration_report.add(
+                "ILLPolicyMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        try:
+            ill_policies = {
+                "a": "Will lend",
+                "b": "Will not lend",
+                "c": "Will lend hard copy only",
+                "l": "Limited lending policy",
+                "u": "Unknown",
+            }
+            mapped_value = ill_policies[value[20]]
+            self.mapper.migration_report.add(
+                "ILLPolicyMapping",
+                i18n.t("%{value} mapped to %{mapped_value}", value=value[20], mapped_value=mapped_value),
+            )
+            ill_policy_id = self.get_ref_data_tuple_by_name(
+                self.ill_policies, "ill_policies", mapped_value
+            )
+            return ill_policy_id[0] if ill_policy_id else ""
+        except Exception:
+            self.mapper.migration_report.add(
+                "ILLPolicyMapping", i18n.t("%{value} not found in map.", value=value[20])
+            )
+            return ""
+
+    def condition_set_digitization_policy(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the digitization policy based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 22:
+            self.mapper.migration_report.add(
+                "DigitizationPolicyMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        try:
+            digitization_policies = {
+                "a": "Will reproduce",
+                "b": "Will not reproduce",
+                "u": "Unknown",
+            }
+            mapped_value = digitization_policies[value[21]]
+            self.mapper.migration_report.add(
+                "DigitizationPolicyMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}",
+                    value=value[21],
+                    mapped_value=mapped_value,
+                ),
+            )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "DigitizationPolicyMapping", i18n.t("%{value} not found in map.", value=value[21])
+            )
+            return ""