folio-migration-tools 1.9.2__tar.gz → 1.9.4__tar.gz

{folio_migration_tools-1.9.2 → folio_migration_tools-1.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.2
+Version: 1.9.4
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -19,6 +19,7 @@ Requires-Dist: argparse-prompt (>=0.0.5,<0.0.6)
 Requires-Dist: art (>=6.5,<7.0)
 Requires-Dist: deepdiff (>=6.2.3,<7.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
+Requires-Dist: folio-data-import (>=0.3.2,<0.4.0)
 Requires-Dist: folio-uuid (>=0.2.8,<0.3.0)
 Requires-Dist: folioclient (>=0.70.1,<0.71.0)
 Requires-Dist: pyaml (>=21.10.1,<22.0.0)

{folio_migration_tools-1.9.2 → folio_migration_tools-1.9.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.2"
+version = "1.9.4"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -56,6 +56,7 @@ deepdiff = "^6.2.3"
 pyaml = "^21.10.1"
 python-i18n = "^0.3.9"
 art = "^6.5"
+folio-data-import = "^0.3.2"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.1.3"
@@ -63,24 +64,31 @@ lxml = ">4.9"
 coverage = {extras = ["toml"], version = "^6.5.0"}
 pytest-cov = "^4.0.0"
 black = "^22.10.0"
-flake8 = "^5.0.4"
+flake8 = "^7.3.0"
 mypy = "^0.982"
-flake8-black = "^0.3.3"
-flake8-bugbear = "^22.9.23"
+flake8-black = "^0.3.6"
+flake8-bugbear = "^24.12.12"
 flake8-bandit = "^4.1.1"
-flake8-isort = "^5.0.0"
+flake8-isort = "^6.1.2"
 flake8-docstrings = "^1.6.0"
 darglint = "^1.8.1"
-sphinx = "^5.3.0"
-sphinx-autodoc-typehints = "^1.19.4"
-myst-parser = "^0.18.1"
 types-requests = "^2.28.11.17"
 types-python-dateutil = "^2.8.19.11"
 ipykernel = "^6.29.5"
 pytest-asyncio = "^0.23.0"
+sphinx-autobuild = "^2024.10.3"
+
+[tool.poetry.group.docs.dependencies]
+m2r = "^0.2.1"
+myst-parser = "<4.0.0"
+sphinx = ">=6.0.0, <8.0.0"
+sphinx-autodoc-typehints = "^1.17.0"
+sphinx-rtd-theme = "^1.0.0"
+toml = "^0.10.2"
+sphinx-book-theme = "^1.1.4"
 
 [tool.poetry.extras]
-docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml"]
+docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml",]
 
 [tool.poetry.requires-plugins]
 poetry-plugin-export = ">=1.8"

{folio_migration_tools-1.9.2 → folio_migration_tools-1.9.4}/src/folio_migration_tools/mapping_file_transformation/user_mapper.py

@@ -59,8 +59,8 @@ class UserMapper(MappingFileMapperBase):
                 True,
             )
             self.notes_mapper.migration_report = self.migration_report
-            self.setup_departments_mapping(departments_mapping)
-            self.setup_groups_mapping(groups_map)
+            self.departments_mapping = self.setup_departments_mapping(departments_mapping)
+            self.groups_mapping = self.setup_groups_mapping(groups_map)
 
             for m in self.record_map["data"]:
                 if m["folio_field"].startswith("customFields"):
@@ -120,7 +120,8 @@ class UserMapper(MappingFileMapperBase):
 
         return clean_folio_object
 
-    def get_users(self, source_file, file_format: str):
+    @staticmethod
+    def get_users(source_file, file_format: str):
         csv.register_dialect("tsv", delimiter="\t")
         if file_format == "tsv":
             reader = csv.DictReader(source_file, dialect="tsv")
@@ -156,12 +157,25 @@ class UserMapper(MappingFileMapperBase):
                     "No Departments mapping set up. Set up a departments mapping file "
                     " or remove the mapping of the Departments field",
                 )
-            return self.get_mapped_name(
-                self.departments_mapping,
-                legacy_user,
-                index_or_id,
-                True,
-            )
+            if len(self.departments_mapping.mapped_legacy_keys) == 1 and self.library_configuration.multi_field_delimiter in legacy_user.get(self.departments_mapping.mapped_legacy_keys[0], ""):
+                split_departments = legacy_user.get(self.departments_mapping.mapped_legacy_keys[0], "").split(
+                    self.library_configuration.multi_field_delimiter
+                )
+                return self.library_configuration.multi_field_delimiter.join([
+                    self.get_mapped_name(
+                        self.departments_mapping,
+                        {self.departments_mapping.mapped_legacy_keys[0]: dept},
+                        index_or_id,
+                        True,
+                    ) for dept in split_departments
+                ])
+            else:
+                return self.get_mapped_name(
+                    self.departments_mapping,
+                    legacy_user,
+                    index_or_id,
+                    True,
+                )
         elif folio_prop_name in ["expirationDate", "enrollmentDate", "personal.dateOfBirth"]:
             return self.get_parsed_date(mapped_value, folio_prop_name)
         return mapped_value
@@ -184,27 +198,21 @@ class UserMapper(MappingFileMapperBase):
             return ""
 
     def setup_groups_mapping(self, groups_map):
-        if groups_map:
-            self.groups_mapping = RefDataMapping(
-                self.folio_client,
-                "/groups",
-                "usergroups",
-                groups_map,
-                "group",
-                "UserGroupMapping",
-            )
-        else:
-            self.groups_mapping = None
+        return RefDataMapping(
+            self.folio_client,
+            "/groups",
+            "usergroups",
+            groups_map,
+            "group",
+            "UserGroupMapping",
+        ) if groups_map else None
 
     def setup_departments_mapping(self, departments_mapping):
-        if departments_mapping:
-            self.departments_mapping = RefDataMapping(
-                self.folio_client,
-                "/departments",
-                "departments",
-                departments_mapping,
-                "name",
-                "DepartmentsMapping",
-            )
-        else:
-            self.departments_mapping = None
+        return RefDataMapping(
+            self.folio_client,
+            "/departments",
+            "departments",
+            departments_mapping,
+            "name",
+            "DepartmentsMapping",
+        ) if departments_mapping else None

{folio_migration_tools-1.9.2 → folio_migration_tools-1.9.4}/src/folio_migration_tools/marc_rules_transformation/conditions.py

@@ -90,6 +90,7 @@ class Conditions:
         logging.info("%s\tholding_note_types", len(self.folio.holding_note_types)) # type: ignore
         logging.info("%s\tcall_number_types", len(self.folio.call_number_types)) # type: ignore
         self.setup_and_validate_holdings_types()
+        self.ill_policies = self.folio.folio_get_all("/ill-policies", "illPolicies")
         # Raise for empty settings
         if not self.folio.holding_note_types:
             raise TransformationProcessError("", "No holding_note_types in FOLIO")
@@ -466,36 +467,6 @@ class Conditions:
     def condition_char_select(self, legacy_id, value, parameter, marc_field: field.Field):
         return value[parameter["from"] : parameter["to"]]
 
-    def condition_set_receipt_status(self, legacy_id, value, parameter, marc_field: field.Field):
-        if len(value) < 7:
-            self.mapper.migration_report.add(
-                "ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
-            )
-            return ""
-        try:
-            status_map = {
-                "0": "Unknown",
-                "1": "Other receipt or acquisition status",
-                "2": "Received and complete or ceased",
-                "3": "On order",
-                "4": "Currently received",
-                "5": "Not currently received",
-                "6": "External access",
-            }
-            mapped_value = status_map[value[6]]
-            self.mapper.migration_report.add(
-                "ReceiptStatusMapping",
-                i18n.t(
-                    "%{value} mapped to %{mapped_value}", value=value[6], mapped_value=mapped_value
-                ),
-            )
-
-            return
-        except Exception:
-            self.mapper.migration_report.add(
-                "ReceiptStatusMapping", i18n.t("%{value} not found in map.", value=value)
-            )
-            return "Unknown"
 
     def condition_set_identifier_type_id_by_name(
         self, legacy_id, value, parameter, marc_field: field.Field
@@ -915,3 +886,241 @@ class Conditions:
                 legacy_id,
                 f"Subject source not found for {value} {marc_field}",
             )
+
+    def condition_set_receipt_status(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the receipt status based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and 
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 7:
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        
+        status_map = {
+            "0": "Unknown",
+            "1": "Other receipt or acquisition status",
+            "2": "Received and complete or ceased",
+            "3": "On order",
+            "4": "Currently received",
+            "5": "Not currently received",
+            "6": "External access",
+        }
+
+        try:
+            mapped_value = status_map[value[6]]
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}",
+                    value=value[6],
+                    mapped_value=mapped_value,
+                ),
+            )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping", i18n.t("%{value} not found in map.", value=value[6])
+            )
+            return ""
+
+    def condition_set_acquisition_method(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the acquisition method based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and 
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 8:
+            self.mapper.migration_report.add(
+                "ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+
+        try:
+            acq_methods = {
+                "c": "Cooperative or consortial purchase",
+                "d": "Deposit",
+                "e": "Exchange",
+                "f": "Free",
+                "g": "Gift",
+                "l": "Legal deposit",
+                "m": "Membership",
+                "n": "Non-library purchase",
+                "p": "Purchase",
+                "q": "Lease",
+                "u": "Unknown",
+                "z": "Other method of acquisition",
+            }
+            mapped_value = acq_methods[value[7]]
+            self.mapper.migration_report.add(
+                "MethodOfAcquisitionMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}", value=value[7], mapped_value=mapped_value
+                ),
+            )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "MethodOfAcquisitionMapping", i18n.t("%{value} not found in map.", value=value[8])
+            )
+            return ""
+
+    def condition_set_retention_policy(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the retention policy based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 13:
+            self.mapper.migration_report.add(
+                "RetentionPolicyMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        value = value.replace("|", " ").replace("#", " ") # Replace pipe with space for mapping consistency
+        try:
+            retention_policies = {
+                "0": "Unknown",
+                "1": "Other general retention policy",
+                "2": "Retained except as replaced by updates",
+                "3": "Sample issue retained",
+                "4": "Retained until replaced by microform",
+                "5": "Retained until replaced by cumulation, replacement volume, or revision",
+                "6": "Retained for a limited period",
+                "7": "Not retained",
+                "8": "Permanently retained",
+            }
+            mapped_value = retention_policies[value[12]]
+            self.mapper.migration_report.add(
+                "RetentionPolicyMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}",
+                    value=value[12],
+                    mapped_value=mapped_value,
+                ),
+            )
+            if value[12] == "6":
+                policy_types = {
+                    "l": "Latest",
+                    "p": "Previous",
+                }
+                unit_types = {
+                    "m": "Day",
+                    "w": "Month",
+                    "y": "Year",
+                    "e": "Edition",
+                    "i": "Issue",
+                    "s": "Supplement"
+                }
+                try:
+                    specific_retention_policy = ""
+                    if value[13].strip() or value[15].strip():
+                        if value[14].strip() and int(value[14]) > 1:
+                            specific_retention_policy = f"{policy_types.get(value[13], '')} {value[14]} {unit_types.get(value[15], '')}s retained".strip()
+                        else:
+                            specific_retention_policy = f"{policy_types.get(value[13], '')} {unit_types.get(value[15], '')} retained".strip()
+                    if specific_retention_policy:
+                        self.mapper.migration_report.add(
+                            "RetentionPolicyMapping",
+                            i18n.t(
+                                "Retention policy 6 indicates a limited period. Specific retention period will be mapped from 008/13-15",
+                            )
+                        )
+                        return specific_retention_policy
+                    else:
+                        raise ValueError(
+                            "Specific retention policy is empty or invalid in 008/13-15"
+                        )
+                except ValueError:
+                    self.mapper.migration_report.add(
+                        "RetentionPolicyMapping",
+                        i18n.t("Invalid specific retention policy in 008/13-15: %{value}", value=value[13:16]),
+                    )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "RetentionPolicyMapping", i18n.t("%{value} not found in map.", value=value[12])
+            )
+            return ""
+
+    def condition_set_ill_policy(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the ILL policy based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition."""
+        if len(value) < 21:
+            self.mapper.migration_report.add(
+                "ILLPolicyMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        try:
+            ill_policies = {
+                "a": "Will lend",
+                "b": "Will not lend",
+                "c": "Will lend hard copy only",
+                "l": "Limited lending policy",
+                "u": "Unknown",
+            }
+            mapped_value = ill_policies[value[20]]
+            self.mapper.migration_report.add(
+                "ILLPolicyMapping",
+                i18n.t("%{value} mapped to %{mapped_value}", value=value[20], mapped_value=mapped_value),
+            )
+            ill_policy_id = self.get_ref_data_tuple_by_name(
+                self.ill_policies, "ill_policies", mapped_value
+            )
+            return ill_policy_id[0] if ill_policy_id else ""
+        except Exception:
+            self.mapper.migration_report.add(
+                "ILLPolicyMapping", i18n.t("%{value} not found in map.", value=value[20])
+            )
+            return ""
+
+    def condition_set_digitization_policy(
+        self, legacy_id, value, parameter, marc_field: field.Field
+    ):
+        """
+        This method maps the digitization policy based on the 008 field.
+        This condition is not available in FOLIO's MARC mapping engine and
+        will require use of a supplemental mapping rules file in the
+        HoldingsMarcTransformer task definition.
+        """
+        if len(value) < 22:
+            self.mapper.migration_report.add(
+                "DigitizationPolicyMapping", i18n.t("008 is too short") + f": {value}"
+            )
+            return ""
+        try:
+            digitization_policies = {
+                "a": "Will reproduce",
+                "b": "Will not reproduce",
+                "u": "Unknown",
+            }
+            mapped_value = digitization_policies[value[21]]
+            self.mapper.migration_report.add(
+                "DigitizationPolicyMapping",
+                i18n.t(
+                    "%{value} mapped to %{mapped_value}",
+                    value=value[21],
+                    mapped_value=mapped_value,
+                ),
+            )
+            return mapped_value
+        except Exception:
+            self.mapper.migration_report.add(
+                "DigitizationPolicyMapping", i18n.t("%{value} not found in map.", value=value[21])
+            )
+            return ""