folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/mapping_file_transformation/organization_mapper.py

@@ -0,0 +1,389 @@
+import json
+import logging
+import os
+import re
+import sys
+
+import httpx
+from folio_uuid.folio_uuid import FOLIONamespaces
+from folioclient import FolioClient
+
+from folio_migration_tools.library_configuration import LibraryConfiguration
+from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
+    MappingFileMapperBase,
+)
+from folio_migration_tools.mapping_file_transformation.notes_mapper import NotesMapper
+from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
+    RefDataMapping,
+)
+
+
+class OrganizationMapper(MappingFileMapperBase):
+    def __init__(
+        self,
+        folio_client: FolioClient,
+        library_configuration: LibraryConfiguration,
+        task_config,
+        organization_map: dict,
+        organization_types_map,
+        address_categories_map,
+        email_categories_map,
+        phone_categories_map,
+    ):
+        # Build composite organization schema
+        if os.environ.get("GITHUB_TOKEN"):
+            logging.info("Using GITHUB_TOKEN environment variable for GitHub API Access")
+        organization_schema = OrganizationMapper.get_latest_acq_schemas_from_github(
+            "folio-org", "mod-organizations-storage", "mod-orgs", "organization"
+        )
+
+        super().__init__(
+            folio_client,
+            organization_schema,
+            organization_map,
+            None,
+            FOLIONamespaces.organizations,
+            library_configuration,
+            task_config,
+        )
+        self.organization_schema = organization_schema
+        # Set up reference data maps
+        self.set_up_reference_data_mapping(
+            organization_types_map,
+            address_categories_map,
+            email_categories_map,
+            phone_categories_map,
+        )
+
+        self.folio_client: FolioClient = folio_client
+        self.notes_mapper: NotesMapper = NotesMapper(
+            library_configuration,
+            None,
+            self.folio_client,
+            organization_map,
+            FOLIONamespaces.note,
+            True,
+        )
+        self.notes_mapper.migration_report = self.migration_report
+
+    # Commence the mapping work
+    def get_prop(self, legacy_organization, folio_prop_name, index_or_id, schema_default_value):
+        value_tuple = (
+            legacy_organization,
+            index_or_id,
+            folio_prop_name,
+        )
+
+        # Perfrom reference data mappings
+        if folio_prop_name == "organizationTypes":
+            return self.get_mapped_ref_data_value(
+                self.organization_types_map,
+                *value_tuple,
+                False,
+            )
+
+        elif re.compile(r"addresses\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
+            return self.get_mapped_ref_data_value(
+                self.address_categories_map,
+                *value_tuple,
+                False,
+            )
+
+        elif re.compile(r"emails\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
+            return self.get_mapped_ref_data_value(
+                self.email_categories_map,
+                *value_tuple,
+                False,
+            )
+
+        elif re.compile(r"phoneNumbers\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
+            return self.get_mapped_ref_data_value(
+                self.phone_categories_map,
+                *value_tuple,
+                False,
+            )
+
+        elif re.compile(r"interfaces\[(\d+)\]\.interfaceCredential.interfaceId").fullmatch(
+            folio_prop_name
+        ):
+            return "replace_with_interface_id"
+
+        return super().get_prop(
+            legacy_organization, folio_prop_name, index_or_id, schema_default_value
+        )
+
+    def set_up_reference_data_mapping(
+        self,
+        organization_types_map,
+        address_categories_map,
+        email_categories_map,
+        phone_categories_map,
+    ):
+        """
+
+        Args:
+            organization_types_map (_type_): _description_
+            address_categories_map (_type_): _description_
+            email_categories_map (_type_): _description_
+            phone_categories_map (_type_): _description_
+        """
+
+        categories_shared_args = (
+            self.folio_client,
+            "/organizations-storage/categories",
+            "categories",
+        )
+
+        if address_categories_map:
+            self.address_categories_map = RefDataMapping(
+                *categories_shared_args, address_categories_map, "value", "CategoriesMapping"
+            )
+        else:
+            self.address_categories_map = None
+
+        if email_categories_map:
+            self.email_categories_map = RefDataMapping(
+                *categories_shared_args, email_categories_map, "value", "CategoriesMapping"
+            )
+        else:
+            self.email_categories_map = None
+
+        if phone_categories_map:
+            self.phone_categories_map = RefDataMapping(
+                *categories_shared_args, phone_categories_map, "value", "CategoriesMapping"
+            )
+        else:
+            self.phone_categories_map = None
+
+        if organization_types_map:
+            self.organization_types_map = RefDataMapping(
+                self.folio_client,
+                "/organizations-storage/organization-types",
+                "organizationTypes",
+                organization_types_map,
+                "name",
+                "OrganizationTypeMapping",
+            )
+        else:
+            self.organization_types_map = None
+
+    @staticmethod
+    def get_latest_acq_schemas_from_github(owner, repo, module, object):
+        """
+        Given a repository owner, a repository, a module name and the name
+        of a FOLIO acquisition object, returns a schema for that object that
+        also includes the schemas of any other referenced acq objects.
+
+        Args:
+            owner (_type_): _description_
+            repo (_type_): _description_
+            module (_type_): _description_
+            object (_type_): _description_
+
+        Returns:
+            _type_: _description_
+        """
+        try:
+            # Authenticate when calling GitHub, using an API key stored in .env
+            github_headers = {
+                "content-type": "application/json",
+                "User-Agent": "FOLIO Migration Tools (https://github.com/FOLIO-FSE/folio_migration_tools/)",  # noqa:E501,B950
+            }
+
+            if os.environ.get("GITHUB_TOKEN"):
+                github_headers["authorization"] = f"token {os.environ.get('GITHUB_TOKEN')}"
+
+            # Start talkign to GitHub...
+            github_path = "https://raw.githubusercontent.com"
+            submodules = OrganizationMapper.get_submodules_of_latest_release(
+                owner, repo, github_headers
+            )
+
+            # Get the sha's of sunmodules acq-models and raml_utils
+            acq_models_sha = next(
+                (item["sha"] for item in submodules if item["path"] == "acq-models")
+            )
+
+            # # TODO Maybe - fetch raml_utils schemas if deemed necessary
+            # raml_utils_sha = next((item["sha"] for item in submodules
+            # if item["path"] == "raml-utils"))
+
+            acq_models_path = f"{github_path}/{owner}/acq-models/{acq_models_sha}/{module}/schemas"
+
+            req = httpx.get(f"{acq_models_path}/{object}.json", headers=github_headers)
+            req.raise_for_status()
+
+            object_schema = json.loads(req.text)
+
+            # Fetch referenced schemas
+            extended_object_schema = OrganizationMapper.build_extended_object(
+                object_schema, acq_models_path, github_headers
+            )
+
+            return extended_object_schema
+
+        except httpx.HTTPError as http_error:
+            logging.critical(f"Halting! \t{http_error}")
+            sys.exit(2)
+
+        except json.decoder.JSONDecodeError as json_error:
+            logging.critical(json_error)
+            sys.exit(2)
+
+    @staticmethod
+    def get_submodules_of_latest_release(owner, repo, github_headers):
+        """
+        Given a repository owner and a repository, identifies the latest
+        release of the repository and returns the submodules associated with
+        this release.
+
+        Args:
+            owner (_type_): _description_
+            repo (_type_): _description_
+            github_headers (_type_): _description_
+
+        Returns:
+            _type_: _description_
+        """
+
+        github_path = "https://api.github.com/repos"
+
+        # Get metadata for the latest release
+        latest_release_path = f"{github_path}/{owner}/{repo}/releases/latest"
+        req = httpx.get(f"{latest_release_path}", headers=github_headers, timeout=None)
+        req.raise_for_status()
+        latest_release = json.loads(req.text)
+
+        # Get the tag assigned to the latest release
+        release_tag = latest_release["tag_name"]
+        logging.info(f"Using schemas from latest {repo} release: {release_tag}")
+
+        # Get the tree for the latest release
+        tree_path = f"{github_path}/{owner}/{repo}/git/trees/{release_tag}"
+        req = httpx.get(tree_path, headers=github_headers, timeout=None)
+        req.raise_for_status()
+        release_tree = json.loads(req.text)
+
+        # Loop through the tree to get the sha of the folder with path "ramls"
+        ramls_sha = next((item["sha"] for item in release_tree["tree"] if item["path"] == "ramls"))
+
+        # Get the tree for the ramls folder
+        ramls_path = f"{github_path}/{owner}/{repo}/git/trees/{ramls_sha}"
+        req = httpx.get(ramls_path, headers=github_headers, timeout=None)
+        req.raise_for_status()
+        ramls_tree = json.loads(req.text)
+
+        # Loop through the tree to get the sha of submodules
+        submodules = [item for item in ramls_tree["tree"] if item["mode"] == "160000"]
+
+        return submodules
+
+    @staticmethod
+    def build_extended_object(object_schema, submodule_path, github_headers):
+        """
+        Takes an object schema (for example an organization) and the path to a
+        submodule repository and returns the same schema with the full schemas
+        of subordinate objects (for example aliases).
+
+        Args:
+            object_schema (_type_): _description_
+            submodule_path (_type_): _description_
+            github_headers (_type_): _description_
+
+        Returns:
+            _type_: _description_
+        """
+
+        supported_types = ["string", "boolean", "number", "integer", "text", "object", "array"]
+
+        try:
+            for property_name_level1, property_level1 in object_schema["properties"].items():
+                # For now, treat references to UUIDs like strings
+                # It's not great practice, but it's the way FOLIO mostly handles it
+                if "../../common/schemas/uuid.json" in property_level1.get("$ref", ""):
+                    property_level1["type"] = "string"
+
+                # Preliminary implementation of tags
+                # https://github.com/folio-org/raml/blob/master/schemas/tags.schema
+                elif property_name_level1 == "tags":
+                    property_level1["properties"] = {
+                        "tagList": {
+                            "description": "List of tags",
+                            "type": "array",
+                            "items": {"type": "string"},
+                        }
+                    }
+
+                elif property_name_level1 == "contacts":
+                    contact_schema = OrganizationMapper.fetch_additional_schema("contact")
+                    property_level1["items"] = contact_schema
+
+                elif property_name_level1 == "interfaces":
+                    interface_schema = OrganizationMapper.fetch_additional_schema("interface")
+                    interface_schema["required"] = ["name"]
+
+                    # Temporarily add the credential object as a subproperty
+                    interface_credential_schema = OrganizationMapper.fetch_additional_schema(
+                        "interface_credential"
+                    )
+                    interface_credential_schema["required"] = (
+                        ["username", "password", "interfaceId"],
+                    )
+
+                    interface_schema["properties"][
+                        "interfaceCredential"
+                    ] = interface_credential_schema
+
+                    property_level1["items"] = interface_schema
+
+                elif (
+                    property_level1.get("type") == "array"
+                    and property_level1.get("items").get("$ref")
+                    == "../../common/schemas/uuid.json"
+                ):
+                    property_level1["items"]["type"] = "string"
+
+                # Report and discard unhandled properties
+                elif (
+                    property_level1.get("type") not in supported_types
+                    or "../" in property_level1.get("$ref", "")
+                    or "../" in property_level1.get("items", {}).get("$ref", "")
+                ):
+                    logging.info(f"Property not yet supported: {property_name_level1}")
+
+                # Handle object properties
+                elif property_level1.get("type") == "object" and property_level1.get("$ref"):
+                    logging.info("Fecthing referenced schema for object %s", property_name_level1)
+
+                    ref_object = property_level1["$ref"]
+                    schema_url = f"{submodule_path}/{ref_object}"
+
+                    req = httpx.get(schema_url, headers=github_headers, timeout=None)
+                    req.raise_for_status()
+
+                    property_level1 = dict(property_level1, **json.loads(req.text))
+
+                elif property_level1.get("type") == "array" and property_level1.get("items").get(
+                    "$ref"
+                ):
+                    ref_object = property_level1["items"]["$ref"]
+                    schema_url = f"{submodule_path}/{ref_object}"
+
+                    req = httpx.get(schema_url, headers=github_headers, timeout=None)
+                    req.raise_for_status()
+
+                    property_level1["items"] = dict(
+                        property_level1["items"], **json.loads(req.text)
+                    )
+
+            return object_schema
+
+        except httpx.HTTPError as he:
+            logging.error(he)
+
+    @staticmethod
+    def fetch_additional_schema(folio_object):
+        additional_schema = OrganizationMapper.get_latest_acq_schemas_from_github(
+            "folio-org", "mod-organizations-storage", "mod-orgs", folio_object
+        )
+        return additional_schema

folio_migration_tools/mapping_file_transformation/ref_data_mapping.py

@@ -3,8 +3,8 @@ import logging
 import sys
 
 from folioclient import FolioClient
+
 from folio_migration_tools.custom_exceptions import TransformationProcessError
-from folio_migration_tools.report_blurbs import Blurbs
 
 
 class RefDataMapping(object):
@@ -15,18 +15,16 @@ class RefDataMapping(object):
         array_name,
         the_map,
         key_type,
-        blurb: Blurbs,
+        blurb_id,
     ):
         self.name = array_name
-        self.cache = {}
-        self.blurb = blurb
+        self.cache: dict = {}
+        self.blurb_id = blurb_id
         logging.info("%s reference data mapping. Initializing", self.name)
         logging.info("Fetching %s reference data from FOLIO", self.name)
-        self.ref_data = list(
-            folio_client.folio_get_all(ref_data_path, array_name, "", 1000)
-        )
+        self.ref_data = list(folio_client.folio_get_all(ref_data_path, array_name, "", 1000))
         self.map = the_map
-        self.regular_mappings = []
+        self.regular_mappings: list = []
         self.key_type = key_type
         self.hybrid_mappings = []
         self.mapped_legacy_keys = []
@@ -45,6 +43,9 @@ class RefDataMapping(object):
         return self.cached_dict.get(key_value.lower().strip(), ())
 
     def setup_mappings(self):
+        if not self.map:
+            logging.info("%s legacy map file is empty or not provided", self.name)
+            return
         self.pre_validate_map()
         for idx, mapping in enumerate(self.map):
             try:
@@ -64,7 +65,8 @@ class RefDataMapping(object):
                         x = mapping.get(f"folio_{self.key_type}", "")
                         raise TransformationProcessError(
                             "",
-                            f"No {self.name} - {x} - set up in map or tenant. Check for inconstencies in {self.name} naming."
+                            f"No {self.name} - {x} - set up in map or tenant. Check for "
+                            f"inconstencies in {self.name} naming. "
                             f"Add a row to mapping file with *:s and a valid {self.name}",
                         )
                 else:
@@ -74,9 +76,7 @@ class RefDataMapping(object):
                         self.regular_mappings.append(mapping)
                     t = self.get_ref_data_tuple(mapping[f"folio_{self.key_type}"])
                     if not t:
-                        raise TransformationProcessError(
-                            "", f"Mapping not found for {mapping}"
-                        )
+                        raise TransformationProcessError("", f"Mapping not found for {mapping}")
                     mapping["folio_id"] = t[0]
             except TransformationProcessError as transformation_process_error:
                 raise transformation_process_error from transformation_process_error
@@ -90,10 +90,12 @@ class RefDataMapping(object):
 
         self.post_validate_map()
         logging.info(
-            f"Loaded {len(self.regular_mappings)} mappings for {len(self.ref_data)} {self.name} in FOLIO"
+            f"Loaded {len(self.regular_mappings)} mappings for {len(self.ref_data)} {self.name} "
+            "in FOLIO"
         )
         logging.info(
-            f"loaded {len(self.hybrid_mappings)} hybrid mappings for {len(self.ref_data)} {self.name} in FOLIO"
+            f"loaded {len(self.hybrid_mappings)} hybrid mappings for {len(self.ref_data)} "
+            f"{self.name} in FOLIO"
         )
 
     def get_hybrid_mapping(self, legacy_object):
@@ -128,24 +130,18 @@ class RefDataMapping(object):
             return self.cache[obj_key]
         prepped_props = {k: legacy_object[k].strip() for k in self.mapped_legacy_keys}
         for mapping in self.regular_mappings:
-            match_number = sum(
-                prepped_props[k] == mapping[k] for k in self.mapped_legacy_keys
-            )
+            match_number = sum(prepped_props[k] == mapping[k] for k in self.mapped_legacy_keys)
             if match_number == len(self.mapped_legacy_keys):
                 self.cache[obj_key] = mapping
                 return mapping
         return None
 
     def is_hybrid_default_mapping(self, mapping):
-        legacy_values = [
-            value for key, value in mapping.items() if key in self.mapped_legacy_keys
-        ]
+        legacy_values = [value for key, value in mapping.items() if key in self.mapped_legacy_keys]
         return "*" in legacy_values and not self.is_default_mapping(mapping)
 
     def is_default_mapping(self, mapping):
-        legacy_values = [
-            value for key, value in mapping.items() if key in self.mapped_legacy_keys
-        ]
+        legacy_values = [value for key, value in mapping.items() if key in self.mapped_legacy_keys]
         return all(f == "*" for f in legacy_values)
 
     def pre_validate_map(self):
@@ -163,12 +159,17 @@ class RefDataMapping(object):
         )
         if any(folio_values_not_in_map):
             logging.info(
-                f"Values from {self.name} ref data in FOLIO that are not in the map: {folio_values_not_in_map}"
+                "Values from %s ref data in FOLIO that are not in the map: %s",
+                self.name,
+                folio_values_not_in_map,
             )
         if any(map_values_not_in_folio):
             raise TransformationProcessError(
                 "",
-                f"Values ({self.key_type}) from {self.name} map are not in FOLIO: {map_values_not_in_folio}",
+                (
+                    f"Values ({self.key_type}) from {self.name} map are not in "
+                    f"FOLIO: {map_values_not_in_folio}"
+                ),
             )
 
     def post_validate_map(self):
@@ -176,7 +177,8 @@ class RefDataMapping(object):
             raise TransformationProcessError(
                 "",
                 f"No fallback {self.name} set up in map."
-                f"Add a row to mapping file with *:s in all legacy columns and a valid {self.name} value",
+                f"Add a row to mapping file with *:s in all legacy "
+                f"columns and a valid {self.name} value",
             )
         for mapping in self.map:
             if f"folio_{self.key_type}" not in mapping:
@@ -209,5 +211,14 @@ def get_mapped_legacy_keys(mapping):
         k.strip()
         for k in mapping.keys()
         if k
-        not in ["folio_group", "folio_code", "folio_id", "folio_name", "legacy_code"]
+        not in [
+            "folio_group",
+            "folio_code",
+            "folio_id",
+            "folio_name",
+            "legacy_code",
+            "folio_value",
+            "folio_owner",
+            "folio_feeFineType",
+        ]
     ]