folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/mapping_file_transformation/courses_mapper.py

@@ -0,0 +1,203 @@
+import i18n
+from typing import Any
+from typing import Dict
+
+from folio_uuid.folio_uuid import FOLIONamespaces
+from folio_uuid.folio_uuid import FolioUUID
+from folioclient import FolioClient
+
+from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
+from folio_migration_tools.library_configuration import LibraryConfiguration
+from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
+    MappingFileMapperBase,
+)
+from folio_migration_tools.mapping_file_transformation.notes_mapper import NotesMapper
+from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
+    RefDataMapping,
+)
+
+
+class CoursesMapper(MappingFileMapperBase):
+    def __init__(
+        self,
+        folio_client: FolioClient,
+        course_map,
+        terms_map,
+        departments_map,
+        library_configuration: LibraryConfiguration,
+        task_configuration,
+    ):
+        self.folio_client: FolioClient = folio_client
+        self.user_cache: dict = {}
+        self.notes_mapper: NotesMapper = NotesMapper(
+            library_configuration,
+            None,
+            self.folio_client,
+            course_map,
+            FOLIONamespaces.note,
+            True,
+        )
+        self.composite_course_schema = self.get_composite_course_schema()
+        super().__init__(
+            folio_client,
+            self.composite_course_schema,
+            course_map,
+            None,
+            FOLIONamespaces.course,
+            library_configuration,
+            task_configuration
+        )
+        self.course_map = course_map
+        if terms_map:
+            self.terms_map = RefDataMapping(
+                self.folio_client,
+                "/coursereserves/terms",
+                "terms",
+                terms_map,
+                "name",
+                "TermsMapping",
+            )
+        else:
+            self.terms_map = None
+
+        if departments_map:
+            self.departments_map = RefDataMapping(
+                self.folio_client,
+                "/coursereserves/departments",
+                "departments",
+                departments_map,
+                "name",
+                "DepartmentsMapping",
+            )
+        else:
+            self.departments_map = None
+
+    def store_objects(self, composite_course):
+        try:
+            self.extradata_writer.write("courselisting", composite_course[0]["courselisting"])
+            self.migration_report.add_general_statistics(i18n.t("Stored courselistings"))
+            self.extradata_writer.write("course", composite_course[0]["course"])
+            self.migration_report.add_general_statistics(i18n.t("Stored courses"))
+            if "instructors" in composite_course[0] and any(composite_course[0]["instructors"]):
+                for instructor in composite_course[0]["instructors"]:
+                    self.extradata_writer.write("instructor", instructor)
+                    self.migration_report.add_general_statistics(i18n.t("Stored instructors"))
+
+        except Exception as ee:
+            raise TransformationRecordFailedError(
+                composite_course[1], "Failed when storing", ee
+            ) from ee
+
+    def perform_additional_mappings(self, composite_course):
+        try:
+            # Assign deterministic ids to every object
+            composite_course[0]["course"]["id"] = self.get_uuid(
+                composite_course, FOLIONamespaces.course
+            )
+            composite_course[0]["courselisting"]["id"] = self.get_uuid(
+                composite_course, FOLIONamespaces.course_listing
+            )
+            if "instructors" in composite_course[0] and any(composite_course[0]["instructors"]):
+                for idx, instructor in enumerate(composite_course[0]["instructors"]):
+                    instructor["id"] = self.get_uuid(
+                        composite_course, FOLIONamespaces.instructor, idx
+                    )
+                    # Link instructor to course listing
+                    instructor["courseListingId"] = composite_course[0]["courselisting"]["id"]
+                    if self.task_configuration.look_up_instructor:
+                        self.populate_instructor_from_users(instructor)
+            else:
+                self.migration_report.add_general_statistics(i18n.t("Missing Instructors"))
+
+            # Link course to courselisting
+            composite_course[0]["course"]["courseListingId"] = composite_course[0][
+                "courselisting"
+            ]["id"]
+
+        except Exception as ee:
+            raise TransformationRecordFailedError(
+                composite_course[1], "Failed when creating and linking ids", ee
+            ) from ee
+
+    def get_uuid(self, composite_course, object_type: FOLIONamespaces, idx: int = 0):
+        return str(
+            FolioUUID(
+                self.base_string_for_folio_uuid,
+                object_type,
+                composite_course[1] if idx == 0 else f"{composite_course[1]}_{idx}",
+            )
+        )
+
+    def populate_instructor_from_users(self, instructor: dict):
+        if instructor["userId"] not in self.user_cache:
+            path = "/users"
+            query = f'?query=(externalSystemId=="{instructor["userId"]}")'
+            if user := next(self.folio_client.folio_get_all(path, "users", query), None):
+                self.user_cache[instructor["userId"]] = user
+        if user := self.user_cache.get(instructor["userId"], {}):
+            instructor["userId"] = user.get("id", "")
+            instructor["barcode"] = user.get("barcode", "")
+            instructor["patronGroup"] = user.get("patronGroup", "")
+        else:
+            del instructor["userId"]
+
+    def get_prop(self, legacy_item, folio_prop_name, index_or_id, schema_default_value):
+        if folio_prop_name == "courselisting.termId":
+            return self.get_mapped_ref_data_value(
+                self.terms_map,
+                legacy_item,
+                folio_prop_name,
+                index_or_id,
+                False,
+            )
+        elif folio_prop_name == "course.departmentId":
+            return self.get_mapped_ref_data_value(
+                self.departments_map,
+                legacy_item,
+                folio_prop_name,
+                index_or_id,
+                False,
+            )
+        elif mapped_value := super().get_prop(
+            legacy_item, folio_prop_name, index_or_id, schema_default_value
+        ):
+            return mapped_value
+        else:
+            self.migration_report.add("UnmappedProperties", f"{folio_prop_name}")
+            return ""
+
+    def get_composite_course_schema(self) -> Dict[str, Any]:
+        if self:
+            return {
+                "properties": {
+                    "course": self.folio_client.get_from_github(
+                        "folio-org", "mod-courses", "/ramls/course.json"
+                    ),
+                    "courselisting": self.folio_client.get_from_github(
+                        "folio-org", "mod-courses", "/ramls/courselisting.json"
+                    ),
+                    "instructors": {
+                        "type": "array",
+                        "items": self.folio_client.get_from_github(
+                            "folio-org", "mod-courses", "/ramls/instructor.json"
+                        ),
+                    },
+                }
+            }
+        else:
+            return {
+                "properties": {
+                    "course": FolioClient.get_latest_from_github(
+                        "folio-org", "mod-courses", "/ramls/course.json"
+                    ),
+                    "courselisting": FolioClient.get_latest_from_github(
+                        "folio-org", "mod-courses", "/ramls/courselisting.json"
+                    ),
+                    "instructors": {
+                        "type": "array",
+                        "items": FolioClient.get_latest_from_github(
+                            "folio-org", "mod-courses", "/ramls/instructor.json"
+                        ),
+                    },
+                }
+            }

folio_migration_tools/mapping_file_transformation/holdings_mapper.py

@@ -1,17 +1,23 @@
 import ast
+import json
+import logging
 
-from folioclient import FolioClient
+import i18n
 from folio_uuid.folio_uuid import FOLIONamespaces
-from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
-from folio_migration_tools.library_configuration import LibraryConfiguration
+from folioclient import FolioClient
+
+from folio_migration_tools.custom_exceptions import TransformationProcessError, TransformationRecordFailedError
+from folio_migration_tools.library_configuration import (
+    FileDefinition,
+    LibraryConfiguration,
+)
 from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
     MappingFileMapperBase,
 )
 from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
     RefDataMapping,
 )
-from folio_migration_tools.report_blurbs import Blurbs
-
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 class HoldingsMapper(MappingFileMapperBase):
     def __init__(
@@ -22,6 +28,7 @@ class HoldingsMapper(MappingFileMapperBase):
         call_number_type_map,
         instance_id_map,
         library_configuration: LibraryConfiguration,
+        task_config: AbstractTaskConfiguration,
         statistical_codes_map=None,
     ):
         holdings_schema = folio_client.get_holdings_schema()
@@ -33,6 +40,7 @@ class HoldingsMapper(MappingFileMapperBase):
             statistical_codes_map,
             FOLIONamespaces.holdings,
             library_configuration,
+            task_config
         )
         self.holdings_map = holdings_map
 
@@ -42,7 +50,7 @@ class HoldingsMapper(MappingFileMapperBase):
             "locations",
             location_map,
             "code",
-            Blurbs.LocationMapping,
+            "LocationMapping",
         )
         if call_number_type_map:
             self.call_number_mapping = RefDataMapping(
@@ -51,71 +59,77 @@ class HoldingsMapper(MappingFileMapperBase):
                 "callNumberTypes",
                 call_number_type_map,
                 "name",
-                Blurbs.CallNumberTypeMapping,
+                "CallNumberTypeMapping",
             )
+        self.holdings_sources = self.get_holdings_sources()
 
-    def get_prop(self, legacy_item, folio_prop_name, index_or_id):
-
-        # Legacy contstruct
-        if not self.use_map:
-            return legacy_item[folio_prop_name]
-
-        legacy_item_keys = self.mapped_from_legacy_data.get(folio_prop_name, [])
+    def get_holdings_sources(self):
+        res = {}
+        holdings_sources = list(
+            self.folio_client.folio_get_all("/holdings-sources", "holdingsRecordsSources")
+        )
+        logging.info("Fetched %s holdingsRecordsSources from tenant", len(holdings_sources))
+        res = {n["name"].upper(): n["id"] for n in holdings_sources}
+        if "FOLIO" not in res:
+            raise TransformationProcessError("", "No holdings source with name FOLIO in tenant")
+        if "MARC" not in res:
+            raise TransformationProcessError("", "No holdings source with name MARC in tenant")
+        logging.info(json.dumps(res, indent=4))
+        return res
 
-        # IF there is a value mapped, return that one
-        if len(legacy_item_keys) == 1 and folio_prop_name in self.mapped_from_values:
-            value = self.mapped_from_values.get(folio_prop_name, "")
-            self.migration_report.add(
-                Blurbs.DefaultValuesAdded, f"{value} added to {folio_prop_name}"
-            )
-            return value
+    def perform_additional_mappings(self, legacy_ids, folio_rec, file_def):
+        self.handle_suppression(folio_rec, file_def)
+        self.map_statistical_codes(folio_rec, file_def)
+        self.map_statistical_code_ids(legacy_ids, folio_rec)
 
-        legacy_values = MappingFileMapperBase.get_legacy_vals(
-            legacy_item, legacy_item_keys
+    def handle_suppression(self, folio_record, file_def: FileDefinition):
+        folio_record["discoverySuppress"] = file_def.discovery_suppressed
+        self.migration_report.add(
+            "Suppression",
+            i18n.t("Suppressed from discovery") + f' = {folio_record["discoverySuppress"]}',
         )
-        legacy_value = " ".join(legacy_values).strip()
 
+    def get_prop(self, legacy_item, folio_prop_name, index_or_id, schema_default_value):
         if folio_prop_name == "permanentLocationId":
             return self.get_location_id(legacy_item, index_or_id, folio_prop_name)
-        elif folio_prop_name == "temporaryLocationId":
-            return self.get_location_id(legacy_item, index_or_id, folio_prop_name, True)
-        elif folio_prop_name == "callNumber":
-            if legacy_value.startswith("[") and len(legacy_value.split(",")) > 1:
-                self.migration_report.add_general_statistics(
-                    "Bound-with items callnumber identified"
-                )
-                self.migration_report.add(
-                    Blurbs.BoundWithMappings,
-                    (
-                        f"Number of bib-level callnumbers in record: "
-                        f"{len(legacy_value.split(','))}"
-                    ),
-                )
-            return legacy_value.removeprefix("[").removesuffix("]")
         elif folio_prop_name == "callNumberTypeId":
-            return self.get_call_number_type_id(
-                legacy_item, folio_prop_name, index_or_id
-            )
-        elif folio_prop_name == "statisticalCodeIds":
-            return self.get_statistical_codes(
-                legacy_values, folio_prop_name, index_or_id
-            )
+            return self.get_call_number_type_id(legacy_item, folio_prop_name, index_or_id)
+        # elif folio_prop_name.startswith("statisticalCodeIds"):
+        #     return self.get_statistical_code(legacy_item, folio_prop_name, index_or_id)
+
+        mapped_value = super().get_prop(
+            legacy_item, folio_prop_name, index_or_id, schema_default_value
+        )
+        if folio_prop_name == "callNumber":
+            return self.get_call_number(mapped_value)
         elif folio_prop_name == "instanceId":
-            return self.get_instance_ids(legacy_value, index_or_id)
-        elif any(legacy_item_keys):
-            if len(legacy_item_keys) > 1:
-                self.migration_report.add(
-                    Blurbs.Details, f"{legacy_item_keys} were concatenated"
-                )
-            return legacy_value
+            return self.get_instance_ids(mapped_value, index_or_id)
+        elif mapped_value:
+            return mapped_value
         else:
-            # edge case
+            self.migration_report.add("UnmappedProperties", f"{folio_prop_name}")
             return ""
 
+    def get_call_number(self, legacy_value):
+        if legacy_value.startswith("[") and len(legacy_value.split(",")) > 1:
+            self.migration_report.add_general_statistics(
+                i18n.t("Bound-with items callnumber identified")
+            )
+            self.migration_report.add(
+                "BoundWithMappings",
+                (f"Number of bib-level callnumbers in record: {len(legacy_value.split(','))}"),
+            )
+        if legacy_value.startswith("[") and len(legacy_value.split(",")) == 1:
+            try:
+                legacy_value = ast.literal_eval(str(legacy_value))[0]
+            except (SyntaxError, ValueError):
+                return legacy_value
+        return legacy_value
+
     def get_location_id(
         self, legacy_item: dict, id_or_index, folio_prop_name, prevent_default=False
     ):
-        return self.get_mapped_value(
+        return self.get_mapped_ref_data_value(
             self.location_mapping,
             legacy_item,
             id_or_index,
@@ -125,10 +139,10 @@ class HoldingsMapper(MappingFileMapperBase):
 
     def get_call_number_type_id(self, legacy_item, folio_prop_name: str, id_or_index):
         if self.call_number_mapping:
-            return self.get_mapped_value(
+            return self.get_mapped_ref_data_value(
                 self.call_number_mapping, legacy_item, id_or_index, folio_prop_name
             )
-        self.migration_report.add(Blurbs.CallNumberTypeMapping, "No mapping")
+        self.migration_report.add("CallNumberTypeMapping", i18n.t("No Call Number Type Mapping"))
         return ""
 
     def get_instance_ids(self, legacy_value: str, index_or_id: str):
@@ -136,8 +150,8 @@ class HoldingsMapper(MappingFileMapperBase):
         return_ids = []
         legacy_bib_ids = self.get_legacy_bib_ids(legacy_value, index_or_id)
         self.migration_report.add(
-            Blurbs.BoundWithMappings,
-            f"Number of bib records referenced in item: {len(legacy_bib_ids)}",
+            "BoundWithMappings",
+            i18n.t("Number of bib records referenced in item") + f": {len(legacy_bib_ids)}",
         )
         for legacy_instance_id in legacy_bib_ids:
             new_legacy_value = (
@@ -150,18 +164,18 @@ class HoldingsMapper(MappingFileMapperBase):
                 and legacy_instance_id not in self.instance_id_map
             ):
                 self.migration_report.add_general_statistics(
-                    "Records not matched to Instances"
+                    i18n.t("Records not matched to Instances")
                 )
                 s = "Bib id not in instance id map."
                 raise TransformationRecordFailedError(index_or_id, s, new_legacy_value)
             else:
                 self.migration_report.add_general_statistics(
-                    "Records matched to Instances"
+                    i18n.t("Records matched to Instances")
+                )
+                entry = self.instance_id_map.get(new_legacy_value, "") or self.instance_id_map.get(
+                    legacy_instance_id
                 )
-                entry = self.instance_id_map.get(
-                    new_legacy_value, ""
-                ) or self.instance_id_map.get(legacy_instance_id)
-                return_ids.append(entry["folio_id"])
+                return_ids.append(entry[1])
         if any(return_ids):
             return return_ids
         else:
@@ -177,11 +191,11 @@ class HoldingsMapper(MappingFileMapperBase):
             new_value_len = len(new_legacy_values)
             if new_value_len > 1:
                 self.migration_report.add_general_statistics(
-                    "Bound-with items identified by bib id"
+                    i18n.t("Bound-with items identified by bib id")
                 )
                 self.migration_report.add(
-                    Blurbs.GeneralStatistics,
-                    "Bib ids referenced in bound-with items",
+                    "GeneralStatistics",
+                    i18n.t("Bib ids referenced in bound-with items"),
                     new_value_len,
                 )
             return new_legacy_values
@@ -190,4 +204,4 @@ class HoldingsMapper(MappingFileMapperBase):
                 index_or_id,
                 f"Instance ID could not get parsed to array of strings {error}",
                 legacy_value,
-            )
+            ) from error