folio-migration-tools 1.9.0rc7__tar.gz → 1.9.0rc9__tar.gz

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc7
+Version: 1.9.0rc9
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: docs
 Requires-Dist: argparse-prompt (>=0.0.5,<0.0.6)
+Requires-Dist: art (>=6.5,<7.0)
 Requires-Dist: deepdiff (>=6.2.3,<7.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
 Requires-Dist: folio-uuid (>=0.2.8,<0.3.0)

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc7"
+version = "1.9.0rc9"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -55,10 +55,11 @@ argparse-prompt = "^0.0.5"
 deepdiff = "^6.2.3"
 pyaml = "^21.10.1"
 python-i18n = "^0.3.9"
+art = "^6.5"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.1.3"
-lxml = "^4.9.1"
+lxml = ">4.9"
 coverage = {extras = ["toml"], version = "^6.5.0"}
 pytest-cov = "^4.0.0"
 black = "^22.10.0"
@@ -73,7 +74,6 @@ darglint = "^1.8.1"
 sphinx = "^5.3.0"
 sphinx-autodoc-typehints = "^1.19.4"
 myst-parser = "^0.18.1"
-pandas = "^1.5.3"
 types-requests = "^2.28.11.17"
 types-python-dateutil = "^2.8.19.11"
 ipykernel = "^6.29.5"

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/__main__.py

@@ -62,6 +62,22 @@ def parse_args(args):
     )
     return parser.parse_args(args)
 
+def prep_library_config(args):
+        config_file_humped = merge_load(args.configuration_path)
+        config_file_humped["libraryInformation"]["okapiPassword"] = args.okapi_password
+        config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
+        config_file = humps.decamelize(config_file_humped)
+        library_config = LibraryConfiguration(**config_file["library_information"])
+        if library_config.ecs_tenant_id:
+            library_config.is_ecs = True
+        if library_config.ecs_tenant_id and not library_config.ecs_central_iteration_identifier:
+            print(
+                "ECS tenant ID is set, but no central iteration identifier is provided. "
+                "Please provide the central iteration identifier in the configuration file."
+            )
+            sys.exit("ECS Central Iteration Identifier Not Found")
+        return config_file, library_config
+
 
 def main():
     try:
@@ -79,11 +95,7 @@ def main():
         except i18n.I18nFileLoadError:
             i18n.load_config(Path(__file__).parent / "i18n_config.py")
         i18n.set("locale", args.report_language)
-        config_file_humped = merge_load(args.configuration_path)
-        config_file_humped["libraryInformation"]["okapiPassword"] = args.okapi_password
-        config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
-        config_file = humps.decamelize(config_file_humped)
-        library_config = LibraryConfiguration(**config_file["library_information"])
+        config_file, library_config = prep_library_config(args)
         try:
             migration_task_config = next(
                 t for t in config_file["migration_tasks"] if t["name"] == args.task_name

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/folder_structure.py

@@ -120,6 +120,9 @@ class FolderStructure:
         self.id_map_path = (
             self.results_folder / f"{str(self.object_type.name).lower()}_id_map.json"
         )
+        self.boundwith_relationships_map_path = (
+            self.results_folder / "boundwith_relationships_map.json"
+        )
         # Mapping files
         self.material_type_map_path = self.mapping_files_folder / "material_types.tsv"
         self.loan_type_map_path = self.mapping_files_folder / "loan_types.tsv"
@@ -139,6 +142,8 @@ class FolderStructure:
 def verify_git_ignore(gitignore: Path):
     with open(gitignore, "r+") as f:
         contents = f.read()
+        if "reports/" not in contents:
+            f.write("reports/\n")
         if "results/" not in contents:
             f.write("results/\n")
         if "archive/" not in contents:

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/library_configuration.py

@@ -90,13 +90,20 @@ class LibraryConfiguration(BaseModel):
     )
     multi_field_delimiter: Optional[str] = "<delimiter>"
     failed_records_threshold: Annotated[
-        int, Field(description=("Number of failed records until the process shuts down"))
+        int,
+        Field(description=("Number of failed records until the process shuts down")),
     ] = 5000
     failed_percentage_threshold: Annotated[
-        int, Field(description=("Percentage of failed records until the process shuts down"))
+        int,
+        Field(
+            description=("Percentage of failed records until the process shuts down")
+        ),
     ] = 20
     generic_exception_threshold: Annotated[
-        int, Field(description=("Number of generic exceptions until the process shuts down"))
+        int,
+        Field(
+            description=("Number of generic exceptions until the process shuts down")
+        ),
     ] = 50
     library_name: str
     log_level_debug: bool
@@ -111,3 +118,34 @@ class LibraryConfiguration(BaseModel):
     add_time_stamp_to_file_names: Annotated[
         bool, Field(title="Add time stamp to file names")
     ] = False
+    use_gateway_url_for_uuids: Annotated[
+        bool,
+        Field(
+            title="Use gateway URL for UUIDs",
+            description=(
+                "If set to true, folio_uuid will use the gateway URL when generating deterministic UUIDs for FOLIO records. "
+                "If set to false (default), the UUIDs will be generated using the tenant_id (or ecs_tenant_id)."
+            ),
+        ),
+    ] = False
+    is_ecs: Annotated[
+        bool,
+        Field(
+            title="Library is running ECS FOLIO",
+            description=(
+                "If set to true, the migration is running in an ECS environment. "
+                "If set to false (default), the migration is running in a non-ECS environment. "
+                "If ecs_tenant_id is set, this will be set to true, regardless of the value here."
+            ),
+        ),
+    ] = False
+    ecs_central_iteration_identifier: Annotated[
+        str,
+        Field(
+            title="ECS central iteration identifier",
+            description=(
+                "The iteration_identifier value from the central tenant configuration that corresponds "
+                "to this configuration's iteration_identifier. Used to access the central instances_id_map."
+            ),
+        ),
+    ] = ""

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/mapper_base.py

@@ -293,42 +293,6 @@ class MapperBase:
             )
             sys.exit(1)
 
-    def setup_boundwith_relationship_map(self, boundwith_relationship_map):
-        new_map = {}
-        for entry in boundwith_relationship_map:
-            if "MFHD_ID" not in entry or not entry.get("MFHD_ID", ""):
-                raise TransformationProcessError(
-                    "", "Column MFHD_ID missing from Boundwith relationship map", ""
-                )
-            if "BIB_ID" not in entry or not entry.get("BIB_ID", ""):
-                raise TransformationProcessError(
-                    "", "Column BIB_ID missing from Boundwith relationship map", ""
-                )
-            instance_uuid = str(
-                FolioUUID(
-                    str(self.folio_client.okapi_url),
-                    FOLIONamespaces.instances,
-                    entry["BIB_ID"],
-                )
-            )
-            mfhd_uuid = str(
-                FolioUUID(
-                    str(self.folio_client.okapi_url),
-                    FOLIONamespaces.holdings,
-                    entry["MFHD_ID"],
-                )
-            )
-            if entry["BIB_ID"] in self.parent_id_map:
-                new_map[mfhd_uuid] = new_map.get(mfhd_uuid, []) + [instance_uuid]
-            else:
-                raise TransformationRecordFailedError(
-                    entry["MFHD_ID"],
-                    "Boundwith relationship map contains a BIB_ID id not in the instance id map. No boundwith holdings created.",
-                    entry["BIB_ID"],
-                )
-
-        return new_map
-
     def save_id_map_file(self, path, legacy_map: dict):
         with open(path, "w") as legacy_map_file:
             for id_string in legacy_map.values():
@@ -417,7 +381,7 @@ class MapperBase:
             "holdingsRecordId": bound_with_holding_uuid,
             "itemId": str(
                 FolioUUID(
-                    self.folio_client.okapi_url,
+                    self.base_string_for_folio_uuid,
                     FOLIONamespaces.items,
                     legacy_item_id,
                 )
@@ -470,12 +434,21 @@ class MapperBase:
     def generate_boundwith_holding_uuid(self, holding_uuid, instance_uuid):
         return str(
             FolioUUID(
-                self.folio_client.okapi_url,
+                self.base_string_for_folio_uuid,
                 FOLIONamespaces.holdings,
                 f"{holding_uuid}-{instance_uuid}",
             )
         )
 
+    @property
+    def base_string_for_folio_uuid(self):
+        if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
+            return str(self.folio_client.okapi_url)
+        elif self.library_configuration.ecs_tenant_id:
+            return str(self.library_configuration.ecs_tenant_id)
+        else:
+            return str(self.library_configuration.tenant_id)
+
     @staticmethod
     def validate_location_map(location_map: List[Dict], locations: List[Dict]) -> List[Dict]:
         mapped_codes = [x['folio_code'] for x in location_map]

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/mapping_file_transformation/courses_mapper.py

@@ -121,7 +121,7 @@ class CoursesMapper(MappingFileMapperBase):
     def get_uuid(self, composite_course, object_type: FOLIONamespaces, idx: int = 0):
         return str(
             FolioUUID(
-                self.folio_client.okapi_url,
+                self.base_string_for_folio_uuid,
                 object_type,
                 composite_course[1] if idx == 0 else f"{composite_course[1]}_{idx}",
             )

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/mapping_file_transformation/item_mapper.py

@@ -42,7 +42,6 @@ class ItemMapper(MappingFileMapperBase):
         temporary_loan_type_mapping,
         temporary_location_mapping,
         library_configuration: LibraryConfiguration,
-        boundwith_relationship_map,
         task_configuration: AbstractTaskConfiguration,
     ):
         item_schema = folio_client.get_item_schema()
@@ -99,9 +98,6 @@ class ItemMapper(MappingFileMapperBase):
             "name",
             "PermanentLoanTypeMapping",
         )
-        self.boundwith_relationship_map = self.setup_boundwith_relationship_map(
-            boundwith_relationship_map
-        )
 
         self.material_type_mapping = RefDataMapping(
             self.folio_client,

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py

@@ -208,7 +208,7 @@ class MappingFileMapperBase(MapperBase):
             )
         generated_id = str(
             FolioUUID(
-                self.folio_client.okapi_url,
+                self.base_string_for_folio_uuid,
                 object_type,
                 legacy_id,
             )

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/mapping_file_transformation/user_mapper.py

@@ -158,7 +158,7 @@ class UserMapper(MappingFileMapperBase):
                 self.departments_mapping,
                 legacy_user,
                 index_or_id,
-                False,
+                True,
             )
         elif folio_prop_name in ["expirationDate", "enrollmentDate", "personal.dateOfBirth"]:
             return self.get_parsed_date(mapped_value, folio_prop_name)

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py

@@ -133,7 +133,7 @@ class AuthorityMapper(RulesMapperBase):
         folio_authority = {}
         folio_authority["id"] = str(
             FolioUUID(
-                str(self.folio_client.okapi_url),
+                self.base_string_for_folio_uuid,
                 FOLIONamespaces.authorities,
                 str(legacy_ids[-1]),
             )

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_base.py

@@ -810,8 +810,8 @@ class RulesMapperBase(MapperBase):
             )
         data_import_marc_file.write(marc_record.as_marc())
 
-    @staticmethod
     def save_source_record(
+        self,
         srs_records_file,
         record_type: FOLIONamespaces,
         folio_client: FolioClient,
@@ -831,7 +831,7 @@ class RulesMapperBase(MapperBase):
             legacy_ids (List[str]): _description_
             suppress (bool): _description_
         """
-        srs_id = RulesMapperBase.create_srs_id(record_type, folio_client.okapi_url, legacy_ids[-1])
+        srs_id = self.create_srs_id(record_type, legacy_ids[-1])
 
         marc_record.add_ordered_field(
             Field(
@@ -850,7 +850,7 @@ class RulesMapperBase(MapperBase):
             logging.exception(
                 "Something is wrong with the marc record's leader: %s, %s", marc_record.leader, ee
             )
-        srs_record_string = RulesMapperBase.get_srs_string(
+        srs_record_string = self.get_srs_string(
             marc_record,
             folio_record,
             srs_id,
@@ -859,8 +859,7 @@ class RulesMapperBase(MapperBase):
         )
         srs_records_file.write(f"{srs_record_string}\n")
 
-    @staticmethod
-    def create_srs_id(record_type, okapi_url: str, legacy_id: str):
+    def create_srs_id(self, record_type, legacy_id: str):
         srs_types = {
             FOLIONamespaces.holdings: FOLIONamespaces.srs_records_holdingsrecord,
             FOLIONamespaces.instances: FOLIONamespaces.srs_records_bib,
@@ -868,7 +867,13 @@ class RulesMapperBase(MapperBase):
             FOLIONamespaces.edifact: FOLIONamespaces.srs_records_edifact,
         }
 
-        return str(FolioUUID(okapi_url, srs_types.get(record_type), legacy_id))
+        return str(
+            FolioUUID(
+                self.base_string_for_folio_uuid,
+                srs_types.get(record_type),
+                legacy_id
+            )
+        )
 
     @staticmethod
     def get_bib_id_from_907y(marc_record: Record, index_or_legacy_id):

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -71,7 +71,7 @@ class BibsRulesMapper(RulesMapperBase):
         folio_instance = {}
         folio_instance["id"] = str(
             FolioUUID(
-                str(self.folio_client.okapi_url),
+                self.base_string_for_folio_uuid,
                 FOLIONamespaces.instances,
                 str(legacy_ids[-1]),
             )

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -211,7 +211,7 @@ class RulesMapperHoldings(RulesMapperBase):
         folio_holding: dict = {}
         folio_holding["id"] = str(
             FolioUUID(
-                str(self.folio_client.okapi_url),
+                self.base_string_for_folio_uuid,
                 FOLIONamespaces.holdings,
                 str(legacy_ids[0]),
             )
@@ -461,3 +461,54 @@ class RulesMapperHoldings(RulesMapperBase):
                 idx, f"No legacy id found in record from {marc_path}", ""
             )
         return results
+
+    def verity_boundwith_map_entry(self, entry):
+        if "MFHD_ID" not in entry or not entry.get("MFHD_ID", ""):
+            raise TransformationProcessError(
+                "", "Column MFHD_ID missing from Boundwith relationship map", ""
+            )
+        if "BIB_ID" not in entry or not entry.get("BIB_ID", ""):
+            raise TransformationProcessError(
+                "", "Column BIB_ID missing from Boundwith relationship map", ""
+            )
+
+    def setup_boundwith_relationship_map(self, boundwith_relationship_map):
+        """
+        Creates a map of MFHD_ID to BIB_ID for boundwith relationships.
+
+        Arguments:
+            boundwith_relationship_map: A list of dictionaries containing the MFHD_ID and BIB_ID.
+
+        Returns:
+            A dictionary mapping MFHD_ID to a list of BIB_IDs.
+
+        Raises:
+            TransformationProcessError: If MFHD_ID or BIB_ID is missing from the entry or if the instance_uuid is not in the parent_id_map.
+            TransformationRecordFailedError: If BIB_ID is not in the instance id map.
+        """
+        new_map = {}
+        for idx, entry in enumerate(boundwith_relationship_map):
+            self.verity_boundwith_map_entry(entry)
+            mfhd_uuid = str(
+                FolioUUID(
+                    self.base_string_for_folio_uuid,
+                    FOLIONamespaces.holdings,
+                    entry["MFHD_ID"],
+                )
+            )
+            try:
+                parent_id_tuple = self.get_bw_instance_id_map_tuple(entry)
+                new_map[mfhd_uuid] = new_map.get(mfhd_uuid, []) + [parent_id_tuple[1]]
+            except TransformationRecordFailedError as trfe:
+                self.handle_transformation_record_failed_error(idx, trfe)
+        return new_map
+
+    def get_bw_instance_id_map_tuple(self, entry):
+        try:
+            return self.parent_id_map[entry["BIB_ID"]]
+        except KeyError:
+            raise TransformationRecordFailedError(
+                entry["MFHD_ID"],
+                "Boundwith relationship map contains a BIB_ID id not in the instance id map. No boundwith holdings created for this BIB_ID.",
+                entry["BIB_ID"],
+            )

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/holdings_csv_transformer.py

@@ -182,7 +182,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
                 self.load_mapped_fields(),
                 self.load_location_map(),
                 self.load_call_number_type_map(),
-                self.load_id_map(self.folder_structure.instance_id_map_path, True),
+                self.load_instance_id_map(True),
                 library_config,
             )
             self.holdings = {}
@@ -296,18 +296,6 @@ class HoldingsCsvTransformer(MigrationTaskBase):
             )
             return holdings_map
 
-    def load_instance_id_map(self):
-        res = {}
-        with open(self.folder_structure.instance_id_map_path, "r") as instance_id_map_file:
-            for index, json_string in enumerate(instance_id_map_file):
-                # Format:{"legacy_id", "folio_id","instanceLevelCallNumber"}
-                if index % 500000 == 0:
-                    print(f"{index} instance ids loaded to map", end="\r")
-                map_object = json.loads(json_string)
-                res[map_object["legacy_id"]] = map_object
-        logging.info("Loaded %s migrated instance IDs", (index + 1))
-        return res
-
     def do_work(self):
         logging.info("Starting....")
         for file_def in self.task_config.files:

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/holdings_marc_transformer.py

@@ -229,9 +229,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         self.check_source_files(
             self.folder_structure.legacy_records_folder, self.task_config.files
         )
-        self.instance_id_map = self.load_id_map(
-            self.folder_structure.instance_id_map_path, True
-        )
+        self.instance_id_map = self.load_instance_id_map(True)
         self.mapper = RulesMapperHoldings(
             self.folio_client,
             self.location_map,
@@ -283,6 +281,17 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         logging.info("Done. Transformer Wrapping up...")
         self.extradata_writer.flush()
         self.processor.wrap_up()
+        if self.mapper.boundwith_relationship_map:
+            with open(
+                self.folder_structure.boundwith_relationships_map_path, "w+"
+            ) as boundwith_relationship_file:
+                logging.info(
+                    "Writing boundwiths relationship map to %s",
+                    boundwith_relationship_file.name,
+                )
+                for key, val in self.mapper.boundwith_relationship_map.items():
+                    boundwith_relationship_file.write(json.dumps((key, val)) + "\n")
+
         with open(self.folder_structure.migration_reports_file, "w+") as report_file:
             self.mapper.migration_report.write_migration_report(
                 i18n.t("Bibliographic records transformation report"),

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -233,7 +233,8 @@ class ItemsTransformer(MigrationTaskBase):
         ).is_file():
             temporary_loan_type_mapping = self.load_ref_data_mapping_file(
                 "temporaryLoanTypeId",
-                self.folder_structure.temp_loan_type_map_path,
+                self.folder_structure.mapping_files_folder
+                / self.task_config.temp_loan_types_map_file_name,
                 self.folio_keys,
             )
         else:
@@ -243,20 +244,9 @@ class ItemsTransformer(MigrationTaskBase):
             )
             temporary_loan_type_mapping = None
         # Load Boundwith relationship map
-        self.boundwith_relationship_map = []
+        self.boundwith_relationship_map = {}
         if self.task_config.boundwith_relationship_file_path:
-            with open(
-                self.folder_structure.data_folder
-                / FOLIONamespaces.holdings.name
-                / self.task_config.boundwith_relationship_file_path
-            ) as boundwith_relationship_file:
-                self.boundwith_relationship_map = list(
-                    csv.DictReader(boundwith_relationship_file, dialect="tsv")
-                )
-            logging.info(
-                "Rows in Bound with relationship map: %s", len(self.boundwith_relationship_map)
-            )
-
+            self.load_boundwith_relationships()
         if (
             self.folder_structure.mapping_files_folder
             / self.task_config.temp_location_map_file_name
@@ -312,7 +302,6 @@ class ItemsTransformer(MigrationTaskBase):
             temporary_loan_type_mapping,
             temporary_location_mapping,
             self.library_configuration,
-            self.boundwith_relationship_map,
             self.task_configuration
         )
         if (
@@ -367,9 +356,9 @@ class ItemsTransformer(MigrationTaskBase):
                     self.mapper.perform_additional_mappings(folio_rec, file_def)
                     self.handle_circiulation_notes(folio_rec, self.folio_client.current_user)
                     self.handle_notes(folio_rec)
-                    if folio_rec["holdingsRecordId"] in self.mapper.boundwith_relationship_map:
+                    if folio_rec["holdingsRecordId"] in self.boundwith_relationship_map:
                         for idx_, instance_id in enumerate(
-                            self.mapper.boundwith_relationship_map.get(
+                            self.boundwith_relationship_map.get(
                                 folio_rec["holdingsRecordId"]
                             )
                         ):
@@ -456,6 +445,22 @@ class ItemsTransformer(MigrationTaskBase):
         else:
             del folio_rec["circulationNotes"]
 
+    def load_boundwith_relationships(self):
+        try:
+            with open(
+                    self.folder_structure.boundwith_relationships_map_path
+            ) as boundwith_relationship_file:
+                self.boundwith_relationship_map = dict(
+                        json.loads(x) for x in boundwith_relationship_file
+                    )
+            logging.info(
+                    "Rows in Bound with relationship map: %s", len(self.boundwith_relationship_map)
+                )
+        except FileNotFoundError:
+            raise TransformationProcessError(
+                    "", "Boundwith relationship file specified, but relationships file from holdings transformation not found. ", self.folder_structure.boundwith_relationships_map_path
+            )
+
     def wrap_up(self):
         logging.info("Done. Transformer wrapping up...")
         self.extradata_writer.flush()

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/loans_migrator.py

@@ -14,12 +14,12 @@ from pydantic import Field
 import i18n
 from dateutil import parser as du_parser
 from folio_uuid.folio_namespaces import FOLIONamespaces
+from art import tprint
 
 from folio_migration_tools.circulation_helper import CirculationHelper
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
     FileDefinition,
-    FolioRelease,
     LibraryConfiguration,
 )
 from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
@@ -781,11 +781,4 @@ def timings(t0, t0func, num_objects):
 
 
 def print_smtp_warning():
-    s = r"""
-          _____   __  __   _____   ______   ___
-         / ____| |  \/  | |_   _| |  __  | |__ \\
-        | (___   | \  / |   | |   | |__|_|    ) |
-         \___ \  | |\/| |   | |   | |        / /
-        |_____/  |_|  |_|   |_|   |_|       (_)
-    """  # noqa: E501, W605
-    print(s)
+    tprint("\nSMTP?\n", space=2)

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/migration_task_base.py

@@ -9,6 +9,7 @@ from abc import abstractmethod
 from datetime import datetime, timezone
 from genericpath import isfile
 from pathlib import Path
+from typing import Optional
 
 import folioclient
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -54,6 +55,15 @@ class MigrationTaskBase:
             {"x-okapi-tenant": self.ecs_tenant_id} if self.ecs_tenant_id else {}
         )
         self.folio_client.okapi_headers.update(self.ecs_tenant_header)
+        self.central_folder_structure: Optional[FolderStructure] = None
+        if library_configuration.is_ecs and library_configuration.ecs_central_iteration_identifier:
+            self.central_folder_structure = FolderStructure(
+                library_configuration.base_folder,
+                FOLIONamespaces.instances,
+                task_configuration.name,
+                library_configuration.ecs_central_iteration_identifier,
+                library_configuration.add_time_stamp_to_file_names,
+            )
         self.folder_structure: FolderStructure = FolderStructure(
             library_configuration.base_folder,
             self.get_object_type(),
@@ -66,6 +76,8 @@ class MigrationTaskBase:
         self.object_type = self.get_object_type()
         try:
             self.folder_structure.setup_migration_file_structure()
+            if self.central_folder_structure:
+                self.central_folder_structure.setup_migration_file_structure()
             # Initiate Worker
         except FileNotFoundError as fne:
             logging.error(fne)
@@ -143,15 +155,48 @@ class MigrationTaskBase:
         for filename in files:
             logging.info("\t%s", filename)
 
+    def load_instance_id_map(self, raise_if_empty=True) -> dict:
+        """
+        This method handles loading instance id maps for holdings and other transformations that require it.
+        This is in the base class because multiple tasks need it. It exists because instances in an ECS environment
+        are transformed for the central and data tenants separately, but the data tenants need to know about
+        the central tenant instance ids. This is a bit of a hack, but it works for now.
+        """
+        map_files = []
+        instance_id_map = {}
+        if self.library_configuration.is_ecs and self.central_folder_structure:
+            logging.info(
+                "Loading ECS central tenant instance id map from %s", self.central_folder_structure.instance_id_map_path
+            )
+            instance_id_map = self.load_id_map(
+                self.central_folder_structure.instance_id_map_path,
+                raise_if_empty=False,
+            )
+            map_files.append(str(self.central_folder_structure.instance_id_map_path))
+            logging.info(
+                "Loading member tenant isntance id map from %s",
+                self.folder_structure.instance_id_map_path
+            )
+        instance_id_map = self.load_id_map(
+            self.folder_structure.instance_id_map_path,
+            raise_if_empty=False,
+            existing_id_map=instance_id_map,
+        )
+        map_files.append(str(self.folder_structure.instance_id_map_path))
+        if not any(instance_id_map) and raise_if_empty:
+            map_file_paths = ", ".join(map_files)
+            raise TransformationProcessError("", "Instance id map is empty", map_file_paths)
+        return instance_id_map
+
     @staticmethod
-    def load_id_map(map_path, raise_if_empty=False):
+    def load_id_map(map_path, raise_if_empty=False, existing_id_map={}):
         if not isfile(map_path):
-            logging.warn(
+            logging.warning(
                 "No legacy id map found at %s. Will build one from scratch", map_path
             )
             return {}
-        id_map = {}
-        loaded_rows = 0
+        id_map = existing_id_map
+        loaded_rows = len(id_map)
         with open(map_path) as id_map_file:
             for index, json_string in enumerate(id_map_file, start=1):
                 loaded_rows = index
@@ -159,12 +204,12 @@ class MigrationTaskBase:
                 map_tuple = json.loads(json_string)
                 if loaded_rows % 500000 == 0:
                     print(
-                        f"{loaded_rows + 1} ids loaded to map. Last Id: {map_tuple[0]}",
+                        f"{loaded_rows + 1} ids loaded to map. Last Id: {map_tuple[0]}          ",
                         end="\r",
                     )
 
                 id_map[map_tuple[0]] = map_tuple
-        logging.info("Loaded %s migrated IDs", loaded_rows)
+        logging.info("Loaded %s migrated IDs from %s", loaded_rows, id_map_file.name)
         if not any(id_map) and raise_if_empty:
             raise TransformationProcessError("", "Legacy id map is empty", map_path)
         return id_map

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/orders_transformer.py

@@ -177,7 +177,7 @@ class OrdersTransformer(MigrationTaskBase):
             self.library_configuration,
             self.orders_map,
             self.load_id_map(self.folder_structure.organizations_id_map_path, True),
-            self.load_id_map(self.folder_structure.instance_id_map_path, True),
+            self.load_instance_id_map(True),
             self.load_ref_data_mapping_file(
                 "acquisitionMethod",
                 self.folder_structure.mapping_files_folder

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/migration_tasks/user_transformer.py

@@ -6,6 +6,7 @@ from pydantic import Field
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
+from art import tprint
 
 from folio_migration_tools.custom_exceptions import (
     TransformationProcessError,
@@ -282,16 +283,7 @@ class UserTransformer(MigrationTaskBase):
 
 
 def print_email_warning():
-    s = (
-        "  ______   __  __              _____   _         _____     ___  \n"  # noqa: E501, W605
-        " |  ____| |  \\/  |     /\\     |_   _| | |       / ____|   |__ \\ \n"  # noqa: E501, W605
-        " | |__    | \\  / |    /  \\      | |   | |      | (___        ) |\n"  # noqa: E501, W605
-        " |  __|   | |\\/| |   / /\\ \\     | |   | |       \\___ \\      / / \n"  # noqa: E501, W605
-        " |______| |_|  |_| /_/    \\_\\ |_____| |______| |_____/     (_)  \n"  # noqa: E501, W605
-        "                                                                \n"  # noqa: E501, W605
-        "                                                       \n"
-    )
-    print(s)
+    tprint("\nEMAILS?\n", space=2)
 
 
 def remove_empty_addresses(folio_user):

{folio_migration_tools-1.9.0rc7 → folio_migration_tools-1.9.0rc9}/src/folio_migration_tools/test_infrastructure/mocked_classes.py

@@ -12,6 +12,10 @@ from folio_migration_tools.mapping_file_transformation.holdings_mapper import (
     HoldingsMapper,
 )
 from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.library_configuration import (
+    LibraryConfiguration,
+    FolioRelease,
+)
 
 
 def mocked_holdings_mapper() -> Mock:
@@ -242,3 +246,62 @@ def folio_get_single_object_mocked(*args, **kwargs):
 
 def folio_get_from_github(owner, repo, file_path):
     return FolioClient.get_latest_from_github(owner, repo, file_path, "")
+
+OKAPI_URL = "http://localhost:9130"
+LIBRARY_NAME = "Test Library"
+
+def get_mocked_library_config():
+    return LibraryConfiguration(
+        okapi_url=OKAPI_URL,
+        tenant_id="test_tenant",
+        okapi_username="test_user",
+        okapi_password="test_password",
+        base_folder=Path("."),
+        library_name=LIBRARY_NAME,
+        log_level_debug=False,
+        folio_release=FolioRelease.sunflower,
+        iteration_identifier="test_iteration"
+    )
+
+def get_mocked_ecs_central_libarary_config():
+    return LibraryConfiguration(
+        okapi_url=OKAPI_URL,
+        tenant_id="test_tenant",
+        okapi_username="test_user",
+        okapi_password="test_password",
+        base_folder=Path("."),
+        library_name=LIBRARY_NAME,
+        log_level_debug=False,
+        folio_release=FolioRelease.sunflower,
+        iteration_identifier="central_iteration",
+        is_ecs=True,
+    )
+
+def get_mocked_ecs_member_libarary_config():
+    return LibraryConfiguration(
+        okapi_url=OKAPI_URL,
+        tenant_id="test_tenant",
+        ecs_tenant_id="test_ecs_tenant",
+        okapi_username="test_user",
+        okapi_password="test_password",
+        base_folder=Path("."),
+        library_name=LIBRARY_NAME,
+        log_level_debug=False,
+        folio_release=FolioRelease.sunflower,
+        iteration_identifier="member_iteration",
+        ecs_central_iteration_identifier="central_iteration",
+        is_ecs=True,
+    )
+
+def get_mocked_folder_structure():
+    mock_fs = MagicMock()
+    mock_fs.mapping_files = Path("mapping_files")
+    mock_fs.results_folder = Path("results")
+    mock_fs.legacy_records_folder = Path("source_files")
+    mock_fs.logs_folder = Path("logs")
+    mock_fs.migration_reports_file = Path("/dev/null")
+    mock_fs.transformation_extra_data_path = Path("transformation_extra_data")
+    mock_fs.transformation_log_path = Path("/dev/null")
+    mock_fs.data_issue_file_path = Path("/dev/null")
+    mock_fs.failed_marc_recs_file = Path("failed_marc_recs.txt")
+    return mock_fs