folio-migration-tools 1.10.0__tar.gz → 1.10.0b1__tar.gz

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: folio-migration-tools
-Version: 1.10.0
+Version: 1.10.0b1
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
 Author: Theodor Tolstoy, Lisa Sjögren, Brooks Travis, Jeremy Nelson, Clinton Bradford
 Author-email: Theodor Tolstoy <github.teddes@tolstoy.se>, Brooks Travis <brooks.travis@gmail.com>
 License-Expression: MIT
-Requires-Dist: folioclient>=1.0.4
+Requires-Dist: folioclient>=1.0.1
 Requires-Dist: pyhumps>=3.7.3,<4.0.0
 Requires-Dist: defusedxml>=0.7.1,<1.0.0
 Requires-Dist: python-dateutil>=2.8.2,<3.0.0
@@ -18,7 +18,6 @@ Requires-Dist: deepdiff>=6.2.3,<7.0.0
 Requires-Dist: pyaml>=21.10.1,<22.0.0
 Requires-Dist: python-i18n>=0.3.9,<1.0.0
 Requires-Dist: art>=6.5,<7.0
-Requires-Dist: folio-data-import>=0.5.0,<0.6.0
 Requires-Python: >=3.10, <4.0
 Project-URL: Documentation, https://folio-migration-tools.readthedocs.io/en/latest/
 Project-URL: Homepage, https://github.com/folio-fse/folio_migration_tools

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.10.0"
+version = "1.10.0b1"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -14,19 +14,19 @@ readme = "README.md"
 keywords = ["FOLIO", "ILS", "LSP", "Library Systems", "MARC21", "Library data"]
 requires-python = ">=3.10,<4.0"
 dependencies = [
-    "folioclient>=1.0.4",
-    "pyhumps>=3.7.3,<4.0.0",
-    "defusedxml>=0.7.1,<1.0.0",
-    "python-dateutil>=2.8.2,<3.0.0",
-    "folio-uuid>=1.0.0,<2.0.0",
-    "pymarc>=5.2.3,<6.0.0",
-    "pydantic>=2.12.3,<3.0.0",
-    "argparse-prompt>=0.0.5,<1.0.0",
-    "deepdiff>=6.2.3,<7.0.0",
-    "pyaml>=21.10.1,<22.0.0",
-    "python-i18n>=0.3.9,<1.0.0",
-    "art>=6.5,<7.0",
-    "folio-data-import>=0.5.0,<0.6.0",
+    "folioclient>=1.0.1",
+	"pyhumps>=3.7.3,<4.0.0",
+	"defusedxml>=0.7.1,<1.0.0",
+	"python-dateutil>=2.8.2,<3.0.0",
+	"folio-uuid>=1.0.0,<2.0.0",
+	"pymarc>=5.2.3,<6.0.0",
+	"pydantic>=2.12.3,<3.0.0",
+	"argparse-prompt>=0.0.5,<1.0.0",
+	"deepdiff>=6.2.3,<7.0.0",
+	"pyaml>=21.10.1,<22.0.0",
+	"python-i18n>=0.3.9,<1.0.0",
+	"art>=6.5,<7.0",
+	# "folio-data-import>=0.4.1"
 ]
 
 [dependency-groups]
@@ -48,8 +48,8 @@ dev = [
 	"ipykernel>=6.29.5,<7.0.0",
 	"pytest-asyncio>=0.23.0,<0.24.0",
 	"lxml>=6.0.2",
+	"ty>=0.0.1a27",
 	"pyrefly>=0.43.1",
-	"ruff>=0.14.13",
 ]
 docs = [
 	"m2r>=0.2.1,<0.3.0",

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/__main__.py

@@ -4,7 +4,6 @@ import logging
 import sys
 from os import environ
 from pathlib import Path
-from warnings import warn
 
 import httpx
 import humps
@@ -127,14 +126,6 @@ def main():
         i18n.set("locale", args.report_language)
         config_file, library_config = prep_library_config(args)
         try:
-            if args.task_name == "AuthorityTransformer":
-                warn(
-                    "The AuthorityTransformer has been removed."
-                    " Please update your configuration accordingly."
-                    " Use Data Import to load authority records.",
-                    DeprecationWarning,
-                    stacklevel=2,
-                )
             migration_task_config = next(
                 t for t in config_file["migration_tasks"] if t["name"] == args.task_name
             )

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/circulation_helper.py

@@ -11,7 +11,6 @@ import i18n
 from folioclient import FolioClient, FolioClientError, FolioConnectionError, FolioValidationError
 
 from folio_migration_tools.helper import Helper
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.migration_report import MigrationReport
 from folio_migration_tools.transaction_migration.legacy_loan import LegacyLoan
 from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
@@ -38,7 +37,7 @@ class CirculationHelper:
     def get_user_by_barcode(self, user_barcode):
         if user_barcode in self.missing_patron_barcodes:
             self.migration_report.add_general_statistics(
-                i18n_t("Users already detected as missing")
+                i18n.t("Users already detected as missing")
             )
             logging.info("User is already detected as missing")
             return {}
@@ -56,7 +55,7 @@ class CirculationHelper:
     def get_item_by_barcode(self, item_barcode):
         if item_barcode in self.missing_item_barcodes:
             self.migration_report.add_general_statistics(
-                i18n_t("Items already detected as missing")
+                i18n.t("Items already detected as missing")
             )
             logging.info("Item is already detected as missing")
             return {}
@@ -141,7 +140,7 @@ class CirculationHelper:
         path = "/circulation/check-out-by-barcode"
         try:
             if legacy_loan.patron_barcode in self.missing_patron_barcodes:
-                error_message = i18n_t("Patron barcode already detected as missing")
+                error_message = i18n.t("Patron barcode already detected as missing")
                 logging.error(
                     f"{error_message} Patron barcode: {legacy_loan.patron_barcode} "
                     f"Item Barcode:{legacy_loan.item_barcode}"
@@ -190,7 +189,7 @@ class CirculationHelper:
             elif "find user with matching barcode" in error_message_from_folio:
                 self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
                 error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
-                stat_message = i18n_t("Patron barcode not in FOLIO")
+                stat_message = i18n.t("Patron barcode not in FOLIO")
                 return TransactionResult(
                     False,
                     False,
@@ -249,7 +248,7 @@ class CirculationHelper:
                 False,
                 None,
                 "Connection error",
-                i18n_t("Connection error during checkout"),
+                i18n.t("Connection error during checkout"),
             )
 
     @staticmethod

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/folder_structure.py

@@ -22,8 +22,6 @@ class FolderStructure:
         self.add_time_stamp_to_file_names = add_time_stamp_to_file_names
         self.iteration_identifier = iteration_identifier
         self.base_folder = Path(base_path)
-        # Ensure the base folder exists and is a directory. This differs from other folders, which
-        # are created if missing.
         if not self.base_folder.is_dir():
             logging.critical("Base Folder Path is not a folder. Exiting.")
             sys.exit(1)
@@ -45,10 +43,6 @@ class FolderStructure:
         self.reports_folder = self.iteration_folder / "reports"
         self.verify_folder(self.reports_folder)
 
-        # Raw migration reports directory
-        self.raw_reports_folder = self.reports_folder / ".raw"
-        self.verify_folder(self.raw_reports_folder)
-
     def log_folder_structure(self):
         logging.info("Mapping files folder is %s", self.mapping_files_folder)
         logging.info("Git ignore is set up correctly")
@@ -104,10 +98,6 @@ class FolderStructure:
 
         self.migration_reports_file = self.reports_folder / f"report{self.file_template}.md"
 
-        self.migration_reports_raw_file = (
-            self.raw_reports_folder / f"raw_report{self.file_template}.json"
-        )
-
         self.srs_records_path = (
             self.results_folder / f"folio_srs_{object_type_string}{self.file_template}.json"
         )
@@ -120,6 +110,9 @@ class FolderStructure:
         self.instance_id_map_path = (
             self.results_folder / f"{str(FOLIONamespaces.instances.name).lower()}_id_map.json"
         )
+        self.auth_id_map_path = (
+            self.results_folder / f"{str(FOLIONamespaces.authorities.name).lower()}_id_map.json"
+        )
 
         self.holdings_id_map_path = (
             self.results_folder / f"{str(FOLIONamespaces.holdings.name).lower()}_id_map.json"
@@ -138,13 +131,10 @@ class FolderStructure:
         self.item_statuses_map_path = self.mapping_files_folder / "item_statuses.tsv"
 
     def verify_folder(self, folder_path: Path):
-        if folder_path.exists() and not folder_path.is_dir():
-            logging.critical("Path exists but is not a directory: %s", folder_path)
+        if not folder_path.is_dir():
+            logging.critical("There is no folder located at %s. Exiting.", folder_path)
+            logging.critical("Create a folder by calling\n\tmkdir %s", folder_path)
             sys.exit(1)
-
-        if not folder_path.exists():
-            logging.info("Creating missing folder %s", folder_path)
-            folder_path.mkdir(parents=True, exist_ok=True)
         else:
             logging.info("Located %s", folder_path)
 

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/helper.py

@@ -1,7 +1,6 @@
 import json
 import logging
-
-from folio_migration_tools.i18n_cache import i18n_t
+import i18n
 
 
 class Helper:
@@ -10,15 +9,15 @@ class Helper:
         report_file, total_records: int, mapped_folio_fields, mapped_legacy_fields
     ):
         details_start = (
-            "<details><summary>" + i18n_t("Click to expand field report") + "</summary>\n\n"
+            "<details><summary>" + i18n.t("Click to expand field report") + "</summary>\n\n"
         )
         details_end = "</details>\n"
-        report_file.write("\n## " + i18n_t("Mapped FOLIO fields") + "\n")
+        report_file.write("\n## " + i18n.t("Mapped FOLIO fields") + "\n")
         # report_file.write(f"{blurbs[header]}\n")
 
         d_sorted = {k: mapped_folio_fields[k] for k in sorted(mapped_folio_fields)}
         report_file.write(details_start)
-        columns = [i18n_t("FOLIO Field"), i18n_t("Mapped"), i18n_t("Unmapped")]
+        columns = [i18n.t("FOLIO Field"), i18n.t("Mapped"), i18n.t("Unmapped")]
         report_file.write(" | ".join(columns) + "\n")
         report_file.write("|".join(len(columns) * ["---"]) + "\n")
         for k, v in d_sorted.items():
@@ -33,12 +32,12 @@ class Helper:
             )
         report_file.write(details_end)
 
-        report_file.write("\n## " + i18n_t("Mapped Legacy fields") + "\n")
+        report_file.write("\n## " + i18n.t("Mapped Legacy fields") + "\n")
         # report_file.write(f"{blurbs[header]}\n")
 
         d_sorted = {k: mapped_legacy_fields[k] for k in sorted(mapped_legacy_fields)}
         report_file.write(details_start)
-        columns = [i18n_t("Legacy Field"), i18n_t("Present"), i18n_t("Mapped"), i18n_t("Unmapped")]
+        columns = [i18n.t("Legacy Field"), i18n.t("Present"), i18n.t("Mapped"), i18n.t("Unmapped")]
         report_file.write("|".join(columns) + "\n")
         report_file.write("|".join(len(columns) * ["---"]) + "\n")
         for k, v in d_sorted.items():

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/holdings_helper.py

@@ -5,7 +5,6 @@ from uuid import uuid4
 
 from folio_migration_tools import custom_exceptions
 from folio_migration_tools import helper
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.migration_report import MigrationReport
 
 
@@ -55,7 +54,7 @@ class HoldingsHelper:
                 values.append(str(uuid4()))
                 migration_report.add(
                     "HoldingsMerging",
-                    i18n_t("Holding prevented from merging by holdingsTypeId"),
+                    i18n.t("Holding prevented from merging by holdingsTypeId"),
                 )
             return "-".join(values)
         except Exception as exception:
@@ -100,12 +99,12 @@ class HoldingsHelper:
                     )
                     migration_report.add(
                         "HoldingsMerging",
-                        i18n_t("Duplicate key based on current merge criteria. Records merged"),
+                        i18n.t("Duplicate key based on current merge criteria. Records merged"),
                     )
                 else:
                     migration_report.add(
                         "HoldingsMerging",
-                        i18n_t("Previously transformed holdings record loaded"),
+                        i18n.t("Previously transformed holdings record loaded"),
                     )
                     prev_holdings[stored_key] = stored_holding
             return prev_holdings

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/library_configuration.py

@@ -62,18 +62,6 @@ class FileDefinition(BaseModel):
             ),
         ),
     ] = True
-    data_import_marc: Annotated[
-        bool,
-        Field(
-            title="Data import MARC",
-            description=(
-                "If set to true, successfully processed MARC records from this file will "
-                "be included in the MARC file for data import generated by the parent task for "
-                "inventory records created from this file. Only applied for MARC-based "
-                "transformations."
-            ),
-        ),
-    ] = True
 
 
 class IlsFlavour(str, Enum):

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/mapper_base.py

@@ -21,7 +21,6 @@ from folio_migration_tools.custom_exceptions import (
 )
 from folio_migration_tools.extradata_writer import ExtradataWriter
 from folio_migration_tools.helper import Helper
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.library_configuration import FileDefinition, LibraryConfiguration
 from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
     RefDataMapping,
@@ -235,10 +234,10 @@ class MapperBase:
         self.migration_report.add("FieldMappingErrors", error)
         error.id = error.id or index_or_id
         error.log_it()
-        self.migration_report.add_general_statistics(i18n_t("Field Mapping Errors found"))
+        self.migration_report.add_general_statistics(i18n.t("Field Mapping Errors found"))
 
     def handle_transformation_process_error(self, idx, error: TransformationProcessError):
-        self.migration_report.add_general_statistics(i18n_t("Transformation process error"))
+        self.migration_report.add_general_statistics(i18n.t("Transformation process error"))
         logging.critical("%s\t%s", idx, error)
         print(f"\n{error.message}: {error.data_value}")
         sys.exit(1)
@@ -247,7 +246,7 @@ class MapperBase:
         self, records_processed: int, error: TransformationRecordFailedError
     ):
         self.migration_report.add(
-            "GeneralStatistics", i18n_t("FAILED Records failed due to an error")
+            "GeneralStatistics", i18n.t("FAILED Records failed due to an error")
         )
         error.index_or_id = error.index_or_id or records_processed
         error.log_it()
@@ -302,7 +301,7 @@ class MapperBase:
             for id_string in legacy_map.values():
                 legacy_map_file.write(f"{json.dumps(id_string)}\n")
                 self.migration_report.add(
-                    "GeneralStatistics", i18n_t("Unique ID:s written to legacy map")
+                    "GeneralStatistics", i18n.t("Unique ID:s written to legacy map")
                 )
         logging.info("Wrote legacy id map to %s", path)
 
@@ -358,7 +357,7 @@ class MapperBase:
     def add_legacy_id_to_admin_note(self, folio_record: dict, legacy_id: str):
         if not legacy_id:
             raise TransformationFieldMappingError(
-                legacy_id, i18n_t("Legacy id is empty"), legacy_id
+                legacy_id, i18n.t("Legacy id is empty"), legacy_id
             )
         if "administrativeNotes" not in folio_record:
             folio_record["administrativeNotes"] = []
@@ -498,7 +497,7 @@ class MapperBase:
             )
         self.migration_report.add(
             "StatisticalCodeMapping",
-            i18n_t("Mapping not set up"),
+            i18n.t("Mapping not set up"),
         )
         return ""
 

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/mapping_file_transformation/user_mapper.py

@@ -115,10 +115,6 @@ class UserMapper(MappingFileMapperBase):
 
         if self.task_config.remove_request_preferences:
             del clean_folio_object["requestPreference"]
-
-        if self.task_config.remove_username:
-            del clean_folio_object["username"]
-
         self.report_folio_mapping_no_schema(clean_folio_object)
         self.report_legacy_mapping_no_schema(legacy_user)
 

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/marc_rules_transformation/conditions.py

@@ -47,6 +47,8 @@ class Conditions:
         if object_type == "bibs":
             self.setup_reference_data_for_all()
             self.setup_reference_data_for_bibs()
+        elif object_type == "auth":
+            self.setup_reference_data_for_auth()
         else:
             self.setup_reference_data_for_all()
             self.setup_reference_data_for_items_and_holdings(default_call_number_type_name)
@@ -148,6 +150,15 @@ class Conditions:
         if not self.folio.class_types:
             raise TransformationProcessError("", "No class_types in FOLIO")
 
+    def setup_reference_data_for_auth(self):
+        self.authority_note_types = list(
+            self.folio.folio_get_all(
+                "/authority-note-types", "authorityNoteTypes", self.folio.cql_all, 1000
+            )
+        )
+        logging.info(f"{len(self.authority_note_types)} \tAuthority note types")
+        logging.info(f"{len(self.folio.identifier_types)} \tidentifier types")  # type: ignore
+
     def get_condition(
         self, name, legacy_id, value, parameter=None, marc_field: field.Field | None = None
     ):
@@ -419,6 +430,24 @@ class Conditions:
                 parameter.get("name", ""),
             ) from ee
 
+    def condition_set_authority_note_type_id(
+        self, legacy_id, _, parameter, marc_field: field.Field
+    ):
+        try:
+            t = self.get_ref_data_tuple_by_name(
+                self.authority_note_types, "authority_note_types", parameter["name"]
+            )
+            self.mapper.migration_report.add("MappedNoteTypes", t[1])
+            return t[0]
+        except Exception as ee:
+            logging.error(ee)
+            raise TransformationProcessError(
+                legacy_id,
+                f"Authority note type mapping error.\tParameter: {parameter.get('name', '')}\t"
+                f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
+                parameter.get("name", ""),
+            ) from ee
+
     def condition_set_classification_type_id(
         self, legacy_id, value, parameter, marc_field: field.Field
     ):

{folio_migration_tools-1.10.0 → folio_migration_tools-1.10.0b1}/src/folio_migration_tools/marc_rules_transformation/marc_file_processor.py

@@ -93,8 +93,8 @@ class MarcFileProcessor:
                             legacy_ids,
                             self.object_type,
                         )
-
-                    self.save_marc_record(marc_record, file_def, folio_rec, self.object_type)
+                    if getattr(self.mapper.task_configuration, "data_import_marc", False):
+                        self.save_marc_record(marc_record, folio_rec, self.object_type)
                 Helper.write_to_file(self.created_objects_file, folio_rec)
                 self.mapper.migration_report.add_general_statistics(
                     i18n.t("Inventory records written to disk")
@@ -131,23 +131,13 @@ class MarcFileProcessor:
                     ):
                         self.mapper.remove_from_id_map(folio_rec.get("formerIds", []))
 
-    def save_marc_record(
-        self,
-        marc_record: Record,
-        file_def: FileDefinition,
-        folio_rec: Dict,
-        object_type: FOLIONamespaces,
-    ):
-        if (
-            getattr(self.mapper.task_configuration, "data_import_marc", False)
-            and file_def.data_import_marc
-        ):
-            self.mapper.save_data_import_marc_record(
-                self.data_import_marc_file,
-                object_type,
-                marc_record,
-                folio_rec,
-            )
+    def save_marc_record(self, marc_record: Record, folio_rec: Dict, object_type: FOLIONamespaces):
+        self.mapper.save_data_import_marc_record(
+            self.data_import_marc_file,
+            object_type,
+            marc_record,
+            folio_rec,
+        )
 
     def save_srs_record(
         self,