folio-migration-tools 1.9.0rc9__tar.gz → 1.9.0rc11__tar.gz

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0rc9
+Version: 1.9.0rc11
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -20,7 +20,7 @@ Requires-Dist: art (>=6.5,<7.0)
 Requires-Dist: deepdiff (>=6.2.3,<7.0.0)
 Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
 Requires-Dist: folio-uuid (>=0.2.8,<0.3.0)
-Requires-Dist: folioclient (>=0.61.2,<0.62.0)
+Requires-Dist: folioclient (==0.70.1)
 Requires-Dist: pyaml (>=21.10.1,<22.0.0)
 Requires-Dist: pydantic (>=1.10.2,<2.0.0)
 Requires-Dist: pyhumps (>=3.7.3,<4.0.0)

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.9.0rc9"
+version = "1.9.0rc11"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -44,7 +44,7 @@ profile = "black"
 
 [tool.poetry.dependencies]
 python = "^3.9"
-folioclient = "^0.61.2"
+folioclient = "0.70.1"
 pyhumps = "^3.7.3"
 defusedxml = "^0.7.1"
 python-dateutil = "^2.8.2"

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/__main__.py

@@ -1,3 +1,4 @@
+from importlib import metadata
 import json
 import logging
 import sys
@@ -38,7 +39,7 @@ def parse_args(args):
         default=environ.get("FOLIO_MIGRATION_TOOLS_TASK_NAME"),
     )
     parser.add_argument(
-        "--okapi_password",
+        "--folio_password", "--okapi_password",
         help="password for the tenant in the configuration file",
         prompt="FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD" not in environ,
         default=environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
@@ -60,11 +61,17 @@ def parse_args(args):
         default=environ.get("FOLIO_MIGRATION_TOOLS_REPORT_LANGUAGE", "en"),
         prompt=False,
     )
+    parser.add_argument(
+        "--version", "-V",
+        help="Show the version of the FOLIO Migration Tools",
+        action="store_true",
+        prompt=False,
+    )
     return parser.parse_args(args)
 
 def prep_library_config(args):
         config_file_humped = merge_load(args.configuration_path)
-        config_file_humped["libraryInformation"]["okapiPassword"] = args.okapi_password
+        config_file_humped["libraryInformation"]["okapiPassword"] = args.folio_password
         config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
         config_file = humps.decamelize(config_file_humped)
         library_config = LibraryConfiguration(**config_file["library_information"])
@@ -78,11 +85,22 @@ def prep_library_config(args):
             sys.exit("ECS Central Iteration Identifier Not Found")
         return config_file, library_config
 
+def print_version(args):
+    if "-V" in args or "--version" in args:
+        print(
+            f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}"
+        )
+        sys.exit(0)
+    return None
+
 
 def main():
     try:
         task_classes = list(inheritors(migration_task_base.MigrationTaskBase))
+        # Check if the script is run with the --version or -V flag
+        print_version(sys.argv)
 
+        # Parse command line arguments
         args = parse_args(sys.argv[1:])
         try:
             i18n.load_config(
@@ -124,10 +142,10 @@ def main():
         try:
             logging.getLogger("httpx").setLevel(logging.WARNING) # Exclude info messages from httpx
             with FolioClient(
-                library_config.okapi_url,
+                library_config.gateway_url,
                 library_config.tenant_id,
-                library_config.okapi_username,
-                library_config.okapi_password,
+                library_config.folio_username,
+                library_config.folio_password,
             ) as folio_client:
                 task_config = task_class.TaskConfiguration(**migration_task_config)
                 task_obj = task_class(task_config, library_config, folio_client)

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/circulation_helper.py

@@ -138,7 +138,7 @@ class CirculationHelper:
         if legacy_loan.proxy_patron_barcode:
             data.update({"proxyUserBarcode": legacy_loan.proxy_patron_barcode})
         path = "/circulation/check-out-by-barcode"
-        url = f"{self.folio_client.okapi_url}{path}"
+        url = f"{self.folio_client.gateway_url}{path}"
         try:
             if legacy_loan.patron_barcode in self.missing_patron_barcodes:
                 error_message = i18n.t("Patron barcode already detected as missing")
@@ -249,7 +249,7 @@ class CirculationHelper:
     ):
         try:
             path = "/circulation/requests"
-            url = f"{folio_client.okapi_url}{path}"
+            url = f"{folio_client.gateway_url}{path}"
             data = legacy_request.serialize()
             data["requestProcessingParameters"] = {
                 "overrideBlocks": {
@@ -313,7 +313,7 @@ class CirculationHelper:
             del loan_to_put["metadata"]
             loan_to_put["dueDate"] = extension_due_date.isoformat()
             loan_to_put["loanDate"] = extend_out_date.isoformat()
-            url = f"{folio_client.okapi_url}/circulation/loans/{loan_to_put['id']}"
+            url = f"{folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
 
             req = httpx.put(
                 url, headers=folio_client.okapi_headers, json=loan_to_put, timeout=None

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/library_configuration.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Annotated, Optional
+from typing import Annotated
 
 from pydantic import BaseModel, Field
 from pydantic.types import DirectoryPath
@@ -68,8 +68,28 @@ class FolioRelease(str, Enum):
 
 
 class LibraryConfiguration(BaseModel):
-    okapi_url: str
-    tenant_id: str
+    gateway_url: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway URL",
+            description=(
+                "The URL of the FOLIO API gateway instance. "
+                "You can find this in Settings > Software versions > API gateway services."
+            ),
+            alias="okapi_url"
+        ),
+    ]
+    tenant_id: Annotated[
+        str,
+        Field(
+            title="FOLIO tenant ID",
+            description=(
+                "The ID of the FOLIO tenant instance. "
+                "You can find this in Settings > Software versions > API gateway services. "
+                "In an ECS environment, this is the ID of the central tenant, for all configurations."
+            ),
+        ),
+    ]
     ecs_tenant_id: Annotated[
         str,
         Field(
@@ -80,15 +100,43 @@ class LibraryConfiguration(BaseModel):
             ),
         ),
     ] = ""
-    okapi_username: str
-    okapi_password: str
+    folio_username: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway username",
+            description=(
+                "The username for the FOLIO user account performing the migration. "
+                "User should have a full admin permissions/roles in FOLIO. "
+            ),
+            alias="okapi_username"
+        ),
+    ]
+    folio_password: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway password",
+            description=(
+                "The password for the FOLIO user account performing the migration. "
+            ),
+            alias="okapi_password"
+        )
+    ]
     base_folder: DirectoryPath = Field(
         description=(
             "The base folder for migration. "
             "Should ideally be a github clone of the migration_repo_template"
         )
     )
-    multi_field_delimiter: Optional[str] = "<delimiter>"
+    multi_field_delimiter: Annotated[
+        str,
+        Field(
+            title="Multi field delimiter",
+            description=(
+                "The delimiter used to separate multiple values in a single field. "
+                "This is used for delimited text (CSV/TSV) fields with multiple sub-delimited values."
+            ),
+        ),
+    ] = "<delimiter>"
     failed_records_threshold: Annotated[
         int,
         Field(description=("Number of failed records until the process shuts down")),

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/mapper_base.py

@@ -272,7 +272,7 @@ class MapperBase:
             [
                 object_type == FOLIONamespaces.instances,
                 (not getattr(self.task_configuration, "data_import_marc", False)),
-                getattr(self.task_configuration, "create_source_records", True),
+                getattr(self, "create_source_records", True),
             ]
         ):
             return (legacy_id, folio_record["id"], folio_record["hrid"])
@@ -443,7 +443,7 @@ class MapperBase:
     @property
     def base_string_for_folio_uuid(self):
         if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
-            return str(self.folio_client.okapi_url)
+            return str(self.folio_client.gateway_url)
         elif self.library_configuration.ecs_tenant_id:
             return str(self.library_configuration.ecs_tenant_id)
         else:

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/marc_rules_transformation/hrid_handler.py

@@ -160,7 +160,7 @@ class HRIDHandler:
             self.hrid_settings["instances"]["startNumber"] = self.instance_hrid_counter
             self.hrid_settings["holdings"]["startNumber"] = self.holdings_hrid_counter
             self.hrid_settings["items"]["startNumber"] = self.items_hrid_counter
-            url = self.folio_client.okapi_url + self.hrid_path
+            url = self.folio_client.gateway_url + self.hrid_path
             resp = httpx.put(
                 url,
                 json=self.hrid_settings,

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/marc_rules_transformation/marc_file_processor.py

@@ -3,7 +3,7 @@ import os
 import sys
 import time
 import traceback
-from typing import List
+from typing import BinaryIO, Dict, List, Set, TextIO
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
@@ -24,24 +24,26 @@ from folio_migration_tools.migration_report import MigrationReport
 
 class MarcFileProcessor:
     def __init__(
-        self, mapper: RulesMapperBase, folder_structure: FolderStructure, created_objects_file
+        self, mapper: RulesMapperBase, folder_structure: FolderStructure, created_objects_file: TextIO
     ):
         self.object_type: FOLIONamespaces = folder_structure.object_type
         self.folder_structure: FolderStructure = folder_structure
         self.mapper: RulesMapperBase = mapper
-        self.created_objects_file = created_objects_file
-        if mapper.task_configuration.create_source_records:
-            self.srs_records_file = open(self.folder_structure.srs_records_path, "w+")
+        self.created_objects_file: TextIO = created_objects_file
+        if mapper.create_source_records and any(
+            x.create_source_records for x in mapper.task_configuration.files
+        ):
+            self.srs_records_file: TextIO = open(self.folder_structure.srs_records_path, "w+")
         if getattr(mapper.task_configuration, "data_import_marc", False):
-            self.data_import_marc_file = open(self.folder_structure.data_import_marc_path, "wb+")
-        self.unique_001s: set = set()
+            self.data_import_marc_file: BinaryIO = open(self.folder_structure.data_import_marc_path, "wb+")
+        self.unique_001s: Set[str] = set()
         self.failed_records_count: int = 0
         self.records_count: int = 0
         self.start: float = time.time()
-        self.legacy_ids: set = set()
+        self.legacy_ids: Set[str] = set()
         if (
             self.object_type == FOLIONamespaces.holdings
-            and self.mapper.task_configuration.create_source_records
+            and self.mapper.create_source_records
         ):
             logging.info("Loading Parent HRID map for SRS creation")
             self.parent_hrids = {entity[1]: entity[2] for entity in mapper.parent_id_map.values()}
@@ -78,7 +80,7 @@ class MarcFileProcessor:
 
                     if (
                         file_def.create_source_records
-                        and self.mapper.task_configuration.create_source_records
+                        and self.mapper.create_source_records
                     ):
                         self.save_srs_record(
                             marc_record,
@@ -132,7 +134,7 @@ class MarcFileProcessor:
     def save_marc_record(
         self,
         marc_record: Record,
-        folio_rec: dict,
+        folio_rec: Dict,
         object_type: FOLIONamespaces
     ):
         self.mapper.save_data_import_marc_record(
@@ -146,14 +148,10 @@ class MarcFileProcessor:
         self,
         marc_record: Record,
         file_def: FileDefinition,
-        folio_rec,
+        folio_rec: Dict,
         legacy_ids: List[str],
         object_type: FOLIONamespaces,
     ):
-        if not all(
-            [file_def.create_source_records, self.mapper.task_configuration.create_source_records]
-        ):
-            return
         if object_type in [FOLIONamespaces.holdings]:
             if "008" in marc_record and len(marc_record["008"].data) > 32:
                 remain, rest = (
@@ -188,7 +186,7 @@ class MarcFileProcessor:
         )
         self.mapper.migration_report.add_general_statistics(i18n.t("SRS records written to disk"))
 
-    def add_mapped_location_code_to_record(self, marc_record, folio_rec):
+    def add_mapped_location_code_to_record(self, marc_record: Record, folio_rec: Dict):
         location_code = next(
             (
                 location["code"]
@@ -225,9 +223,9 @@ class MarcFileProcessor:
 
     @staticmethod
     def get_valid_folio_record_ids(
-        legacy_ids, folio_record_identifiers, migration_report: MigrationReport
-    ):
-        new_ids = set()
+        legacy_ids: List[str], folio_record_identifiers: Set[str], migration_report: MigrationReport
+    ) -> List[str]:
+        new_ids: Set[str] = set()
         for legacy_id in legacy_ids:
             if legacy_id not in folio_record_identifiers:
                 new_ids.add(legacy_id)
@@ -266,12 +264,12 @@ class MarcFileProcessor:
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
-        if self.mapper.task_configuration.create_source_records:
+        if hasattr(self, "srs_records_file"):
             self.srs_records_file.seek(0)
             if not self.srs_records_file.seek(0):
                 os.remove(self.srs_records_file.name)
             self.srs_records_file.close()
-        if getattr(self.mapper.task_configuration, "data_import_marc", False):
+        if hasattr(self, "data_import_marc_file"):
             self.data_import_marc_file.seek(0)
             if not self.data_import_marc_file.read(1):
                 os.remove(self.data_import_marc_file.name)
@@ -281,7 +279,7 @@ class MarcFileProcessor:
         logging.info("Transformation report written to %s", report_file.name)
         logging.info("Processor is done.")
 
-    def add_legacy_ids_to_map(self, folio_rec, filtered_legacy_ids):
+    def add_legacy_ids_to_map(self, folio_rec: Dict, filtered_legacy_ids: List[str]):
         for legacy_id in filtered_legacy_ids:
             self.legacy_ids.add(legacy_id)
             if legacy_id not in self.mapper.id_map:

{folio_migration_tools-1.9.0rc9 → folio_migration_tools-1.9.0rc11}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_base.py

@@ -51,6 +51,9 @@ class RulesMapperBase(MapperBase):
         self.item_json_schema = ""
         self.mappings: dict = {}
         self.schema_properties = None
+        self.create_source_records = all(
+            [self.task_configuration.create_source_records, (not getattr(self.task_configuration, "data_import_marc", False))]
+        )
         if hasattr(self.task_configuration, "hrid_handling"):
             self.hrid_handler = HRIDHandler(
                 folio_client,