folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl

folio_migration_tools/holdings_helper.py

@@ -1,12 +1,11 @@
 import json
-import sys
 import logging
+import i18n
 from uuid import uuid4
 
 from folio_migration_tools import custom_exceptions
 from folio_migration_tools import helper
 from folio_migration_tools.migration_report import MigrationReport
-from folio_migration_tools.report_blurbs import Blurbs
 
 
 class HoldingsHelper:
@@ -20,20 +19,20 @@ class HoldingsHelper:
         """Creates a key from values determined by the fields_crieteria in a holding
         record to determine uniquenes
 
-        fields_criterias are limited to the strings and UUID properties on the first level of the object.
-        If the property is not found, or empty, it will be ignored
+        fields_criterias are limited to the strings and UUID properties on the first level of
+        the object. If the property is not found, or empty, it will be ignored
 
         IF the holdings type id is matched to holdings_type_id_to_exclude_from_merging,
         the key will be added with a uuid to prevent merging of this holding
 
         Args:
             holdings_record (dict): The Holdingsrecord
-            fields_criteria (list[str]): names of the properties of the holdingsrecord.
+            fields_criterias (list[str]): names of the properties of the holdingsrecord.
             migration_report (MigrationReport): Report to help reporting merge
             holdings_type_id_to_exclude_from_merging: (str): the holdings type UUID to exclude
 
         Raises:
-            exception: _description_
+            Exception: _description_
 
         Returns:
             str: _description_
@@ -44,18 +43,18 @@ class HoldingsHelper:
                 v = holdings_record.get(fields_criteria, "")
                 if not v:
                     migration_report.add(
-                        Blurbs.HoldingsMerging, f"{fields_criteria} empty or not set"
+                        "HoldingsMerging",
+                        i18n.t(
+                            "%{fields_criteria} empty or not set", fields_criteria=fields_criteria
+                        ),
                     )
                 values.append(v)
 
-            if (
-                holdings_record.get("holdingsTypeId")
-                == holdings_type_id_to_exclude_from_merging
-            ):
+            if holdings_record.get("holdingsTypeId") == holdings_type_id_to_exclude_from_merging:
                 values.append(str(uuid4()))
                 migration_report.add(
-                    Blurbs.HoldingsMerging,
-                    "Holding prevented from merging by holdingsTypeId",
+                    "HoldingsMerging",
+                    i18n.t("Holding prevented from merging by holdingsTypeId"),
                 )
             return "-".join(values)
         except Exception as exception:
@@ -89,43 +88,55 @@ class HoldingsHelper:
                 )
                 if stored_key in prev_holdings:
                     message = (
-                        f"Previously stored holdings key already exists in the "
-                        f"list of previously stored Holdings. You have likely not used the same "
-                        f"matching criterias ({fields_criteria}) as you did in the previous process"
-                    )
-                    helper.Helper.log_data_issue(
-                        stored_holding["formerIds"], message, stored_key
+                        f"Previously stored holdings key already exists in the list of previously"
+                        f" stored Holdings. You have likely not used the same matching criterias"
+                        f" ({fields_criteria}) as you did in the previous process"
                     )
+                    helper.Helper.log_data_issue(stored_holding["formerIds"], message, stored_key)
                     logging.warn(message)
                     prev_holdings[stored_key] = HoldingsHelper.merge_holding(
                         prev_holdings[stored_key], stored_holding
                     )
                     migration_report.add(
-                        Blurbs.HoldingsMerging,
-                        "Duplicate key based on current merge criteria. Records merged",
+                        "HoldingsMerging",
+                        i18n.t("Duplicate key based on current merge criteria. Records merged"),
                     )
                 else:
                     migration_report.add(
-                        Blurbs.HoldingsMerging,
-                        "Previously transformed holdings record loaded",
+                        "HoldingsMerging",
+                        i18n.t("Previously transformed holdings record loaded"),
                     )
                     prev_holdings[stored_key] = stored_holding
             return prev_holdings
 
     @staticmethod
     def merge_holding(holdings_record: dict, incoming_holdings: dict) -> dict:
-        extend_list("holdingsStatementsForIndexes", holdings_record, incoming_holdings)
-        extend_list("holdingsStatements", holdings_record, incoming_holdings)
-        extend_list(
-            "holdingsStatementsForSupplements", holdings_record, incoming_holdings
-        )
+        extend_list("holdingsStatementsForIndexes", holdings_record, incoming_holdings, True)
+        extend_list("holdingsStatements", holdings_record, incoming_holdings, True)
+        extend_list("holdingsStatementsForSupplements", holdings_record, incoming_holdings, True)
         extend_list("notes", holdings_record, incoming_holdings)
         holdings_record["notes"] = dedupe(holdings_record.get("notes", []))
         extend_list("formerIds", holdings_record, incoming_holdings)
-        holdings_record["formerIds"] = list(set(holdings_record["formerIds"]))
         extend_list("electronicAccess", holdings_record, incoming_holdings)
+        HoldingsHelper.remove_empty_holdings_statements(holdings_record)
+        merge_boolean("discoverySuppress", holdings_record, incoming_holdings)
         return holdings_record
 
+    @staticmethod
+    def remove_empty_holdings_statements(holdings_record: dict):
+        keys = [
+            "holdingsStatements",
+            "holdingsStatementsForIndexes",
+            "holdingsStatementsForSupplements",
+        ]
+
+        for key in keys:
+            if key in holdings_record:
+                temp_l = [stmt for stmt in holdings_record[key] if any(stmt.values())]
+                holdings_record[key] = temp_l
+            if key in holdings_record and not holdings_record.get(key, []):
+                del holdings_record[key]
+
     @staticmethod
     def handle_notes(folio_object):
         if folio_object.get("notes", []):
@@ -145,15 +156,27 @@ class HoldingsHelper:
                 del folio_object["notes"]
 
 
-def extend_list(prop_name: str, holdings_record: dict, incoming_holdings: dict):
-
+def extend_list(
+    prop_name: str, holdings_record: dict, incoming_holdings: dict, accept_dupe_items: bool = False
+):
     temp = holdings_record.get(prop_name, [])
+    all_already_in = all(i in temp for i in incoming_holdings.get(prop_name, []))
     for f in incoming_holdings.get(prop_name, []):
-        if f not in temp:
+        if not all_already_in and (accept_dupe_items or f not in temp):
             temp.append(f)
-    holdings_record[prop_name] = temp
+    if temp:
+        holdings_record[prop_name] = temp
 
 
 def dedupe(list_of_dicts):
-    # TODO: Move to interface or parent class
     return [dict(t) for t in {tuple(d.items()) for d in list_of_dicts}]
+
+
+def merge_boolean(prop_name: str, holdings_record: dict, incoming_holdings: dict):
+    if (
+        holdings_record.get(prop_name, False) is True
+        and incoming_holdings.get(prop_name, False) is True
+    ):
+        holdings_record[prop_name] = True
+    else:
+        holdings_record[prop_name] = False

folio_migration_tools/i18n_config.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+settings = {
+    "file_format": "json",
+    "skip_locale_root_data": True,
+    "fallback": "en",
+    "filename_format": "{locale}.{format}",
+    "load_path": [Path(__file__).parent / "translations"],
+}

folio_migration_tools/library_configuration.py

@@ -1,17 +1,69 @@
-from typing import Optional
+from enum import Enum
+from typing import Annotated
+
 from pydantic import BaseModel, Field
 from pydantic.types import DirectoryPath
-from enum import Enum
 
 
 class HridHandling(str, Enum):
+    """Enum determining how the HRID generation should be handled.
+        - default: Enumerates the HRID, building on the current value in the HRID settings
+        - preserve001: Takes the 001 and uses this as the HRID.
+
+    Args:
+        str (_type_): _description_
+        Enum (_type_): _description_
+    """
+
     default = "default"
     preserve001 = "preserve001"
 
 
 class FileDefinition(BaseModel):
-    file_name: str
-    suppressed: Optional[bool] = False
+    file_name: Annotated[
+        str,
+        Field(
+            title="File name",
+            description=(
+                "Name of the file to be processed. "
+                "The location of the file depends on the context"
+            ),
+        ),
+    ] = ""
+    discovery_suppressed: Annotated[bool, Field(title="Discovery suppressed")] = False
+    staff_suppressed: Annotated[bool, Field(title="Staff suppressed")] = False
+    service_point_id: Annotated[
+        str,
+        Field(
+            title="Service point ID",
+            description=(
+                "Service point to be used for "
+                "transactions created from this file (Loans-only)."
+            ),
+        )
+    ] = ""
+    statistical_code: Annotated[
+        str,
+        Field(
+            title="Statistical code",
+            description=(
+                "Statistical code (code) to be used inventory records created from "
+                "this file (Instances, Holdings, Items). Specify multiple codes using "
+                "multi_field_delimiter."
+            ),
+        )
+    ] = ""
+    create_source_records: Annotated[
+        bool,
+        Field(
+            title="Create source records",
+            description=(
+                "If set to true, the source records will be created in FOLIO. "
+                "If set to false, the source records will not be created in FOLIO. "
+                "Only applied for MARC-based transformations."
+            ),
+        ),
+    ] = True
 
 
 class IlsFlavour(str, Enum):
@@ -25,27 +77,102 @@ class IlsFlavour(str, Enum):
     tag907y = "tag907y"
     tag001 = "tag001"
     tagf990a = "tagf990a"
+    custom = "custom"
     none = "none"
 
 
 class FolioRelease(str, Enum):
-    kiwi = "kiwi"
-    juniper = "juniper"
+    ramsons = "ramsons"
+    sunflower = "sunflower"
+    trillium = "trillium"
 
 
 class LibraryConfiguration(BaseModel):
-    okapi_url: str
-    tenant_id: str
-    okapi_username: str
-    okapi_password: str
+    gateway_url: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway URL",
+            description=(
+                "The URL of the FOLIO API gateway instance. "
+                "You can find this in Settings > Software versions > API gateway services."
+            ),
+            alias="okapi_url"
+        ),
+    ]
+    tenant_id: Annotated[
+        str,
+        Field(
+            title="FOLIO tenant ID",
+            description=(
+                "The ID of the FOLIO tenant instance. "
+                "You can find this in Settings > Software versions > API gateway services. "
+                "In an ECS environment, this is the ID of the central tenant, for all configurations."
+            ),
+        ),
+    ]
+    ecs_tenant_id: Annotated[
+        str,
+        Field(
+            title="ECS tenant ID",
+            description=(
+                "For use in ECS environments when the configuration file is meant to target a ",
+                "data tenant. Set to the tenant ID of the data tenant.",
+            ),
+        ),
+    ] = ""
+    folio_username: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway username",
+            description=(
+                "The username for the FOLIO user account performing the migration. "
+                "User should have a full admin permissions/roles in FOLIO. "
+            ),
+            alias="okapi_username"
+        ),
+    ]
+    folio_password: Annotated[
+        str,
+        Field(
+            title="FOLIO API Gateway password",
+            description=(
+                "The password for the FOLIO user account performing the migration. "
+            ),
+            alias="okapi_password"
+        )
+    ]
     base_folder: DirectoryPath = Field(
         description=(
             "The base folder for migration. "
             "Should ideally be a github clone of the migration_repo_template"
         )
     )
-    failed_records_threshold: Optional[int] = 5000
-    failed_percentage_threshold: Optional[int] = 20
+    multi_field_delimiter: Annotated[
+        str,
+        Field(
+            title="Multi field delimiter",
+            description=(
+                "The delimiter used to separate multiple values in a single field. "
+                "This is used for delimited text (CSV/TSV) fields with multiple sub-delimited values."
+            ),
+        ),
+    ] = "<delimiter>"
+    failed_records_threshold: Annotated[
+        int,
+        Field(description=("Number of failed records until the process shuts down")),
+    ] = 5000
+    failed_percentage_threshold: Annotated[
+        int,
+        Field(
+            description=("Percentage of failed records until the process shuts down")
+        ),
+    ] = 20
+    generic_exception_threshold: Annotated[
+        int,
+        Field(
+            description=("Number of generic exceptions until the process shuts down")
+        ),
+    ] = 50
     library_name: str
     log_level_debug: bool
     folio_release: FolioRelease = Field(
@@ -56,4 +183,37 @@ class LibraryConfiguration(BaseModel):
         )
     )
     iteration_identifier: str
-    add_time_stamp_to_file_names: Optional[bool] = False
+    add_time_stamp_to_file_names: Annotated[
+        bool, Field(title="Add time stamp to file names")
+    ] = False
+    use_gateway_url_for_uuids: Annotated[
+        bool,
+        Field(
+            title="Use gateway URL for UUIDs",
+            description=(
+                "If set to true, folio_uuid will use the gateway URL when generating deterministic UUIDs for FOLIO records. "
+                "If set to false (default), the UUIDs will be generated using the tenant_id (or ecs_tenant_id)."
+            ),
+        ),
+    ] = False
+    is_ecs: Annotated[
+        bool,
+        Field(
+            title="Library is running ECS FOLIO",
+            description=(
+                "If set to true, the migration is running in an ECS environment. "
+                "If set to false (default), the migration is running in a non-ECS environment. "
+                "If ecs_tenant_id is set, this will be set to true, regardless of the value here."
+            ),
+        ),
+    ] = False
+    ecs_central_iteration_identifier: Annotated[
+        str,
+        Field(
+            title="ECS central iteration identifier",
+            description=(
+                "The iteration_identifier value from the central tenant configuration that corresponds "
+                "to this configuration's iteration_identifier. Used to access the central instances_id_map."
+            ),
+        ),
+    ] = ""