folio_data_import 0.6.0__tar.gz → 0.6.2__tar.gz

{folio_data_import-0.6.0 → folio_data_import-0.6.2}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: folio_data_import
-Version: 0.6.0
+Version: 0.6.2
 Summary: A python module to perform bulk import of data into a FOLIO environment. Currently supports MARC and user data import.
 Author: Brooks Travis
 Author-email: Brooks Travis <brooks.travis@gmail.com>
 License-Expression: MIT
-Requires-Dist: folioclient>=1.0.3,<2.0
+Requires-Dist: folioclient>=1.0.8,<2.0
 Requires-Dist: pymarc>=5.2.2
 Requires-Dist: pyhumps>=3.8.0
 Requires-Dist: tabulate>=0.9.0

{folio_data_import-0.6.0 → folio_data_import-0.6.2}/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "folio_data_import"
-version = "0.6.0"
+version = "0.6.2"
 description = "A python module to perform bulk import of data into a FOLIO environment. Currently supports MARC and user data import."
 authors = [{ name = "Brooks Travis", email = "brooks.travis@gmail.com" }]
 license = "MIT"
 readme = "README.md"
 requires-python = ">=3.10,<4.0"
 dependencies = [
-    "folioclient>=1.0.3,<2.0",
+    "folioclient>=1.0.8,<2.0",
     "pymarc>=5.2.2",
     "pyhumps>=3.8.0",
     "tabulate>=0.9.0",

{folio_data_import-0.6.0 → folio_data_import-0.6.2}/src/folio_data_import/UserImport.py

@@ -152,6 +152,15 @@ class UserImporter:  # noqa: R0902
                 description="Whether to delete existing users, rather than create/update.",
             ),
         ] = False
+        delete_pus: Annotated[
+            bool,
+            Field(
+                title="Delete permission user objects when deleting users",
+                description=(
+                    "Whether to delete associated permission user objects when deleting users."
+                ),
+            ),
+        ] = False
 
     logfile: AsyncTextIOWrapper
     errorfile: AsyncTextIOWrapper
@@ -174,6 +183,7 @@ class UserImporter:  # noqa: R0902
             )
         else:
             self.reporter = reporter
+        self.delete_pus = config.delete_pus
         self.limit_simultaneous_requests = asyncio.Semaphore(config.limit_simultaneous_requests)
         # Build reference data maps (these need processing)
         self.patron_group_map: dict = self.build_ref_data_id_map(
@@ -800,36 +810,54 @@ class UserImporter:  # noqa: R0902
                 "skipping deletion\n"
             )
             return
-        try:
-            if existing_user:
+        if existing_user:
+            try:
                 await self.folio_client.folio_delete_async(
                     f"/users/{existing_user['id']}",
                 )
-            if existing_rp:
-                await self.folio_client.folio_delete_async(
-                    f"/request-preference-storage/request-preference/{existing_rp.get('id', '')}"
-                )
-            if existing_pu:
-                await self.folio_client.folio_delete_async(
-                    f"/perms/users/{existing_pu.get('id', '')}"
-                )
-            if existing_spu:
-                await self.folio_client.folio_delete_async(
-                    f"/service-points-users/{existing_spu.get('id', '')}"
-                )
-
-            if existing_user:
                 logger.debug(f"Row {line_number}: Deleted user {existing_user['id']}\n")
 
-            async with self.lock:
                 self.stats.deleted += 1
-        except folioclient.FolioError as ee:
-            logger.error(
-                f"Row {line_number}: Failed to delete user {existing_user['id']}: "
-                f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
-            )
-            async with self.lock:
+            except folioclient.FolioError as ee:
+                logger.exception(
+                    f"Row {line_number}: Failed to delete user {existing_user['id']}: "
+                    f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
+                )
                 self.stats.failed += 1
+            try:
+                if existing_rp:
+                    await self.folio_client.folio_delete_async(
+                        "/request-preference-storage/request-preference/"
+                        f"{existing_rp.get('id', '')}"
+                    )
+            except folioclient.FolioError as ee:
+                logger.exception(
+                    f"Row {line_number}: Failed to delete request preference for user "
+                    f"{existing_user['id']}: "
+                    f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
+                )
+            try:
+                if existing_pu and self.delete_pus:
+                    await self.folio_client.folio_delete_async(
+                        f"/perms/users/{existing_pu.get('id', '')}"
+                    )
+            except folioclient.FolioError as ee:
+                logger.exception(
+                    f"Row {line_number}: Failed to delete permission user for user "
+                    f"{existing_user['id']}: "
+                    f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
+                )
+            try:
+                if existing_spu:
+                    await self.folio_client.folio_delete_async(
+                        f"/service-points-users/{existing_spu.get('id', '')}"
+                    )
+            except folioclient.FolioError as ee:
+                logger.exception(
+                    f"Row {line_number}: Failed to delete service points user for user "
+                    f"{existing_user['id']}: "
+                    f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
+                )
 
     async def process_line(
         self,

{folio_data_import-0.6.0 → folio_data_import-0.6.2}/src/folio_data_import/marc_preprocessors/_preprocessors.py

@@ -2,6 +2,7 @@ import importlib
 import logging
 import re
 import sys
+from datetime import datetime
 from typing import Callable, Dict, List, Tuple, Union
 
 import pymarc
@@ -106,6 +107,29 @@ class MARCPreprocessor:
         return record
 
 
+def remove_sudoc_prefixes(record: Record) -> Record:
+    """Remove common ABES/SUDOC prefixes from the 001 of a record.
+    Useful when importing records from the ABES SUDOC catalog that
+    you want to match to existing records in FOLIO without the prefix.
+
+    Args:
+    record (Record): The MARC record to preprocess.
+    """
+    sudoc_prefixes = ["PPN", "ABES"]
+    if "001" in record:
+        if any(record["001"].data.startswith(prefix) for prefix in sudoc_prefixes):
+            logger.debug("Field '001' starts with a known SUDOC prefix. Removing prefix from 001.")
+            for prefix in sudoc_prefixes:
+                record["001"].data = record["001"].data.replace(prefix, "").strip()
+        else:
+            logger.debug(
+                "Field '001' does not start with a known SUDOC prefix. No changes made to 001."
+            )
+    else:
+        logger.warning("Field '001' not found in record. Skipping prefix removal.")
+    return record
+
+
 def prepend_prefix_001(record: Record, prefix: str) -> Record:
     """
     Prepend a prefix to the record's 001 field.
@@ -117,9 +141,11 @@ def prepend_prefix_001(record: Record, prefix: str) -> Record:
     Returns:
         Record: The preprocessed MARC record.
     """
-    if "001" in record:
+    if "001" in record and record["001"].data:
         record["001"].data = (
-            f"({prefix})" + record["001"].data if record["001"].data else f"({prefix})"
+            f"({prefix})" + record["001"].data
+            if not record["001"].data.startswith(f"({prefix})")
+            else record["001"].data
         )
     else:
         logger.warning("Field '001' not found in record. Skipping prefix prepend.")
@@ -426,6 +452,35 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
     return record
 
 
+def clean_empty_contributors(record: Record, **kwargs) -> Record:
+    """
+    Remove contributor fields (100, 110, 111, 700, 710, 711) that have empty subfield a. These can
+    cause data import mapping issues in FOLIO. Removals are logged at custom log level 26, which is
+    used by folio_migration_tools to populate the data issues report.
+
+    Args:
+        record (Record): The MARC record to preprocess.
+
+    Returns:
+        Record: The preprocessed MARC record.
+    """
+    for field in record.get_fields("100", "110", "111", "700", "710", "711"):
+        if all(
+            not subfield.value.strip()
+            for subfield in field.subfields
+            if subfield.code in ["a", "b", "c", "d", "q"]
+        ):
+            logger.log(
+                26,
+                "DATA ISSUE\t%s\t%s\t%s",
+                record["001"].value(),
+                f"{field.tag} contributor field has empty name subfields, removing field",
+                field,
+            )
+            record.remove_field(field)
+    return record
+
+
 def fix_bib_leader(record: Record, **kwargs) -> Record:
     """
     Fixes the leader of the record by setting the record status to 'c' (modified
@@ -546,6 +601,51 @@ def remove_non_numeric_fields(record: Record, **kwargs) -> Record:
     return record
 
 
+def populate_blank_008_0_5(record: Record, **kwargs) -> Record:
+    """
+    Populate the first 6 characters of the 008 with today's date in the format YYMMDD if they are
+    blank. Workaround for FOLIO issue: https://folio-org.atlassian.net/browse/MODQM-515
+    """
+    if "008" in record and record["008"].data:
+        if len(record["008"].data) >= 6 and record["008"].data[:6] == "      ":
+            today = datetime.today().strftime("%y%m%d")
+            record["008"].data = today + record["008"].data[6:]
+            logger.log(
+                26,
+                "DATA ISSUE\t%s\t%s\t%s",
+                record["001"].value(),
+                "First 6 characters of 008 were blank, populated with today's date",
+                record["008"],
+            )
+    else:
+        logger.warning("Field '008' not found in record. Skipping 008 population.")
+    return record
+
+
+def move_856z_to_856y(record: Record, **kwargs) -> Record:
+    """
+    Move subfield z from 856 fields to subfield y. This is useful when
+    importing records from the ABES SUDOC catalog.
+
+    Args:
+        record (Record): The MARC record to preprocess.
+    Returns:
+        Record: The preprocessed MARC record.
+    """
+    for field in record.get_fields("856"):
+        if "z" in field:
+            field.add_subfield("y", field["z"])
+            field.delete_subfield("z")
+            logger.log(
+                26,
+                "DATA ISSUE\t%s\t%s\t%s",
+                record["001"].value(),
+                f"Subfield z moved to subfield y in {field.tag}",
+                field,
+            )
+    return record
+
+
 def ordinal(n: int) -> str:
     s = ("th", "st", "nd", "rd") + ("th",) * 10
     v = n % 100