folio_data_import 0.6.2__tar.gz → 0.6.4__tar.gz

{folio_data_import-0.6.2 → folio_data_import-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: folio_data_import
-Version: 0.6.2
+Version: 0.6.4
 Summary: A python module to perform bulk import of data into a FOLIO environment. Currently supports MARC and user data import.
 Author: Brooks Travis
 Author-email: Brooks Travis <brooks.travis@gmail.com>

{folio_data_import-0.6.2 → folio_data_import-0.6.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_data_import"
-version = "0.6.2"
+version = "0.6.4"
 description = "A python module to perform bulk import of data into a FOLIO environment. Currently supports MARC and user data import."
 authors = [{ name = "Brooks Travis", email = "brooks.travis@gmail.com" }]
 license = "MIT"

{folio_data_import-0.6.2 → folio_data_import-0.6.4}/src/folio_data_import/BatchPoster.py

@@ -848,6 +848,7 @@ class BatchPoster:
         except folioclient.FolioClientError as e:
             logger.error(f"Batch failed: {e} - {e.response.text}")
             self.stats.records_failed += len(batch)
+            self.stats.batches_failed += 1
             self._write_failed_batch(batch)
 
             # Update progress bar if available
@@ -864,6 +865,7 @@ class BatchPoster:
         except folioclient.FolioConnectionError as e:
             logger.error(f"Batch failed due to connection error: {e}")
             self.stats.records_failed += len(batch)
+            self.stats.batches_failed += 1
             self._write_failed_batch(batch)
 
             # Update progress bar if available

{folio_data_import-0.6.2 → folio_data_import-0.6.4}/src/folio_data_import/UserImport.py

@@ -42,6 +42,7 @@ PREFERRED_CONTACT_TYPES_MAP = {
     "004": "phone",
     "005": "mobile",
 }
+PREFERRED_CONTACT_TYPES_NAME_TO_ID = {v: k for k, v in PREFERRED_CONTACT_TYPES_MAP.items()}
 
 
 USER_MATCH_KEYS = ["username", "barcode", "externalSystemId"]
@@ -527,6 +528,8 @@ class UserImporter:  # noqa: R0902
         Raises:
             HTTPError: If the HTTP request to create the user fails.
         """
+        # Normalize preferred contact type to numeric ID before create.
+        await self.set_preferred_contact_type(user_obj, user_obj)
         response = await self.http_client.post(
             "/users",
             headers=self.folio_client.okapi_headers,
@@ -544,35 +547,48 @@ class UserImporter:  # noqa: R0902
         user object has a valid preferred contact type set. In that case, the existing preferred
         contact type is used.
         """
+        default_preferred_contact_type = self.normalize_preferred_contact_type(
+            self.config.default_preferred_contact_type
+        )
         if "personal" in user_obj and "preferredContactTypeId" in user_obj["personal"]:
             current_pref_contact = user_obj["personal"].get("preferredContactTypeId", "")
-            if mapped_contact_type := {v: k for k, v in PREFERRED_CONTACT_TYPES_MAP.items()}.get(
-                current_pref_contact,
-                "",
-            ):
-                existing_user["personal"]["preferredContactTypeId"] = mapped_contact_type
-            else:
-                existing_user["personal"]["preferredContactTypeId"] = (
-                    current_pref_contact
-                    if current_pref_contact in PREFERRED_CONTACT_TYPES_MAP
-                    else self.config.default_preferred_contact_type
-                )
+            if "personal" not in existing_user:
+                existing_user["personal"] = {}
+            existing_user["personal"]["preferredContactTypeId"] = (
+                self.normalize_preferred_contact_type(current_pref_contact)
+            )
         else:
             logger.warning(
                 f"Preferred contact type not provided or is not a valid option: "
                 f"{PREFERRED_CONTACT_TYPES_MAP} Setting preferred contact type to "
                 f"{self.config.default_preferred_contact_type} or using existing value"
             )
-            mapped_contact_type = (
-                existing_user.get("personal", {}).get("preferredContactTypeId", "")
-                or self.config.default_preferred_contact_type
+            existing_contact_type = existing_user.get("personal", {}).get(
+                "preferredContactTypeId", ""
             )
             if "personal" not in existing_user:
                 existing_user["personal"] = {}
             existing_user["personal"]["preferredContactTypeId"] = (
-                mapped_contact_type or self.config.default_preferred_contact_type
+                self.normalize_preferred_contact_type(existing_contact_type)
+                if existing_contact_type
+                else default_preferred_contact_type
             )
 
+    def normalize_preferred_contact_type(self, preferred_contact_type: str) -> str:
+        """Normalize preferred contact type names/IDs to a valid numeric ID."""
+        if preferred_contact_type in PREFERRED_CONTACT_TYPES_MAP:
+            return preferred_contact_type
+
+        normalized_value = (preferred_contact_type or "").strip().lower()
+        if mapped_contact_type := PREFERRED_CONTACT_TYPES_NAME_TO_ID.get(normalized_value):
+            return mapped_contact_type
+
+        default_preferred_contact_type = self.config.default_preferred_contact_type
+        if default_preferred_contact_type in PREFERRED_CONTACT_TYPES_MAP:
+            return default_preferred_contact_type
+
+        return PREFERRED_CONTACT_TYPES_NAME_TO_ID.get(default_preferred_contact_type, "002")
+
     async def create_or_update_user(
         self, user_obj, existing_user, protected_fields, line_number: int
     ) -> dict:
@@ -621,7 +637,7 @@ class UserImporter:  # noqa: R0902
 
     async def process_user_obj(self, user: str) -> dict:
         """
-        Process a user object. If not type is found in the source object, type is set to "patron".
+        Process a user object. If type is not found in the source object, type is set to "patron".
 
         Args:
             user (str): The user data to be processed, as a json string.

{folio_data_import-0.6.2 → folio_data_import-0.6.4}/src/folio_data_import/marc_preprocessors/_preprocessors.py

@@ -11,6 +11,33 @@ from pymarc.record import Record
 logger = logging.getLogger("folio_data_import.MARCDataImport")
 
 
+def _get_record_id(
+    record: Record,
+    record_id_field: str = "001",
+    record_id_subfield: str | None = None,
+    **kwargs,
+) -> str:
+    """
+    Extract a record identifier from the specified field and optional subfield.
+    Returns 'UNKNOWN' if the field is absent or the subfield value is empty.
+
+    Args:
+        record (Record): The MARC record.
+        record_id_field (str): The tag of the field to use as the identifier. Defaults to '001'.
+        record_id_subfield (str | None): The subfield code to use. If None, uses the full field
+            value (appropriate for control fields). Defaults to None.
+
+    Returns:
+        str: The record identifier, or 'UNKNOWN' if not found.
+    """
+    field = record.get(record_id_field)
+    if field is None:
+        return "UNKNOWN"
+    if record_id_subfield is not None:
+        return field.get(record_id_subfield) or "UNKNOWN"
+    return field.value() or "UNKNOWN"
+
+
 class MARCPreprocessor:
     """
     A class to preprocess MARC records for data import into FOLIO.
@@ -234,7 +261,7 @@ def clean_non_ff_999_fields(record: Record, **kwargs) -> Record:
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                _get_record_id(record, **kwargs),
                 "Record contains a 999 field with non-ff indicators: Moving field to a 945 with"
                 ' indicators "99"',
                 field,
@@ -395,6 +422,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
         "856": ["u", "y", "z"],
     }
 
+    record_id = _get_record_id(record, **kwargs)
     for field in record.get_fields(*MAPPED_FIELDS.keys()):
         len_subs = len(field.subfields)
         subfield_value = (
@@ -404,7 +432,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                record_id,
                 f"{field.tag} is empty, removing field",
                 field,
             )
@@ -413,7 +441,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                record_id,
                 f"{field.tag}${field.subfields[0].code} is empty,"
                 " no other subfields present, removing field",
                 field,
@@ -424,7 +452,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
                 logger.log(
                     26,
                     "DATA ISSUE\t%s\t%s\t%s",
-                    record["001"].value(),
+                    record_id,
                     f"{field.tag}$a is empty, removing subfield",
                     field,
                 )
@@ -434,7 +462,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
                     logger.log(
                         26,
                         "DATA ISSUE\t%s\t%s\t%s",
-                        record["001"].value(),
+                        record_id,
                         f"{field.tag}${subfield.code} ({ordinal(idx)} subfield) is empty, but "
                         "other subfields have values, removing subfield",
                         field,
@@ -444,7 +472,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
                 logger.log(
                     26,
                     "DATA ISSUE\t%s\t%s\t%s",
-                    record["001"].value(),
+                    record_id,
                     f"{field.tag} has no non-empty subfields after cleaning, removing field",
                     field,
                 )
@@ -473,7 +501,7 @@ def clean_empty_contributors(record: Record, **kwargs) -> Record:
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                _get_record_id(record, **kwargs),
                 f"{field.tag} contributor field has empty name subfields, removing field",
                 field,
             )
@@ -494,11 +522,12 @@ def fix_bib_leader(record: Record, **kwargs) -> Record:
     """
     VALID_STATUSES = ["a", "c", "d", "n", "p"]
     VALID_TYPES = ["a", "c", "d", "e", "f", "g", "i", "j", "k", "m", "o", "p", "r", "t"]
+    record_id = _get_record_id(record, **kwargs)
     if record.leader[5] not in VALID_STATUSES:
         logger.log(
             26,
             "DATA ISSUE\t%s\t%s\t%s",
-            record["001"].value(),
+            record_id,
             f"Invalid record status: {record.leader[5]}, setting to 'c'",
             record.leader,
         )
@@ -507,7 +536,7 @@ def fix_bib_leader(record: Record, **kwargs) -> Record:
         logger.log(
             26,
             "DATA ISSUE\t%s\t%s\t%s",
-            record["001"].value(),
+            record_id,
             f"Invalid record type: {record.leader[6]}, setting to 'a'",
             record.leader,
         )
@@ -556,7 +585,7 @@ def move_authority_subfield_9_to_0_all_controllable_fields(record: Record, **kwa
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                _get_record_id(record, **kwargs),
                 f"Subfield 9 moved to subfield 0 in {field.tag}",
                 field,
             )
@@ -587,13 +616,14 @@ def remove_non_numeric_fields(record: Record, **kwargs) -> Record:
     Returns:
         Record: The preprocessed MARC record.
     """
+    record_id = _get_record_id(record, **kwargs)
     for field in record.get_fields():
         if not re.fullmatch(r"\d{3}", field.tag) or field.tag == "000":
             reason = "invalid tag 000" if field.tag == "000" else f"non-numeric tag {field.tag}"
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                record_id,
                 f"Field removed: {reason}",
                 field,
             )
@@ -613,7 +643,7 @@ def populate_blank_008_0_5(record: Record, **kwargs) -> Record:
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                _get_record_id(record, **kwargs),
                 "First 6 characters of 008 were blank, populated with today's date",
                 record["008"],
             )
@@ -639,13 +669,72 @@ def move_856z_to_856y(record: Record, **kwargs) -> Record:
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
-                record["001"].value(),
+                _get_record_id(record, **kwargs),
                 f"Subfield z moved to subfield y in {field.tag}",
                 field,
             )
     return record
 
 
+def normalize_subfield_codes(record: Record, **kwargs) -> Record:
+    """
+    Normalize subfield codes to lowercase. Fixes common import error in FOLIO. Preserves the order
+    of subfields in each field.
+
+    Args:
+        record (Record): The MARC record to preprocess.
+    Returns:
+        Record: The preprocessed MARC record.
+    """
+    record_id = _get_record_id(record, **kwargs)
+    for field in record.get_fields():
+        if not any(sf.code != sf.code.lower() for sf in field.subfields):
+            continue
+        normalized_subfields = []
+        for subfield in field.subfields:
+            if subfield.code != subfield.code.lower():
+                logger.log(
+                    26,
+                    "DATA ISSUE\t%s\t%s\t%s",
+                    record_id,
+                    f"Invalid subfield code {field.tag}${subfield.code}:"
+                    f" normalizing to ${subfield.code.lower()}",
+                    field,
+                )
+            normalized_subfields.append(
+                pymarc.field.Subfield(subfield.code.lower(), subfield.value)
+            )
+        field.subfields = normalized_subfields
+    return record
+
+
+def copy_240_to_245_if_no_245(record: Record, **kwargs) -> Record:
+    """
+    Copy subfields from 240 fields to 245 fields. This is a workaround for FOLIO requiring 245s.
+
+    Args:
+        record (Record): The MARC record to preprocess.
+    Returns:
+        Record: The preprocessed MARC record.
+    """
+    if "245" not in record and "240" in record:
+        logger.log(
+            26,
+            "DATA ISSUE\t%s\t%s\t%s",
+            _get_record_id(record, **kwargs),
+            "No 245 field found: copying subfields from 240 to 245",
+            record,
+        )
+        field_240 = record.get_fields("240")[0]
+        field_245 = pymarc.Field(
+            tag="245",
+            indicators=["0", "0"],
+            subfields=field_240.subfields,
+        )
+        record.add_ordered_field(field_245)
+    return record
+
+
 def ordinal(n: int) -> str:
     s = ("th", "st", "nd", "rd") + ("th",) * 10
     v = n % 100