rara-tools 0.7.10__tar.gz → 0.7.12__tar.gz

{rara_tools-0.7.10/rara_tools.egg-info → rara_tools-0.7.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.10
+Version: 0.7.12
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.7.12/VERSION

@@ -0,0 +1 @@
+0.7.12

{rara_tools-0.7.10 → rara_tools-0.7.12}/rara_tools/normalizers/authorities.py

@@ -59,6 +59,9 @@ class AuthoritiesRecordNormalizer(RecordNormalizer):
             record, "046", "f", formatted_birth_date)
         death_date = self.get_subfield(
             record, "046", "g", formatted_death_date)
+        
+        if not birth_date and not death_date:
+            return
 
         subfields_046 = [
             Subfield("f", birth_date),
@@ -117,7 +120,9 @@ class AuthoritiesRecordNormalizer(RecordNormalizer):
         self._add_author(record, viaf_record)
 
     def _normalize_record(self, record: Record, sierraID: str,
-                          viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record:
+                          viaf_record: VIAFRecord, 
+                          is_editing_existing_record: bool,
+                          original_entity: str) -> Record:
         
         self._normalize_sierra(record, sierraID)
         self._normalize_viaf(record, viaf_record)

{rara_tools-0.7.10 → rara_tools-0.7.12}/rara_tools/normalizers/base.py

@@ -213,13 +213,33 @@ class RecordNormalizer:
         )
 
     def _add_fields_to_record(self, record: Record, fields: List[Field]) -> Record:
-        # filter out subfields that are empty, or 0, as VIAF returns 0 for unknown dates
+        cleaned_fields = []
+        
         for field in fields:
-            field.subfields = [sub for sub in field.subfields if sub.value and sub.value not in ["0", 0]]
             
-        self._handle_repeatable_fields(record, *fields)
-        self._handle_editable_fields(record, *fields)
-        self._handle_default_fields(record, *fields)
+            # Always assume control fields cleaned
+            if field.tag < "010" and field.tag.isdigit():
+                cleaned_fields.append(field)
+                continue
+            
+            # filter out subfields that are empty or 0 (VIAF returns 0 for unknown dates)
+            field.subfields = [
+                sub for sub in field.subfields 
+                if sub.value and sub.value not in ["0", 0]
+            ]
+            
+            # only keep the field if it still has subfields left
+            if field.subfields:
+                cleaned_fields.append(field)
+
+        if not cleaned_fields:
+            return record
+        
+        self._handle_repeatable_fields(record, *cleaned_fields)
+        self._handle_editable_fields(record, *cleaned_fields)
+        self._handle_default_fields(record, *cleaned_fields)
+        
+        return record
 
     def _add_author(self, record: Record, viaf_record: VIAFRecord) -> Optional[Field]:
 
@@ -360,7 +380,7 @@ class RecordNormalizer:
         return viaf_record
 
     def _normalize_record(self, record: Record, sierraID: str,
-                          viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record:
+                          viaf_record: VIAFRecord, is_editing_existing_record: bool, original_entity: str) -> Record:
         return record
 
     @property
@@ -374,7 +394,11 @@ class RecordNormalizer:
                 logger.error(f"Failed to normalize record: {e}")
                 continue
         return result
-
+    
+    @property
+    def first(self) -> Record:
+        return next(iter(self))
+    
     def __iter__(self) -> Iterator:
         viaf_id_path = "viaf.queryResult.records.record.0.recordData.VIAFCluster.viafID"
         sierra_id_path = "sierraID"
@@ -391,7 +415,7 @@ class RecordNormalizer:
             record = self._normalize_common(record, is_editing_existing_record)
 
             normalized_record = self._normalize_record(
-                record, sierra_id, viaf_record, is_editing_existing_record)
+                record, sierra_id, viaf_record, is_editing_existing_record, original_entity=entity)
 
             normalized_record.fields.sort(key=lambda field: field.tag)
 

rara_tools-0.7.12/rara_tools/normalizers/bibs.py

@@ -0,0 +1,111 @@
+from pymarc import (Field, Subfield, Record)
+from typing import List, Optional
+
+from rara_tools.constants import EMPTY_INDICATORS
+from rara_tools.normalizers.viaf import VIAFRecord
+from rara_tools.normalizers import RecordNormalizer
+
+from typing import List
+
+
+class BibRecordNormalizer(RecordNormalizer):
+    """ Normalize bib records. """
+
+    def __init__(self, linking_results: List[dict] = [], sierra_data: List[dict] = [],
+                 ALLOW_EDIT_FIELDS: List[str] = ["008", "925"],
+                 REPEATABLE_FIELDS: List[str] = ["667"]):
+        super().__init__(linking_results, sierra_data)
+        self.DEFAULT_LEADER = "00399nz  a2200145n  4500" # must be 24 digits
+        self.ALLOW_EDIT_FIELDS = ALLOW_EDIT_FIELDS
+        self.REPEATABLE_FIELDS = REPEATABLE_FIELDS
+        
+        self.records_extra_data = []
+        self.sierra_data = sierra_data
+        self.records = self._setup_records(linking_results, sierra_data)
+
+    def _normalize_sierra(self, record: Record) -> Record:
+        
+        suffix_008 = "|||aznnnaabn          || |||      "
+        
+        fields = [
+            Field(
+                tag="008",
+                data=f"{self.current_timestamp()}{suffix_008}"
+            ),
+        ]
+
+        self._add_fields_to_record(record, fields)
+        
+    def _include_name_variations(self, record: Record, viaf_record: VIAFRecord) -> None:
+        """ Include name variations from VIAF record as 400|t fields """
+        
+        if not viaf_record or not viaf_record.name_variations:
+            return
+        
+        existing_name_variations = record.get_fields("400")
+        existing_variations = [sf.value for field in existing_name_variations for sf in field.get_subfields("t")]
+        
+        fields = []
+        
+        for variation in viaf_record.name_variations:
+            if variation not in existing_variations:
+                fields.append(
+                    Field(
+                        tag="400",
+                        indicators=EMPTY_INDICATORS,
+                        subfields=[
+                            Subfield("t", variation)
+                        ]
+                    )
+                )
+        
+        self._add_fields_to_record(record, fields)
+        
+    def _add_author(self, record: Record, viaf_record: Optional[VIAFRecord], original_entity: str) -> Optional[Field]:
+        if record.get("100") or record.get("110") or record.get("111"):
+            return record
+
+        type_map = {
+            "Personal": "100",
+            "Corporate": "110",
+            "Collective": "111"
+        }
+        
+        tag = type_map.get(getattr(viaf_record, "name_type", None), "100")
+        title = getattr(viaf_record, "name", None) or original_entity
+
+        fields = [Field(tag=tag, indicators=EMPTY_INDICATORS, subfields=[Subfield("t", title)])]
+        
+        self._add_fields_to_record(record, fields)
+        
+        if viaf_record:
+            self._include_name_variations(record, viaf_record)
+
+    def _normalize_viaf(self, record: Record, viaf_record: VIAFRecord, original_entity: str) -> None:
+        
+        if not viaf_record:
+            # viaf record not found, include original entity as 100|t
+            self._add_author(record, viaf_record=None, original_entity=original_entity)
+            return record
+
+        viaf_id = viaf_record.viaf_id
+        fields = [
+            Field(
+                tag="035",
+                indicators=EMPTY_INDICATORS,
+                subfields=[
+                    Subfield("a", viaf_id)
+                ]
+            )
+        ]
+
+        self._add_fields_to_record(record, fields)
+        self._add_author(record, viaf_record, original_entity=original_entity)
+
+    def _normalize_record(self, record: Record, sierraID: str,
+                          viaf_record: VIAFRecord, is_editing_existing_record: bool, original_entity: str) -> Record:
+
+        self._normalize_sierra(record)
+        self._normalize_viaf(record, viaf_record, original_entity=original_entity)
+
+        return record

{rara_tools-0.7.10 → rara_tools-0.7.12/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.10
+Version: 0.7.12
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.7.10 → rara_tools-0.7.12}/tests/test_normalization.py

@@ -404,6 +404,17 @@ def test_add_birth_and_death_dates():
     # empty indicators represented with \
     assert field_046 == "=046  \\\\$f19700817"
     
+    # Case 3 - viaf record has no birth or death date
+    viaf_record = normalizer._get_viaf_record(
+        record,
+        entity="Eesti Interlingvistika Selts"      
+    )
+    record = Record()
+    normalizer._add_birth_and_death_dates(record, viaf_record)
+    # should not add 046 field
+    fields_046 = record.get_fields("046")
+    assert len(fields_046) == 0
+    
 def test_add_nationality():
     """ Test adding nationality from VIAF record to 043 field """
     
@@ -456,7 +467,7 @@ def test_add_nationality():
     # Case 4 - 043 field already exists - should not get edited (not in ALLOW_EDIT_FIELDS)
     linking_results = [{
         "original_entity": "Eduard Vilde",
-        "entity_type": "PER",
+        "entity_type": EntityType.PER,
         "linked_info": [
             {
                 "json": {
@@ -482,12 +493,7 @@ def test_add_nationality():
     normalizer = AuthoritiesRecordNormalizer(
         linking_results=linking_results,
     )
-    records = JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )
-    
-    record = next(iter(records))
-        
+    record = normalizer.first
     # mock run add nationality with foreign VIAF record
     viaf_record = normalizer._get_viaf_record(
         record,
@@ -519,9 +525,7 @@ def test_create_new_normrecord():
     data = normalizer.data
     assert len(data) == 1
     
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     leader = str(record.leader)
     assert leader == "01682nz  a2200349n  4500"
     assert len(leader) == 24
@@ -534,9 +538,7 @@ def test_create_new_normrecord():
     normalizer = AuthoritiesRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
     assert len(data) == 1
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
     _validate_new_record_notes(record)
     # validate leader
@@ -552,9 +554,7 @@ def test_create_new_normrecord():
     normalizer = BibRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
     assert len(data) == 1
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
     _validate_new_record_notes(record)
     # validate leader
@@ -571,9 +571,7 @@ def test_create_new_normrecord():
     normalizer = BibRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
     assert len(data) == 1
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
 def test_680_field_on_existing_record_moved_to_667():
     """ 680 Should not be added for new, if exists on existing record, should be moved to 667 """
@@ -604,9 +602,7 @@ def test_680_field_on_existing_record_moved_to_667():
     normalizer = AuthoritiesRecordNormalizer(
         linking_results=linking_results
     )
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
     fields_680 = record.get_fields("680")
     assert len(fields_680) == 0
@@ -630,4 +626,43 @@ def test_date_formatting():
     # invalid date formats - should return empty string
     invalid_dates = ["abcd", "199A0101"]
     for date in invalid_dates:
-        assert normalizer._format_date(date) == ""
+        assert normalizer._format_date(date) == ""
+        
+def test_new_bibrecord_title_included():
+    """ normrecord for bibs has to always have the 1XX|t field filled """
+    
+    # Case 1 No linker response, & Viaf record found
+    linking_results = [{
+        "original_entity": "Lord of the Rings",
+        "entity_type": EntityType.TITLE,
+        "linked_info": []
+    }]
+
+    normalizer = BibRecordNormalizer(
+        linking_results=linking_results,
+    )
+    data = normalizer.data
+    assert len(data) == 1  # should enrich existing record
+    record = normalizer.first
+    
+    _validate_new_record_notes(record)
+    fields_100 = record.get_fields("100")
+    assert len(fields_100) == 1
+    assert fields_100[0].get_subfields("t")[0] == "Lord of the rings"
+    
+    # Case 2 - Viaf record not found - should use original entity
+    linking_results = [{
+        "original_entity": "Roolijoodiku katastroofiline jõulusõit",
+        "entity_type": EntityType.TITLE,
+        "linked_info": []
+    }]
+    normalizer = BibRecordNormalizer(
+        linking_results=linking_results,
+    )
+    record = normalizer.first
+    data = normalizer.data
+    assert len(data) == 1  # should enrich existing record
+    
+    fields_100 = record.get_fields("100")
+    assert len(fields_100) == 1
+    assert fields_100[0].get_subfields("t")[0] == "Roolijoodiku katastroofiline jõulusõit"

rara_tools-0.7.10/VERSION

@@ -1 +0,0 @@
-0.7.10

rara_tools-0.7.10/rara_tools/normalizers/bibs.py

@@ -1,63 +0,0 @@
-from pymarc import (Field, Subfield, Record)
-
-from rara_tools.constants import EMPTY_INDICATORS
-from rara_tools.normalizers.viaf import VIAFRecord
-from rara_tools.normalizers import RecordNormalizer
-
-from typing import List
-
-
-class BibRecordNormalizer(RecordNormalizer):
-    """ Normalize bib records. """
-
-    def __init__(self, linking_results: List[dict] = [], sierra_data: List[dict] = [],
-                 ALLOW_EDIT_FIELDS: List[str] = ["008", "925"],
-                 REPEATABLE_FIELDS: List[str] = ["667"]):
-        super().__init__(linking_results, sierra_data)
-        self.DEFAULT_LEADER = "00399nz  a2200145n  4500" # must be 24 digits
-        self.ALLOW_EDIT_FIELDS = ALLOW_EDIT_FIELDS
-        self.REPEATABLE_FIELDS = REPEATABLE_FIELDS
-        
-        self.records_extra_data = []
-        self.sierra_data = sierra_data
-        self.records = self._setup_records(linking_results, sierra_data)
-
-    def _normalize_sierra(self, record: Record) -> Record:
-        
-        suffix_008 = "|||aznnnaabn          || |||      "
-        
-        fields = [
-            Field(
-                tag="008",
-                data=f"{self.current_timestamp()}{suffix_008}"
-            ),
-        ]
-
-        self._add_fields_to_record(record, fields)
-
-    def _normalize_viaf(self, record: Record, viaf_record: VIAFRecord) -> None:
-
-        if not viaf_record:
-            return record
-
-        viaf_id = viaf_record.viaf_id
-        fields = [
-            Field(
-                tag="035",
-                indicators=EMPTY_INDICATORS,
-                subfields=[
-                    Subfield("a", viaf_id)
-                ]
-            )
-        ]
-
-        self._add_fields_to_record(record, fields)
-        self._add_author(record, viaf_record)
-
-    def _normalize_record(self, record: Record, sierraID: str,
-                          viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record:
-
-        self._normalize_sierra(record)
-        self._normalize_viaf(record, viaf_record)
-
-        return record