rara-tools 0.7.9__tar.gz → 0.7.11__tar.gz

{rara_tools-0.7.9/rara_tools.egg-info → rara_tools-0.7.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.9
+Version: 0.7.11
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.7.11/VERSION

@@ -0,0 +1 @@
+0.7.11

{rara_tools-0.7.9 → rara_tools-0.7.11}/rara_tools/normalizers/authorities.py

@@ -69,7 +69,6 @@ class AuthoritiesRecordNormalizer(RecordNormalizer):
             record, [Field(tag="046", indicators=EMPTY_INDICATORS, subfields=subfields_046)])
 
     def _add_viaf_url_or_isni(self, record: Record, viaf_record: VIAFRecord) -> None:
-        # TODO 024. will be used to store KRATT KATA ID. Just generate one?
         viaf_url = f"https://viaf.org/viaf/{viaf_record.viaf_id}"
 
         subfields = [Subfield("0", self.get_subfield(
@@ -109,7 +108,6 @@ class AuthoritiesRecordNormalizer(RecordNormalizer):
         100, 110, 111 - non-repeatable field, attempts to add author type, if missing.
 
         """
-        # TODO: include KRATT KATA ID to 024 and remove on delete. Increment last elastic ID?
         if not viaf_record:
             return
 
@@ -119,7 +117,9 @@ class AuthoritiesRecordNormalizer(RecordNormalizer):
         self._add_author(record, viaf_record)
 
     def _normalize_record(self, record: Record, sierraID: str,
-                          viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record:
+                          viaf_record: VIAFRecord, 
+                          is_editing_existing_record: bool,
+                          original_entity: str) -> Record:
         
         self._normalize_sierra(record, sierraID)
         self._normalize_viaf(record, viaf_record)

{rara_tools-0.7.9 → rara_tools-0.7.11}/rara_tools/normalizers/base.py

@@ -147,21 +147,26 @@ class RecordNormalizer:
         return filter(lambda field: not self._field_in_record(field, record), fields)
     
     def _format_date(self, value: str) -> str:
-        if value is None:
+        
+        if not value:
             return ""
 
         if isinstance(value, (datetime, date)):
-            return value.strftime(YYMMDD_FORMAT)
+            return value.strftime("%Y%m%d")
+
+        val = str(value).strip()
 
         try:
-            dt = parser.parse(str(value), fuzzy=True)
-            formatted_date = dt.strftime(YYMMDD_FORMAT)
-            logger.info(f"Formatted date '{formatted_date}' from value '{value}'")
-            return formatted_date
-        except Exception as e:
-            logger.info(f"Failed to format date string '{value}': {e}")
+            dt = parser.parse(val, fuzzy=False, default=datetime(1, 1, 1))
+        except Exception:
             return ""
 
+        if len(val) == 4 and val.isdigit():
+            return dt.strftime("%Y")  # YYYY
+        if len(val) in (6, 7):  # YYYYMM or YYYY-MM
+            return dt.strftime("%Y%m")  # YYYYMM
+        return dt.strftime("%Y%m%d")  # YYYYMMDD
+
     def get_subfield(self, record: Record, tag: str, subfield: str, default: str) -> str:
         """ get record existing subfield value or assign a fallback value. """
 
@@ -355,7 +360,7 @@ class RecordNormalizer:
         return viaf_record
 
     def _normalize_record(self, record: Record, sierraID: str,
-                          viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record:
+                          viaf_record: VIAFRecord, is_editing_existing_record: bool, original_entity: str) -> Record:
         return record
 
     @property
@@ -369,7 +374,11 @@ class RecordNormalizer:
                 logger.error(f"Failed to normalize record: {e}")
                 continue
         return result
-
+    
+    @property
+    def first(self) -> Record:
+        return next(iter(self))
+    
     def __iter__(self) -> Iterator:
         viaf_id_path = "viaf.queryResult.records.record.0.recordData.VIAFCluster.viafID"
         sierra_id_path = "sierraID"
@@ -386,7 +395,7 @@ class RecordNormalizer:
             record = self._normalize_common(record, is_editing_existing_record)
 
             normalized_record = self._normalize_record(
-                record, sierra_id, viaf_record, is_editing_existing_record)
+                record, sierra_id, viaf_record, is_editing_existing_record, original_entity=entity)
 
             normalized_record.fields.sort(key=lambda field: field.tag)
 

rara_tools-0.7.11/rara_tools/normalizers/bibs.py

@@ -0,0 +1,111 @@
+from pymarc import (Field, Subfield, Record)
+from typing import List, Optional
+
+from rara_tools.constants import EMPTY_INDICATORS
+from rara_tools.normalizers.viaf import VIAFRecord
+from rara_tools.normalizers import RecordNormalizer
+
+from typing import List
+
+
+class BibRecordNormalizer(RecordNormalizer):
+    """ Normalize bib records. """
+
+    def __init__(self, linking_results: List[dict] = [], sierra_data: List[dict] = [],
+                 ALLOW_EDIT_FIELDS: List[str] = ["008", "925"],
+                 REPEATABLE_FIELDS: List[str] = ["667"]):
+        super().__init__(linking_results, sierra_data)
+        self.DEFAULT_LEADER = "00399nz  a2200145n  4500" # must be 24 digits
+        self.ALLOW_EDIT_FIELDS = ALLOW_EDIT_FIELDS
+        self.REPEATABLE_FIELDS = REPEATABLE_FIELDS
+        
+        self.records_extra_data = []
+        self.sierra_data = sierra_data
+        self.records = self._setup_records(linking_results, sierra_data)
+
+    def _normalize_sierra(self, record: Record) -> Record:
+        
+        suffix_008 = "|||aznnnaabn          || |||      "
+        
+        fields = [
+            Field(
+                tag="008",
+                data=f"{self.current_timestamp()}{suffix_008}"
+            ),
+        ]
+
+        self._add_fields_to_record(record, fields)
+        
+    def _include_name_variations(self, record: Record, viaf_record: VIAFRecord) -> None:
+        """ Include name variations from VIAF record as 400|t fields """
+        
+        if not viaf_record or not viaf_record.name_variations:
+            return
+        
+        existing_name_variations = record.get_fields("400")
+        existing_variations = [sf.value for field in existing_name_variations for sf in field.get_subfields("t")]
+        
+        fields = []
+        
+        for variation in viaf_record.name_variations:
+            if variation not in existing_variations:
+                fields.append(
+                    Field(
+                        tag="400",
+                        indicators=EMPTY_INDICATORS,
+                        subfields=[
+                            Subfield("t", variation)
+                        ]
+                    )
+                )
+        
+        self._add_fields_to_record(record, fields)
+        
+    def _add_author(self, record: Record, viaf_record: Optional[VIAFRecord], original_entity: str) -> Optional[Field]:
+        if record.get("100") or record.get("110") or record.get("111"):
+            return record
+
+        type_map = {
+            "Personal": "100",
+            "Corporate": "110",
+            "Collective": "111"
+        }
+        
+        tag = type_map.get(getattr(viaf_record, "name_type", None), "100")
+        title = getattr(viaf_record, "name", None) or original_entity
+
+        fields = [Field(tag=tag, indicators=EMPTY_INDICATORS, subfields=[Subfield("t", title)])]
+        
+        self._add_fields_to_record(record, fields)
+        
+        if viaf_record:
+            self._include_name_variations(record, viaf_record)
+
+    def _normalize_viaf(self, record: Record, viaf_record: VIAFRecord, original_entity: str) -> None:
+        
+        if not viaf_record:
+            # viaf record not found, include original entity as 100|t
+            self._add_author(record, viaf_record=None, original_entity=original_entity)
+            return record
+
+        viaf_id = viaf_record.viaf_id
+        fields = [
+            Field(
+                tag="035",
+                indicators=EMPTY_INDICATORS,
+                subfields=[
+                    Subfield("a", viaf_id)
+                ]
+            )
+        ]
+
+        self._add_fields_to_record(record, fields)
+        self._add_author(record, viaf_record, original_entity=original_entity)
+
+    def _normalize_record(self, record: Record, sierraID: str,
+                          viaf_record: VIAFRecord, is_editing_existing_record: bool, original_entity: str) -> Record:
+
+        self._normalize_sierra(record)
+        self._normalize_viaf(record, viaf_record, original_entity=original_entity)
+
+        return record

{rara_tools-0.7.9 → rara_tools-0.7.11/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.9
+Version: 0.7.11
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.7.9 → rara_tools-0.7.11}/tests/test_normalization.py

@@ -385,7 +385,7 @@ def test_add_birth_and_death_dates():
     normalizer._add_birth_and_death_dates(record, viaf_record)
     
     field_046 = str(record.get_fields("046")[0])
-    assert field_046 == "=046  \\\\$f160107$g750605"
+    assert field_046 == "=046  \\\\$f19160107$g19750605"
     
     # Case two: viaf record has birth date, but no death date (author still alive)
     viaf_record = normalizer._get_viaf_record(
@@ -402,7 +402,7 @@ def test_add_birth_and_death_dates():
     
     field_046 = str(record.get_fields("046")[0])
     # empty indicators represented with \
-    assert field_046 == "=046  \\\\$f700817"
+    assert field_046 == "=046  \\\\$f19700817"
     
 def test_add_nationality():
     """ Test adding nationality from VIAF record to 043 field """
@@ -456,7 +456,7 @@ def test_add_nationality():
     # Case 4 - 043 field already exists - should not get edited (not in ALLOW_EDIT_FIELDS)
     linking_results = [{
         "original_entity": "Eduard Vilde",
-        "entity_type": "PER",
+        "entity_type": EntityType.PER,
         "linked_info": [
             {
                 "json": {
@@ -482,12 +482,7 @@ def test_add_nationality():
     normalizer = AuthoritiesRecordNormalizer(
         linking_results=linking_results,
     )
-    records = JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )
-    
-    record = next(iter(records))
-        
+    record = normalizer.first
     # mock run add nationality with foreign VIAF record
     viaf_record = normalizer._get_viaf_record(
         record,
@@ -519,9 +514,7 @@ def test_create_new_normrecord():
     data = normalizer.data
     assert len(data) == 1
     
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     leader = str(record.leader)
     assert leader == "01682nz  a2200349n  4500"
     assert len(leader) == 24
@@ -534,9 +527,7 @@ def test_create_new_normrecord():
     normalizer = AuthoritiesRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
     assert len(data) == 1
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
     _validate_new_record_notes(record)
     # validate leader
@@ -552,9 +543,7 @@ def test_create_new_normrecord():
     normalizer = BibRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
     assert len(data) == 1
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
     _validate_new_record_notes(record)
     # validate leader
@@ -571,9 +560,7 @@ def test_create_new_normrecord():
     normalizer = BibRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
     assert len(data) == 1
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
 def test_680_field_on_existing_record_moved_to_667():
     """ 680 Should not be added for new, if exists on existing record, should be moved to 667 """
@@ -604,15 +591,67 @@ def test_680_field_on_existing_record_moved_to_667():
     normalizer = AuthoritiesRecordNormalizer(
         linking_results=linking_results
     )
-    record = next(iter(JSONReader(
-            json.dumps(normalizer.data, ensure_ascii=False)
-        )))
+    record = normalizer.first
     
     fields_680 = record.get_fields("680")
     assert len(fields_680) == 0
     fields_667 = record.get_fields("667")
     assert len(fields_667) == 3  # original + moved from 680 + new note    
 
+def test_date_formatting():
+    normalizer = AuthoritiesRecordNormalizer()
+    
+    dates = {
+        "19700712": "19700712",
+        "1970": "1970",
+        "1970-07": "197007",
+        "2001-12-31": "20011231",
+        "1999-01": "199901",
+    }
+    
+    for input_date, expected in dates.items():
+        assert normalizer._format_date(input_date) == expected
+    
+    # invalid date formats - should return empty string
+    invalid_dates = ["abcd", "199A0101"]
+    for date in invalid_dates:
+        assert normalizer._format_date(date) == ""
+        
+def test_new_bibrecord_title_included():
+    """ normrecord for bibs has to always have the 1XX|t field filled """
+    
+    # Case 1 No linker response, & Viaf record found
+    linking_results = [{
+        "original_entity": "Lord of the Rings",
+        "entity_type": EntityType.TITLE,
+        "linked_info": []
+    }]
 
-
-    
+    normalizer = BibRecordNormalizer(
+        linking_results=linking_results,
+    )
+    data = normalizer.data
+    assert len(data) == 1  # should enrich existing record
+    record = normalizer.first
+    
+    _validate_new_record_notes(record)
+    fields_100 = record.get_fields("100")
+    assert len(fields_100) == 1
+    assert fields_100[0].get_subfields("t")[0] == "Lord of the rings"
+    
+    # Case 2 - Viaf record not found - should use original entity
+    linking_results = [{
+        "original_entity": "Roolijoodiku katastroofiline jõulusõit",
+        "entity_type": EntityType.TITLE,
+        "linked_info": []
+    }]
+    normalizer = BibRecordNormalizer(
+        linking_results=linking_results,
+    )
+    record = normalizer.first
+    data = normalizer.data
+    assert len(data) == 1  # should enrich existing record
+    
+    fields_100 = record.get_fields("100")
+    assert len(fields_100) == 1
+    assert fields_100[0].get_subfields("t")[0] == "Roolijoodiku katastroofiline jõulusõit"

rara_tools-0.7.9/VERSION

@@ -1 +0,0 @@
-0.7.9

rara_tools-0.7.9/rara_tools/normalizers/bibs.py

@@ -1,63 +0,0 @@
-from pymarc import (Field, Indicators, Subfield, Record)
-
-from rara_tools.constants import EMPTY_INDICATORS
-from rara_tools.normalizers.viaf import VIAFRecord
-from rara_tools.normalizers import RecordNormalizer
-
-from typing import List
-
-
-class BibRecordNormalizer(RecordNormalizer):
-    """ Normalize bib records. """
-
-    def __init__(self, linking_results: List[dict] = [], sierra_data: List[dict] = [],
-                 ALLOW_EDIT_FIELDS: List[str] = ["008", "925"],
-                 REPEATABLE_FIELDS: List[str] = ["667"]):
-        super().__init__(linking_results, sierra_data)
-        self.DEFAULT_LEADER = "00399nz  a2200145n  4500" # must be 24 digits
-        self.ALLOW_EDIT_FIELDS = ALLOW_EDIT_FIELDS
-        self.REPEATABLE_FIELDS = REPEATABLE_FIELDS
-        
-        self.records_extra_data = []
-        self.sierra_data = sierra_data
-        self.records = self._setup_records(linking_results, sierra_data)
-
-    def _normalize_sierra(self, record: Record) -> Record:
-        
-        suffix_008 = "|||aznnnaabn          || |||      "
-        
-        fields = [
-            Field(
-                tag="008",
-                data=f"{self.current_timestamp()}{suffix_008}"
-            ),
-        ]
-
-        self._add_fields_to_record(record, fields)
-
-    def _normalize_viaf(self, record: Record, viaf_record: VIAFRecord) -> None:
-
-        if not viaf_record:
-            return record
-
-        viaf_id = viaf_record.viaf_id
-        fields = [
-            Field(
-                tag="035",
-                indicators=EMPTY_INDICATORS,
-                subfields=[
-                    Subfield("a", viaf_id)
-                ]
-            )
-        ]
-
-        self._add_fields_to_record(record, fields)
-        self._add_author(record, viaf_record)
-
-    def _normalize_record(self, record: Record, sierraID: str,
-                          viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record:
-
-        self._normalize_sierra(record)
-        self._normalize_viaf(record, viaf_record)
-
-        return record