rara-tools 0.7.16__py3-none-any.whl → 0.7.18__py3-none-any.whl

rara_tools/digar_schema_converter.py

@@ -77,7 +77,7 @@ class PageSchema:
             self.__schema = {
                 "@type": "CreativeWork",  # CONSTANT for pages
                 "@id": self.page_id,
-                "hasPart": []
+                "dcterms:hasPart": []
             }
             text_schemas = [
                 TextPageSchema(page).schema
@@ -91,7 +91,7 @@ class PageSchema:
             page_schemas = text_schemas + image_schemas
             page_schemas_with_ids = self._add_segment_ids(page_schemas)
 
-            self.__schema["hasPart"].extend(page_schemas_with_ids)
+            self.__schema["dcterms:hasPart"].extend(page_schemas_with_ids)
 
         return self.__schema
 

rara_tools/normalizers/base.py

@@ -3,6 +3,8 @@ from pymarc import (Field, Subfield, JSONReader, Record)
 from typing import List, Optional, Iterator
 from rara_tools.normalizers.reader import SafeJSONReader
 
+from rara_tools.parsers.tools.validators import filter_names
+
 from rara_tools.normalizers.viaf import VIAFRecord, VIAFClient
 from rara_tools.constants.normalizers import (
     DEFAULT_VIAF_FIELD, ALLOWED_VIAF_FIELDS, ALLOWED_VIAF_WIKILINK_LANGS,
@@ -311,25 +313,34 @@ class RecordNormalizer:
         if viaf_record:
             self._include_name_variations(record, viaf_record)
             
-    def _include_name_variations(self, record: Record, viaf_record: VIAFRecord) -> None:
+    def _include_name_variations(self, record: Record, viaf_record: VIAFRecord, filter_variations=True) -> None:
         """ Include name variations from VIAF record as 400|t fields """
         
         if not viaf_record or not viaf_record.name_variations:
             return
         
         existing_name_variations = record.get_fields("400")
-        existing_variations = [sf.value for field in existing_name_variations for sf in field.get_subfields("t")]
+        existing_variations = [sf.value for field in existing_name_variations for sf in field.get_subfields("a")]
+        
+        if filter_variations:
+            allowed_variations = filter_names(viaf_record.name_variations)
+            logger.debug(
+                f"filtered out {len(viaf_record.name_variations) - len(allowed_variations)} name variations for '{viaf_record.name}'"
+            )
+            
+        else:
+            allowed_variations = viaf_record.name_variations
         
         fields = []
         
-        for variation in viaf_record.name_variations:
+        for variation in allowed_variations:
             if variation not in existing_variations:
                 fields.append(
                     Field(
                         tag="400",
                         indicators=EMPTY_INDICATORS,
                         subfields=[
-                            Subfield("t", variation)
+                            Subfield("a", variation)
                         ]
                     )
                 )

rara_tools/normalizers/bibs.py

@@ -73,26 +73,13 @@ class BibRecordNormalizer(RecordNormalizer):
             
           
     def _normalize_viaf(self, record: Record, viaf_record: VIAFRecord, original_entity: str) -> None:
-        
         if not viaf_record:
             # viaf record not found, include original entity as 100|t
             self._add_author(record, viaf_record=None, original_entity=original_entity)
             return record
-
-        viaf_id = viaf_record.viaf_id
-        fields = [
-            Field(
-                tag="035",
-                indicators=EMPTY_INDICATORS,
-                subfields=[
-                    Subfield("a", viaf_id)
-                ]
-            )
-        ]
-
-        self._add_fields_to_record(record, fields)
+        
         self._add_author(record, viaf_record, original_entity=original_entity)
-
+        
     def _normalize_record(self, record: Record, sierraID: str,
                           viaf_record: VIAFRecord, is_editing_existing_record: bool, original_entity: str) -> Record:
 

rara_tools/parsers/tools/validators.py

@@ -0,0 +1,54 @@
+import regex as re
+from typing import List
+
+def has_valid_chars(entity: str, allow_cyrillic: bool = True) -> bool:
+    """ Checks if entity contains any valid characters in latin
+    or in cyrillic, if the latter is enabled
+
+    Parameters
+    ------------
+    entity: str
+        String to validate.
+    allow_cyrillic: bool
+        Allow strings in cyrillic?
+
+    Returns
+    ------------
+    bool
+        Boolean value indicating, if the string
+        contains any valid characters.
+
+    """
+    # Check for latin characters
+    is_valid = bool(re.search(r"[a-züõöäA-ZÜÕÖÄ]", entity))
+
+    if allow_cyrillic and not is_valid:
+        # If cyrillic characters are allowed,
+        # check for them as well
+        is_valid = bool(re.search(r"[а-яА-Я]", entity))
+
+    return is_valid
+
+
+def filter_names(names: List[str], allow_cyrillic: bool = True) -> List[str]:
+    """ Filters out names not in allowed encodings (latin / cyrillic).
+
+    Parameters
+    ------------
+    names: List[str]
+        Names to filters.
+    allow_cyrillic: bool
+        Allow strings in cyrillic?
+
+    Returns
+    ------------
+    List[str]
+        List of filtered names.
+
+    """
+    filtered_names = [
+        name for name in names
+        if has_valid_chars(entity=name, allow_cyrillic=allow_cyrillic)
+    ]
+    return filtered_names
+    

{rara_tools-0.7.16.dist-info → rara_tools-0.7.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.16
+Version: 0.7.18
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.7.16.dist-info → rara_tools-0.7.18.dist-info}/RECORD

@@ -1,6 +1,6 @@
 rara_tools/converters.py,sha256=a1dEMa0TwcO9UmjuSBkiuc7LGmH0d_dB6wwoTLpdZhI,4040
 rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
-rara_tools/digar_schema_converter.py,sha256=usrNwlbN63wTE5U56vbmyzT_SxGLXO6ZF4JwY3Lnkqg,15061
+rara_tools/digar_schema_converter.py,sha256=wd6QeSxC1nfiH5tDogfNl0zO1VnS5IiPZ5Y2UIrjOL4,15077
 rara_tools/elastic.py,sha256=4D9yoyMy6AJIKwhSi2H1usffDHAh2A_IZfv5BtYnBKg,13992
 rara_tools/exceptions.py,sha256=YQyaueUbXeTkJYFDEuN6iWTXMI3eCv5l7PxGp87vg5I,550
 rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
@@ -21,8 +21,8 @@ rara_tools/core_formatters/formatted_meta.py,sha256=WEnMs8K0YeTLGjXn_mxQTpshxcz5
 rara_tools/core_formatters/formatted_object.py,sha256=7a499ZmcZXOqtlwxDi6FWHWF5a6HdCsduS22wV3uHIE,5656
 rara_tools/normalizers/__init__.py,sha256=_NqpS5w710DhaURytHq9JpEt8HgYpSPfRDcOtOymJgE,193
 rara_tools/normalizers/authorities.py,sha256=iW3cYOqqVJKy4CcnG9_T6dN-1bBT1e-0jtLYvco-MyQ,5311
-rara_tools/normalizers/base.py,sha256=DhMicY5p_N2SC_E3lbWUvSM77AOy_pBjQpbLSvYWDxM,20488
-rara_tools/normalizers/bibs.py,sha256=s8NGoieCjiftASUb--1YvYZ0VzW6uBt2ZidhLi_wP9A,3938
+rara_tools/normalizers/base.py,sha256=tw64ZK7KXg9O2IPMxICMogYHAG6il10qQqCd4fIjQL0,20941
+rara_tools/normalizers/bibs.py,sha256=5pOw8RsQ4eDwbREbYySeI_b7dQyGlJnfMRSS-tWGJ9c,3632
 rara_tools/normalizers/reader.py,sha256=GYCkAtnsNx135w5lD-_MqCZzdHQHHPDF-pDxYj839Vo,1595
 rara_tools/normalizers/viaf.py,sha256=C-NfbvL83ZcHVB9ICMw43wAMYKTqDTHU3ZT2mXKec00,24288
 rara_tools/parsers/marc_parsers/base_parser.py,sha256=Kdw4aivJf2FkWgIK7pJtHtVXF_G1pjHVQ7IcFItSqy8,1649
@@ -39,8 +39,9 @@ rara_tools/parsers/marc_records/title_record.py,sha256=XrtJ4gj7wzSaGxNaPtPuawmqq
 rara_tools/parsers/tools/entity_normalizers.py,sha256=VyCy_NowCLpOsL0luQ55IW-Qi-J5oBH0Ofzr7HRFBhM,8949
 rara_tools/parsers/tools/marc_converter.py,sha256=LgSHe-7n7aiDrw2bnsB53r3fXTRFjZXTwBYfTpL0pfs,415
 rara_tools/parsers/tools/russian_transliterator.py,sha256=5ZU66iTqAhr7pmfVqXPAI_cidF43VqqmuN4d7H4_JuA,9770
-rara_tools-0.7.16.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.7.16.dist-info/METADATA,sha256=yBmOUFVy7V6RyZdN_qBGkY7M8zH7H7h7v_QLQeZ3bAM,4080
-rara_tools-0.7.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rara_tools-0.7.16.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.7.16.dist-info/RECORD,,
+rara_tools/parsers/tools/validators.py,sha256=JTGbfAWcLldlZrX0nb343P9RJ8QwSh3455fYap3UxxY,1335
+rara_tools-0.7.18.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.7.18.dist-info/METADATA,sha256=9AR_e8-yNVW_qp6Iaxp0IP2_HxV_NU87DE_I2GQOuJg,4080
+rara_tools-0.7.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rara_tools-0.7.18.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.7.18.dist-info/RECORD,,