rara-tools 0.5.2__tar.gz → 0.6.0__tar.gz

{rara_tools-0.5.2/rara_tools.egg-info → rara_tools-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.5.2
+Version: 0.6.0
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -18,6 +18,7 @@ Requires-Dist: nltk
 Requires-Dist: jsonlines
 Requires-Dist: requests
 Requires-Dist: iso639-lang
+Requires-Dist: jellyfish
 Requires-Dist: pymarc
 Requires-Dist: regex
 Requires-Dist: glom

rara_tools-0.6.0/VERSION

@@ -0,0 +1 @@
+0.6.0

{rara_tools-0.5.2 → rara_tools-0.6.0}/rara_tools/constants/digitizer.py

@@ -22,4 +22,4 @@ class Queue:
 
 
 class Tasks:
-    MODEL_UPDATE = "component_model_update"
+    START_DIGITIZER_PIPELINE = "start_digitizer_pipeline"

{rara_tools-0.5.2 → rara_tools-0.6.0}/rara_tools/constants/general.py

@@ -11,5 +11,8 @@ class Queue:
     CORE = "core"
 
 
-class Task:
+class Tasks:
     SEND_VERSION = "send_version_to_core"
+    UPDATE_TASK_STATUS = "update_task_status"
+    UPDATE_TASK_VALUES = "update_task_values"
+    MODEL_UPDATE = "component_model_update"

rara_tools-0.6.0/rara_tools/constants/language_evaluator.py

@@ -0,0 +1,9 @@
+COMPONENT_KEY = "language_evaluator"
+
+
+class Tasks:
+    EVALUATE = "text_evaluator"
+
+
+class Queue:
+    EVALUATE = "text_evaluator"

rara_tools-0.6.0/rara_tools/constants/normalizers.py

@@ -0,0 +1,44 @@
+from pymarc import Indicators
+import os
+
+class EntityType:
+    PER = "PER"
+    ORG = "ORG"
+    KEYWORD = "EMS_KEYWORD"
+    LOC = "LOC"
+    TITLE = "TITLE"
+    UNK = "UNKNOWN"
+
+
+EMPTY_INDICATORS = Indicators(" ", " ")
+VIAF_ALLOWED_SOURCES = ["LC", "DNB", "LNB", "NLL",
+                        "ERRR", "J9U"]
+
+DEFAULT_VIAF_FIELD = "local.names"
+
+ALLOWED_VIAF_FIELDS = [
+    "cql.any",                          # All fields
+    "local.names",                      # All headings
+    "local.personalNames",              # Personal names
+    "local.corporateNames",             # Corporate names
+    "local.geographicNames",            # Geographic names
+    "local.uniformTitleWorks",          # Works
+    "local.uniformTitleExpressions",    # Expressions
+    "local.mainHeadingEl",              # Preferred headings
+    "Xlocal.names",                     # Exact headings
+    "local.title"                       # Bibliographic titles
+]
+
+# For mapping rara-linker's entity type's to corresponding VIAF fields
+VIAF_ENTITY_MAP = {
+    EntityType.PER: "local.personalNames",
+    EntityType.ORG: "local.corporateNames",
+    EntityType.LOC: "loca.geographicNames",
+    EntityType.TITLE: "local.uniformTitleWorks"
+
+
+}
+ALLOWED_VIAF_WIKILINK_LANGS = ["en", "et"]
+VIAF_SIMILARITY_THRESHOLD = 0.92
+VERIFY_VIAF_RECORD = True
+MAX_VIAF_RECORDS_TO_VERIFY = 10

rara_tools-0.6.0/rara_tools/constants/subject_indexer.py

@@ -0,0 +1,9 @@
+COMPONENT_KEY = "subject_indexer"
+
+
+class Tasks:
+    PIPELINE  = "run_subject_indexer_pipeline"
+
+
+class Queue:
+    MAIN = "subject-indexer"

{rara_tools-0.5.2 → rara_tools-0.6.0}/rara_tools/normalizers/base.py

@@ -4,7 +4,10 @@ from typing import List, Optional, Iterator
 
 from rara_tools.constants import EMPTY_INDICATORS
 from rara_tools.normalizers.viaf import VIAFRecord, VIAFClient
-
+from rara_tools.constants.normalizers import (
+    DEFAULT_VIAF_FIELD, ALLOWED_VIAF_FIELDS, ALLOWED_VIAF_WIKILINK_LANGS,
+    VIAF_SIMILARITY_THRESHOLD, VERIFY_VIAF_RECORD, MAX_VIAF_RECORDS_TO_VERIFY
+)
 from glom import glom
 import logging
 import json
@@ -187,7 +190,7 @@ class RecordNormalizer:
             "Collective": "111"
         }
 
-        author_type = viaf_record.author_type
+        author_type = viaf_record.name_type
         tag = type_map.get(author_type, "100")
 
         fields = [
@@ -195,9 +198,9 @@ class RecordNormalizer:
                 tag=tag,
                 indicators=EMPTY_INDICATORS,
                 subfields=[
-                    Subfield("a", viaf_record.author),
-                    Subfield("b", viaf_record.author_type),
-                    Subfield("c", viaf_record.author_type)
+                    Subfield("a", viaf_record.name),
+                    Subfield("b", viaf_record.name_type), # Is this correct??
+                    Subfield("c", viaf_record.name_type)  # Is this correct??
                 ]
             )
         ]
@@ -231,32 +234,45 @@ class RecordNormalizer:
         if entity:
             return entity
         else:
-            return record.author
+            return record.name
+
+    def _get_viaf_record(self, record: Record, viaf_id: Optional[int] = None,
+            entity: Optional[str] = None, viaf_field: str = DEFAULT_VIAF_FIELD,
+            threshold: float = VIAF_SIMILARITY_THRESHOLD, verify: bool = VERIFY_VIAF_RECORD,
+            max_records: int = MAX_VIAF_RECORDS_TO_VERIFY
+    ) -> Optional[VIAFRecord]:
+        viaf_record = None
 
-    def _get_viaf_record(self, record: Record, viaf_id: Optional[int] = None, entity: Optional[str] = None) -> Optional[VIAFRecord]:
         try:
             viaf_client = VIAFClient()
 
             if viaf_id:
-                viaf_info = viaf_client.get_records_by_viaf_id(viaf_id).json()
-                return VIAFRecord(viaf_info)
-
-            search_term = self._get_viaf_search_term(record, entity)
-
-            results = viaf_client.get_records_by_search_term(
-                search_term).json()
-
-            num_records = glom(
-                results, "queryResult.numberOfRecords.value", default=0)
-
-            if num_records == 1:
-                return VIAFRecord(results)
-
-            logger.warning(
-                f"Multiple VIAF records found for {search_term}: {num_records}. Skipping.")
+                viaf_records = viaf_client.get_normalized_data_by_ids([viaf_id])
+                if viaf_records:
+                    viaf_record = viaf_records[0]
+            else:
+                search_term = self._get_viaf_search_term(record, entity)
+                if not verify:
+                    logger.warning(
+                        f"Record verification is turned off. If multiple records are " \
+                        f"detected for search term '{search_term}', the first " \
+                        f"result is automatically returned. This might lead to " \
+                        f"some inaccuracies!"
+                    )
+
+                viaf_record = viaf_client.get_normalized_data_by_search_term(
+                    search_term=search_term,
+                    field=viaf_field,
+                    max_records=max_records,
+                    verify=verify,
+                    threshold=threshold
+                )
 
         except Exception as e:
-            logger.error(f"Error fetching VIAF record: {e}")
+            logger.error(
+                f"Error fetching VIAF record with ID={viaf_id} / entity='{entity}': {e}"
+            )
+        return viaf_record
 
     def _normalize_record(self, record: Record, sierraID: str,
                           viaf_record: VIAFRecord, is_editing_existing_record: bool) -> Record: