rara-tools 0.7.0__py3-none-any.whl → 0.7.2__py3-none-any.whl

rara_tools/constants/digitizer.py

@@ -20,7 +20,9 @@ class Queue:
     DOWNLOAD = "download"
     FINISH = "finish"
     OCR = "ocr"
+    UTILITY = "digitizer-utility"
 
 
 class Tasks:
     START_DIGITIZER_PIPELINE = "start_digitizer_pipeline"
+    PURGE_MODELS = "purge_unused_digitizer_models"

rara_tools/constants/general.py

@@ -20,6 +20,7 @@ class Tasks:
     UPDATE_TASK_VALUES = "update_task_values"
     MODEL_UPDATE = "component_model_update"
     RUN_POST_TASK_COMPLETION_TASKS = "run_post_task_completion_tasks"
+    PURGE_MODELS = "purge_unused_models"
 
 
 class Models:

rara_tools/constants/subject_indexer.py

@@ -6,10 +6,12 @@ COMPONENT_KEY = "subject_indexer"
 class Tasks:
     SINGLE = "run_subject_indexer_process"
     PIPELINE = "run_subject_indexer_with_core_logic"
+    PURGE_MODELS = "purge_unused_subjectindexer_models"
 
 
 class Queue:
     MAIN = "subject-indexer"
+    UTILITY = "subjectindexer-utility"
 
 
 class StatusKeys:
@@ -21,6 +23,7 @@ class URLSource:
     SIERRA = "Sierra"
     EMS = "EMS"
 
+
 class KeywordType:
     LOC = "Kohamärksõnad"
     TIME = "Ajamärksõnad"
@@ -45,6 +48,7 @@ class KeywordMARC:
     EVENT = 611
     TITLE = 630
 
+
 class KeywordSource:
     EMS = "EMS"
     SIERRA = "SIERRA"

rara_tools/normalizers/base.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from pymarc import (Field, Subfield, JSONReader, Record)
 from typing import List, Optional, Iterator
+from rara_tools.normalizers.reader import SafeJSONReader
 
 from rara_tools.normalizers.viaf import VIAFRecord, VIAFClient
 from rara_tools.constants.normalizers import (
@@ -35,7 +36,6 @@ class RecordNormalizer:
         self.ALLOW_EDIT_FIELDS = ALLOW_EDIT_FIELDS
         # include, if should be added alongside existing fields
         self.REPEATABLE_FIELDS = REPEATABLE_FIELDS
-
         self.records_extra_data = []
         self.records = self._setup_records(linking_results, sierra_data)
         self.sierra_data = sierra_data
@@ -92,8 +92,56 @@ class RecordNormalizer:
 
         all_records = linked_records + (sierra_data or [])
 
-        return JSONReader(json.dumps(all_records,
-                                     ensure_ascii=False), stream=False)
+        return SafeJSONReader(
+            json.dumps(all_records, ensure_ascii=False),
+        )
+
+    def _setup_records(self, linking_results: List[dict], sierra_data: List[dict]) -> JSONReader:
+        """Setup initial MARC records and data.
+
+        For linked entities:
+            1. Try to get single linked normalized record from KATA elastic. If more than one found, skip.
+            2. If 0 matches, search from VIAF and if 1 result found, create a new authority record from the data.
+            3. If none or more than one responses found, use only Classificator data (coming from Linker?).
+        """
+        linked_records = []
+
+        for linked in linking_results or []:
+            if not isinstance(linked, dict):
+                continue
+
+            entity = linked.get("original_entity")
+            linked_info = linked.get("linked_info", [])
+
+            if not isinstance(linked_info, list) or not linked_info:
+                continue
+
+            if len(linked_info) == 1:
+                linked_item = linked_info[0]
+                if not isinstance(linked_item, dict):
+                    continue
+
+                linked_records.append(linked_item.get("json", {}))
+                self.records_extra_data.append({
+                    "entity": entity,
+                    "viaf": linked_item.get("viaf", {}),
+                    "type": "linked",
+                    "edited": True
+                })
+
+        self.records_extra_data.extend(
+            {
+                "sierraID": obj.get("sierraID"),
+                "type": "sierra",
+                "edited": True
+            }
+            for obj in (sierra_data or [])
+            if isinstance(obj, dict)
+        )
+
+        all_records = linked_records + (sierra_data or [])
+
+        return SafeJSONReader(json.dumps(all_records, ensure_ascii=False))
 
     @staticmethod
     def current_timestamp():
@@ -230,11 +278,15 @@ class RecordNormalizer:
 
     def _get_viaf_search_term(self, record: Record, entity: Optional[str]) -> Optional[str]:
         """ prioritize entity name, if not available, use author name. """
-
         if entity:
             return entity
-        else:
-            return record.name
+
+        author_field = record.get("100") or record.get("110") or record.get("111")
+        if author_field:
+            return author_field.get_subfields("a")[0] if author_field.get_subfields("a") else None
+
+        logger.warning(
+            "No entity or author name found for VIAF search. Skipping VIAF enrichment.")
 
     def _get_viaf_record(self, record: Record, viaf_id: Optional[int] = None,
             entity: Optional[str] = None, viaf_field: str = DEFAULT_VIAF_FIELD,
@@ -252,21 +304,26 @@ class RecordNormalizer:
                     viaf_record = viaf_records[0]
             else:
                 search_term = self._get_viaf_search_term(record, entity)
-                if not verify:
-                    logger.warning(
-                        f"Record verification is turned off. If multiple records are " \
-                        f"detected for search term '{search_term}', the first " \
-                        f"result is automatically returned. This might lead to " \
-                        f"some inaccuracies!"
-                    )
 
-                viaf_record = viaf_client.get_normalized_data_by_search_term(
-                    search_term=search_term,
-                    field=viaf_field,
-                    max_records=max_records,
-                    verify=verify,
-                    threshold=threshold
-                )
+                if search_term:
+                    logger.info(
+                        f"Searching for VIAF record with search term: {search_term}")
+
+                    if not verify:
+                        logger.warning(
+                            f"Record verification is turned off. If multiple records are " \
+                            f"detected for search term '{search_term}', the first " \
+                            f"result is automatically returned. This might lead to " \
+                            f"some inaccuracies!"
+                        )
+
+                    viaf_record = viaf_client.get_normalized_data_by_search_term(
+                        search_term=search_term,
+                        field=viaf_field,
+                        max_records=max_records,
+                        verify=verify,
+                        threshold=threshold
+                    )
 
         except Exception as e:
             logger.error(

rara_tools/normalizers/reader.py

@@ -0,0 +1,45 @@
+from pymarc import Record, Field, Subfield, Leader, JSONReader
+import logging
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_LEADER = '01682nz a2200349n 4500'
+
+class SafeJSONReader(JSONReader):
+
+    def __next__(self):
+        while True:
+            try:
+                jobj = next(self.iter)
+                rec = Record()
+
+                # Use custom default leader if missing
+                leader_str = jobj.get("leader")
+                if leader_str:
+                    rec.leader = Leader(leader_str)
+                else:
+                    logger.warning("Missing leader in record. Using DEFAULT_LEADER.")
+                    rec.leader = Leader(DEFAULT_LEADER)
+
+                for field in jobj["fields"]:
+                    k, v = list(field.items())[0]
+
+                    if isinstance(v, dict) and "subfields" in v:
+                        subfields = []
+                        for sub in v["subfields"]:
+                            for code, value in sub.items():
+                                subfields.append(Subfield(code, value))
+                        ind1 = v.get("ind1", " ")
+                        ind2 = v.get("ind2", " ")
+                        fld = Field(tag=k, indicators=[ind1, ind2], subfields=subfields)
+                    else:
+                        fld = Field(tag=k, data=v)
+                    rec.add_field(fld)
+
+                return rec
+
+            except StopIteration:
+                raise
+            except Exception as e:
+                logger.error(f"Skipping invalid record: {e}")
+                continue

{rara_tools-0.7.0.dist-info → rara_tools-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.0
+Version: 0.7.2
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.7.0.dist-info → rara_tools-0.7.2.dist-info}/RECORD

@@ -7,22 +7,23 @@ rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
 rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
 rara_tools/utils.py,sha256=1UrxOzo3cxe4juMkDlKWv1VKWMYay5v1pivGci1ajiM,3003
 rara_tools/constants/__init__.py,sha256=r78laM9vyRDAvzDhPvzDlhaX6qPwUUBBtwf1WosrW3o,27
-rara_tools/constants/digitizer.py,sha256=A7FfqqEB4hGJ9t3z8gTFK7hkzCxz44rCOSWx6Pzvwjs,548
-rara_tools/constants/general.py,sha256=jE1aIir_eKbka_q1iCJWRtmyz_xpnTPntbshiWo9eTA,1061
+rara_tools/constants/digitizer.py,sha256=9aQkJj8C5a_HLgCayrz3PpGYJMLoO4Ph9_U28Q-C1T4,633
+rara_tools/constants/general.py,sha256=uCSIfE0o-drRDDqnaJALMGHecUOQIRhdr2g5M5NQqMY,1102
 rara_tools/constants/language_evaluator.py,sha256=3sCSaoS-zXQRY0vJ7UUMuZqbtYQD_quVVbdpgvJjE7I,124
 rara_tools/constants/linker.py,sha256=yBN9NpUhB3ENz8BapoIfpSHY_xNqwYdqutgQFdc_Cd8,3240
 rara_tools/constants/meta_extractor.py,sha256=adYH8cQqH0ZWYO7clGMiObclXRTGsxWgk3pC1oiHxHE,242
 rara_tools/constants/normalizers.py,sha256=Xs3anDwJHpHeniwx3xoIZyqdEXtO3eb7ouGLLr0CpHw,1344
 rara_tools/constants/parsers.py,sha256=L6nh1Itget9_9DMsliDkh6T25z78eMFPWVkbaU08DwU,5561
-rara_tools/constants/subject_indexer.py,sha256=E2D7pylH6Yey9h2TAvAWQiX5JtKKagsZx2E1Fz_afMI,1967
+rara_tools/constants/subject_indexer.py,sha256=i0xRdqwasyb6d6WZZKPgyuEUd2JeO_qwWYoG6UeBo5U,2064
 rara_tools/core_formatters/core_formatter.py,sha256=HJX7jOi9kaFie_zm0Wzjk0nKF8dRleJpVWbCplFFquo,2760
 rara_tools/core_formatters/formatted_keyword.py,sha256=1-B9IQTycFt69pTy8WZNnfJ2WIMRow3kpEub6igyNQc,7865
 rara_tools/core_formatters/formatted_meta.py,sha256=Zd0oQFLbn6m_wHaWtgxBsu9J7wGyWIpZxb2-8PrR3Wk,5240
 rara_tools/core_formatters/formatted_object.py,sha256=7a499ZmcZXOqtlwxDi6FWHWF5a6HdCsduS22wV3uHIE,5656
 rara_tools/normalizers/__init__.py,sha256=_NqpS5w710DhaURytHq9JpEt8HgYpSPfRDcOtOymJgE,193
 rara_tools/normalizers/authorities.py,sha256=IDtcm0yNZNhv1f-WcdqWFSRzZk_CoKuBFsk6hEPddWM,4513
-rara_tools/normalizers/base.py,sha256=6tLfNdF6FZo8M6j_Q61lXoaF1HdIB1c0SKMatTc-Z64,12014
+rara_tools/normalizers/base.py,sha256=e2Ql_JNrpRiVDeb9Iqf6mug3BHNjKasjwmiIh5oWgWc,14276
 rara_tools/normalizers/bibs.py,sha256=4DTS6k37z8qR5B3n7aiCXsT5Z49rLTvQ60lKKr5dyLs,2352
+rara_tools/normalizers/reader.py,sha256=_usCY2jDC_1KvEJEdthPEldVz-v4-2-S8deNJCzQVoI,1573
 rara_tools/normalizers/viaf.py,sha256=LIeqbJoKtVt_0H1o7XMmhSE0BjF4l-jdAJgX_8Gg9Z4,24218
 rara_tools/parsers/marc_parsers/base_parser.py,sha256=Kdw4aivJf2FkWgIK7pJtHtVXF_G1pjHVQ7IcFItSqy8,1649
 rara_tools/parsers/marc_parsers/ems_parser.py,sha256=LFuhZcVwmHMcJknX9p4ZkO8RdjPdQZ4APGbw8KV6BIs,2024
@@ -38,8 +39,8 @@ rara_tools/parsers/marc_records/title_record.py,sha256=XrtJ4gj7wzSaGxNaPtPuawmqq
 rara_tools/parsers/tools/entity_normalizers.py,sha256=VyCy_NowCLpOsL0luQ55IW-Qi-J5oBH0Ofzr7HRFBhM,8949
 rara_tools/parsers/tools/marc_converter.py,sha256=LgSHe-7n7aiDrw2bnsB53r3fXTRFjZXTwBYfTpL0pfs,415
 rara_tools/parsers/tools/russian_transliterator.py,sha256=5ZU66iTqAhr7pmfVqXPAI_cidF43VqqmuN4d7H4_JuA,9770
-rara_tools-0.7.0.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.7.0.dist-info/METADATA,sha256=UmA4_431SdsYqP6IrEv8mr4yL9OoFVGHvpG-mQgs_g8,4079
-rara_tools-0.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rara_tools-0.7.0.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.7.0.dist-info/RECORD,,
+rara_tools-0.7.2.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.7.2.dist-info/METADATA,sha256=x8GYrcQQCpMdhCUlfLr0Nxd1a2fNH3Fg5sUq2nK5x6Q,4079
+rara_tools-0.7.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rara_tools-0.7.2.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.7.2.dist-info/RECORD,,