rara-tools 0.6.12__py3-none-any.whl → 0.6.14__py3-none-any.whl

rara_tools/constants/subject_indexer.py

@@ -1,3 +1,5 @@
+from rara_tools.constants.normalizers import EntityType
+
 COMPONENT_KEY = "subject_indexer"
 
 
@@ -11,4 +13,73 @@ class Queue:
 
 
 class StatusKeys:
-    EXTRACT_KEYWORDS = "extract_keywords"
+    EXTRACT_KEYWORDS = "extract_keywords"
+
+
+class URLSource:
+    VIAF = "VIAF"
+    SIERRA = "Sierra"
+    EMS = "EMS"
+
+class KeywordType:
+    LOC = "Kohamärksõnad"
+    TIME = "Ajamärksõnad"
+    TOPIC = "Teemamärksõnad"
+    GENRE = "Vormimärksõnad"
+    TITLE = "Teose pealkiri"
+    PER = "Isikunimi"
+    ORG = "Kollektiivi nimi"
+    EVENT = "Ajutine kollektiiv või sündmus"
+    CATEGORY = "Valdkonnamärksõnad"
+    UDC = "UDC Summary"
+    UDK = "UDK Rahvusbibliograafia"
+
+
+class KeywordMARC:
+    PER = 600
+    ORG = 610
+    TOPIC = 650
+    GENRE = 655
+    TIME = 648
+    LOC = 651
+    EVENT = 611
+    TITLE = 630
+
+class KeywordSource:
+    EMS = "EMS"
+    SIERRA = "SIERRA"
+    VIAF = "VIAF"
+    AI = "AI"
+
+
+KEYWORD_TYPE_MAP = {
+    KeywordType.TIME: EntityType.KEYWORD,
+    KeywordType.GENRE: EntityType.KEYWORD,
+    KeywordType.LOC: EntityType.LOC,
+    KeywordType.PER: EntityType.PER,
+    KeywordType.ORG: EntityType.ORG,
+    KeywordType.TOPIC: EntityType.KEYWORD,
+    KeywordType.TITLE: EntityType.TITLE,
+    KeywordType.EVENT: EntityType.ORG
+}
+
+KEYWORD_MARC_MAP = {
+    KeywordType.LOC: KeywordMARC.LOC,
+    KeywordType.TIME: KeywordMARC.TIME,
+    KeywordType.TOPIC: KeywordMARC.TOPIC,
+    KeywordType.GENRE: KeywordMARC.GENRE,
+    KeywordType.TITLE: KeywordMARC.TITLE,
+    KeywordType.ORG: KeywordMARC.ORG,
+    KeywordType.PER: KeywordMARC.PER,
+    KeywordType.EVENT: KeywordMARC.EVENT
+}
+
+KEYWORD_TYPES_TO_IGNORE = [
+    KeywordType.CATEGORY,
+    KeywordType.UDC,
+    KeywordType.UDK
+]
+
+EMS_ENTITY_TYPES = [EntityType.KEYWORD, EntityType.LOC]
+SIERRA_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]
+VIAF_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]

rara_tools/formatters.py

@@ -0,0 +1,106 @@
+from typing import List, Tuple, Any
+from rara_tools.constants.subject_indexer import (
+    EntityType, KeywordType, KeywordMARC, KeywordSource, URLSource,
+    KEYWORD_TYPE_MAP, KEYWORD_MARC_MAP, KEYWORD_TYPES_TO_IGNORE,
+    EMS_ENTITY_TYPES, SIERRA_ENTITY_TYPES, VIAF_ENTITY_TYPES
+)
+
+def _get_keyword_source(linked_doc: Any, entity_type: str, is_linked: bool
+) -> str:
+    """ Find keyword source.
+    """
+    if not is_linked:
+        source = KeywordSource.AI
+    elif entity_type in EMS_ENTITY_TYPES:
+        source = KeywordSource.EMS
+    elif entity_type in SIERRA_ENTITY_TYPES:
+        if linked_doc and linked_doc.elastic:
+            source = KeywordSource.SIERRA
+        elif linked_doc and linked_doc.viaf:
+            source = KeywordSource.VIAF
+        else:
+            source = KeywordSource.AI
+    else:
+        source = KeywordSource.AI
+    return source
+
+def _find_indicators(entity_type: str, entity: str,
+        is_linked: bool
+) -> Tuple[str, str]:
+    """ Find MARC indicators 1 and 2.
+    """
+    ind1 = " "
+    ind2 = " "
+    if entity_type in SIERRA_ENTITY_TYPES:
+        if entity_type == EntityType.PER:
+            if "," in entity:
+                ind1 = "1"
+            else:
+                ind1 = "0"
+        else:
+            # 1 märksõna esimeseks elemendiks võimupiirkonna nimi, nt:
+            #    (a) Eesti (b) Riigikogu - raske automaatselt määrata
+            # 2 märksõna esimeseks elemendiks nimi pärijärjestuses
+            ind1 = "2"
+        if not is_linked:
+            ind2 = "4"
+    elif entity_type in EMS_ENTITY_TYPES:
+        ind2 = "4"
+    return (ind1, ind2)
+
+
+def format_keywords(flat_keywords: List[dict]) -> dict:
+    """ Formats unlinked keywords for Kata CORE.
+    """
+    ignored_keywords = []
+    filtered_keywords = []
+
+    for keyword_dict in flat_keywords:
+        keyword_type = keyword_dict.get("entity_type")
+        if keyword_type in KEYWORD_TYPES_TO_IGNORE:
+            ignored_keywords.append(keyword_dict)
+        else:
+            filtered_keywords.append(keyword_dict)
+
+    formatted_keywords = {
+        "keywords": [],
+        "other": ignored_keywords
+    }
+
+    for keyword_dict in filtered_keywords:
+        original_keyword = keyword_dict.get("keyword")
+        keyword_type = keyword_dict.get("entity_type")
+        entity_type = KEYWORD_TYPE_MAP.get(keyword_type, "")
+        marc_field = KEYWORD_MARC_MAP.get(str(keyword_type), "")
+        lang = keyword_dict.get("language", "")
+
+        ind1, ind2 = _find_indicators(
+            entity_type=entity_type,
+            entity=original_keyword,
+            is_linked=False
+        )
+        keyword_source = _get_keyword_source(
+            linked_doc=None,
+            is_linked=False,
+            entity_type=entity_type
+        )
+        new_keyword_dict = {
+            "dates": "",
+            "indicator1": ind1,
+            "indicator2": ind2,
+            "is_linked": False,
+            "keyword_source": keyword_source,
+            "lang": lang,
+            "location": "",
+            "marc_field": marc_field,
+            "numeration": "",
+            "organisation_sub_unit": "",
+            "original_keyword": original_keyword,
+            "persons_title": "",
+            "url": "",
+            "url_source": ""
+        }
+        new_keyword_dict.update(keyword_dict)
+        formatted_keywords["keywords"].append(new_keyword_dict)
+
+    return formatted_keywords

rara_tools/parsers/marc_records/organization_record.py

@@ -6,13 +6,8 @@ import regex as re
 import json
 
 # TODO: indikaatorid ind1 väljadel 100 ja 400?
-"""
-|c asutuse konverentsi toimumise koht (MK)
 
-|d asutuse konverentsi toimumise aeg (K)
 
-|n asutuse konverentsi järjenumber (K)
-"""
 class OrganizationRecord(BaseRecord):
     """ Generates a simplified organization JSON record
     from a pymarc MARC record.
@@ -45,6 +40,7 @@ class OrganizationRecord(BaseRecord):
         self.__name_specification: str = ""
         self.__dates: str = ""
         self.__location: str = ""
+        self.__numeration: str = ""
         self.__name_variations: List[str] = []
         self.__source: str = ""
         self.__description: str = ""
@@ -61,7 +57,10 @@ class OrganizationRecord(BaseRecord):
 
 
     def _clean_value(self, value: str) -> str:
-        cleaned_value = value.strip("., ")
+        try:
+            cleaned_value = value.strip("., ")
+        except Exception as e:
+            cleaned_value = ""
         return cleaned_value
 
     def _merge_and_clean(self, value: dict, keys: List[str]) -> str:
@@ -103,10 +102,10 @@ class OrganizationRecord(BaseRecord):
         if not self.__dates:
             values = self.get_values(
                 marc_ids=self.__name_field_id,
-                subfield_id=["d"]
+                subfield_id="d"
             )
             if values:
-                self.__dates = self.__clean_value(values[0])
+                self.__dates = self._clean_value(values[0])
         return self.__dates
 
     @property
@@ -114,12 +113,23 @@ class OrganizationRecord(BaseRecord):
         if not self.__location:
             values = self.get_values(
                 marc_ids=self.__name_field_id,
-                subfield_id=["c"]
+                subfield_id="c"
             )
             if values:
-                self.__location = self.__clean_value(values[0])
+                self.__location = self._clean_value(values[0])
         return self.__location
 
+    @property
+    def numeration(self) -> str:
+        if not self.__numeration:
+            values = self.get_values(
+                marc_ids=self.__name_field_id,
+                subfield_id="n"
+            )
+            if values:
+                self.__numeration = self._clean_value(values[0])
+        return self.__numeration
+
     @property
     def acronyms(self) -> List[str]:
         if not self.__acronyms:

{rara_tools-0.6.12.dist-info → rara_tools-0.6.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.12
+Version: 0.6.14
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.6.12.dist-info → rara_tools-0.6.14.dist-info}/RECORD

@@ -3,6 +3,7 @@ rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
 rara_tools/digar_schema_converter.py,sha256=k95U2iRlEA3sh772-v6snhHW6fju6qSTMnvWJ6DpzZk,14254
 rara_tools/elastic.py,sha256=7HvDmFKpQbGnnzYyiCKOg0uvubnv2TpCASRrnPP8DcQ,13540
 rara_tools/exceptions.py,sha256=YQyaueUbXeTkJYFDEuN6iWTXMI3eCv5l7PxGp87vg5I,550
+rara_tools/formatters.py,sha256=LTliadjIPZTO4s-44NsumaUdlQlEvqetvWz4bEvwf90,3418
 rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
 rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
 rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
@@ -14,7 +15,7 @@ rara_tools/constants/linker.py,sha256=TQaigi7AUNOqmQPPz3hM8_xXgofrhoQ1taln79LhXQ
 rara_tools/constants/meta_extractor.py,sha256=adYH8cQqH0ZWYO7clGMiObclXRTGsxWgk3pC1oiHxHE,242
 rara_tools/constants/normalizers.py,sha256=Xs3anDwJHpHeniwx3xoIZyqdEXtO3eb7ouGLLr0CpHw,1344
 rara_tools/constants/parsers.py,sha256=L6nh1Itget9_9DMsliDkh6T25z78eMFPWVkbaU08DwU,5561
-rara_tools/constants/subject_indexer.py,sha256=C-Hi4fJ8YKyXB1L-hSKX0plw1ghMkpk61eDhFOqZw2c,250
+rara_tools/constants/subject_indexer.py,sha256=E2D7pylH6Yey9h2TAvAWQiX5JtKKagsZx2E1Fz_afMI,1967
 rara_tools/normalizers/__init__.py,sha256=_NqpS5w710DhaURytHq9JpEt8HgYpSPfRDcOtOymJgE,193
 rara_tools/normalizers/authorities.py,sha256=IDtcm0yNZNhv1f-WcdqWFSRzZk_CoKuBFsk6hEPddWM,4513
 rara_tools/normalizers/base.py,sha256=LbS7Y7CEL-C-ynT-WPc-eCLkNeMO9BI9qtBm-W1skGM,11790
@@ -28,14 +29,14 @@ rara_tools/parsers/marc_parsers/person_parser.py,sha256=iMycHSlgfvgB0axE_rneB5sI
 rara_tools/parsers/marc_parsers/title_parser.py,sha256=uZiYb_aZWzv_xLEBSZmFt2vN6UIauNSFRCkNG_ZKL10,1570
 rara_tools/parsers/marc_records/base_record.py,sha256=yllX2ArjBm9PfUnH6dk3__Rb2LQuEGCYqZGVKBzqSl0,4673
 rara_tools/parsers/marc_records/ems_record.py,sha256=B2YZLEeDd-GmmYqxhczbMsSEB7-x6ZLjB8OeDnzOxww,9376
-rara_tools/parsers/marc_records/organization_record.py,sha256=i1bYVxHkC7Gc7tG7on4LJ83skI0gLiKsU1QD9gSi8Mo,10044
+rara_tools/parsers/marc_records/organization_record.py,sha256=WFmmMBiZUhaIMh-j06ChH37JLT7yFG7ZDc_0keqjIYo,10355
 rara_tools/parsers/marc_records/person_record.py,sha256=AtGESwFmN5YvrBES0BsfTgOZbroB4l0SuFRznumfmJA,7867
 rara_tools/parsers/marc_records/title_record.py,sha256=XrtJ4gj7wzSaGxNaPtPuawmqqkXsVX5HAAKfXTSo4mA,6855
 rara_tools/parsers/tools/entity_normalizers.py,sha256=VyCy_NowCLpOsL0luQ55IW-Qi-J5oBH0Ofzr7HRFBhM,8949
 rara_tools/parsers/tools/marc_converter.py,sha256=LgSHe-7n7aiDrw2bnsB53r3fXTRFjZXTwBYfTpL0pfs,415
 rara_tools/parsers/tools/russian_transliterator.py,sha256=5ZU66iTqAhr7pmfVqXPAI_cidF43VqqmuN4d7H4_JuA,9770
-rara_tools-0.6.12.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.6.12.dist-info/METADATA,sha256=N-5ZB05VDM0rjb6pxV88lD7jIdwYl1FxRf84E05-Wr0,4080
-rara_tools-0.6.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rara_tools-0.6.12.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.6.12.dist-info/RECORD,,
+rara_tools-0.6.14.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.6.14.dist-info/METADATA,sha256=kGpk4MmwncYdzkqpFT64vu9ZgI_oQj4hOzAb6NzpyJs,4080
+rara_tools-0.6.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rara_tools-0.6.14.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.6.14.dist-info/RECORD,,