rara-tools 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

rara_tools/parsers/marc_records/ems_record.py

@@ -0,0 +1,267 @@
+from typing import List, NoReturn
+from pymarc.record import Record
+from rara_tools.parsers.tools.entity_normalizers import KeywordNormalizer
+from rara_tools.parsers.marc_records.base_record import BaseRecord
+from rara_tools.constants.parsers import (
+    EMSMarcIDs, KeywordType,
+    EN_SUBJECT_FIELDS, ET_SUBJECT_FIELDS
+)
+import regex as re
+import json
+
+
+
+class EMSRecord(BaseRecord):
+    """ Generates a simplified EMS JSON record
+    from a pymarc MARC record.
+    """
+    def __init__(self, record: Record, add_variations: bool = False) -> NoReturn:
+        """ Initializes EMSRecord object.
+
+        Parameters
+        -----------
+        record: Record
+            pymarc.record.Record object.
+        add_variations: bool
+            If enabled, constructs an additional variations field, which
+            combines the content of multiple fields + adds some generated
+            variations. If the output is uploaded into Elastic and used
+            via rara-norm-linker, it is necessary to enable this.
+        """
+        super().__init__(record=record, add_variations=add_variations)
+        self.__en_subject_field_mapping: dict = EN_SUBJECT_FIELDS
+        self.__et_subject_field_mapping: dict = ET_SUBJECT_FIELDS
+        self.__keyword: str = ""
+        self.__keyword_en: str = ""
+        self.__keyword_type: str = ""
+        self.__keyword_variations: List[str] = []
+        self.__keyword_fields = {
+            EMSMarcIDs.TIME_KEYWORD: KeywordType.TIME,
+            EMSMarcIDs.TOPIC_KEYWORD: KeywordType.TOPIC,
+            EMSMarcIDs.LOC_KEYWORD: KeywordType.LOC,
+            EMSMarcIDs.GENRE_KEYWORD: KeywordType.GENRE
+        }
+
+        self.__ems_url_ids = EMSMarcIDs.URL
+        self.__synonym_ids = EMSMarcIDs.SYNONYMS
+        self.__related_ids = EMSMarcIDs.RELATED
+        self.__category_ids = EMSMarcIDs.CATEGORY
+        self.__notes_ids = EMSMarcIDs.NOTES
+        self.__synonyms: List[str] = []
+        self.__synonyms_en: List[str] = []
+        self.__subject_field_ids: List[str] = []
+
+        self.__subject_fields_et: List[str] = []
+        self.__subject_fields_en: List[str] = []
+        self.__ems_url: str = ""
+        self.__narrower: List[str] = []
+        self.__broader: List[str] = []
+        self.__related: List[str] = []
+        self.__narrower_ems_urls: List[str] = []
+        self.__broader_ems_urls: List[str] = []
+        self.__related_ems_urls: List[str] = []
+        self.__variations: List[str] = []
+        self.__variations_en: List[str] = []
+        self.__use_with_others: bool | None = None
+        self.__full_record: dict = {}
+
+    @property
+    def keyword(self) -> str:
+        if not self.__keyword:
+            self.__keyword = self.get_values(
+                marc_ids=self.__keyword_fields,
+                subfield_id="a"
+            )[0]
+        return self.__keyword
+
+    @property
+    def keyword_en(self) -> str:
+        if not self.__keyword_en:
+            self.__keyword_en = self.synonyms_en[0] if self.synonyms_en else ""
+        return self.__keyword_en
+
+    @property
+    def synonyms(self) -> List[str]:
+        if not self.__synonyms:
+            self.__synonyms = self.get_values(
+                marc_ids=self.__synonym_ids,
+                subfield_id="a"
+            )
+        return self.__synonyms
+
+    @property
+    def synonyms_en(self) -> List[str]:
+        if not self.__synonyms_en:
+            self.__synonyms_en = self.get_values(
+                marc_ids=self.__synonym_ids,
+                subfield_id="a", ind2="9"
+            )
+        return self.__synonyms_en
+
+    @property
+    def subject_field_ids(self) -> List[str]:
+        if not self.__subject_field_ids:
+            self.__subject_field_ids = self.get_values(
+                marc_ids=self.__category_ids,
+                subfield_id="a",
+                ind2="7"
+            )
+        return self.__subject_field_ids
+
+    @property
+    def subject_fields_et(self) -> List[str]:
+        if not self.__subject_fields_et:
+            self.__subject_fields_et = [
+                self.__et_subject_field_mapping[_id]
+                for _id in self.subject_field_ids
+            ]
+        return self.__subject_fields_et
+
+    @property
+    def subject_fields_en(self) -> List[str]:
+        if not self.__subject_fields_en:
+            self.__subject_fields_en = [
+                self.__en_subject_field_mapping[_id]
+                for _id in self.subject_field_ids
+            ]
+        return self.__subject_fields_en
+
+
+    @property
+    def ems_url(self) -> str:
+        if not self.__ems_url:
+            self.__ems_url = self.get_values(
+                marc_ids=self.__ems_url_ids,
+                subfield_id="0",
+                ind2="8"
+            )[0]
+        return self.__ems_url
+
+    @property
+    def broader(self) -> List[str]:
+        if not self.__broader:
+            self.__broader = self.get_values(
+                marc_ids=self.__related_ids,
+                subfield_id="a",
+                subfield_restriction = ("w", "g")
+            )
+        return self.__broader
+
+    @property
+    def narrower(self) -> List[str]:
+        if not self.__narrower:
+            self.__narrower = self.get_values(
+                marc_ids=self.__related_ids,
+                subfield_id="a",
+                subfield_restriction = ("w", "h")
+            )
+        return self.__narrower
+
+    @property
+    def related(self) -> List[str]:
+        if not self.__related:
+            self.__related = self.get_values(
+                marc_ids=self.__related_ids,
+                subfield_id="a",
+                subfield_to_ignore ="w"
+            )
+        return self.__related
+
+    @property
+    def broader_ems_urls(self) -> List[str]:
+        if not self.__broader_ems_urls:
+            self.__broader_ems_urls = self.get_values(
+                marc_ids=self.__related_ids,
+                subfield_id="0",
+                subfield_restriction = ("w", "g")
+            )
+        return self.__broader_ems_urls
+
+    @property
+    def narrower_ems_urls(self) -> List[str]:
+        if not self.__narrower_ems_urls:
+            self.__narrower_ems_urls = self.get_values(
+                marc_ids=self.__related_ids,
+                subfield_id="0",
+                subfield_restriction = ("w", "h")
+            )
+        return self.__narrower_ems_urls
+
+    @property
+    def related_ems_urls(self) -> List[str]:
+        if not self.__related_ems_urls:
+            self.__related_ems_urls = self.get_values(
+                marc_ids=self.__related_ids,
+                subfield_id="0",
+                subfield_to_ignore ="w"
+            )
+        return self.__related_ems_urls
+
+    @property
+    def keyword_type(self) -> str:
+        if not self.__keyword_type:
+            for field in self.dict_record:
+                field_id = list(field.keys())[0]
+                if field_id in self.__keyword_fields:
+                    self.__keyword_type = self.__keyword_fields[field_id]
+        return self.__keyword_type
+
+    @property
+    def use_with_others(self) -> bool:
+        if self.__use_with_others == None:
+            notes = self.get_values(marc_ids=self.__notes_ids, subfield_id="i")
+            self.__use_with_others = False
+            if notes:
+                if re.search(r"Kasutada koos teise", notes[0]):
+                    self.__use_with_others = True
+
+        return self.__use_with_others
+
+    @property
+    def variations(self) -> List[str]:
+        if not self.__variations:
+            original_variations = self.synonyms + [self.keyword]
+            variations = []
+            for kw in original_variations:
+                variations_ = KeywordNormalizer(kw, keyword_type=self.keyword_type).variations
+                variations.extend(variations_)
+            self.__variations = [v.lower() for v in list(set(variations))]
+        return self.__variations
+
+    @property
+    def variations_en(self) -> List[str]:
+        if not self.__variations_en:
+            pass
+        return self.__variations_en
+
+
+    @property
+    def full_record(self) -> dict:
+        if not self.__full_record:
+            self.__full_record = {
+                "keyword": self.keyword,
+                "keyword_en": self.keyword_en,
+                "keyword_type": self.keyword_type,
+                "use_with_others": self.use_with_others,
+                "subject_field_ids": self.subject_field_ids,
+                "subject_fields_et": self.subject_fields_et,
+                "subject_fields_en": self.subject_fields_en,
+                "synonyms": self.synonyms,
+                "synonyms_en": self.synonyms_en,
+                "narrower": self.narrower,
+                "broader": self.broader,
+                "related": self.related,
+                "narrower_ems_urls": self.narrower_ems_urls,
+                "broader_ems_urls": self.broader_ems_urls,
+                "related_ems_urls": self.related_ems_urls,
+                "ems_id": self.identifier,
+                "ems_url": self.ems_url,
+                "identifier_source": self.identifier_source,
+                "full_record_marc": str(self.marc_record),
+                "full_record_json": json.dumps(self.marc_json_record)
+            }
+            if self.add_variations:
+                self.__full_record.update(
+                    {"link_variations": self.variations}
+                )
+        return self.__full_record

rara_tools/parsers/marc_records/organization_record.py

@@ -0,0 +1,245 @@
+from typing import List, NoReturn
+from pymarc.record import Record
+from rara_tools.parsers.marc_records.base_record import BaseRecord
+from rara_tools.constants.parsers import OrganizationMarcIDs
+import regex as re
+import json
+
+# TODO: indikaatorid ind1 väljadel 100 ja 400?
+
+
+class OrganizationRecord(BaseRecord):
+    """ Generates a simplified organization JSON record
+    from a pymarc MARC record.
+    """
+    def __init__(self, record: Record, add_variations: bool = False) -> NoReturn:
+        """ Initializes OrganizationRecord object.
+
+        Parameters
+        -----------
+        record: Record
+            pymarc.record.Record object.
+        add_variations: bool
+            If enabled, constructs an additional variations field, which
+            combines the content of multiple fields + adds some generated
+            variations. If the output is uploaded into Elastic and used
+            via rara-norm-linker, it is necessary to enable this.
+        """
+        super().__init__(record=record, add_variations=add_variations)
+
+        self.__name_field_id: List[str] = OrganizationMarcIDs.NAME
+        self.__name_variations_field_id: List[str] = OrganizationMarcIDs.NAME_VARIATIONS
+        self.__related_names_field_id: List[str] = OrganizationMarcIDs.RELATED_NAMES
+        self.__source_field_id: List[str] = OrganizationMarcIDs.SOURCE
+        self.__description_field_id: List[str] = OrganizationMarcIDs.DESCRIPTION
+        self.__area_code_id: List[str] = OrganizationMarcIDs.AREA_CODE
+        self.__default_year: int | None = None
+
+        self.__name: str = ""
+        self.__original_name: dict = {}
+        self.__name_specification: str = ""
+        self.__life_years: str = ""
+        self.__birth_year: int = -1
+        self.__death_year: int = -1
+        self.__name_variations: List[str] = []
+        self.__source: str = ""
+        self.__description: str = ""
+        self.__area_code: str = ""
+        self.__acronyms: List[str] = []
+        self.__alternative_names: List[str] = []
+        self.__related_acronyms: List[str] = []
+        self.__old_names: List[str] = []
+        self.__new_names: List[str] = []
+        self.__related_old_names: List[str] = []
+        self.__related_new_names: List[str] = []
+        self.__full_record: dict = {}
+        self.__variations: List[str] = []
+
+
+    def _clean_value(self, value: str) -> str:
+        cleaned_value = value.strip("., ")
+        return cleaned_value
+
+    def _merge_and_clean(self, value: dict, keys: List[str]) -> str:
+        _merged = []
+        for key in keys:
+            _value = self._clean_value(value.get(key, ""))
+            if _value:
+                _merged.append(_value)
+        merged = " ".join(_merged)
+        return merged
+
+    @property
+    def original_name(self) -> str:
+        if not self.__original_name:
+            values = self.get_values(
+                marc_ids=self.__name_field_id,
+                subfield_id=["a", "b"]
+            )
+            if values:
+                self.__original_name = {
+                    "a": self._clean_value(values[0].get("a", "")),
+                    "b": self._clean_value(values[0].get("b", ""))
+                }
+            else:
+                print(self.marc_record)
+        return self.__original_name
+
+    @property
+    def name(self) -> str:
+        if not self.__name:
+            self.__name = self._merge_and_clean(self.original_name, ["a", "b"])
+        return self.__name
+
+    @property
+    def acronyms(self) -> List[str]:
+        if not self.__acronyms:
+            values = self.get_values(
+                marc_ids=self.__name_variations_field_id,
+                subfield_id="a",
+                subfield_restriction = ("w", "d")
+            )
+            self.__acronyms = [self._clean_value(value) for value in values]
+        return self.__acronyms
+
+    @property
+    def new_names(self) -> List[str]:
+        if not self.__new_names:
+            values = self.get_values(
+                marc_ids=self.__name_variations_field_id,
+                subfield_id=["a", "b"],
+                subfield_restriction = ("w", "b")
+            )
+            self.__new_names = [self._merge_and_clean(value, ["a", "b"]) for value in values]
+        return self.__new_names
+
+    @property
+    def old_names(self) -> List[str]:
+        if not self.__old_names:
+            values = self.get_values(
+                marc_ids=self.__name_variations_field_id,
+                subfield_id=["a", "b"],
+                subfield_restriction = ("w", "a")
+            )
+            self.__old_names = [self._merge_and_clean(value, ["a", "b"]) for value in values]
+        return self.__old_names
+
+    @property
+    def alternative_names(self) -> List[str]:
+        if not self.__alternative_names:
+            values = self.get_values(
+                marc_ids=self.__name_variations_field_id,
+                subfield_id=["a", "b"],
+                subfield_to_ignore="w"
+            )
+            self.__alternative_names = [self._merge_and_clean(value, ["a", "b"]) for value in values]
+        return self.__alternative_names
+
+
+    @property
+    def related_acronyms(self) -> List[str]:
+        if not self.__related_acronyms:
+            values = self.get_values(
+                marc_ids=self.__related_names_field_id,
+                subfield_id="a",
+                subfield_restriction = ("w", "d")
+            )
+            self.__related_acronyms = [self._clean_value(value) for value in values]
+        return self.__related_acronyms
+
+    @property
+    def related_new_names(self) -> List[str]:
+        if not self.__related_new_names:
+            values = self.get_values(
+                marc_ids=self.__related_names_field_id,
+                subfield_id=["a", "b"],
+                subfield_restriction = ("w", "b")
+            )
+            self.__related_new_names = [self._merge_and_clean(value, ["a", "b"]) for value in values]
+        return self.__related_new_names
+
+    @property
+    def related_old_names(self) -> List[str]:
+        if not self.__related_old_names:
+            values = self.get_values(
+                marc_ids=self.__related_names_field_id,
+                subfield_id=["a", "b"],
+                subfield_restriction = ("w", "a")
+            )
+            self.__related_old_names = [self._merge_and_clean(value, ["a", "b"]) for value in values]
+        return self.__related_old_names
+
+
+    @property
+    def source(self) -> str:
+        if not self.__source:
+            values = self.get_values(
+                marc_ids=self.__source_field_id,
+                subfield_id="a"
+            )
+            self.__source = self._clean_value(values[0]) if values else ""
+        return self.__source
+
+
+    @property
+    def area_code(self) -> str:
+        if not self.__area_code:
+            values = self.get_values(
+                marc_ids=self.__area_code_id,
+                subfield_id="c"
+            )
+            self.__area_code = self._clean_value(values[0]) if values else ""
+        return self.__area_code
+
+    @property
+    def description(self) -> str:
+        if not self.__description:
+            values = self.get_values(
+                marc_ids=self.__description_field_id,
+                subfield_id="i"
+            )
+            self.__description = self._clean_value(values[0]) if values else ""
+        return self.__description
+
+    @property
+    def variations(self) -> List[str]:
+        if not self.__variations:
+            _variations = [self.name]
+            _variations.extend(self.new_names)
+            _variations.extend(self.old_names)
+            _variations.extend(self.alternative_names)
+            _variations.extend(self.related_old_names)
+            _variations.extend(self.related_new_names)
+            self.__variations = [v.lower() for v in list(set(_variations))]
+
+        return self.__variations
+
+    @property
+    def full_record(self) -> dict:
+        if not self.__full_record:
+            self.__full_record = {
+                "name": self.name,
+                "original_name": self.original_name,
+                "acronyms": self.acronyms,
+                "new_names": self.new_names,
+                "old_names": self.old_names,
+                "source": self.source,
+                "description": self.description,
+                "area_code": self.area_code,
+                "alternative_names": self.alternative_names,
+                "related_acryonyms": self.related_acronyms,
+                "related_new_names": self.related_new_names,
+                "related_old_names": self.related_old_names,
+                "identifier": self.identifier,
+                "identifier_source": self.identifier_source,
+                "full_record_marc": str(self.marc_record),
+                "full_record_json": json.dumps(self.marc_json_record)
+            }
+            if self.add_variations:
+                self.__full_record.update(
+                    {
+                        "link_variations": self.variations,
+                        "link_acronyms": [a.lower() for a in self.acronyms]
+                    }
+                )
+        return self.__full_record