rara-tools 0.0.4__py3-none-any.whl → 0.0.9__py3-none-any.whl

rara_tools/constants/digitizer.py

@@ -0,0 +1,13 @@
+class StatusKeys:
+    CLEAN_UP = "digitizer_clean_up"
+    ELASTICSEARCH_UPLOAD = "digitizer_elasticsearch_upload"
+    UPLOAD = "s3_upload"
+    DOWNLOAD = "digitizer_s3_download"
+    OCR = "digitizer_ocr"
+
+
+class Queue:
+    IO = "io"
+    DOWNLOAD = "download"
+    FINISH = "finish"
+    OCR = "ocr"

rara_tools/constants/general.py

@@ -0,0 +1,10 @@
+class Status:
+    FAILED = "FAILED"
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    COMPLETED = "COMPLETED"
+    RETRYING = "RETRYING"
+
+
+class Queue:
+    CORE = "core"

rara_tools/converters.py

@@ -0,0 +1,41 @@
+from .exceptions import SierraResponseConverterException
+
+
+class SierraResponseConverter:
+    """  Takes a JSON response from the Sierra API (https://tester.ester.ee/iii/sierra-api/swagger/index.html)
+    and converts it to MARC-in-JSON format.
+    
+    """
+    
+    def __init__(self, response: dict):
+        if not isinstance(response, dict):
+            raise SierraResponseConverterException("Please provide a valid JSON response.")
+        self.response = response
+        
+    def _map_field_data(self, field):
+        tag = field.get("tag")
+        if not tag:
+            raise SierraResponseConverterException("Field is missing a valid 'tag'.")
+        data = field.get("data", {})
+        return {tag: data}
+        
+    def _convert_response(self):
+        response = self.response
+        
+        entries = response.get("entries")
+        if not entries:
+            raise SierraResponseConverterException("No entries found in the response.")
+        
+        try:
+            fields = [self._map_field_data(f) for e in entries for f in e["marc"]["fields"]]
+        except KeyError as e:
+            raise SierraResponseConverterException(f"Missing expected MARC fields in the response: {e}")
+        
+        return {"fields": fields}
+            
+    def convert(self):
+        """Runner method, converts the response to MARC-in-JSON format with error handling."""
+        try:
+            return self._convert_response()
+        except Exception as e:
+            raise SierraResponseConverterException(f"An unexpected error occurred during conversion: {e}")

rara_tools/decorators.py

@@ -1,12 +1,12 @@
 import functools
+from typing import Any, Callable
+
 from elasticsearch import AuthenticationException
 from elasticsearch import ConnectionError as ElasticsearchConnectionError
 from elasticsearch import ConnectionTimeout, NotFoundError, RequestError
-from typing import Any, Callable
 
 from .exceptions import ElasticsearchException
 
-
 ELASTIC_NOT_FOUND_MESSAGE = 'Could not find specified data from Elasticsearch!'
 ELASTIC_REQUEST_ERROR_MESSAGE = 'Error executing Elasticsearch query! Bad query?'
 ELASTIC_CONNECTION_TIMEOUT_MESSAGE = 'Connection to Elasticsearch took too long, please try again later!'
@@ -39,4 +39,4 @@ def _elastic_connection(func: Callable) -> Callable:
 			raise ElasticsearchException(ELASTIC_CONNECTION_ERROR_MESSAGE) from exception
 		except Exception as exception:
 			raise ElasticsearchException(ELASTIC_UNKNOWN_ERROR_MESSAGE) from exception
-	return wrapper
+	return wrapper

rara_tools/elastic.py

@@ -1,6 +1,10 @@
-from typing import Dict, Optional, List
+from typing import Any, Dict, Iterator, Optional
+
+import elasticsearch_dsl
+from elastic_transport import ObjectApiResponse
 from elasticsearch import Elasticsearch
-from elasticsearch_dsl import Search
+from elasticsearch.helpers import bulk
+from elasticsearch_dsl import Index
 
 from .decorators import _elastic_connection
 
@@ -8,11 +12,54 @@ from .decorators import _elastic_connection
 class KataElastic:
     """A class to manage all required Elasticsearch operations for Kata.
     """
+
+    TYPE_MAPPING = {
+        "keyword": elasticsearch_dsl.Keyword,
+        "text": elasticsearch_dsl.Text,
+        "float": elasticsearch_dsl.Float,
+        "integer": elasticsearch_dsl.Integer,
+        "date": elasticsearch_dsl.Date,
+    }
+
+    DEFAULT_MAPPING = {
+        "text": "keyword",
+        "parent_id": "keyword",
+        "text_quality": "float",
+        "n_chars": "integer",
+        "n_words": "integer",
+        "language": "keyword",
+        "end_page": "integer",
+        "start_page": "integer",
+        "sequence_nr": "integer",
+        "section_title": "keyword",
+        "section_type": "keyword",
+        "section_meta": "keyword",
+    }
+
     def __init__(self, elasticsearch_url: str, timeout: Optional[int] = None):
         self.timeout = timeout
         self.elasticsearch_url = elasticsearch_url
         self.elasticsearch = Elasticsearch(self.elasticsearch_url, request_timeout=self.timeout)
 
+    def _produce_rollover_index(self, index_prefix: str, rollover_limit: int) -> str:
+        indices = self.elasticsearch.indices.get(index=f"{index_prefix}-*", expand_wildcards="open")
+        sorted_indices = sorted([(k, v["settings"]["index"]["creation_date"]) for k, v in indices.items()], key=lambda x: x[1], reverse=True)
+        sorted_indices = [i[0] for i in sorted_indices]
+
+        # new index name if none exist
+        if not len(sorted_indices):
+            last_index_name = f"{index_prefix}-0"
+            last_index_count = 0
+        else:
+            last_index_name = sorted_indices[0]
+            last_index_count = self.elasticsearch.count(index=last_index_name)["count"]
+        # check the size of the last index of the pipeline
+        if last_index_count >= rollover_limit:
+            new_index_number = int(last_index_name[-1]) + 1
+            last_index_name = f"{index_prefix}-{new_index_number}"
+
+        return last_index_name
+
     @_elastic_connection
     def check(self) -> bool:
         """Checks Elasticsearch connection.
@@ -22,25 +69,42 @@ class KataElastic:
             return True
         return False
 
+    def generate_mapping(self, schema: dict | None = None) -> dict:
+        mapping_dsl = elasticsearch_dsl.Mapping()
+        mapping = schema or self.DEFAULT_MAPPING
+        for field_name, field_type in mapping.items():
+            if field_type in self.TYPE_MAPPING:
+                # We instantiate the class stored in the type mapping.
+                mapping_dsl.field(field_name, self.TYPE_MAPPING[field_type]())
+        return mapping_dsl.to_dict()
+
+    @_elastic_connection
+    def add_mapping(self, index_name: str, schema: dict):
+        index = Index(name=index_name)
+        return index.put_mapping(body=schema, using=self.elasticsearch)
+
     @_elastic_connection
     def create_index(
             self,
             index: str,
             shards: int = 3,
             replicas: int = 1,
-            settings: Optional[dict] = None
-    ) -> Dict:
+            settings: Optional[dict] = None,
+    ) -> Dict | None:
         """Creates empty index.
         :param: index str: Name of the index to create.
         :param: shards int: Number of shards for the index.
         :param: replicas int: Number of replicas of the index.
         :param: settings dict: Overwrite settings for the index.
         """
-        body = settings or {
-            "number_of_shards": shards,
-            "number_of_replicas": replicas,
-        }
-        return self.elasticsearch.indices.create(index=index, settings=body)
+
+        index_exists = self.elasticsearch.indices.exists(index=index).body
+        if index_exists is False:
+            setting_body = settings or {
+                "number_of_shards": shards,
+                "number_of_replicas": replicas,
+            }
+            return self.elasticsearch.indices.create(index=index, settings=setting_body)
 
     @_elastic_connection
     def delete_index(self, index: str, ignore: Optional[bool] = True) -> Dict:
@@ -49,11 +113,11 @@ class KataElastic:
         :param: ignore bool: Ignore errors because of closed/deleted index.
         :return: Dict of Elastic's acknowledgement of the action.
         """
-        response = self.elasticsearch.indices.delete(index=index, ignore_unavailable=ignore)
+        response = self.elasticsearch.indices.delete(index=index, ignore_unavailable=ignore, expand_wildcards="open")
         return response
 
     @_elastic_connection
-    def delete_document(self, index: str, document_id: str) -> Dict:
+    def delete_document(self, index: str, document_id: str) -> ObjectApiResponse[Any]:
         """Deletes document fom index.
         :param: document_id str: ID of the document to be deleted.
         :param: index str: Index where the document is to be found.
@@ -63,6 +127,25 @@ class KataElastic:
         response = self.elasticsearch.delete(id=document_id, index=index)
         return response
 
+    @_elastic_connection
+    def bulk_index(
+            self,
+            documents: Iterator[dict],
+            index_prefix: str,
+            rollover_limit: int,
+            refresh="false",
+            create_index: bool = True
+    ) -> (int, int):
+        last_index_name = self._produce_rollover_index(index_prefix, rollover_limit)
+        if create_index:
+            response = self.create_index(index=last_index_name)
+            response = self.add_mapping(index_name=last_index_name, schema=self.generate_mapping())
+            pass
+
+        actions = [{"_index": last_index_name, "_source": document} for document in documents]
+        successful_count, error_count = bulk(actions=actions, client=self.elasticsearch, max_retries=3, refresh=refresh)
+        return successful_count, error_count
+
     @_elastic_connection
     def index_document(self, index: str, body: dict, document_id: Optional[str] = None) -> Dict:
         """Indexes document.
@@ -78,15 +161,15 @@ class KataElastic:
         return indexed
 
     @_elastic_connection
-    def get_documents_by_key(self, index: str, document_key: str) -> List:
-        """This method is for retrieving all texts/pages of the original document.
-        :param: index str: Index to search the documents from.
-        :param: document_key str: parent_id field that connects pages of document together.
-        :return: List of matching documents. 
-        """
-        s = Search(using=self.elasticsearch, index=index)
-        docs = s.query("match", parent_id=document_key).execute()
-        return docs
+    def get_documents_by_key(self, index: str, document_key: str, sort_fields=("start_page", "end_page", "sequence_nr",)):
+        index = f"{index}-*"
+        s = elasticsearch_dsl.Search(using=self.elasticsearch, index=index)
+        s = s.query("match", parent_id=document_key).sort(*sort_fields)
+        # Since scan doesn't allow for sorting, we do it manually after fetching the documents.
+        documents = sorted(
+            s.scan(), key=lambda doc: [getattr(doc, field) for field in sort_fields]
+        )
+        return documents
 
     def __str__(self) -> str:
-         return self.elasticsearch_url
+        return self.elasticsearch_url

rara_tools/exceptions.py

@@ -12,3 +12,6 @@ class ElasticsearchException(Exception):
 
 class TaskReporterException(Exception):
     """Raised TaskReporter Error."""
+
+class SierraResponseConverterException(Exception):
+    """Raised SierraResponseConverter Error."""

rara_tools/s3.py

@@ -1,10 +1,11 @@
 import os
 import uuid
-from typing import Optional, List, Generator, Any
+from typing import Any, Generator, List, Optional
 
 from minio import Minio
 
-from .exceptions import S3InitException, S3ConnectionException, S3InputException
+from .exceptions import (S3ConnectionException, S3InitException,
+                         S3InputException)
 
 
 class S3Files:

{rara_tools-0.0.4.dist-info → rara_tools-0.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rara-tools
-Version: 0.0.4
+Version: 0.0.9
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -28,25 +28,28 @@ Requires-Dist: pytest-order; extra == "testing"
 
 ---
 
-## ✨ Features  
+## ✨ Features
 
 - Elasticsearch index & document operations
 - S3 file management operations
 - Task reporting to Core API
+- Converting SIERRA API responses to Pymarc compatible JSON
+
 ---
 
-## ⚡ Quick Start  
+## ⚡ Quick Start
 
 Get started with `rara-tools` in just a few steps:
 
 1. **Install the Package**  
-   Ensure you're using Python 3.10 or above, then run:  
+   Ensure you're using Python 3.10 or above, then run:
+
    ```bash
    pip install rara-tools
    ```
 
 2. **Import and Use**  
-   Example usage to download a folder from S3:  
+   Example usage to download a folder from S3:
 
    ```python
    from rara_tools.s3 import S3Files
@@ -77,22 +80,25 @@ Follow the steps below to install the `rara-tools` package, either via `pip` or
    Create or activate a Python environment using Python **3.10** or above.
 
 2. **Install the Package**  
-   Run the following command:  
+    Run the following command:
+
    ```bash
    pip install rara-tools
    ```
-</details>
+
+   </details>
 
 ---
 
 ### Local Installation
 
-Follow these steps to install the `rara-tools` package locally:  
+Follow these steps to install the `rara-tools` package locally:
 
 <details><summary>Click to expand</summary>
 
 1. **Clone the Repository**  
-   Clone the repository and navigate into it:  
+   Clone the repository and navigate into it:
+
    ```bash
    git clone <repository-url>
    cd <repository-directory>
@@ -100,25 +106,29 @@ Follow these steps to install the `rara-tools` package locally:
 
 2. **Set Up Python Environment**  
    Create or activate a Python environment using Python 3.10 or above. E.g:
+
    ```bash
    conda create -n py310 python==3.10
    conda activate py310
    ```
 
 3. **Install Build Package**  
-   Install the `build` package to enable local builds:  
+   Install the `build` package to enable local builds:
+
    ```bash
    pip install build
    ```
 
 4. **Build the Package**  
-   Run the following command inside the repository:  
+   Run the following command inside the repository:
+
    ```bash
    python -m build
    ```
 
 5. **Install the Package**  
-   Install the built package locally:  
+   Install the built package locally:
+
    ```bash
    pip install .
    ```
@@ -131,13 +141,13 @@ Follow these steps to install the `rara-tools` package locally:
 
 Follow these steps to test the `rara-tools` package.
 
-
 ### How to Test
 
 <details><summary>Click to expand</summary>
 
 1. **Clone the Repository**  
-   Clone the repository and navigate into it:  
+   Clone the repository and navigate into it:
+
    ```bash
    git clone <repository-url>
    cd <repository-directory>
@@ -147,25 +157,29 @@ Follow these steps to test the `rara-tools` package.
    Create or activate a Python environment using Python 3.10 or above.
 
 3. **Install Build Package**  
-   Install the `build` package:  
+   Install the `build` package:
+
    ```bash
    pip install build
    ```
 
 4. **Build the Package**  
-   Build the package inside the repository:  
+   Build the package inside the repository:
+
    ```bash
    python -m build
    ```
 
 5. **Install with Testing Dependencies**  
-   Install the package along with its testing dependencies:  
+   Install the package along with its testing dependencies:
+
    ```bash
    pip install .[testing]
    ```
 
 6. **Run Tests**  
-   Run the test suite from the repository root:  
+   Run the test suite from the repository root:
+
    ```bash
    python -m pytest -v tests
    ```

rara_tools-0.0.9.dist-info/RECORD

@@ -0,0 +1,14 @@
+rara_tools/converters.py,sha256=JcS74VzV6jm12l3C6aqMJBY9nuVW_aevQeCe32KmfrE,1576
+rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
+rara_tools/elastic.py,sha256=vEvrbIPRtdqTdrNrPH2cewHLMfOTSf87a4JOiRQgYyA,7146
+rara_tools/exceptions.py,sha256=BwNh4qWxau_ylr9RqZoYwd1KnExI6oWWWDno3jkh8q4,474
+rara_tools/s3.py,sha256=uNDu2HzMYHAWh33RcHeyPFK7gdQfQPxsdfohyIKezEY,4467
+rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
+rara_tools/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rara_tools/constants/digitizer.py,sha256=gJ3jOMwuZfKcLqgOAxTyB266VYsskLabJiMUiSz3xX4,297
+rara_tools/constants/general.py,sha256=E9Jaw-YxocS_tOZw9QBoxO3e9KK5EMbLoM0R7D4Iflw,171
+rara_tools-0.0.9.dist-info/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.0.9.dist-info/METADATA,sha256=HhxVd2e_lhAizmc9p88dOVuaCygVRH5tDv3xrPZXVmk,3867
+rara_tools-0.0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+rara_tools-0.0.9.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.0.9.dist-info/RECORD,,

rara_tools-0.0.4.dist-info/RECORD

@@ -1,10 +0,0 @@
-rara_tools/decorators.py,sha256=rYDk5CEHhCZvqeFaHku8qLMv7G7NTMWppHwLg3ZeVj4,2186
-rara_tools/elastic.py,sha256=MVqai6wDQlDQeHQzAKsRpxOchI29y3W1UiridgfH6d4,3718
-rara_tools/exceptions.py,sha256=FtuHG-2snaEfADA25HjjutGNQzNo6sTdSfqk9VrzOuE,374
-rara_tools/s3.py,sha256=eqMiOKbjXvXY04JJV68gmOU-4DUnwEaeYdhjQSI6crU,4440
-rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
-rara_tools-0.0.4.dist-info/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.0.4.dist-info/METADATA,sha256=onb2qPn7IXknjCM09yxKfuEZYJlTlUwetCrS2ZXlPio,3820
-rara_tools-0.0.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-rara_tools-0.0.4.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.0.4.dist-info/RECORD,,