rara-tools 0.0.13__py3-none-any.whl → 0.1.0__py3-none-any.whl

rara_tools/constants/digitizer.py

@@ -1,4 +1,12 @@
+COMPONENT_KEY = "digitizer"
+
+
+class ModelTypes:
+    IMAGE_PROCESSOR = "image_processor"
+
+
 class StatusKeys:
+    DOWNLOAD_MODELS = "digitizer_download_models"
     CLEAN_UP = "digitizer_clean_up"
     ELASTICSEARCH_UPLOAD = "digitizer_elasticsearch_upload"
     UPLOAD = "s3_upload"
@@ -11,3 +19,7 @@ class Queue:
     DOWNLOAD = "download"
     FINISH = "finish"
     OCR = "ocr"
+
+
+class Tasks:
+    MODEL_UPDATE = "component_model_update"

rara_tools/constants/general.py

@@ -1,4 +1,5 @@
 class Status:
+    SKIPPED = "SKIPPED"
     FAILED = "FAILED"
     PENDING = "PENDING"
     RUNNING = "RUNNING"

rara_tools/exceptions.py

@@ -7,6 +7,10 @@ class S3InitException(Exception):
 class S3ConnectionException(Exception):
     """Raised S3 Bucket/Connection Error."""
 
+class S3DownloadException(Exception):
+    """Raised S3 Download Error."""
+
+
 class ElasticsearchException(Exception):
     """Raised Elasticsearch Error."""
 

rara_tools/s3.py

@@ -1,11 +1,20 @@
+import logging
 import os
+import pathlib
+import time
 import uuid
 from typing import Any, Generator, List, Optional
 
-from minio import Minio
+from minio import Minio, S3Error
 
-from .exceptions import (S3ConnectionException, S3InitException,
-                         S3InputException)
+from .exceptions import (
+    S3ConnectionException,
+    S3InitException,
+    S3InputException,
+    S3DownloadException
+)
+
+logger = logging.getLogger("tools.s3")
 
 
 class S3Files:
@@ -76,9 +85,49 @@ class S3Files:
         list_of_objects = list(self.minio_client.list_objects(self.bucket, prefix=path, recursive=True))
         for minio_object in list_of_objects:
             full_path = os.path.join(download_dir, minio_object.object_name)
-            self.minio_client.fget_object(self.bucket, minio_object.object_name, full_path)
+            self._download_file(minio_object.object_name, full_path)
             yield full_path
 
+    def _download_file(self, path, download_dir=".", max_retries=3) -> str:
+        """Download a single file with retry and resume support."""
+        attempts = 0
+
+        while attempts < max_retries:
+            try:
+                stat = self.minio_client.stat_object(self.bucket, path)
+                file_size = stat.size
+                temp_path = download_dir + ".part"
+                pathlib.Path(temp_path).parent.mkdir(parents=True, exist_ok=True)
+
+                # Check if a partial file exists
+                downloaded_size = os.path.getsize(temp_path) if os.path.exists(temp_path) else 0
+
+                if downloaded_size >= file_size:
+                    os.rename(temp_path, download_dir)  # Rename to final filename
+                    logger.info(f"Completed: {path}")
+                    return str(pathlib.Path(download_dir) / path)
+
+                logger.info(f"Downloading {path} ({downloaded_size}/{file_size} bytes)...")
+
+                # Open file in append mode to resume download
+                with open(temp_path, "ab") as f:
+                    response = self.minio_client.get_object(self.bucket, path, offset=downloaded_size)
+                    for data in response.stream(32 * 1024):  # 32KB chunks
+                        f.write(data)
+                    response.close()
+                    response.release_conn()
+
+                os.rename(temp_path, download_dir)  # Rename temp to final
+                logger.info(f"Downloaded: {path}")
+                return str(pathlib.Path(download_dir) / path)
+
+            except S3Error as e:
+                logger.info(f"Error downloading {path}, attempt {attempts + 1}: {e}")
+                attempts += 1
+                time.sleep(2 ** attempts)  # Exponential backoff
+
+        raise S3DownloadException(f"Failed to download {path} after {max_retries} attempts.")
+
     def upload(self, path: str, prefix: Optional[str] = "") -> str:
         """Uploads file or folder to S3 bucket.
         :param: path str: Path to the file to upload in local file system.

{rara_tools-0.0.13.dist-info → rara_tools-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.0.13
+Version: 0.1.0
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -18,6 +18,7 @@ Requires-Dist: iso639-lang
 Provides-Extra: testing
 Requires-Dist: pytest>=8.0; extra == "testing"
 Requires-Dist: pytest-order; extra == "testing"
+Dynamic: license-file
 
 # RaRa Tools
 

rara_tools-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+rara_tools/converters.py,sha256=O769zNjde1VCfEUF2VU_49IAbm8NT-cG-VR0uPxixtE,2687
+rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
+rara_tools/digar_schema_converter.py,sha256=k95U2iRlEA3sh772-v6snhHW6fju6qSTMnvWJ6DpzZk,14254
+rara_tools/elastic.py,sha256=MgPHxZ3UbSTIL8_sT9gU5V4PLKJjo3aQ8CGyhXjRz6M,13065
+rara_tools/exceptions.py,sha256=YQyaueUbXeTkJYFDEuN6iWTXMI3eCv5l7PxGp87vg5I,550
+rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
+rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
+rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
+rara_tools/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rara_tools/constants/digitizer.py,sha256=MND0dUQySBAOVWzuUBxQGZWv_Ckdz2jCp25F2_oHGi8,496
+rara_tools/constants/general.py,sha256=aVUQTMss89atAkTDZKJXNdnsBHPX-RSrlBOtt-wdPGU,195
+rara_tools-0.1.0.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.1.0.dist-info/METADATA,sha256=TM89xbTBJiyz3ZNXbYzLwYi84ChYLDN9Lyo2XZ-posE,3916
+rara_tools-0.1.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rara_tools-0.1.0.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.1.0.dist-info/RECORD,,

{rara_tools-0.0.13.dist-info → rara_tools-0.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.1)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

rara_tools-0.0.13.dist-info/RECORD

@@ -1,16 +0,0 @@
-rara_tools/converters.py,sha256=O769zNjde1VCfEUF2VU_49IAbm8NT-cG-VR0uPxixtE,2687
-rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
-rara_tools/digar_schema_converter.py,sha256=k95U2iRlEA3sh772-v6snhHW6fju6qSTMnvWJ6DpzZk,14254
-rara_tools/elastic.py,sha256=MgPHxZ3UbSTIL8_sT9gU5V4PLKJjo3aQ8CGyhXjRz6M,13065
-rara_tools/exceptions.py,sha256=BwNh4qWxau_ylr9RqZoYwd1KnExI6oWWWDno3jkh8q4,474
-rara_tools/s3.py,sha256=uNDu2HzMYHAWh33RcHeyPFK7gdQfQPxsdfohyIKezEY,4467
-rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
-rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
-rara_tools/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rara_tools/constants/digitizer.py,sha256=gJ3jOMwuZfKcLqgOAxTyB266VYsskLabJiMUiSz3xX4,297
-rara_tools/constants/general.py,sha256=E9Jaw-YxocS_tOZw9QBoxO3e9KK5EMbLoM0R7D4Iflw,171
-rara_tools-0.0.13.dist-info/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.0.13.dist-info/METADATA,sha256=0Aipkuodi_CzCTUMkVqKOI__n5mN2r8hEGJ49-MjpMo,3895
-rara_tools-0.0.13.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
-rara_tools-0.0.13.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.0.13.dist-info/RECORD,,