my-aws-helpers 5.7.0__tar.gz → 5.9.0__tar.gz

{my_aws_helpers-5.7.0 → my_aws_helpers-5.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: my_aws_helpers
-Version: 5.7.0
+Version: 5.9.0
 Summary: AWS Helpers
 Home-page: https://github.com/JarrodMccarthy/aws_helpers.git
 Author: Jarrod McCarthy

{my_aws_helpers-5.7.0 → my_aws_helpers-5.9.0}/my_aws_helpers/s3.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import boto3
 import io
 import json
+from concurrent.futures import Future, ThreadPoolExecutor
 from abc import ABC, abstractmethod
 from typing import Optional, Any, Dict, List
 from datetime import datetime, date
@@ -100,11 +101,35 @@ class S3:
         )
 
     def list_objects_by_prefix(self, bucket_name: str, prefix: str) -> List[S3Location]:
+        """
+        list objects by prefix gets 1000 items at a time, if theres more, I want em
+        """
+        objects = list()
         try:
-            response = self.client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
-            contents = response["Contents"]
+            continuation_token = None
+            while True:
+                if continuation_token:
+                    response = self.client.list_objects_v2(
+                        Bucket=bucket_name,
+                        Prefix=prefix,
+                        ContinuationToken=continuation_token
+                    )
+                else:
+                    response = self.client.list_objects_v2(
+                        Bucket=bucket_name,
+                        Prefix=prefix
+                    )
+
+                # Append current batch
+                objects.extend(response.get("Contents", []))
+
+                # Check if more results exist
+                if response.get("IsTruncated"):  # True if more pages available
+                    continuation_token = response["NextContinuationToken"]
+                else:
+                    break
             locations = [
-                S3Location(bucket=bucket_name, file_name=c["Key"]) for c in contents
+                S3Location(bucket=bucket_name, file_name=c["Key"]) for c in objects
             ]
             return locations
         except Exception as e:
@@ -296,6 +321,23 @@ class BaseS3Queries:
         self.s3_client = s3_client
         self.bucket_name = bucket_name
 
+    def _concurrent_s3_dict_read(
+        self, locations: List[S3Location], max_workers: int = 10
+    ) -> List[BaseS3Object]:
+        results: List[BaseS3Object] = list()
+        futures: List[Future] = list()
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            for location in locations:
+                future = executor.submit(
+                    self.s3_client.read_dict_from_s3,
+                    s3_location=location,
+                )
+                futures.append(future)
+        for f in futures:
+            results.append(f.result())
+        results = [r for r in results if r is not None]
+        return results
+
     def save_s3_object_to_s3(self, object: BaseS3Object) -> Optional[S3Location]:
         try:
             obj = object.to_s3_representation()

{my_aws_helpers-5.7.0 → my_aws_helpers-5.9.0}/my_aws_helpers.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: my-aws-helpers
-Version: 5.7.0
+Version: 5.9.0
 Summary: AWS Helpers
 Home-page: https://github.com/JarrodMccarthy/aws_helpers.git
 Author: Jarrod McCarthy

{my_aws_helpers-5.7.0 → my_aws_helpers-5.9.0}/setup.py

@@ -3,7 +3,7 @@ from setuptools import find_namespace_packages, setup
 
 base_path = os.path.abspath(os.path.dirname(__file__))
 
-version = "5.7.0"
+version = "5.9.0"
 
 core = [
     "boto3==1.34.36",