datakit-data 0.6.1__tar.gz → 0.6.2__tar.gz

{datakit_data-0.6.1 → datakit_data-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datakit-data
-Version: 0.6.1
+Version: 0.6.2
 Summary: A datakit plugin to manage data assets on AWS S3.
 Author: Larry Fenn
 Author-email: Larry Fenn <lfenn@ap.org>

{datakit_data-0.6.1 → datakit_data-0.6.2}/datakit_data/__init__.py

@@ -1,6 +1,6 @@
 __author__ = """Larry Fenn"""
 __email__ = 'lfenn@ap.org'
-__version__ = '0.6.1'
+__version__ = '0.6.2'
 
 from .commands.init import Init
 from .commands.push import Push

{datakit_data-0.6.1 → datakit_data-0.6.2}/datakit_data/commands/pull.py

@@ -36,7 +36,7 @@ class Pull(ProjectMixin, CommandHelpers, Command):
         if bucket == "":
             self.log.info("No bucket specified in config - no data pulled")
             return
-        s3 = S3(user_profile, bucket, max_workers=self.project_configs.get('max_workers'))
+        s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
         if getattr(parsed_args, 'force', False) is True and '--force' not in clean_flags:
             clean_flags.append('--force')

{datakit_data-0.6.1 → datakit_data-0.6.2}/datakit_data/commands/push.py

@@ -44,7 +44,7 @@ class Push(ProjectMixin, CommandHelpers, Command):
         if bucket == "":
             self.log.info("No bucket specified in config - no data pushed")
             return
-        s3 = S3(user_profile, bucket, max_workers=self.project_configs.get('max_workers'))
+        s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
         if getattr(parsed_args, 'force', False) is True and '--force' not in clean_flags:
             clean_flags.append('--force')

{datakit_data-0.6.1 → datakit_data-0.6.2}/datakit_data/s3.py

@@ -1,12 +1,9 @@
 import os
 import logging
 from collections import namedtuple
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from logging import NullHandler
 
 import boto3
-from boto3.s3.transfer import TransferConfig
-from botocore.config import Config
 from botocore.exceptions import BotoCoreError, ClientError
 
 
@@ -35,26 +32,9 @@ class S3:
     # record is the one S3 itself reports, so it stays comparable to a later head/list probe.
     MULTIPART_THRESHOLD = 8 * 1024 * 1024
 
-    # Per-file transfers (push uploads, pull downloads) are network-latency-bound, so we run them
-    # across a thread pool. boto3 clients are thread-safe for these calls and release the GIL during
-    # network I/O, so threads give near-linear speedup for the many-small-files case without the
-    # complexity of asyncio. Callers can override via the max_workers arg (e.g. a config key).
-    MAX_WORKERS = 16
-
-    # boto3's managed transfers (upload_file/download_file) each spin up their own internal thread
-    # pool (TransferConfig.max_concurrency, default 10), every thread of which borrows a connection
-    # from the shared pool. Because we already parallelize across files in our own thread pool, that
-    # inner concurrency is redundant and multiplies connection demand to max_workers * 10 — far past
-    # the max_pool_connections=max_workers pool, triggering urllib3 "connection pool is full"
-    # warnings and pathological TLS-handshake churn. Disabling inner threads makes each managed
-    # transfer use a single connection, so total demand equals the worker count and the pool matches.
-    # Large files still upload/download via multipart (parts just go sequentially) for resilience.
-    _TRANSFER_CONFIG = TransferConfig(use_threads=False)
-
-    def __init__(self, aws_user_profile, s3_bucket, max_workers=None):
+    def __init__(self, aws_user_profile, s3_bucket):
         self.user_profile = aws_user_profile
         self.bucket = s3_bucket
-        self.max_workers = max_workers or self.MAX_WORKERS
 
     def push(self, data_dir, s3_path='', extra_flags=None, sync_status_dir=None):
         extra_flags = extra_flags or []
@@ -68,21 +48,20 @@ class S3:
         client = self._client()
         failures = 0
         local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
-        # Decide what to transfer (cheap, local-only checks) and log intent up front so the log
-        # stays deterministic; the transfers themselves run concurrently below.
-        to_upload = []
         for rel_path, local_path in sorted(local_files.items()):
             key = prefix + rel_path
             if not force and self._marker_is_fresh(local_path, rel_path, sync_status_dir):
                 logger.info(f"skipped: {local_path}")
                 continue
             logger.info(f"upload: {local_path} to s3://{self.bucket}/{key}")
-            to_upload.append((rel_path, local_path, key))
-        if not dryrun:
-            failures += self._run_transfers(
-                to_upload,
-                lambda item: self._push_one(client, item, sync_status_dir),
-            )
+            if not dryrun:
+                try:
+                    etag = self._upload(client, local_path, key, sync_status_dir is not None)
+                    if sync_status_dir is not None:
+                        self._create_sync_marker(rel_path, sync_status_dir, etag)
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
+                    logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
             remote_keys = self._list_s3_keys(client, prefix)
             remote_rel = {k[len(prefix):] for k in remote_keys}
@@ -105,9 +84,6 @@ class S3:
         client = self._client()
         failures = 0
         remote_objects = self._list_s3_objects(client, prefix)
-        # Decide what to transfer (cheap, local-only checks) and log intent up front so the log
-        # stays deterministic; the transfers themselves run concurrently below.
-        to_download = []
         for rel_path in sorted(remote_objects):
             key = prefix + rel_path
             remote_etag = remote_objects[rel_path].etag
@@ -118,12 +94,15 @@ class S3:
                     logger.info(f"skipped: s3://{self.bucket}/{key}")
                     continue
             logger.info(f"download: s3://{self.bucket}/{key} to {local_path}")
-            to_download.append((rel_path, key, remote_etag, local_path))
-        if not dryrun:
-            failures += self._run_transfers(
-                to_download,
-                lambda item: self._pull_one(client, item, sync_status_dir),
-            )
+            if not dryrun:
+                os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
+                try:
+                    client.download_file(self.bucket, key, local_path)
+                    if sync_status_dir is not None:
+                        self._create_sync_marker(rel_path, sync_status_dir, remote_etag)
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
+                    logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
             local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
             remote_rel = set(remote_objects)
@@ -140,10 +119,7 @@ class S3:
 
     def _client(self):
         session = boto3.Session(profile_name=self.user_profile)
-        # Size the connection pool to the worker count so concurrent transfers don't queue on a
-        # too-small pool (botocore's default is 10) or emit "connection pool is full" warnings.
-        config = Config(max_pool_connections=self.max_workers)
-        return session.client('s3', config=config)
+        return session.client('s3')
 
     @staticmethod
     def _normalize_etag(etag):
@@ -164,39 +140,9 @@ class S3:
             with open(local_path, 'rb') as body:
                 response = client.put_object(Bucket=self.bucket, Key=key, Body=body)
             return self._normalize_etag(response.get('ETag'))
-        client.upload_file(local_path, self.bucket, key, Config=self._TRANSFER_CONFIG)
+        client.upload_file(local_path, self.bucket, key)
         return self._object_etag(client, key) if need_etag else None
 
-    def _run_transfers(self, items, transfer):
-        # Run transfer(item) for each item across a thread pool, returning the failure count.
-        # Per-item ClientError/BotoCoreError are logged and counted (matching the sequential
-        # behavior); all logging happens here on the main thread, so the workers do pure I/O.
-        if not items:
-            return 0
-        failures = 0
-        with ThreadPoolExecutor(max_workers=min(self.max_workers, len(items))) as pool:
-            futures = [pool.submit(transfer, item) for item in items]
-            for future in as_completed(futures):
-                try:
-                    future.result()
-                except (ClientError, BotoCoreError) as e:
-                    failures += 1
-                    logger.info(f"\n*** Error ***\n{e}\n")
-        return failures
-
-    def _push_one(self, client, item, sync_status_dir):
-        rel_path, local_path, key = item
-        etag = self._upload(client, local_path, key, sync_status_dir is not None)
-        if sync_status_dir is not None:
-            self._create_sync_marker(rel_path, sync_status_dir, etag)
-
-    def _pull_one(self, client, item, sync_status_dir):
-        rel_path, key, remote_etag, local_path = item
-        os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
-        client.download_file(self.bucket, key, local_path, Config=self._TRANSFER_CONFIG)
-        if sync_status_dir is not None:
-            self._create_sync_marker(rel_path, sync_status_dir, remote_etag)
-
     def _normalize_prefix(self, s3_path):
         # Returns '' for falsy input so callers can concatenate key segments without a leading slash.
         if not s3_path:

{datakit_data-0.6.1 → datakit_data-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "datakit-data"
-version = "0.6.1"
+version = "0.6.2"
 description = "A datakit plugin to manage data assets on AWS S3."
 authors = [
     {name = "Larry Fenn", email = "lfenn@ap.org"},