datakit-data 0.6.0__tar.gz → 0.6.2__tar.gz

{datakit_data-0.6.0 → datakit_data-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datakit-data
-Version: 0.6.0
+Version: 0.6.2
 Summary: A datakit plugin to manage data assets on AWS S3.
 Author: Larry Fenn
 Author-email: Larry Fenn <lfenn@ap.org>

{datakit_data-0.6.0 → datakit_data-0.6.2}/datakit_data/__init__.py

@@ -1,6 +1,6 @@
 __author__ = """Larry Fenn"""
 __email__ = 'lfenn@ap.org'
-__version__ = '0.6.0'
+__version__ = '0.6.2'
 
 from .commands.init import Init
 from .commands.push import Push

{datakit_data-0.6.0 → datakit_data-0.6.2}/datakit_data/commands/pull.py

@@ -36,7 +36,7 @@ class Pull(ProjectMixin, CommandHelpers, Command):
         if bucket == "":
             self.log.info("No bucket specified in config - no data pulled")
             return
-        s3 = S3(user_profile, bucket, max_workers=self.project_configs.get('max_workers'))
+        s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
         if getattr(parsed_args, 'force', False) is True and '--force' not in clean_flags:
             clean_flags.append('--force')

{datakit_data-0.6.0 → datakit_data-0.6.2}/datakit_data/commands/push.py

@@ -44,7 +44,7 @@ class Push(ProjectMixin, CommandHelpers, Command):
         if bucket == "":
             self.log.info("No bucket specified in config - no data pushed")
             return
-        s3 = S3(user_profile, bucket, max_workers=self.project_configs.get('max_workers'))
+        s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
         if getattr(parsed_args, 'force', False) is True and '--force' not in clean_flags:
             clean_flags.append('--force')

{datakit_data-0.6.0 → datakit_data-0.6.2}/datakit_data/s3.py

@@ -1,11 +1,9 @@
 import os
 import logging
 from collections import namedtuple
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from logging import NullHandler
 
 import boto3
-from botocore.config import Config
 from botocore.exceptions import BotoCoreError, ClientError
 
 
@@ -34,16 +32,9 @@ class S3:
     # record is the one S3 itself reports, so it stays comparable to a later head/list probe.
     MULTIPART_THRESHOLD = 8 * 1024 * 1024
 
-    # Per-file transfers (push uploads, pull downloads) are network-latency-bound, so we run them
-    # across a thread pool. boto3 clients are thread-safe for these calls and release the GIL during
-    # network I/O, so threads give near-linear speedup for the many-small-files case without the
-    # complexity of asyncio. Callers can override via the max_workers arg (e.g. a config key).
-    MAX_WORKERS = 16
-
-    def __init__(self, aws_user_profile, s3_bucket, max_workers=None):
+    def __init__(self, aws_user_profile, s3_bucket):
         self.user_profile = aws_user_profile
         self.bucket = s3_bucket
-        self.max_workers = max_workers or self.MAX_WORKERS
 
     def push(self, data_dir, s3_path='', extra_flags=None, sync_status_dir=None):
         extra_flags = extra_flags or []
@@ -57,21 +48,20 @@ class S3:
         client = self._client()
         failures = 0
         local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
-        # Decide what to transfer (cheap, local-only checks) and log intent up front so the log
-        # stays deterministic; the transfers themselves run concurrently below.
-        to_upload = []
         for rel_path, local_path in sorted(local_files.items()):
             key = prefix + rel_path
             if not force and self._marker_is_fresh(local_path, rel_path, sync_status_dir):
                 logger.info(f"skipped: {local_path}")
                 continue
             logger.info(f"upload: {local_path} to s3://{self.bucket}/{key}")
-            to_upload.append((rel_path, local_path, key))
-        if not dryrun:
-            failures += self._run_transfers(
-                to_upload,
-                lambda item: self._push_one(client, item, sync_status_dir),
-            )
+            if not dryrun:
+                try:
+                    etag = self._upload(client, local_path, key, sync_status_dir is not None)
+                    if sync_status_dir is not None:
+                        self._create_sync_marker(rel_path, sync_status_dir, etag)
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
+                    logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
             remote_keys = self._list_s3_keys(client, prefix)
             remote_rel = {k[len(prefix):] for k in remote_keys}
@@ -94,9 +84,6 @@ class S3:
         client = self._client()
         failures = 0
         remote_objects = self._list_s3_objects(client, prefix)
-        # Decide what to transfer (cheap, local-only checks) and log intent up front so the log
-        # stays deterministic; the transfers themselves run concurrently below.
-        to_download = []
         for rel_path in sorted(remote_objects):
             key = prefix + rel_path
             remote_etag = remote_objects[rel_path].etag
@@ -107,12 +94,15 @@ class S3:
                     logger.info(f"skipped: s3://{self.bucket}/{key}")
                     continue
             logger.info(f"download: s3://{self.bucket}/{key} to {local_path}")
-            to_download.append((rel_path, key, remote_etag, local_path))
-        if not dryrun:
-            failures += self._run_transfers(
-                to_download,
-                lambda item: self._pull_one(client, item, sync_status_dir),
-            )
+            if not dryrun:
+                os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
+                try:
+                    client.download_file(self.bucket, key, local_path)
+                    if sync_status_dir is not None:
+                        self._create_sync_marker(rel_path, sync_status_dir, remote_etag)
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
+                    logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
             local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
             remote_rel = set(remote_objects)
@@ -129,10 +119,7 @@ class S3:
 
     def _client(self):
         session = boto3.Session(profile_name=self.user_profile)
-        # Size the connection pool to the worker count so concurrent transfers don't queue on a
-        # too-small pool (botocore's default is 10) or emit "connection pool is full" warnings.
-        config = Config(max_pool_connections=self.max_workers)
-        return session.client('s3', config=config)
+        return session.client('s3')
 
     @staticmethod
     def _normalize_etag(etag):
@@ -156,36 +143,6 @@ class S3:
         client.upload_file(local_path, self.bucket, key)
         return self._object_etag(client, key) if need_etag else None
 
-    def _run_transfers(self, items, transfer):
-        # Run transfer(item) for each item across a thread pool, returning the failure count.
-        # Per-item ClientError/BotoCoreError are logged and counted (matching the sequential
-        # behavior); all logging happens here on the main thread, so the workers do pure I/O.
-        if not items:
-            return 0
-        failures = 0
-        with ThreadPoolExecutor(max_workers=min(self.max_workers, len(items))) as pool:
-            futures = [pool.submit(transfer, item) for item in items]
-            for future in as_completed(futures):
-                try:
-                    future.result()
-                except (ClientError, BotoCoreError) as e:
-                    failures += 1
-                    logger.info(f"\n*** Error ***\n{e}\n")
-        return failures
-
-    def _push_one(self, client, item, sync_status_dir):
-        rel_path, local_path, key = item
-        etag = self._upload(client, local_path, key, sync_status_dir is not None)
-        if sync_status_dir is not None:
-            self._create_sync_marker(rel_path, sync_status_dir, etag)
-
-    def _pull_one(self, client, item, sync_status_dir):
-        rel_path, key, remote_etag, local_path = item
-        os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
-        client.download_file(self.bucket, key, local_path)
-        if sync_status_dir is not None:
-            self._create_sync_marker(rel_path, sync_status_dir, remote_etag)
-
     def _normalize_prefix(self, s3_path):
         # Returns '' for falsy input so callers can concatenate key segments without a leading slash.
         if not s3_path:

{datakit_data-0.6.0 → datakit_data-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "datakit-data"
-version = "0.6.0"
+version = "0.6.2"
 description = "A datakit plugin to manage data assets on AWS S3."
 authors = [
     {name = "Larry Fenn", email = "lfenn@ap.org"},