datakit-data 0.5.2__tar.gz → 0.6.0__tar.gz

{datakit_data-0.5.2 → datakit_data-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datakit-data
-Version: 0.5.2
+Version: 0.6.0
 Summary: A datakit plugin to manage data assets on AWS S3.
 Author: Larry Fenn
 Author-email: Larry Fenn <lfenn@ap.org>

{datakit_data-0.5.2 → datakit_data-0.6.0}/datakit_data/__init__.py

@@ -1,6 +1,6 @@
 __author__ = """Larry Fenn"""
 __email__ = 'lfenn@ap.org'
-__version__ = '0.5.2'
+__version__ = '0.6.0'
 
 from .commands.init import Init
 from .commands.push import Push

{datakit_data-0.5.2 → datakit_data-0.6.0}/datakit_data/commands/init.py

@@ -28,12 +28,14 @@ class Init(ProjectMixin, CommandHelpers, Command):
             mkdir_p(directory)
         open('data/.gitkeep', 'w').close()
         self.log.info("Created data/ directory")
-        self.create_project_config()
-        self.log.info("Created config/datakit-data.json")
+        if self.create_project_config():
+            self.log.info("Created config/datakit-data.json")
+        else:
+            self.log.info("config/datakit-data.json already exists - leaving it unchanged")
 
     def create_project_config(self):
         if os.path.exists(self.project_config_path):
-            return
+            return False
         if os.path.exists(self.plugin_config_path):
             plugin_configs = read_json(self.plugin_config_path)
             if not plugin_configs.get('s3_bucket'):
@@ -56,6 +58,7 @@ class Init(ProjectMixin, CommandHelpers, Command):
         self._expand_vars(to_write)
         self.finalize_configs(to_write)
         write_json(self.project_config_path, to_write)
+        return True
 
     def _prompt_for_plugin_configs(self):
         print("Please provide the following configuration values (press Enter to use the default):\n")

{datakit_data-0.5.2 → datakit_data-0.6.0}/datakit_data/commands/pull.py

@@ -17,7 +17,13 @@ class Pull(ProjectMixin, CommandHelpers, Command):
         parser.add_argument(
             'args',
             nargs=argparse.REMAINDER,
-            help="One or more boolean S3 sync flags without leading dashes, e.g. delete or dryrun"
+            help="One or more boolean flags without leading dashes: delete, dryrun, force"
+        )
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            default=False,
+            help="Pull every S3 object, ignoring sync status checks"
         )
         return parser
 
@@ -30,12 +36,19 @@ class Pull(ProjectMixin, CommandHelpers, Command):
         if bucket == "":
             self.log.info("No bucket specified in config - no data pulled")
             return
-        s3 = S3(user_profile, bucket)
+        s3 = S3(user_profile, bucket, max_workers=self.project_configs.get('max_workers'))
         clean_flags = ExtraFlags.convert(parsed_args.args)
+        if getattr(parsed_args, 'force', False) is True and '--force' not in clean_flags:
+            clean_flags.append('--force')
+        unsupported = ExtraFlags.unsupported(parsed_args.args)
+        if unsupported:
+            self.log.info(f"Ignoring unsupported flag(s): {', '.join(unsupported)}")
+        sync_status_dir = self.project_configs.get('sync_status_location')
         failures = s3.pull(
             'data/',
             self.project_configs['s3_path'],
-            extra_flags=clean_flags
+            extra_flags=clean_flags,
+            sync_status_dir=sync_status_dir
         )
         if failures:
             self.log.info(f"{failures} file(s) failed to transfer")

{datakit_data-0.5.2 → datakit_data-0.6.0}/datakit_data/commands/push.py

@@ -19,7 +19,13 @@ class Push(ProjectMixin, CommandHelpers, Command):
         parser.add_argument(
             'args',
             nargs=argparse.REMAINDER,
-            help="One or more boolean S3 sync flags without leading dashes, e.g. delete or dryrun"
+            help="One or more boolean flags without leading dashes: delete, dryrun, force"
+        )
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            default=False,
+            help="Push every file, ignoring sync status checks"
         )
         parser.add_argument(
             '--sync-status-in-data',
@@ -38,13 +44,20 @@ class Push(ProjectMixin, CommandHelpers, Command):
         if bucket == "":
             self.log.info("No bucket specified in config - no data pushed")
             return
-        s3 = S3(user_profile, bucket)
+        s3 = S3(user_profile, bucket, max_workers=self.project_configs.get('max_workers'))
         clean_flags = ExtraFlags.convert(parsed_args.args)
+        if getattr(parsed_args, 'force', False) is True and '--force' not in clean_flags:
+            clean_flags.append('--force')
+        unsupported = ExtraFlags.unsupported(parsed_args.args)
+        if unsupported:
+            self.log.info(f"Ignoring unsupported flag(s): {', '.join(unsupported)}")
+        dryrun = '--dryrun' in clean_flags or '--dry-run' in clean_flags
         if parsed_args.sync_status_in_data:
             sync_status_dir = 'data/'
-            configs = self.project_configs.copy()
-            configs['sync_status_location'] = 'data/'
-            write_json(self.project_config_path, configs)
+            if not dryrun:
+                configs = self.project_configs.copy()
+                configs['sync_status_location'] = 'data/'
+                write_json(self.project_config_path, configs)
         else:
             sync_status_dir = self.project_configs.get('sync_status_location')
         failures = s3.push(

{datakit_data-0.5.2 → datakit_data-0.6.0}/datakit_data/commands/status.py

@@ -7,7 +7,6 @@ from datakit.utils import read_json, write_json
 from ..project_mixin import ProjectMixin
 from ..s3 import S3
 
-
 class Status(ProjectMixin, CommandHelpers, Command):
 
     "Show sync status of local data files"
@@ -47,6 +46,7 @@ class Status(ProjectMixin, CommandHelpers, Command):
                 self.project_configs['aws_user_profile'],
                 bucket,
                 self.project_configs['s3_path'],
+                sync_status_dir,
                 filepaths=parsed_args.filepaths,
             )
             return
@@ -60,10 +60,11 @@ class Status(ProjectMixin, CommandHelpers, Command):
                 self.log.info("Added sync_status_location to config/datakit-data.json")
             return
         missing, stale = self._find_unsynced('data/', sync_status_dir)
-        self._log_group("file(s) not yet pushed to S3", missing, parsed_args.filepaths)
-        self._log_group("file(s) modified since last push", stale, parsed_args.filepaths)
 
-    def _report_s3_comparison(self, user_profile, bucket, s3_path, filepaths=False):
+        self._log_group("file(s) missing a .synced file", missing, parsed_args.filepaths)
+        self._log_group("file(s) modified since last sync", stale, parsed_args.filepaths)
+
+    def _report_s3_comparison(self, user_profile, bucket, s3_path, sync_status_dir, filepaths=False):
         s3 = S3(user_profile, bucket)
         client = s3._client()
         prefix = s3._normalize_prefix(s3_path)
@@ -74,18 +75,47 @@ class Status(ProjectMixin, CommandHelpers, Command):
         s3_keys = set(s3_objects)
         only_local = sorted(local_keys - s3_keys)
         only_s3 = sorted(s3_keys - local_keys)
-        newer_local = []
-        newer_s3 = []
+        changed_local = []
+        changed_s3 = []
+        conflict = []
+        differ = []
         for rel_path in local_keys & s3_keys:
-            local_dt = datetime.fromtimestamp(os.path.getmtime(local_files[rel_path]), tz=timezone.utc)
-            if local_dt > s3_objects[rel_path]:
-                newer_local.append(rel_path)
-            elif s3_objects[rel_path] > local_dt:
-                newer_s3.append(rel_path)
+            local_path = local_files[rel_path]
+            marker_etag, marker_mtime = self._read_marker(rel_path, sync_status_dir)
+            if marker_etag is None:
+                differ.append(rel_path)
+                continue
+            s3_changed = s3_objects[rel_path].etag != marker_etag
+            local_mtime = datetime.fromtimestamp(os.path.getmtime(local_path), tz=timezone.utc)
+            local_changed = local_mtime > marker_mtime
+            if local_changed and s3_changed:
+                conflict.append(rel_path)
+            elif local_changed:
+                changed_local.append(rel_path)
+            elif s3_changed:
+                changed_s3.append(rel_path)
+            # else: neither side changed since the last sync -> in sync, nothing to report
         self._log_group("file(s) local but not on S3", only_local, filepaths)
         self._log_group("file(s) on S3 but not local", only_s3, filepaths)
-        self._log_group("file(s) newer locally than on S3", sorted(newer_local), filepaths)
-        self._log_group("file(s) newer on S3 than locally", sorted(newer_s3), filepaths)
+        self._log_group("file(s) changed locally since last sync", sorted(changed_local), filepaths)
+        self._log_group("file(s) changed on S3 since last sync", sorted(changed_s3), filepaths)
+        self._log_group("file(s) changed both locally and on S3 (conflict)", sorted(conflict), filepaths)
+        self._log_group("file(s) differing from S3 (no sync record)", sorted(differ), filepaths)
+
+    def _read_marker(self, rel_path, sync_status_dir):
+        """Return the (etag, mtime) recorded in the .synced marker, or (None, None) when there
+        is no usable record (no location configured, no marker, or a legacy/empty marker)."""
+        if not sync_status_dir:
+            return None, None
+        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
+        if not os.path.exists(marker_path):
+            return None, None
+        with open(marker_path) as f:
+            etag = f.read().strip()
+        if not etag:
+            return None, None
+        mtime = datetime.fromtimestamp(os.path.getmtime(marker_path), tz=timezone.utc)
+        return etag, mtime
 
     def _log_group(self, label, paths, filepaths):
         self.log.info(f"{len(paths)} {label}")

datakit_data-0.6.0/datakit_data/extra_flags.py

@@ -0,0 +1,16 @@
+class ExtraFlags:
+
+    # Flags the plugin actually acts on (boto3 push/pull only honor these).
+    SUPPORTED = ('dryrun', 'dry-run', 'delete', 'force')
+
+    @staticmethod
+    def _normalize(flag):
+        return flag[2:] if flag.startswith('--') else flag
+
+    @classmethod
+    def convert(kls, raw_flags):
+        return [f"--{kls._normalize(flag)}" for flag in raw_flags]
+
+    @classmethod
+    def unsupported(kls, raw_flags):
+        return [flag for flag in raw_flags if kls._normalize(flag) not in kls.SUPPORTED]

datakit_data-0.6.0/datakit_data/s3.py

@@ -0,0 +1,274 @@
+import os
+import logging
+from collections import namedtuple
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from logging import NullHandler
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import BotoCoreError, ClientError
+
+
+logger = logging.getLogger(__name__)
+logger.addHandler(NullHandler())
+
+# Metadata captured per remote object from a list_objects_v2 listing. The ETag identifies the
+# object's content; it is what we compare against the recorded .synced marker to detect whether
+# the object changed on S3 since our last sync.
+S3ObjectInfo = namedtuple('S3ObjectInfo', ['etag'])
+
+EMPTY_PATH_DELETE_MSG = (
+    "\n*** Refusing --delete with an empty s3_path: this would scan and delete "
+    "across the entire bucket. Set s3_path in the project config. ***\n"
+)
+
+
+class S3:
+    """A limited, human-friendly interface to S3."""
+
+    # boto3's managed upload_file transparently switches from a single PutObject to a multipart
+    # upload above this many bytes (its TransferConfig.multipart_threshold default). We mirror it:
+    # files below the threshold take the put_object fast path, whose response carries the ETag, so
+    # we avoid a follow-up head_object; larger files keep upload_file's multipart transfer (and its
+    # part-level resilience) and we read the ETag back with head_object. Either way the ETag we
+    # record is the one S3 itself reports, so it stays comparable to a later head/list probe.
+    MULTIPART_THRESHOLD = 8 * 1024 * 1024
+
+    # Per-file transfers (push uploads, pull downloads) are network-latency-bound, so we run them
+    # across a thread pool. boto3 clients are thread-safe for these calls and release the GIL during
+    # network I/O, so threads give near-linear speedup for the many-small-files case without the
+    # complexity of asyncio. Callers can override via the max_workers arg (e.g. a config key).
+    MAX_WORKERS = 16
+
+    def __init__(self, aws_user_profile, s3_bucket, max_workers=None):
+        self.user_profile = aws_user_profile
+        self.bucket = s3_bucket
+        self.max_workers = max_workers or self.MAX_WORKERS
+
+    def push(self, data_dir, s3_path='', extra_flags=None, sync_status_dir=None):
+        extra_flags = extra_flags or []
+        dryrun = '--dryrun' in extra_flags or '--dry-run' in extra_flags
+        delete = '--delete' in extra_flags
+        force = '--force' in extra_flags
+        prefix = self._normalize_prefix(s3_path)
+        if delete and not prefix:
+            logger.info(EMPTY_PATH_DELETE_MSG)
+            return 1
+        client = self._client()
+        failures = 0
+        local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
+        # Decide what to transfer (cheap, local-only checks) and log intent up front so the log
+        # stays deterministic; the transfers themselves run concurrently below.
+        to_upload = []
+        for rel_path, local_path in sorted(local_files.items()):
+            key = prefix + rel_path
+            if not force and self._marker_is_fresh(local_path, rel_path, sync_status_dir):
+                logger.info(f"skipped: {local_path}")
+                continue
+            logger.info(f"upload: {local_path} to s3://{self.bucket}/{key}")
+            to_upload.append((rel_path, local_path, key))
+        if not dryrun:
+            failures += self._run_transfers(
+                to_upload,
+                lambda item: self._push_one(client, item, sync_status_dir),
+            )
+        if delete:
+            remote_keys = self._list_s3_keys(client, prefix)
+            remote_rel = {k[len(prefix):] for k in remote_keys}
+            to_delete = [prefix + rel_path for rel_path in sorted(remote_rel - set(local_files.keys()))]
+            for key in to_delete:
+                logger.info(f"delete: s3://{self.bucket}/{key}")
+            if not dryrun:
+                failures += self._delete_keys(client, to_delete)
+        return failures
+
+    def pull(self, data_dir, s3_path='', extra_flags=None, sync_status_dir=None):
+        extra_flags = extra_flags or []
+        dryrun = '--dryrun' in extra_flags or '--dry-run' in extra_flags
+        delete = '--delete' in extra_flags
+        force = '--force' in extra_flags
+        prefix = self._normalize_prefix(s3_path)
+        if delete and not prefix:
+            logger.info(EMPTY_PATH_DELETE_MSG)
+            return 1
+        client = self._client()
+        failures = 0
+        remote_objects = self._list_s3_objects(client, prefix)
+        # Decide what to transfer (cheap, local-only checks) and log intent up front so the log
+        # stays deterministic; the transfers themselves run concurrently below.
+        to_download = []
+        for rel_path in sorted(remote_objects):
+            key = prefix + rel_path
+            remote_etag = remote_objects[rel_path].etag
+            local_path = os.path.join(data_dir, rel_path)
+            if not force:
+                marker_etag = self._marker_etag(rel_path, sync_status_dir)
+                if marker_etag is not None and marker_etag == remote_etag and os.path.exists(local_path):
+                    logger.info(f"skipped: s3://{self.bucket}/{key}")
+                    continue
+            logger.info(f"download: s3://{self.bucket}/{key} to {local_path}")
+            to_download.append((rel_path, key, remote_etag, local_path))
+        if not dryrun:
+            failures += self._run_transfers(
+                to_download,
+                lambda item: self._pull_one(client, item, sync_status_dir),
+            )
+        if delete:
+            local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
+            remote_rel = set(remote_objects)
+            for rel_path, local_path in sorted(local_files.items()):
+                if rel_path not in remote_rel:
+                    logger.info(f"delete: {local_path}")
+                    if not dryrun:
+                        try:
+                            os.remove(local_path)
+                        except OSError as e:
+                            failures += 1
+                            logger.info(f"\n*** Error ***\n{e}\n")
+        return failures
+
+    def _client(self):
+        session = boto3.Session(profile_name=self.user_profile)
+        # Size the connection pool to the worker count so concurrent transfers don't queue on a
+        # too-small pool (botocore's default is 10) or emit "connection pool is full" warnings.
+        config = Config(max_pool_connections=self.max_workers)
+        return session.client('s3', config=config)
+
+    @staticmethod
+    def _normalize_etag(etag):
+        # boto3 surfaces the ETag wrapped in literal double quotes; strip them so the value we
+        # store and compare is the bare hash.
+        return etag.strip('"') if etag else etag
+
+    def _object_etag(self, client, key):
+        response = client.head_object(Bucket=self.bucket, Key=key)
+        return self._normalize_etag(response.get('ETag'))
+
+    def _upload(self, client, local_path, key, need_etag):
+        # Upload a local file to `key` and return the object's S3 ETag (quotes stripped). Small
+        # files go via put_object, whose response carries the ETag, so no extra head_object is
+        # needed; larger files keep upload_file's managed multipart transfer and we read the ETag
+        # back with head_object only when a sync marker needs it (need_etag), else return None.
+        if os.path.getsize(local_path) < self.MULTIPART_THRESHOLD:
+            with open(local_path, 'rb') as body:
+                response = client.put_object(Bucket=self.bucket, Key=key, Body=body)
+            return self._normalize_etag(response.get('ETag'))
+        client.upload_file(local_path, self.bucket, key)
+        return self._object_etag(client, key) if need_etag else None
+
+    def _run_transfers(self, items, transfer):
+        # Run transfer(item) for each item across a thread pool, returning the failure count.
+        # Per-item ClientError/BotoCoreError are logged and counted (matching the sequential
+        # behavior); all logging happens here on the main thread, so the workers do pure I/O.
+        if not items:
+            return 0
+        failures = 0
+        with ThreadPoolExecutor(max_workers=min(self.max_workers, len(items))) as pool:
+            futures = [pool.submit(transfer, item) for item in items]
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
+                    logger.info(f"\n*** Error ***\n{e}\n")
+        return failures
+
+    def _push_one(self, client, item, sync_status_dir):
+        rel_path, local_path, key = item
+        etag = self._upload(client, local_path, key, sync_status_dir is not None)
+        if sync_status_dir is not None:
+            self._create_sync_marker(rel_path, sync_status_dir, etag)
+
+    def _pull_one(self, client, item, sync_status_dir):
+        rel_path, key, remote_etag, local_path = item
+        os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
+        client.download_file(self.bucket, key, local_path)
+        if sync_status_dir is not None:
+            self._create_sync_marker(rel_path, sync_status_dir, remote_etag)
+
+    def _normalize_prefix(self, s3_path):
+        # Returns '' for falsy input so callers can concatenate key segments without a leading slash.
+        if not s3_path:
+            return ''
+        return s3_path.strip('/') + '/'
+
+    def _marker_is_fresh(self, local_path, rel_path, sync_status_dir):
+        # True when a .synced marker exists and is at least as new as the data file, i.e. the
+        # file has not been modified on disk since our last push. Mirrors the staleness test in
+        # `data status` (a file is stale when its mtime is strictly newer than its marker's).
+        if not sync_status_dir:
+            return False
+        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
+        if not os.path.exists(marker_path):
+            return False
+        return os.path.getmtime(marker_path) >= os.path.getmtime(local_path)
+
+    def _marker_etag(self, rel_path, sync_status_dir):
+        # Return the ETag recorded in the file's .synced marker, or None when there is no
+        # usable record (no location configured, no marker, or a legacy/empty marker).
+        if not sync_status_dir:
+            return None
+        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
+        if not os.path.exists(marker_path):
+            return None
+        with open(marker_path) as f:
+            etag = f.read().strip()
+        return etag or None
+
+    def _create_sync_marker(self, rel_path, sync_status_dir, etag):
+        # The marker's content is the object's S3 ETag at sync time (the basis for detecting
+        # remote changes); its mtime is the sync time (the basis for detecting local changes).
+        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
+        os.makedirs(os.path.dirname(os.path.abspath(marker_path)), exist_ok=True)
+        with open(marker_path, 'w') as f:
+            f.write(etag or '')
+
+    def _list_local_files(self, data_dir):
+        files = {}
+        if not os.path.isdir(data_dir):
+            return files
+        for root, _, filenames in os.walk(data_dir):
+            for filename in filenames:
+                full_path = os.path.join(root, filename)
+                # The key is used to build/compare S3 keys, which always use '/'. Normalize
+                # the OS separator so keys generated on Windows match remote keys; the value
+                # stays OS-native for filesystem operations.
+                rel_path = os.path.relpath(full_path, data_dir).replace(os.sep, '/')
+                files[rel_path] = full_path
+        return files
+
+    def _delete_keys(self, client, keys):
+        # delete_objects removes up to 1000 keys per request; batch accordingly.
+        failures = 0
+        for start in range(0, len(keys), 1000):
+            batch = keys[start:start + 1000]
+            try:
+                response = client.delete_objects(
+                    Bucket=self.bucket,
+                    Delete={'Objects': [{'Key': key} for key in batch]},
+                )
+            except (ClientError, BotoCoreError) as e:
+                failures += len(batch)
+                logger.info(f"\n*** Error ***\n{e}\n")
+                continue
+            for error in response.get('Errors', []):
+                failures += 1
+                logger.info(f"\n*** Error ***\n{error.get('Key')}: {error.get('Message')}\n")
+        return failures
+
+    def _list_s3_keys(self, client, prefix):
+        keys = []
+        paginator = client.get_paginator('list_objects_v2')
+        for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
+            for obj in page.get('Contents', []):
+                keys.append(obj['Key'])
+        return keys
+
+    def _list_s3_objects(self, client, prefix):
+        objects = {}
+        paginator = client.get_paginator('list_objects_v2')
+        for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
+            for obj in page.get('Contents', []):
+                rel_path = obj['Key'][len(prefix):]
+                objects[rel_path] = S3ObjectInfo(etag=self._normalize_etag(obj.get('ETag')))
+        return objects

{datakit_data-0.5.2 → datakit_data-0.6.0}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["uv-build"]
+requires = ["uv-build>=0.8.18,<0.9.0"]
 build-backend = "uv_build"
 
 [project]
 name = "datakit-data"
-version = "0.5.2"
+version = "0.6.0"
 description = "A datakit plugin to manage data assets on AWS S3."
 authors = [
     {name = "Larry Fenn", email = "lfenn@ap.org"},
@@ -44,7 +44,7 @@ dev = [
 module-root = ""
 
 [tool.tox]
-env_list = ["py311", "py312", "py313"]
+env_list = ["py310", "py311", "py312", "py313"]
 
 [tool.tox.env_run_base]
 dependency_groups = ["dev"]

datakit_data-0.5.2/datakit_data/extra_flags.py

@@ -1,5 +0,0 @@
-class ExtraFlags:
-
-    @classmethod
-    def convert(kls, raw_flags):
-        return [f"--{flag}" for flag in raw_flags]

datakit_data-0.5.2/datakit_data/s3.py

@@ -1,152 +0,0 @@
-import os
-import logging
-from logging import NullHandler
-
-import boto3
-from botocore.exceptions import BotoCoreError, ClientError
-
-
-logger = logging.getLogger(__name__)
-logger.addHandler(NullHandler())
-
-EMPTY_PATH_DELETE_MSG = (
-    "\n*** Refusing --delete with an empty s3_path: this would scan and delete "
-    "across the entire bucket. Set s3_path in the project config. ***\n"
-)
-
-
-class S3:
-    """A limited, human-friendly interface to S3."""
-
-    def __init__(self, aws_user_profile, s3_bucket):
-        self.user_profile = aws_user_profile
-        self.bucket = s3_bucket
-
-    def push(self, data_dir, s3_path='', extra_flags=None, sync_status_dir=None):
-        extra_flags = extra_flags or []
-        dryrun = '--dryrun' in extra_flags or '--dry-run' in extra_flags
-        delete = '--delete' in extra_flags
-        prefix = self._normalize_prefix(s3_path)
-        if delete and not prefix:
-            logger.info(EMPTY_PATH_DELETE_MSG)
-            return 1
-        client = self._client()
-        failures = 0
-        local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
-        for rel_path, local_path in sorted(local_files.items()):
-            key = prefix + rel_path
-            logger.info(f"upload: {local_path} to s3://{self.bucket}/{key}")
-            if not dryrun:
-                try:
-                    client.upload_file(local_path, self.bucket, key)
-                    if sync_status_dir is not None:
-                        self._create_sync_marker(rel_path, sync_status_dir)
-                except (ClientError, BotoCoreError) as e:
-                    failures += 1
-                    logger.info(f"\n*** Error ***\n{e}\n")
-        if delete:
-            remote_keys = self._list_s3_keys(client, prefix)
-            remote_rel = {k[len(prefix):] for k in remote_keys}
-            to_delete = [prefix + rel_path for rel_path in sorted(remote_rel - set(local_files.keys()))]
-            for key in to_delete:
-                logger.info(f"delete: s3://{self.bucket}/{key}")
-            if not dryrun:
-                failures += self._delete_keys(client, to_delete)
-        return failures
-
-    def pull(self, data_dir, s3_path='', extra_flags=None):
-        extra_flags = extra_flags or []
-        dryrun = '--dryrun' in extra_flags or '--dry-run' in extra_flags
-        delete = '--delete' in extra_flags
-        prefix = self._normalize_prefix(s3_path)
-        if delete and not prefix:
-            logger.info(EMPTY_PATH_DELETE_MSG)
-            return 1
-        client = self._client()
-        failures = 0
-        remote_keys = self._list_s3_keys(client, prefix)
-        for key in remote_keys:
-            rel_path = key[len(prefix):]
-            local_path = os.path.join(data_dir, rel_path)
-            logger.info(f"download: s3://{self.bucket}/{key} to {local_path}")
-            if not dryrun:
-                os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
-                try:
-                    client.download_file(self.bucket, key, local_path)
-                except (ClientError, BotoCoreError) as e:
-                    failures += 1
-                    logger.info(f"\n*** Error ***\n{e}\n")
-        if delete:
-            local_files = self._list_local_files(data_dir)
-            remote_rel = {k[len(prefix):] for k in remote_keys}
-            for rel_path, local_path in sorted(local_files.items()):
-                if rel_path not in remote_rel:
-                    logger.info(f"delete: {local_path}")
-                    if not dryrun:
-                        os.remove(local_path)
-        return failures
-
-    def _client(self):
-        session = boto3.Session(profile_name=self.user_profile)
-        return session.client('s3')
-
-    def _normalize_prefix(self, s3_path):
-        # Returns '' for falsy input so callers can concatenate key segments without a leading slash.
-        if not s3_path:
-            return ''
-        return s3_path.strip('/') + '/'
-
-    def _create_sync_marker(self, rel_path, sync_status_dir):
-        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
-        os.makedirs(os.path.dirname(os.path.abspath(marker_path)), exist_ok=True)
-        open(marker_path, 'w').close()
-
-    def _list_local_files(self, data_dir):
-        files = {}
-        if not os.path.isdir(data_dir):
-            return files
-        for root, _, filenames in os.walk(data_dir):
-            for filename in filenames:
-                full_path = os.path.join(root, filename)
-                # The key is used to build/compare S3 keys, which always use '/'. Normalize
-                # the OS separator so keys generated on Windows match remote keys; the value
-                # stays OS-native for filesystem operations.
-                rel_path = os.path.relpath(full_path, data_dir).replace(os.sep, '/')
-                files[rel_path] = full_path
-        return files
-
-    def _delete_keys(self, client, keys):
-        # delete_objects removes up to 1000 keys per request; batch accordingly.
-        failures = 0
-        for start in range(0, len(keys), 1000):
-            batch = keys[start:start + 1000]
-            try:
-                response = client.delete_objects(
-                    Bucket=self.bucket,
-                    Delete={'Objects': [{'Key': key} for key in batch]},
-                )
-            except (ClientError, BotoCoreError) as e:
-                failures += len(batch)
-                logger.info(f"\n*** Error ***\n{e}\n")
-                continue
-            for error in response.get('Errors', []):
-                failures += 1
-                logger.info(f"\n*** Error ***\n{error.get('Key')}: {error.get('Message')}\n")
-        return failures
-
-    def _list_s3_keys(self, client, prefix):
-        keys = []
-        paginator = client.get_paginator('list_objects_v2')
-        for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
-            for obj in page.get('Contents', []):
-                keys.append(obj['Key'])
-        return keys
-
-    def _list_s3_objects(self, client, prefix):
-        objects = {}
-        paginator = client.get_paginator('list_objects_v2')
-        for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
-            for obj in page.get('Contents', []):
-                rel_path = obj['Key'][len(prefix):]
-                objects[rel_path] = obj['LastModified']
-        return objects