datakit-data 0.5.0__tar.gz → 0.5.2__tar.gz

{datakit_data-0.5.0 → datakit_data-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datakit-data
-Version: 0.5.0
+Version: 0.5.2
 Summary: A datakit plugin to manage data assets on AWS S3.
 Author: Larry Fenn
 Author-email: Larry Fenn <lfenn@ap.org>

{datakit_data-0.5.0 → datakit_data-0.5.2}/datakit_data/__init__.py

@@ -1,6 +1,6 @@
 __author__ = """Larry Fenn"""
 __email__ = 'lfenn@ap.org'
-__version__ = '0.5.0'
+__version__ = '0.5.2'
 
 from .commands.init import Init
 from .commands.push import Push

{datakit_data-0.5.0 → datakit_data-0.5.2}/datakit_data/commands/init.py

@@ -40,6 +40,13 @@ class Init(ProjectMixin, CommandHelpers, Command):
                 print(f"\nThere is an issue with {self.plugin_config_path}: `s3_bucket` is missing or empty.")
                 print("Please review and update the file, then re-run `data init`.")
                 raise SystemExit(1)
+            if not plugin_configs.get('sync_status_location'):
+                print(f"\n`sync_status_location` is not configured in {self.plugin_config_path}.")
+                print("Without it, the `status` command will be non-functional for newly created projects.")
+                answer = input("Add sync_status_location = '.sync_status/' to system config? [Y/n]: ").strip().lower()
+                if answer in ('', 'y', 'yes'):
+                    plugin_configs['sync_status_location'] = '.sync_status/'
+                    self.write_configs(plugin_configs)
         else:
             print(f"\nNo system configuration for datakit-data exists at {self.plugin_config_path}.")
             plugin_configs = self._prompt_for_plugin_configs()

{datakit_data-0.5.0 → datakit_data-0.5.2}/datakit_data/commands/pull.py

@@ -28,12 +28,15 @@ class Pull(ProjectMixin, CommandHelpers, Command):
             self.log.info("No config file found - have you run `datakit data init`?")
             return
         if bucket == "":
-            self.log.info("No bucket specified in config - no data pushed")
+            self.log.info("No bucket specified in config - no data pulled")
             return
         s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
-        s3.pull(
+        failures = s3.pull(
             'data/',
             self.project_configs['s3_path'],
             extra_flags=clean_flags
         )
+        if failures:
+            self.log.info(f"{failures} file(s) failed to transfer")
+            return 1

{datakit_data-0.5.0 → datakit_data-0.5.2}/datakit_data/commands/push.py

@@ -47,9 +47,12 @@ class Push(ProjectMixin, CommandHelpers, Command):
             write_json(self.project_config_path, configs)
         else:
             sync_status_dir = self.project_configs.get('sync_status_location')
-        s3.push(
+        failures = s3.push(
             'data/',
             self.project_configs['s3_path'],
             extra_flags=clean_flags,
             sync_status_dir=sync_status_dir
         )
+        if failures:
+            self.log.info(f"{failures} file(s) failed to transfer")
+            return 1

{datakit_data-0.5.0 → datakit_data-0.5.2}/datakit_data/commands/status.py

@@ -2,6 +2,7 @@ import os
 from datetime import datetime, timezone
 from cliff.command import Command
 from datakit import CommandHelpers
+from datakit.utils import read_json, write_json
 
 from ..project_mixin import ProjectMixin
 from ..s3 import S3
@@ -51,6 +52,12 @@ class Status(ProjectMixin, CommandHelpers, Command):
             return
         if not sync_status_dir:
             self.log.info("No sync_status_location configured")
+            answer = input("\nAdd sync_status_location = '.sync_status/' to project config? [Y/n]: ").strip().lower()
+            if answer in ('', 'y', 'yes'):
+                configs = read_json(self.project_config_path)
+                configs['sync_status_location'] = '.sync_status/'
+                write_json(self.project_config_path, configs)
+                self.log.info("Added sync_status_location to config/datakit-data.json")
             return
         missing, stale = self._find_unsynced('data/', sync_status_dir)
         self._log_group("file(s) not yet pushed to S3", missing, parsed_args.filepaths)

{datakit_data-0.5.0 → datakit_data-0.5.2}/datakit_data/s3.py

@@ -3,12 +3,17 @@ import logging
 from logging import NullHandler
 
 import boto3
-from botocore.exceptions import ClientError
+from botocore.exceptions import BotoCoreError, ClientError
 
 
 logger = logging.getLogger(__name__)
 logger.addHandler(NullHandler())
 
+EMPTY_PATH_DELETE_MSG = (
+    "\n*** Refusing --delete with an empty s3_path: this would scan and delete "
+    "across the entire bucket. Set s3_path in the project config. ***\n"
+)
+
 
 class S3:
     """A limited, human-friendly interface to S3."""
@@ -22,7 +27,11 @@ class S3:
         dryrun = '--dryrun' in extra_flags or '--dry-run' in extra_flags
         delete = '--delete' in extra_flags
         prefix = self._normalize_prefix(s3_path)
+        if delete and not prefix:
+            logger.info(EMPTY_PATH_DELETE_MSG)
+            return 1
         client = self._client()
+        failures = 0
         local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
         for rel_path, local_path in sorted(local_files.items()):
             key = prefix + rel_path
@@ -32,23 +41,29 @@ class S3:
                     client.upload_file(local_path, self.bucket, key)
                     if sync_status_dir is not None:
                         self._create_sync_marker(rel_path, sync_status_dir)
-                except ClientError as e:
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
                     logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
             remote_keys = self._list_s3_keys(client, prefix)
             remote_rel = {k[len(prefix):] for k in remote_keys}
-            for rel_path in sorted(remote_rel - set(local_files.keys())):
-                key = prefix + rel_path
+            to_delete = [prefix + rel_path for rel_path in sorted(remote_rel - set(local_files.keys()))]
+            for key in to_delete:
                 logger.info(f"delete: s3://{self.bucket}/{key}")
-                if not dryrun:
-                    client.delete_object(Bucket=self.bucket, Key=key)
+            if not dryrun:
+                failures += self._delete_keys(client, to_delete)
+        return failures
 
     def pull(self, data_dir, s3_path='', extra_flags=None):
         extra_flags = extra_flags or []
         dryrun = '--dryrun' in extra_flags or '--dry-run' in extra_flags
         delete = '--delete' in extra_flags
         prefix = self._normalize_prefix(s3_path)
+        if delete and not prefix:
+            logger.info(EMPTY_PATH_DELETE_MSG)
+            return 1
         client = self._client()
+        failures = 0
         remote_keys = self._list_s3_keys(client, prefix)
         for key in remote_keys:
             rel_path = key[len(prefix):]
@@ -58,7 +73,8 @@ class S3:
                 os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
                 try:
                     client.download_file(self.bucket, key, local_path)
-                except ClientError as e:
+                except (ClientError, BotoCoreError) as e:
+                    failures += 1
                     logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
             local_files = self._list_local_files(data_dir)
@@ -68,6 +84,7 @@ class S3:
                     logger.info(f"delete: {local_path}")
                     if not dryrun:
                         os.remove(local_path)
+        return failures
 
     def _client(self):
         session = boto3.Session(profile_name=self.user_profile)
@@ -91,10 +108,32 @@ class S3:
         for root, _, filenames in os.walk(data_dir):
             for filename in filenames:
                 full_path = os.path.join(root, filename)
-                rel_path = os.path.relpath(full_path, data_dir)
+                # The key is used to build/compare S3 keys, which always use '/'. Normalize
+                # the OS separator so keys generated on Windows match remote keys; the value
+                # stays OS-native for filesystem operations.
+                rel_path = os.path.relpath(full_path, data_dir).replace(os.sep, '/')
                 files[rel_path] = full_path
         return files
 
+    def _delete_keys(self, client, keys):
+        # delete_objects removes up to 1000 keys per request; batch accordingly.
+        failures = 0
+        for start in range(0, len(keys), 1000):
+            batch = keys[start:start + 1000]
+            try:
+                response = client.delete_objects(
+                    Bucket=self.bucket,
+                    Delete={'Objects': [{'Key': key} for key in batch]},
+                )
+            except (ClientError, BotoCoreError) as e:
+                failures += len(batch)
+                logger.info(f"\n*** Error ***\n{e}\n")
+                continue
+            for error in response.get('Errors', []):
+                failures += 1
+                logger.info(f"\n*** Error ***\n{error.get('Key')}: {error.get('Message')}\n")
+        return failures
+
     def _list_s3_keys(self, client, prefix):
         keys = []
         paginator = client.get_paginator('list_objects_v2')

{datakit_data-0.5.0 → datakit_data-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "datakit-data"
-version = "0.5.0"
+version = "0.5.2"
 description = "A datakit plugin to manage data assets on AWS S3."
 authors = [
     {name = "Larry Fenn", email = "lfenn@ap.org"},