datakit-data 0.5.2__tar.gz → 0.5.3__tar.gz

{datakit_data-0.5.2 → datakit_data-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datakit-data
-Version: 0.5.2
+Version: 0.5.3
 Summary: A datakit plugin to manage data assets on AWS S3.
 Author: Larry Fenn
 Author-email: Larry Fenn <lfenn@ap.org>

{datakit_data-0.5.2 → datakit_data-0.5.3}/datakit_data/__init__.py

@@ -1,6 +1,6 @@
 __author__ = """Larry Fenn"""
 __email__ = 'lfenn@ap.org'
-__version__ = '0.5.2'
+__version__ = '0.5.3'
 
 from .commands.init import Init
 from .commands.push import Push

{datakit_data-0.5.2 → datakit_data-0.5.3}/datakit_data/commands/init.py

@@ -28,12 +28,14 @@ class Init(ProjectMixin, CommandHelpers, Command):
             mkdir_p(directory)
         open('data/.gitkeep', 'w').close()
         self.log.info("Created data/ directory")
-        self.create_project_config()
-        self.log.info("Created config/datakit-data.json")
+        if self.create_project_config():
+            self.log.info("Created config/datakit-data.json")
+        else:
+            self.log.info("config/datakit-data.json already exists - leaving it unchanged")
 
     def create_project_config(self):
         if os.path.exists(self.project_config_path):
-            return
+            return False
         if os.path.exists(self.plugin_config_path):
             plugin_configs = read_json(self.plugin_config_path)
             if not plugin_configs.get('s3_bucket'):
@@ -56,6 +58,7 @@ class Init(ProjectMixin, CommandHelpers, Command):
         self._expand_vars(to_write)
         self.finalize_configs(to_write)
         write_json(self.project_config_path, to_write)
+        return True
 
     def _prompt_for_plugin_configs(self):
         print("Please provide the following configuration values (press Enter to use the default):\n")

{datakit_data-0.5.2 → datakit_data-0.5.3}/datakit_data/commands/pull.py

@@ -17,7 +17,7 @@ class Pull(ProjectMixin, CommandHelpers, Command):
         parser.add_argument(
             'args',
             nargs=argparse.REMAINDER,
-            help="One or more boolean S3 sync flags without leading dashes, e.g. delete or dryrun"
+            help="One or more boolean flags without leading dashes: delete, dryrun"
         )
         return parser
 
@@ -32,6 +32,9 @@ class Pull(ProjectMixin, CommandHelpers, Command):
             return
         s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
+        unsupported = ExtraFlags.unsupported(parsed_args.args)
+        if unsupported:
+            self.log.info(f"Ignoring unsupported flag(s): {', '.join(unsupported)}")
         failures = s3.pull(
             'data/',
             self.project_configs['s3_path'],

{datakit_data-0.5.2 → datakit_data-0.5.3}/datakit_data/commands/push.py

@@ -19,7 +19,7 @@ class Push(ProjectMixin, CommandHelpers, Command):
         parser.add_argument(
             'args',
             nargs=argparse.REMAINDER,
-            help="One or more boolean S3 sync flags without leading dashes, e.g. delete or dryrun"
+            help="One or more boolean flags without leading dashes: delete, dryrun"
         )
         parser.add_argument(
             '--sync-status-in-data',
@@ -40,11 +40,16 @@ class Push(ProjectMixin, CommandHelpers, Command):
             return
         s3 = S3(user_profile, bucket)
         clean_flags = ExtraFlags.convert(parsed_args.args)
+        unsupported = ExtraFlags.unsupported(parsed_args.args)
+        if unsupported:
+            self.log.info(f"Ignoring unsupported flag(s): {', '.join(unsupported)}")
+        dryrun = '--dryrun' in clean_flags or '--dry-run' in clean_flags
         if parsed_args.sync_status_in_data:
             sync_status_dir = 'data/'
-            configs = self.project_configs.copy()
-            configs['sync_status_location'] = 'data/'
-            write_json(self.project_config_path, configs)
+            if not dryrun:
+                configs = self.project_configs.copy()
+                configs['sync_status_location'] = 'data/'
+                write_json(self.project_config_path, configs)
         else:
             sync_status_dir = self.project_configs.get('sync_status_location')
         failures = s3.push(

{datakit_data-0.5.2 → datakit_data-0.5.3}/datakit_data/commands/status.py

@@ -1,5 +1,5 @@
 import os
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from cliff.command import Command
 from datakit import CommandHelpers
 from datakit.utils import read_json, write_json
@@ -7,6 +7,10 @@ from datakit.utils import read_json, write_json
 from ..project_mixin import ProjectMixin
 from ..s3 import S3
 
+# Tolerance for clock skew when comparing a local .synced marker (our last-push time)
+# against an object's S3 LastModified, which is stamped by S3's clock.
+SYNC_WINDOW = timedelta(seconds=2)
+
 
 class Status(ProjectMixin, CommandHelpers, Command):
 
@@ -47,6 +51,7 @@ class Status(ProjectMixin, CommandHelpers, Command):
                 self.project_configs['aws_user_profile'],
                 bucket,
                 self.project_configs['s3_path'],
+                sync_status_dir,
                 filepaths=parsed_args.filepaths,
             )
             return
@@ -63,7 +68,7 @@ class Status(ProjectMixin, CommandHelpers, Command):
         self._log_group("file(s) not yet pushed to S3", missing, parsed_args.filepaths)
         self._log_group("file(s) modified since last push", stale, parsed_args.filepaths)
 
-    def _report_s3_comparison(self, user_profile, bucket, s3_path, filepaths=False):
+    def _report_s3_comparison(self, user_profile, bucket, s3_path, sync_status_dir, filepaths=False):
         s3 = S3(user_profile, bucket)
         client = s3._client()
         prefix = s3._normalize_prefix(s3_path)
@@ -74,18 +79,45 @@ class Status(ProjectMixin, CommandHelpers, Command):
         s3_keys = set(s3_objects)
         only_local = sorted(local_keys - s3_keys)
         only_s3 = sorted(s3_keys - local_keys)
-        newer_local = []
-        newer_s3 = []
+        changed_local = []
+        changed_s3 = []
+        conflict = []
+        differ = []
         for rel_path in local_keys & s3_keys:
-            local_dt = datetime.fromtimestamp(os.path.getmtime(local_files[rel_path]), tz=timezone.utc)
-            if local_dt > s3_objects[rel_path]:
-                newer_local.append(rel_path)
-            elif s3_objects[rel_path] > local_dt:
-                newer_s3.append(rel_path)
+            local_path = local_files[rel_path]
+            if os.path.getsize(local_path) == s3_objects[rel_path].size:
+                continue  # same byte size: treat as in sync
+            # Sizes differ. Attribute direction using the .synced marker (our last-push time)
+            # rather than comparing local mtime to S3 LastModified, which measure different things.
+            marker_mtime = self._marker_mtime(rel_path, sync_status_dir)
+            if marker_mtime is None:
+                differ.append(rel_path)
+                continue
+            local_mtime = datetime.fromtimestamp(os.path.getmtime(local_path), tz=timezone.utc)
+            local_changed = local_mtime > marker_mtime + SYNC_WINDOW
+            s3_changed = s3_objects[rel_path].last_modified > marker_mtime + SYNC_WINDOW
+            if local_changed and s3_changed:
+                conflict.append(rel_path)
+            elif local_changed:
+                changed_local.append(rel_path)
+            elif s3_changed:
+                changed_s3.append(rel_path)
+            else:
+                differ.append(rel_path)
         self._log_group("file(s) local but not on S3", only_local, filepaths)
         self._log_group("file(s) on S3 but not local", only_s3, filepaths)
-        self._log_group("file(s) newer locally than on S3", sorted(newer_local), filepaths)
-        self._log_group("file(s) newer on S3 than locally", sorted(newer_s3), filepaths)
+        self._log_group("file(s) changed locally since last push", sorted(changed_local), filepaths)
+        self._log_group("file(s) changed on S3 since last push", sorted(changed_s3), filepaths)
+        self._log_group("file(s) changed both locally and on S3 (conflict)", sorted(conflict), filepaths)
+        self._log_group("file(s) differing from S3 (no sync record)", sorted(differ), filepaths)
+
+    def _marker_mtime(self, rel_path, sync_status_dir):
+        if not sync_status_dir:
+            return None
+        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
+        if not os.path.exists(marker_path):
+            return None
+        return datetime.fromtimestamp(os.path.getmtime(marker_path), tz=timezone.utc)
 
     def _log_group(self, label, paths, filepaths):
         self.log.info(f"{len(paths)} {label}")

datakit_data-0.5.3/datakit_data/extra_flags.py

@@ -0,0 +1,12 @@
+class ExtraFlags:
+
+    # Flags the plugin actually acts on (boto3 push/pull only honor these).
+    SUPPORTED = ('dryrun', 'dry-run', 'delete')
+
+    @classmethod
+    def convert(kls, raw_flags):
+        return [f"--{flag}" for flag in raw_flags]
+
+    @classmethod
+    def unsupported(kls, raw_flags):
+        return [flag for flag in raw_flags if flag not in kls.SUPPORTED]

{datakit_data-0.5.2 → datakit_data-0.5.3}/datakit_data/s3.py

@@ -1,5 +1,6 @@
 import os
 import logging
+from collections import namedtuple
 from logging import NullHandler
 
 import boto3
@@ -9,6 +10,9 @@ from botocore.exceptions import BotoCoreError, ClientError
 logger = logging.getLogger(__name__)
 logger.addHandler(NullHandler())
 
+# Metadata captured per remote object from a list_objects_v2 listing.
+S3ObjectInfo = namedtuple('S3ObjectInfo', ['size', 'last_modified'])
+
 EMPTY_PATH_DELETE_MSG = (
     "\n*** Refusing --delete with an empty s3_path: this would scan and delete "
     "across the entire bucket. Set s3_path in the project config. ***\n"
@@ -77,13 +81,17 @@ class S3:
                     failures += 1
                     logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
-            local_files = self._list_local_files(data_dir)
+            local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
             remote_rel = {k[len(prefix):] for k in remote_keys}
             for rel_path, local_path in sorted(local_files.items()):
                 if rel_path not in remote_rel:
                     logger.info(f"delete: {local_path}")
                     if not dryrun:
-                        os.remove(local_path)
+                        try:
+                            os.remove(local_path)
+                        except OSError as e:
+                            failures += 1
+                            logger.info(f"\n*** Error ***\n{e}\n")
         return failures
 
     def _client(self):
@@ -148,5 +156,5 @@ class S3:
         for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
             for obj in page.get('Contents', []):
                 rel_path = obj['Key'][len(prefix):]
-                objects[rel_path] = obj['LastModified']
+                objects[rel_path] = S3ObjectInfo(size=obj['Size'], last_modified=obj['LastModified'])
         return objects

{datakit_data-0.5.2 → datakit_data-0.5.3}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["uv-build"]
+requires = ["uv-build>=0.8.18,<0.9.0"]
 build-backend = "uv_build"
 
 [project]
 name = "datakit-data"
-version = "0.5.2"
+version = "0.5.3"
 description = "A datakit plugin to manage data assets on AWS S3."
 authors = [
     {name = "Larry Fenn", email = "lfenn@ap.org"},
@@ -44,7 +44,7 @@ dev = [
 module-root = ""
 
 [tool.tox]
-env_list = ["py311", "py312", "py313"]
+env_list = ["py310", "py311", "py312", "py313"]
 
 [tool.tox.env_run_base]
 dependency_groups = ["dev"]

datakit_data-0.5.2/datakit_data/extra_flags.py

@@ -1,5 +0,0 @@
-class ExtraFlags:
-
-    @classmethod
-    def convert(kls, raw_flags):
-        return [f"--{flag}" for flag in raw_flags]