datakit-data 0.6.2__tar.gz → 0.6.3__tar.gz

{datakit_data-0.6.2 → datakit_data-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datakit-data
-Version: 0.6.2
+Version: 0.6.3
 Summary: A datakit plugin to manage data assets on AWS S3.
 Author: Larry Fenn
 Author-email: Larry Fenn <lfenn@ap.org>

{datakit_data-0.6.2 → datakit_data-0.6.3}/datakit_data/__init__.py

@@ -1,6 +1,6 @@
 __author__ = """Larry Fenn"""
 __email__ = 'lfenn@ap.org'
-__version__ = '0.6.2'
+__version__ = '0.6.3'
 
 from .commands.init import Init
 from .commands.push import Push

datakit_data-0.6.3/datakit_data/commands/status.py

@@ -0,0 +1,90 @@
+import os
+from datetime import datetime
+from cliff.command import Command
+from datakit import CommandHelpers
+from datakit.utils import read_json, write_json
+
+from ..project_mixin import ProjectMixin
+from ..s3 import S3, list_local_files
+from ..sync_markers import SyncMarkers
+
+class Status(ProjectMixin, CommandHelpers, Command):
+
+    "Show sync status of local data files"
+
+    def get_parser(self, prog_name):
+        parser = super(Status, self).get_parser(prog_name)
+        parser.add_argument(
+            '--filepaths',
+            action='store_true',
+            default=False,
+            help="List individual file paths instead of just counts"
+        )
+        parser.add_argument(
+            '--all',
+            action='store_true',
+            default=False,
+            help="Query S3 directly for a full comparison of local and remote files"
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        if not os.path.exists("config/datakit-data.json"):
+            self.log.info("No config file found - have you run `datakit data init`?")
+            return
+        sync_status_dir = self.project_configs.get('sync_status_location')
+        markers = SyncMarkers(sync_status_dir)
+        last_push = markers.latest_mtime()
+        if last_push:
+            stamp = datetime.fromtimestamp(last_push).astimezone()
+            self.log.info(f"Last pushed: {stamp.strftime('%Y-%m-%d %H:%M:%S %Z')}")
+        else:
+            self.log.info("Last pushed: never")
+        if getattr(parsed_args, 'all', False):
+            self._report_s3_comparison(sync_status_dir, parsed_args.filepaths)
+            return
+        if not markers.enabled:
+            self.log.info("No sync_status_location configured")
+            answer = input("\nAdd sync_status_location = '.sync_status/' to project config? [Y/n]: ").strip().lower()
+            if answer in ('', 'y', 'yes'):
+                configs = read_json(self.project_config_path)
+                configs['sync_status_location'] = '.sync_status/'
+                write_json(self.project_config_path, configs)
+                self.log.info("Added sync_status_location to config/datakit-data.json")
+            return
+        missing, stale = self._find_unsynced('data/', markers)
+
+        self._log_group("file(s) missing a .synced file", missing, parsed_args.filepaths)
+        self._log_group("file(s) modified since last sync", stale, parsed_args.filepaths)
+
+    def _report_s3_comparison(self, sync_status_dir, filepaths):
+        bucket = self.project_configs['s3_bucket']
+        if bucket == "":
+            self.log.info("No bucket specified in config")
+            return
+        s3 = S3(self.project_configs['aws_user_profile'], bucket)
+        comparison = s3.compare('data/', self.project_configs['s3_path'], sync_status_dir)
+        self._log_group("file(s) local but not on S3", comparison.only_local, filepaths)
+        self._log_group("file(s) on S3 but not local", comparison.only_s3, filepaths)
+        self._log_group("file(s) changed locally since last sync", comparison.changed_local, filepaths)
+        self._log_group("file(s) changed on S3 since last sync", comparison.changed_s3, filepaths)
+        self._log_group("file(s) changed both locally and on S3 (conflict)", comparison.conflict, filepaths)
+        self._log_group("file(s) differing from S3 (no sync record)", comparison.differ, filepaths)
+
+    def _find_unsynced(self, data_dir, markers):
+        # Local-only staleness check against the recorded markers; no S3 round-trips.
+        missing = []
+        stale = []
+        for rel_path, local_path in list_local_files(data_dir).items():
+            _, marker_mtime = markers.read(rel_path)
+            if marker_mtime is None:
+                missing.append(rel_path)
+            elif os.path.getmtime(local_path) > marker_mtime:
+                stale.append(rel_path)
+        return sorted(missing), sorted(stale)
+
+    def _log_group(self, label, paths, filepaths):
+        self.log.info(f"{len(paths)} {label}")
+        if filepaths:
+            for path in paths:
+                self.log.info(f"  {path}")

{datakit_data-0.6.2 → datakit_data-0.6.3}/datakit_data/s3.py

@@ -1,11 +1,14 @@
 import os
 import logging
+import mimetypes
 from collections import namedtuple
 from logging import NullHandler
 
 import boto3
 from botocore.exceptions import BotoCoreError, ClientError
 
+from .sync_markers import SyncMarkers
+
 
 logger = logging.getLogger(__name__)
 logger.addHandler(NullHandler())
@@ -15,12 +18,38 @@ logger.addHandler(NullHandler())
 # the object changed on S3 since our last sync.
 S3ObjectInfo = namedtuple('S3ObjectInfo', ['etag'])
 
+# The result of S3.compare: sorted lists of rel_paths bucketed by how the local copy, the live
+# remote object, and the recorded .synced marker disagree. 'differ' holds files present on both
+# sides whose difference cannot be attributed for lack of a usable sync record.
+SyncComparison = namedtuple(
+    'SyncComparison', ['only_local', 'only_s3', 'changed_local', 'changed_s3', 'conflict', 'differ']
+)
+
 EMPTY_PATH_DELETE_MSG = (
     "\n*** Refusing --delete with an empty s3_path: this would scan and delete "
     "across the entire bucket. Set s3_path in the project config. ***\n"
 )
 
 
+def list_local_files(data_dir):
+    # Map of rel_path -> full path for every data file under data_dir, excluding .synced
+    # markers (which live alongside the data when sync_status_location is data/). The key is
+    # used to build/compare S3 keys, which always use '/'; normalize the OS separator so keys
+    # generated on Windows match remote keys, while the value stays OS-native for filesystem
+    # operations.
+    files = {}
+    if not os.path.isdir(data_dir):
+        return files
+    for root, _, filenames in os.walk(data_dir):
+        for filename in filenames:
+            if filename.endswith(SyncMarkers.SUFFIX):
+                continue
+            full_path = os.path.join(root, filename)
+            rel_path = os.path.relpath(full_path, data_dir).replace(os.sep, '/')
+            files[rel_path] = full_path
+    return files
+
+
 class S3:
     """A limited, human-friendly interface to S3."""
 
@@ -45,20 +74,21 @@ class S3:
         if delete and not prefix:
             logger.info(EMPTY_PATH_DELETE_MSG)
             return 1
+        markers = SyncMarkers(sync_status_dir)
         client = self._client()
         failures = 0
-        local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
+        local_files = list_local_files(data_dir)
         for rel_path, local_path in sorted(local_files.items()):
             key = prefix + rel_path
-            if not force and self._marker_is_fresh(local_path, rel_path, sync_status_dir):
+            if not force and markers.is_fresh(rel_path, local_path):
                 logger.info(f"skipped: {local_path}")
                 continue
             logger.info(f"upload: {local_path} to s3://{self.bucket}/{key}")
             if not dryrun:
                 try:
-                    etag = self._upload(client, local_path, key, sync_status_dir is not None)
-                    if sync_status_dir is not None:
-                        self._create_sync_marker(rel_path, sync_status_dir, etag)
+                    etag = self._upload(client, local_path, key, markers.enabled)
+                    if markers.enabled:
+                        markers.write(rel_path, etag)
                 except (ClientError, BotoCoreError) as e:
                     failures += 1
                     logger.info(f"\n*** Error ***\n{e}\n")
@@ -81,6 +111,7 @@ class S3:
         if delete and not prefix:
             logger.info(EMPTY_PATH_DELETE_MSG)
             return 1
+        markers = SyncMarkers(sync_status_dir)
         client = self._client()
         failures = 0
         remote_objects = self._list_s3_objects(client, prefix)
@@ -89,7 +120,7 @@ class S3:
             remote_etag = remote_objects[rel_path].etag
             local_path = os.path.join(data_dir, rel_path)
             if not force:
-                marker_etag = self._marker_etag(rel_path, sync_status_dir)
+                marker_etag = markers.etag(rel_path)
                 if marker_etag is not None and marker_etag == remote_etag and os.path.exists(local_path):
                     logger.info(f"skipped: s3://{self.bucket}/{key}")
                     continue
@@ -98,13 +129,13 @@ class S3:
                 os.makedirs(os.path.dirname(os.path.abspath(local_path)), exist_ok=True)
                 try:
                     client.download_file(self.bucket, key, local_path)
-                    if sync_status_dir is not None:
-                        self._create_sync_marker(rel_path, sync_status_dir, remote_etag)
+                    if markers.enabled:
+                        markers.write(rel_path, remote_etag)
                 except (ClientError, BotoCoreError) as e:
                     failures += 1
                     logger.info(f"\n*** Error ***\n{e}\n")
         if delete:
-            local_files = {k: v for k, v in self._list_local_files(data_dir).items() if not k.endswith('.synced')}
+            local_files = list_local_files(data_dir)
             remote_rel = set(remote_objects)
             for rel_path, local_path in sorted(local_files.items()):
                 if rel_path not in remote_rel:
@@ -117,6 +148,43 @@ class S3:
                             logger.info(f"\n*** Error ***\n{e}\n")
         return failures
 
+    def compare(self, data_dir, s3_path='', sync_status_dir=None):
+        """Compare local data files against the bucket's live listing.
+
+        Returns a SyncComparison bucketing each rel_path by what changed since the sync
+        recorded in its .synced marker: present on only one side, changed locally, changed
+        on S3, changed on both (conflict), or present on both sides with no usable sync
+        record to attribute the difference (differ). In-sync files are not reported.
+        """
+        markers = SyncMarkers(sync_status_dir)
+        client = self._client()
+        prefix = self._normalize_prefix(s3_path)
+        local_files = list_local_files(data_dir)
+        remote_objects = self._list_s3_objects(client, prefix)
+        changed_local, changed_s3, conflict, differ = [], [], [], []
+        for rel_path in sorted(set(local_files) & set(remote_objects)):
+            marker_etag, marker_mtime = markers.read(rel_path)
+            if marker_etag is None:
+                differ.append(rel_path)
+                continue
+            s3_changed = remote_objects[rel_path].etag != marker_etag
+            local_changed = os.path.getmtime(local_files[rel_path]) > marker_mtime
+            if local_changed and s3_changed:
+                conflict.append(rel_path)
+            elif local_changed:
+                changed_local.append(rel_path)
+            elif s3_changed:
+                changed_s3.append(rel_path)
+            # else: neither side changed since the last sync -> in sync, nothing to report
+        return SyncComparison(
+            only_local=sorted(set(local_files) - set(remote_objects)),
+            only_s3=sorted(set(remote_objects) - set(local_files)),
+            changed_local=changed_local,
+            changed_s3=changed_s3,
+            conflict=conflict,
+            differ=differ,
+        )
+
     def _client(self):
         session = boto3.Session(profile_name=self.user_profile)
         return session.client('s3')
@@ -136,11 +204,14 @@ class S3:
         # files go via put_object, whose response carries the ETag, so no extra head_object is
         # needed; larger files keep upload_file's managed multipart transfer and we read the ETag
         # back with head_object only when a sync marker needs it (need_etag), else return None.
+        content_type, _ = mimetypes.guess_type(local_path)
+        if content_type is None:
+            content_type = 'application/octet-stream'
         if os.path.getsize(local_path) < self.MULTIPART_THRESHOLD:
             with open(local_path, 'rb') as body:
-                response = client.put_object(Bucket=self.bucket, Key=key, Body=body)
+                response = client.put_object(Bucket=self.bucket, Key=key, Body=body, ContentType=content_type)
             return self._normalize_etag(response.get('ETag'))
-        client.upload_file(local_path, self.bucket, key)
+        client.upload_file(local_path, self.bucket, key, ExtraArgs={'ContentType': content_type})
         return self._object_etag(client, key) if need_etag else None
 
     def _normalize_prefix(self, s3_path):
@@ -149,51 +220,6 @@ class S3:
             return ''
         return s3_path.strip('/') + '/'
 
-    def _marker_is_fresh(self, local_path, rel_path, sync_status_dir):
-        # True when a .synced marker exists and is at least as new as the data file, i.e. the
-        # file has not been modified on disk since our last push. Mirrors the staleness test in
-        # `data status` (a file is stale when its mtime is strictly newer than its marker's).
-        if not sync_status_dir:
-            return False
-        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
-        if not os.path.exists(marker_path):
-            return False
-        return os.path.getmtime(marker_path) >= os.path.getmtime(local_path)
-
-    def _marker_etag(self, rel_path, sync_status_dir):
-        # Return the ETag recorded in the file's .synced marker, or None when there is no
-        # usable record (no location configured, no marker, or a legacy/empty marker).
-        if not sync_status_dir:
-            return None
-        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
-        if not os.path.exists(marker_path):
-            return None
-        with open(marker_path) as f:
-            etag = f.read().strip()
-        return etag or None
-
-    def _create_sync_marker(self, rel_path, sync_status_dir, etag):
-        # The marker's content is the object's S3 ETag at sync time (the basis for detecting
-        # remote changes); its mtime is the sync time (the basis for detecting local changes).
-        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
-        os.makedirs(os.path.dirname(os.path.abspath(marker_path)), exist_ok=True)
-        with open(marker_path, 'w') as f:
-            f.write(etag or '')
-
-    def _list_local_files(self, data_dir):
-        files = {}
-        if not os.path.isdir(data_dir):
-            return files
-        for root, _, filenames in os.walk(data_dir):
-            for filename in filenames:
-                full_path = os.path.join(root, filename)
-                # The key is used to build/compare S3 keys, which always use '/'. Normalize
-                # the OS separator so keys generated on Windows match remote keys; the value
-                # stays OS-native for filesystem operations.
-                rel_path = os.path.relpath(full_path, data_dir).replace(os.sep, '/')
-                files[rel_path] = full_path
-        return files
-
     def _delete_keys(self, client, keys):
         # delete_objects removes up to 1000 keys per request; batch accordingly.
         failures = 0

datakit_data-0.6.3/datakit_data/sync_markers.py

@@ -0,0 +1,67 @@
+import os
+
+
+class SyncMarkers:
+    """Bookkeeping for per-file sync markers under a sync status directory.
+
+    For each data file `foo.csv`, a marker `foo.csv.synced` mirrors its path under the
+    sync status directory. The marker's content is the object's S3 ETag at sync time
+    (the basis for detecting remote changes); its mtime is the sync time (the basis for
+    detecting local changes — a file is stale when its mtime is strictly newer than its
+    marker's).
+
+    A falsy sync_status_dir disables bookkeeping: reads report no record and `enabled`
+    is False. Callers must check `enabled` before calling `write`.
+    """
+
+    SUFFIX = '.synced'
+
+    def __init__(self, sync_status_dir):
+        self.sync_status_dir = sync_status_dir
+
+    @property
+    def enabled(self):
+        return bool(self.sync_status_dir)
+
+    def read(self, rel_path):
+        """Return the (etag, mtime) recorded for rel_path. The etag is None when there is
+        no usable record (no location configured, no marker, or a legacy/empty marker);
+        the mtime is None only when the marker itself is missing."""
+        marker_path = self._path(rel_path)
+        if marker_path is None or not os.path.exists(marker_path):
+            return None, None
+        with open(marker_path) as f:
+            etag = f.read().strip()
+        return etag or None, os.path.getmtime(marker_path)
+
+    def etag(self, rel_path):
+        return self.read(rel_path)[0]
+
+    def is_fresh(self, rel_path, local_path):
+        """True when a marker exists and is at least as new as the data file, i.e. the
+        file has not been modified on disk since the last sync."""
+        _, marker_mtime = self.read(rel_path)
+        return marker_mtime is not None and marker_mtime >= os.path.getmtime(local_path)
+
+    def write(self, rel_path, etag):
+        marker_path = self._path(rel_path)
+        os.makedirs(os.path.dirname(os.path.abspath(marker_path)), exist_ok=True)
+        with open(marker_path, 'w') as f:
+            f.write(etag or '')
+
+    def latest_mtime(self):
+        """The mtime of the most recently written marker, or None when there are none."""
+        if not self.enabled or not os.path.isdir(self.sync_status_dir):
+            return None
+        mtimes = [
+            os.path.getmtime(os.path.join(root, filename))
+            for root, _, filenames in os.walk(self.sync_status_dir)
+            for filename in filenames
+            if filename.endswith(self.SUFFIX)
+        ]
+        return max(mtimes, default=None)
+
+    def _path(self, rel_path):
+        if not self.enabled:
+            return None
+        return os.path.join(self.sync_status_dir, rel_path + self.SUFFIX)

{datakit_data-0.6.2 → datakit_data-0.6.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "datakit-data"
-version = "0.6.2"
+version = "0.6.3"
 description = "A datakit plugin to manage data assets on AWS S3."
 authors = [
     {name = "Larry Fenn", email = "lfenn@ap.org"},

datakit_data-0.6.2/datakit_data/commands/status.py

@@ -1,154 +0,0 @@
-import os
-from datetime import datetime, timezone
-from cliff.command import Command
-from datakit import CommandHelpers
-from datakit.utils import read_json, write_json
-
-from ..project_mixin import ProjectMixin
-from ..s3 import S3
-
-class Status(ProjectMixin, CommandHelpers, Command):
-
-    "Show sync status of local data files"
-
-    def get_parser(self, prog_name):
-        parser = super(Status, self).get_parser(prog_name)
-        parser.add_argument(
-            '--filepaths',
-            action='store_true',
-            default=False,
-            help="List individual file paths instead of just counts"
-        )
-        parser.add_argument(
-            '--all',
-            action='store_true',
-            default=False,
-            help="Query S3 directly for a full comparison of local and remote files"
-        )
-        return parser
-
-    def take_action(self, parsed_args):
-        if not os.path.exists("config/datakit-data.json"):
-            self.log.info("No config file found - have you run `datakit data init`?")
-            return
-        sync_status_dir = self.project_configs.get('sync_status_location')
-        last_push = self._last_push_time(sync_status_dir) if sync_status_dir else None
-        if last_push:
-            self.log.info(f"Last pushed: {last_push.strftime('%Y-%m-%d %H:%M:%S %Z')}")
-        else:
-            self.log.info("Last pushed: never")
-        if getattr(parsed_args, 'all', False):
-            bucket = self.project_configs['s3_bucket']
-            if bucket == "":
-                self.log.info("No bucket specified in config")
-                return
-            self._report_s3_comparison(
-                self.project_configs['aws_user_profile'],
-                bucket,
-                self.project_configs['s3_path'],
-                sync_status_dir,
-                filepaths=parsed_args.filepaths,
-            )
-            return
-        if not sync_status_dir:
-            self.log.info("No sync_status_location configured")
-            answer = input("\nAdd sync_status_location = '.sync_status/' to project config? [Y/n]: ").strip().lower()
-            if answer in ('', 'y', 'yes'):
-                configs = read_json(self.project_config_path)
-                configs['sync_status_location'] = '.sync_status/'
-                write_json(self.project_config_path, configs)
-                self.log.info("Added sync_status_location to config/datakit-data.json")
-            return
-        missing, stale = self._find_unsynced('data/', sync_status_dir)
-
-        self._log_group("file(s) missing a .synced file", missing, parsed_args.filepaths)
-        self._log_group("file(s) modified since last sync", stale, parsed_args.filepaths)
-
-    def _report_s3_comparison(self, user_profile, bucket, s3_path, sync_status_dir, filepaths=False):
-        s3 = S3(user_profile, bucket)
-        client = s3._client()
-        prefix = s3._normalize_prefix(s3_path)
-        local_files = {k: v for k, v in s3._list_local_files('data/').items()
-                       if not k.endswith('.synced')}
-        s3_objects = s3._list_s3_objects(client, prefix)
-        local_keys = set(local_files)
-        s3_keys = set(s3_objects)
-        only_local = sorted(local_keys - s3_keys)
-        only_s3 = sorted(s3_keys - local_keys)
-        changed_local = []
-        changed_s3 = []
-        conflict = []
-        differ = []
-        for rel_path in local_keys & s3_keys:
-            local_path = local_files[rel_path]
-            marker_etag, marker_mtime = self._read_marker(rel_path, sync_status_dir)
-            if marker_etag is None:
-                differ.append(rel_path)
-                continue
-            s3_changed = s3_objects[rel_path].etag != marker_etag
-            local_mtime = datetime.fromtimestamp(os.path.getmtime(local_path), tz=timezone.utc)
-            local_changed = local_mtime > marker_mtime
-            if local_changed and s3_changed:
-                conflict.append(rel_path)
-            elif local_changed:
-                changed_local.append(rel_path)
-            elif s3_changed:
-                changed_s3.append(rel_path)
-            # else: neither side changed since the last sync -> in sync, nothing to report
-        self._log_group("file(s) local but not on S3", only_local, filepaths)
-        self._log_group("file(s) on S3 but not local", only_s3, filepaths)
-        self._log_group("file(s) changed locally since last sync", sorted(changed_local), filepaths)
-        self._log_group("file(s) changed on S3 since last sync", sorted(changed_s3), filepaths)
-        self._log_group("file(s) changed both locally and on S3 (conflict)", sorted(conflict), filepaths)
-        self._log_group("file(s) differing from S3 (no sync record)", sorted(differ), filepaths)
-
-    def _read_marker(self, rel_path, sync_status_dir):
-        """Return the (etag, mtime) recorded in the .synced marker, or (None, None) when there
-        is no usable record (no location configured, no marker, or a legacy/empty marker)."""
-        if not sync_status_dir:
-            return None, None
-        marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
-        if not os.path.exists(marker_path):
-            return None, None
-        with open(marker_path) as f:
-            etag = f.read().strip()
-        if not etag:
-            return None, None
-        mtime = datetime.fromtimestamp(os.path.getmtime(marker_path), tz=timezone.utc)
-        return etag, mtime
-
-    def _log_group(self, label, paths, filepaths):
-        self.log.info(f"{len(paths)} {label}")
-        if filepaths:
-            for path in paths:
-                self.log.info(f"  {path}")
-
-    def _last_push_time(self, sync_status_dir):
-        if not os.path.isdir(sync_status_dir):
-            return None
-        latest = None
-        for root, _, filenames in os.walk(sync_status_dir):
-            for filename in filenames:
-                if filename.endswith('.synced'):
-                    mtime = os.path.getmtime(os.path.join(root, filename))
-                    if latest is None or mtime > latest:
-                        latest = mtime
-        return datetime.fromtimestamp(latest).astimezone() if latest is not None else None
-
-    def _find_unsynced(self, data_dir, sync_status_dir):
-        missing = []
-        stale = []
-        if not os.path.isdir(data_dir):
-            return missing, stale
-        for root, _, filenames in os.walk(data_dir):
-            for filename in filenames:
-                full_path = os.path.join(root, filename)
-                rel_path = os.path.relpath(full_path, data_dir)
-                if rel_path.endswith('.synced'):
-                    continue
-                marker_path = os.path.join(sync_status_dir, rel_path + '.synced')
-                if not os.path.exists(marker_path):
-                    missing.append(rel_path)
-                elif os.path.getmtime(full_path) > os.path.getmtime(marker_path):
-                    stale.append(rel_path)
-        return sorted(missing), sorted(stale)