datalake 1.1__tar.gz → 2.2__tar.gz

datalake-2.2/LICENSE

@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 Planet Labs Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

datalake-2.2/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: datalake
+Version: 2.2
+Summary: datalake: a metadata-aware archive
+Home-page: https://github.com/planetlabs/datalake
+Author: Brian Cavagnolo
+Author-email: brian@planet.com
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+License-File: LICENSE
+Requires-Dist: boto3>=1.9.68
+Requires-Dist: memoized_property>=1.0.1
+Requires-Dist: pyblake2>=0.9.3; python_version < "3.6"
+Requires-Dist: click>=4.1
+Requires-Dist: python-dotenv>=0.1.3
+Requires-Dist: requests>=2.5
+Requires-Dist: six>=1.10.0
+Requires-Dist: python-dateutil>=2.4.2
+Requires-Dist: pytz>=2015.4
+Provides-Extra: test
+Requires-Dist: pytest<8.0.0; extra == "test"
+Requires-Dist: moto[s3]<5,>4; extra == "test"
+Requires-Dist: twine<4.0.0; extra == "test"
+Requires-Dist: pip<22.0.0,>=20.0.0; extra == "test"
+Requires-Dist: wheel<0.38.0; extra == "test"
+Requires-Dist: flake8<4.1,>=2.5.0; extra == "test"
+Requires-Dist: responses<0.22.0; extra == "test"
+Provides-Extra: queuable
+Requires-Dist: pyinotify>=0.9.4; extra == "queuable"
+Provides-Extra: sentry
+Requires-Dist: raven>=5.0.0; extra == "sentry"

{datalake-1.1 → datalake-2.2}/datalake/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2022-05-06T09:52:24-0700",
+ "date": "2024-01-10T16:28:12-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "4215d7f1982009e5597f6f8e1423a8e728eac1bf",
- "version": "1.1"
+ "full-revisionid": "b1ffd3c6211b7d759ceb70f8f07bfee4ac21e93a",
+ "version": "2.2"
 }
 '''  # END VERSION_JSON
 

{datalake-1.1 → datalake-2.2}/datalake/archive.py

@@ -28,10 +28,10 @@ from .common import Metadata
 import requests
 from io import BytesIO
 import errno
+from copy import deepcopy
+from datetime import datetime
 
-from boto.s3.connection import S3Connection
-from boto.s3.key import Key
-from boto.s3.connection import NoHostProvided
+import boto3
 import math
 from logging import getLogger
 log = getLogger('datalake-archive')
@@ -210,7 +210,26 @@ class Archive(object):
         return self.url_from_file(f)
 
     def _upload_file(self, f):
-        key = self._s3_key_from_metadata(f)
+
+        # Implementation inspired by https://stackoverflow.com/a/60892027
+        obj = self._s3_object_from_metadata(f)
+
+        # NB: we have an opportunitiy to turn on threading here, which may
+        # improve performance. However, in some cases (i.e., queue-based
+        # uploader) we already use threads. So let's add it later as a
+        # configuration if/when we want to experiment.
+        config = boto3.s3.transfer.TransferConfig(
+            # All sizes are bytes
+            multipart_threshold=CHUNK_SIZE(),
+            use_threads=False,
+            multipart_chunksize=CHUNK_SIZE(),
+        )
+
+        extra = {
+            'Metadata': {
+                METADATA_NAME: json.dumps(f.metadata)
+            }
+        }
 
         spos = f.tell()
         f.seek(0, os.SEEK_END)
@@ -220,38 +239,22 @@ class Archive(object):
 
         num_chunks = int(math.ceil(f_size / float(CHUNK_SIZE())))
         log.info("Uploading {} ({} B / {} chunks)".format(
-            key.name, f_size, num_chunks))
-        if num_chunks <= 1:
-            key.set_metadata(METADATA_NAME, json.dumps(f.metadata))
-            completed_size = key.set_contents_from_file(f)
-            log.info("Upload of {} complete (1 part / {} B).".format(
-                key.name, completed_size))
-            return
-        completed_size = 0
+            obj.key, f_size, num_chunks))
+
         chunk = 0
-        mp = key.bucket.initiate_multipart_upload(
-            key.name, metadata={
-                METADATA_NAME: json.dumps(f.metadata)
-            })
-        try:
-            for chunk in range(1, num_chunks + 1):
-                part = mp.upload_part_from_file(
-                    f, chunk, size=CHUNK_SIZE())
-                completed_size += part.size
-                log.debug("Uploaded chunk {}/{} ({}B)".format(
-                    chunk, num_chunks, part.size))
-        except:  # NOQA
-            # Any exception we want to attempt to cancel_upload, otherwise
-            # AWS will bill us every month indefnitely for storing the
-            # partial-uploaded chunks.
-            log.exception("Upload of {} failed on chunk {}".format(
-                key.name, chunk))
-            mp.cancel_upload()
-            raise
-        else:
-            completed = mp.complete_upload()
-            log.info("Upload of {} complete ({} parts / {} B).".format(
-                completed.key_name, chunk, completed_size))
+
+        def _progress(number_of_bytes):
+            nonlocal chunk
+            log.info("Uploaded chunk {}/{} ({}B)".format(
+                chunk, num_chunks, CHUNK_SIZE()))
+            chunk += 1
+
+        # NB: deep under the hood, upload_fileobj creates a
+        # CreateMultipartUploadTask. And that object cleans up after itself:
+        # https://github.com/boto/s3transfer/blob/develop/s3transfer/tasks.py#L353-L360  # noqa
+        obj.upload_fileobj(f, ExtraArgs=extra, Config=config,
+                           Callback=_progress)
+        obj.wait_until_exists()
 
     def url_from_file(self, f):
         return self._get_s3_url(f)
@@ -279,12 +282,11 @@ class Archive(object):
         return url.startswith('http') and url.endswith('/data')
 
     def _fetch_s3_url(self, url, stream=False):
-        k = self._get_key_from_url(url)
-        m = self._get_metadata_from_key(k)
+        obj, m = self._get_object_from_url(url)
         if stream:
-            return StreamingFile(k, **m)
+            return StreamingFile(obj._datalake_details['Body'], **m)
         fd = BytesIO()
-        k.get_contents_to_file(fd)
+        self._s3_bucket.download_fileobj(obj.key, fd)
         fd.seek(0)
         return File(fd, **m)
 
@@ -323,16 +325,19 @@ class Archive(object):
         example, to store a file based on `what` it is, you could pass
         something like {what}.log. Or if you gathering many `what`'s that
         originate from many `where`'s you might want to use something like
-        {where}/{what}-{start}.log. If filename_template is None (the default),
-        files are stored in the current directory and the filenames are the ids
-        from the metadata.
+        {where}/{what}-{start}.log. Note that the template variables
+        {start_iso} and {end_iso} are also supported and expand to the ISO
+        timestamps with millisecond precision (e.g., 2023-12-19T00:10:22.123).
+
+        If filename_template is None (the default), files are stored in the
+        current directory and the filenames are the ids from the metadata.
 
         Returns the filename written.
+
         '''
         k = None
         if url.startswith('s3://'):
-            k = self._get_key_from_url(url)
-            m = self._get_metadata_from_key(k)
+            obj, m = self._get_object_from_url(url)
         else:
             m = self._get_metadata_from_http_url(url)
         fname = self._get_filename_from_template(filename_template, m)
@@ -357,38 +362,58 @@ class Archive(object):
             else:
                 raise
 
-    def _get_key_from_url(self, url):
+    def _get_object_from_url(self, url):
         self._validate_fetch_url(url)
         key_name = self._get_key_name_from_url(url)
-        k = self._s3_bucket.get_key(key_name)
-        if k is None:
+        obj = self._s3.Object(self._s3_bucket_name, key_name)
+        try:
+            # cache the results of the get on the obj to avoid superfluous
+            # network calls.
+            obj._datalake_details = obj.get()
+            m = obj._datalake_details['Metadata'].get(METADATA_NAME)
+        except self._s3.meta.client.exceptions.NoSuchKey:
             msg = 'Failed to find {} in the datalake.'.format(url)
             raise InvalidDatalakePath(msg)
-        return k
-
-    def _get_metadata_from_key(self, key):
-        m = key.get_metadata(METADATA_NAME)
-        return Metadata.from_json(m)
+        return obj, Metadata.from_json(m)
 
     def _get_filename_from_template(self, template, metadata):
+        template_vars = deepcopy(metadata)
+        template_vars.update(
+            start_iso=self._ms_to_iso(metadata.get('start')),
+            end_iso=self._ms_to_iso(metadata.get('end')),
+        )
         if template is None:
             template = '{id}'
         try:
-            return template.format(**metadata)
+            return template.format(**template_vars)
         except KeyError as e:
-            m = '"{}" does not appear in the datalake metadata'
+            m = '"{}" does not appear to be a supported template variable.'
             m = m.format(str(e))
             raise InvalidDatalakePath(m)
         except ValueError as e:
             raise InvalidDatalakePath(str(e))
 
+    _ISO_FORMAT_MS = '%Y-%m-%dT%H:%M:%S.%f'
+
+    def _ms_to_iso(self, ts):
+        if ts is None:
+            return None
+        d = datetime.utcfromtimestamp(ts/1000.0)
+        # drop to ms precision
+        return d.strftime(self._ISO_FORMAT_MS)[:-3]
+
     def _get_key_name_from_url(self, url):
         parts = urlparse(url)
         if not parts.path:
             msg = '{} is not a valid datalake url'.format(url)
             raise InvalidDatalakePath(msg)
 
-        return parts.path
+        # NB: under boto 2 we didn't used to have to have the lstrip. It seems
+        # that boto2 explicitly stripped these leading slashes for us:
+        # https://groups.google.com/g/boto-users/c/mv--NMPUXoU ...but boto3
+        # does not. So we must take care to strip it whenever we parse a URL to
+        # get a key.
+        return parts.path.lstrip('/')
 
     def _validate_fetch_url(self, url):
         valid_base_urls = (self.storage_url, self.http_url)
@@ -398,9 +423,9 @@ class Archive(object):
             raise InvalidDatalakePath(msg)
 
     def _get_s3_url(self, f):
-        key = self._s3_key_from_metadata(f)
+        obj = self._s3_object_from_metadata(f)
         return self._URL_FORMAT.format(bucket=self._s3_bucket_name,
-                                       key=key.name)
+                                       key=obj.key)
 
     @property
     def _s3_bucket_name(self):
@@ -408,41 +433,37 @@ class Archive(object):
 
     @memoized_property
     def _s3_bucket(self):
-        # Note: we pass validate=False because we may just have push
-        # permissions. If validate is not False, boto tries to list the
-        # bucket. And this will 403.
-        return self._s3_conn.get_bucket(self._s3_bucket_name, validate=False)
+        return self._s3.Bucket(self._s3_bucket_name)
 
     _KEY_FORMAT = '{id}/data'
 
-    def _s3_key_from_metadata(self, f):
-        # For performance reasons, s3 keys should start with a short random
-        # sequence:
-        # https://aws.amazon.com/blogs/aws/amazon-s3-performance-tips-tricks-seattle-hiring-event/
-        # http://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html
+    def _s3_object_from_metadata(self, f):
         key_name = self._KEY_FORMAT.format(**f.metadata)
-        return Key(self._s3_bucket, name=key_name)
+        return self._s3_bucket.Object(key_name)
 
     @property
     def _s3_host(self):
         h = environ.get('AWS_S3_HOST')
         if h is not None:
-            return h
+            return 'https://' + h
         r = environ.get('AWS_REGION') or environ.get('AWS_DEFAULT_REGION')
         if r is not None:
-            return 's3-' + r + '.amazonaws.com'
+            return 'https://s3-' + r + '.amazonaws.com'
         else:
-            return NoHostProvided
+            return None
 
-    @property
-    def _s3_conn(self):
-        if not hasattr(self, '_conn'):
-            k = environ.get('AWS_ACCESS_KEY_ID')
-            s = environ.get('AWS_SECRET_ACCESS_KEY')
-            self._conn = S3Connection(aws_access_key_id=k,
-                                      aws_secret_access_key=s,
-                                      host=self._s3_host)
-        return self._conn
+    @memoized_property
+    def _s3(self):
+        # boto3 uses AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+        # boto3 will use AWS_DEFAULT_REGION if AWS_REGION is not set
+        return boto3.resource('s3',
+                              region_name=environ.get('AWS_REGION'),
+                              endpoint_url=self._s3_host)
+
+    @memoized_property
+    def _s3_client(self):
+        boto_session = boto3.Session()
+        return boto_session.client('s3')
 
     def _requests_get(self, url, **kwargs):
         return self._session.get(url, timeout=TIMEOUT(), **kwargs)

{datalake-1.1 → datalake-2.2}/datalake/common/record.py

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-from . import Metadata, InvalidDatalakeMetadata
+from . import Metadata
 from six.moves.urllib.parse import urlparse
 import json
 import os
@@ -28,8 +28,7 @@ they are unavailable, the affected functions will raise
 InsufficientConfiguration.'''
 has_s3 = True
 try:
-    import boto.s3
-    from boto.exception import S3ResponseError
+    import boto3
 except ImportError:
     has_s3 = False
 
@@ -44,6 +43,11 @@ def requires_s3(f):
     return wrapped
 
 
+# NB: Some time ago we migrated this class from datalake-common to datalake in
+# order to reduce the number of packages that comprise the datalake. As a side
+# effect of this migration this class contains some code duplicated with the
+# datalake.Archive. There is an opportunity to clean this up, but we can take
+# that on after getting off of boto 2.
 class DatalakeRecord(dict):
 
     def __init__(self, url, metadata, time_bucket, create_time, size):
@@ -64,66 +68,58 @@ class DatalakeRecord(dict):
     @requires_s3
     def list_from_url(cls, url):
         '''return a list of DatalakeRecords for the specified url'''
-        key = cls._get_key(url)
-        metadata = cls._get_metadata_from_key(key)
-        ct = cls._get_create_time(key)
+        obj, metadata = cls._get_object(url)
+        ct = cls._get_create_time(obj)
         time_buckets = cls.get_time_buckets_from_metadata(metadata)
-        return [cls(url, metadata, t, ct, key.size) for t in time_buckets]
+
+        return [
+            cls(url, metadata, t, ct, obj.content_length) for t in time_buckets
+        ]
 
     @classmethod
     @requires_s3
     def list_from_metadata(cls, url, metadata):
         '''return a list of DatalakeRecords for the url and metadata'''
-        key = cls._get_key(url)
+        obj, _ = cls._get_object(url)
         metadata = Metadata(**metadata)
-        ct = cls._get_create_time(key)
+        ct = cls._get_create_time(obj)
         time_buckets = cls.get_time_buckets_from_metadata(metadata)
-        return [cls(url, metadata, t, ct, key.size) for t in time_buckets]
+        return [
+            cls(url, metadata, t, ct, obj.content_length) for t in time_buckets
+        ]
 
     @classmethod
-    def _get_create_time(cls, key):
-        return Metadata.normalize_date(key.last_modified)
+    def _get_create_time(cls, obj):
+        return Metadata.normalize_date(obj.last_modified)
 
     @classmethod
-    def _get_key(cls, url):
+    def _get_object(cls, url):
         parsed_url = urlparse(url)
-        bucket = cls._get_bucket(parsed_url.netloc)
-        key = bucket.get_key(parsed_url.path)
-        if key is None:
+        bucket = parsed_url.netloc
+
+        # NB: under boto 2 we didn't used to have to have the lstrip. It seems
+        # that boto2 explicitly stripped these leading slashes for us:
+        # https://groups.google.com/g/boto-users/c/mv--NMPUXoU ...but boto3
+        # does not. So we must take care to strip it whenever we parse a URL to
+        # get a key.
+        key_name = parsed_url.path.lstrip('/')
+        obj = cls._connection().Object(parsed_url.netloc, key_name)
+
+        try:
+            # cache the results of the get on the obj to avoid superfluous
+            # network calls.
+            obj._datalake_details = obj.get()
+            m = obj._datalake_details['Metadata'].get('datalake')
+        except cls._connection().meta.client.exceptions.NoSuchKey:
             msg = '{} does not appear to be in the datalake'
             msg = msg.format(url)
             raise NoSuchDatalakeFile(msg)
-        return key
-
-    @classmethod
-    def _get_metadata_from_key(cls, key):
-        metadata = key.get_metadata('datalake')
-        if not metadata:
-            msg = 'No datalake metadata for s3://{}{}'
-            msg = msg.format(key.bucket.name, key.name)
-            raise InvalidDatalakeMetadata(msg)
-        return Metadata.from_json(metadata)
-
-    _BUCKETS = {}
-
-    @classmethod
-    def _get_bucket(cls, bucket_name):
-        if bucket_name not in cls._BUCKETS:
-            bucket = cls._get_bucket_from_s3(bucket_name)
-            DatalakeRecord._BUCKETS[bucket_name] = bucket
-        return cls._BUCKETS[bucket_name]
+        except cls._connection().meta.client.exceptions.NoSuchBucket:
+            msg = 'Cannot find datalake file (s3 bucket {} does not exist)'
+            msg = msg.format(bucket)
+            raise NoSuchDatalakeFile(msg)
 
-    @classmethod
-    def _get_bucket_from_s3(cls, bucket_name):
-        try:
-            return cls._connection().get_bucket(bucket_name)
-        except S3ResponseError as e:
-            if e.error_code == 'NoSuchBucket':
-                msg = 'Cannot find datalake file (s3 bucket {} does not exist)'
-                msg = msg.format(bucket_name)
-                raise NoSuchDatalakeFile(msg)
-            else:
-                raise
+        return obj, Metadata.from_json(m)
 
     _CONNECTION = None
 
@@ -133,13 +129,24 @@ class DatalakeRecord(dict):
             cls._CONNECTION = cls._prepare_connection()
         return cls._CONNECTION
 
+    @classmethod
+    def _s3_host(cls):
+        h = os.environ.get('AWS_S3_HOST')
+        if h is not None:
+            return 'https://' + h
+        r = (os.environ.get('AWS_REGION') or
+             os.environ.get('AWS_DEFAULT_REGION'))
+        if r is not None:
+            return 'https://s3-' + r + '.amazonaws.com'
+        else:
+            return None
+
     @classmethod
     def _prepare_connection(cls):
-        kwargs = {}
-        s3_host = os.environ.get('AWS_S3_HOST')
-        if s3_host:
-            kwargs['host'] = s3_host
-        return boto.connect_s3(**kwargs)
+
+        return boto3.resource('s3',
+                              region_name=os.environ.get('AWS_REGION'),
+                              endpoint_url=cls._s3_host())
 
     _ONE_DAY_IN_MS = 24*60*60*1000
 

{datalake-1.1 → datalake-2.2}/datalake/logging_helpers.py

@@ -5,7 +5,7 @@ command line client may choose to configure logging in some cases. Users with
 sentry accounts may wish to configure it by installing the sentry extras.
 '''
 import os
-import logging
+import logging.config
 from .common.errors import InsufficientConfiguration
 
 

{datalake-1.1 → datalake-2.2}/datalake/tests/conftest.py

@@ -20,9 +20,8 @@ import six
 
 
 try:
-    from moto import mock_s3_deprecated
-    import boto.s3
-    from boto.s3.key import Key
+    from moto import mock_s3
+    import boto3
     from six.moves.urllib.parse import urlparse
     import json
 except ImportError:
@@ -113,6 +112,18 @@ def tmpfile(tmpdir):
     return get_tmpfile
 
 
+@pytest.fixture
+def tmpfile_maker(tmpdir):
+
+    def get_tmpfile(content):
+        name = random_word(10)
+        f = tmpdir.join(name)
+        f.write(content)
+        return str(f)
+
+    return get_tmpfile
+
+
 @pytest.fixture
 def aws_connector(request):
 
@@ -131,14 +142,17 @@ def aws_connector(request):
 
 @pytest.fixture
 def s3_connection(aws_connector):
-    return aws_connector(mock_s3_deprecated, boto.connect_s3)
+    with mock_s3():
+        yield boto3.resource('s3')
 
 
 @pytest.fixture
 def s3_bucket_maker(s3_connection):
 
     def maker(bucket_name):
-        return s3_connection.create_bucket(bucket_name)
+        b = s3_connection.Bucket(bucket_name)
+        b.create()
+        return b
 
     return maker
 
@@ -148,11 +162,10 @@ def s3_file_maker(s3_bucket_maker):
 
     def maker(bucket, key, content, metadata):
         b = s3_bucket_maker(bucket)
-        k = Key(b)
-        k.key = key
-        if metadata:
-            k.set_metadata('datalake', json.dumps(metadata))
-        k.set_contents_from_string(content)
+        b.Object(key).put(
+            Body=content,
+            Metadata={'datalake': json.dumps(metadata)} if metadata else {}
+        )
 
     return maker
 
@@ -163,6 +176,7 @@ def s3_file_from_metadata(s3_file_maker):
     def maker(url, metadata):
         url = urlparse(url)
         assert url.scheme == 's3'
-        s3_file_maker(url.netloc, url.path, '', metadata)
+        # NB: clean up leading slash.
+        s3_file_maker(url.netloc, url.path.lstrip('/'), '', metadata)
 
     return maker

datalake-2.2/datalake.egg-info/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: datalake
+Version: 2.2
+Summary: datalake: a metadata-aware archive
+Home-page: https://github.com/planetlabs/datalake
+Author: Brian Cavagnolo
+Author-email: brian@planet.com
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+License-File: LICENSE
+Requires-Dist: boto3>=1.9.68
+Requires-Dist: memoized_property>=1.0.1
+Requires-Dist: pyblake2>=0.9.3; python_version < "3.6"
+Requires-Dist: click>=4.1
+Requires-Dist: python-dotenv>=0.1.3
+Requires-Dist: requests>=2.5
+Requires-Dist: six>=1.10.0
+Requires-Dist: python-dateutil>=2.4.2
+Requires-Dist: pytz>=2015.4
+Provides-Extra: test
+Requires-Dist: pytest<8.0.0; extra == "test"
+Requires-Dist: moto[s3]<5,>4; extra == "test"
+Requires-Dist: twine<4.0.0; extra == "test"
+Requires-Dist: pip<22.0.0,>=20.0.0; extra == "test"
+Requires-Dist: wheel<0.38.0; extra == "test"
+Requires-Dist: flake8<4.1,>=2.5.0; extra == "test"
+Requires-Dist: responses<0.22.0; extra == "test"
+Provides-Extra: queuable
+Requires-Dist: pyinotify>=0.9.4; extra == "queuable"
+Provides-Extra: sentry
+Requires-Dist: raven>=5.0.0; extra == "sentry"

{datalake-1.1 → datalake-2.2}/datalake.egg-info/SOURCES.txt

@@ -1,3 +1,4 @@
+LICENSE
 MANIFEST.in
 README.md
 setup.cfg

datalake-2.2/datalake.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+datalake = datalake.scripts.cli:cli

{datalake-1.1 → datalake-2.2}/datalake.egg-info/requires.txt

@@ -1,4 +1,4 @@
-boto>=2.38.0
+boto3>=1.9.68
 memoized_property>=1.0.1
 click>=4.1
 python-dotenv>=0.1.3
@@ -18,7 +18,7 @@ raven>=5.0.0
 
 [test]
 pytest<8.0.0
-moto<3.0.0
+moto[s3]<5,>4
 twine<4.0.0
 pip<22.0.0,>=20.0.0
 wheel<0.38.0

{datalake-1.1 → datalake-2.2}/setup.cfg

@@ -1,5 +1,5 @@
 [versioneer]
-vcs = git
+VCS = git
 style = pep440
 versionfile_source = datalake/_version.py
 versionfile_build = datalake/_version.py

{datalake-1.1 → datalake-2.2}/setup.py

@@ -21,7 +21,7 @@ setup(name='datalake',
       author_email='brian@planet.com',
       packages=find_packages(exclude=['test']),
       install_requires=[
-          'boto>=2.38.0',
+          'boto3>=1.9.68',
           'memoized_property>=1.0.1',
           'pyblake2>=0.9.3; python_version<"3.6"',
           'click>=4.1',
@@ -34,7 +34,7 @@ setup(name='datalake',
       extras_require={
           'test': [
               'pytest<8.0.0',
-              'moto<3.0.0',
+              'moto[s3]>4,<5',
               'twine<4.0.0',
               'pip>=20.0.0,<22.0.0',
               'wheel<0.38.0',

{datalake-1.1 → datalake-2.2}/test/test_archive.py

@@ -16,13 +16,17 @@ import json
 import re
 
 
-def test_push_file(archive, random_metadata, tmpfile, s3_key):
+def _get_contents_as_string(obj):
+    return obj.get()['Body'].read()
+
+
+def test_push_file(archive, random_metadata, tmpfile, s3_object):
     expected_content = 'mwahaha'.encode('utf-8')
     f = tmpfile(expected_content)
     url = archive.prepare_metadata_and_push(f, **random_metadata)
-    from_s3 = s3_key(url)
-    assert from_s3.get_contents_as_string() == expected_content
-    metadata = from_s3.get_metadata('datalake')
+    from_s3 = s3_object(url)
+    assert _get_contents_as_string(from_s3) == expected_content
+    metadata = from_s3.get()['Metadata'].get('datalake')
     assert metadata is not None
     metadata = json.loads(metadata)
     common_keys = set(metadata.keys()).intersection(random_metadata.keys())
@@ -30,14 +34,14 @@ def test_push_file(archive, random_metadata, tmpfile, s3_key):
 
 
 def test_push_large_file(
-        monkeypatch, archive, random_metadata, tmpfile, s3_key):
-    monkeypatch.setenv('DATALAKE_CHUNK_SIZE_MB', 5)
+        monkeypatch, archive, random_metadata, tmpfile, s3_object):
+    monkeypatch.setenv('DATALAKE_CHUNK_SIZE_MB', '5')
     expected_content = ('big data' * 1024 * 1024).encode('utf-8')
     f = tmpfile(expected_content)
     url = archive.prepare_metadata_and_push(f, **random_metadata)
-    from_s3 = s3_key(url)
-    assert from_s3.get_contents_as_string() == expected_content
-    metadata = from_s3.get_metadata('datalake')
+    from_s3 = s3_object(url)
+    assert _get_contents_as_string(from_s3) == expected_content
+    metadata = from_s3.get()['Metadata'].get('datalake')
     assert metadata is not None
     metadata = json.loads(metadata)
     common_keys = set(metadata.keys()).intersection(random_metadata.keys())

{datalake-1.1 → datalake-2.2}/test/test_cli.py

@@ -14,9 +14,9 @@
 
 import pytest
 from test_crtime import crtime_setuid
-from click.testing import CliRunner
 import random
 
+
 def test_cli_without_command_fails(cli_tester):
     cli_tester('', expected_exit=2)
 
@@ -62,17 +62,18 @@ def test_translate_with_good_args_succceeds(cli_tester):
     cli_tester(cmd)
 
 
-@pytest.mark.parametrize("content", [['text'], ['multiple', 'things']])  
-def test_cat(cli_tester, datalake_url_maker, random_metadata, content):
+@pytest.mark.parametrize("content", [['text'], ['multiple', 'things']])
+def test_cat(cli_tester, datalake_url_maker, random_metadata, content,
+             s3_connection):
     metadata1 = random_metadata
     metadata2 = random_metadata.copy()
     metadata2['id'] = ('%0' + str(40) + 'x') % random.randrange(16**40)
     url = ""+datalake_url_maker(metadata=metadata1, content=content[0])
     if content[0] == 'multiple':
-        for i in range (1, len(content)):
-            url = url + " " + datalake_url_maker(metadata=metadata2, content=content[i])
-    cmd = ("cat "+url)
-    print(cmd)
+        for i in range(1, len(content)):
+            url = (url + " " +
+                   datalake_url_maker(metadata=metadata2, content=content[i]))
+    cmd = "cat " + url
     output = cli_tester(cmd)
     output = output.split('\n')
     for i in range(len(content)):

{datalake-1.1 → datalake-2.2}/test/test_fetch.py

@@ -227,3 +227,27 @@ def test_metadata_from_http_url(archive, random_metadata):
     f = archive.fetch(url + '/data')
     assert f.read() == content
     assert f.metadata == random_metadata
+
+
+def test_fetch_template_with_iso(archive, datalake_url_maker, random_metadata,
+                                 tmpdir):
+    random_metadata['start']=1702944622123
+    random_metadata['end']=1702944634456
+    url = datalake_url_maker(metadata=random_metadata)
+    t = os.path.join(str(tmpdir), '{start_iso}-{end_iso}-foobar.log')
+    fname = '2023-12-19T00:10:22.123-2023-12-19T00:10:34.456-foobar.log'
+    expected_path = os.path.join(str(tmpdir), fname)
+    archive.fetch_to_filename(url, filename_template=t)
+    assert os.path.exists(expected_path)
+
+
+def test_fetch_template_with_none(archive, datalake_url_maker, random_metadata,
+                                  tmpdir):
+    random_metadata['start']=1702944622123
+    random_metadata['end']=None
+    url = datalake_url_maker(metadata=random_metadata)
+    t = os.path.join(str(tmpdir), '{start_iso}-{end_iso}-foobar.log')
+    fname = '2023-12-19T00:10:22.123-None-foobar.log'
+    expected_path = os.path.join(str(tmpdir), fname)
+    archive.fetch_to_filename(url, filename_template=t)
+    assert os.path.exists(expected_path)

{datalake-1.1 → datalake-2.2}/test/test_queue.py

@@ -16,7 +16,7 @@ import pytest
 import json
 from threading import Timer
 import os
-from datalake.tests import random_word
+from datalake.tests import random_word, generate_random_metadata
 from datalake.common.errors import InsufficientConfiguration
 from datalake import Enqueuer, Uploader, InvalidDatalakeBundle
 from datalake.queue import has_queue
@@ -53,18 +53,23 @@ def faulty_uploader(archive, queue_dir):
 
 
 @pytest.fixture
-def uploaded_content_validator(s3_key):
+def uploaded_content_validator(s3_object):
 
     def validator(expected_content, expected_metadata=None, compressed=False):
 
-        from_s3 = s3_key()
+        if expected_metadata:
+            from_s3 = s3_object(
+                f's3://datalake-test/{expected_metadata["id"]}/data'
+            )
+        else:
+            from_s3 = s3_object()
         assert from_s3 is not None
-        content = from_s3.get_contents_as_string()
+        content = from_s3.get()['Body'].read()
         if compressed:
             content = zlib.decompress(content, 16 + zlib.MAX_WBITS)
         assert content == expected_content
         if expected_metadata is not None:
-            metadata = json.loads(from_s3.get_metadata('datalake'))
+            metadata = json.loads(from_s3.get()['Metadata'].get('datalake'))
             assert metadata == expected_metadata
 
     return validator
@@ -84,7 +89,7 @@ def uploaded_file_validator(archive, uploaded_content_validator):
 def assert_s3_bucket_empty(s3_bucket):
 
     def asserter():
-        assert len([k for k in s3_bucket.list()]) == 0
+        assert len(list(s3_bucket.objects.all())) == 0
 
     return asserter
 
@@ -95,6 +100,16 @@ def random_file(tmpfile, random_metadata):
     return tmpfile(expected_content)
 
 
+@pytest.fixture
+def random_file_maker(tmpfile_maker):
+
+    def maker():
+        expected_content = random_word(100)
+        return tmpfile_maker(expected_content)
+
+    return maker
+
+
 @pytest.mark.skipif(not has_queue, reason='requires queuable features')
 def test_upload_existing(enqueuer, uploader, random_file, random_metadata,
                          uploaded_file_validator):
@@ -220,7 +235,7 @@ def test_enqueue_compress_cli(cli_tester, uploader, random_file,
 
 
 @pytest.mark.skipif(not has_queue, reason='requires queuable features')
-def test_threaded_upload(enqueuer, uploader, random_file, random_metadata,
+def test_threaded_upload(enqueuer, uploader, random_file_maker,
                          uploaded_file_validator):
 
     # This test does not actually validate that multiple threads are running.
@@ -229,7 +244,8 @@ def test_threaded_upload(enqueuer, uploader, random_file, random_metadata,
     enqueued_files = []
 
     def enqueue():
-        f = enqueuer.enqueue(random_file, **random_metadata)
+        m = generate_random_metadata()
+        f = enqueuer.enqueue(random_file_maker(), **m)
         enqueued_files.append(f)
         if len(enqueued_files) < 3:
             t = Timer(0.1, enqueue)
@@ -238,7 +254,6 @@ def test_threaded_upload(enqueuer, uploader, random_file, random_metadata,
     t = Timer(0.5, enqueue)
     t.start()
     uploader.listen(timeout=1.0, workers=3)
-
     assert len(enqueued_files) == 3
     for f in enqueued_files:
         uploaded_file_validator(f)

datalake-1.1/PKG-INFO

@@ -1,17 +0,0 @@
-Metadata-Version: 2.1
-Name: datalake
-Version: 1.1
-Summary: datalake: a metadata-aware archive
-Home-page: https://github.com/planetlabs/datalake
-Author: Brian Cavagnolo
-Author-email: brian@planet.com
-License: UNKNOWN
-Description: UNKNOWN
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Provides-Extra: test
-Provides-Extra: queuable
-Provides-Extra: sentry

datalake-1.1/datalake.egg-info/PKG-INFO

@@ -1,17 +0,0 @@
-Metadata-Version: 2.1
-Name: datalake
-Version: 1.1
-Summary: datalake: a metadata-aware archive
-Home-page: https://github.com/planetlabs/datalake
-Author: Brian Cavagnolo
-Author-email: brian@planet.com
-License: UNKNOWN
-Description: UNKNOWN
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Provides-Extra: test
-Provides-Extra: queuable
-Provides-Extra: sentry

datalake-1.1/datalake.egg-info/entry_points.txt

@@ -1,4 +0,0 @@
-
-      [console_scripts]
-      datalake=datalake.scripts.cli:cli
-