MapProxy 2.1.0__py3-none-any.whl

mapproxy/cache/azureblob.py

@@ -0,0 +1,145 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2022 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import calendar
+import hashlib
+import threading
+from io import BytesIO
+
+from mapproxy.cache import path
+from mapproxy.cache.base import tile_buffer, TileCacheBase
+from mapproxy.image import ImageSource
+from mapproxy.util import async_
+
+try:
+    from azure.storage.blob import BlobServiceClient, ContentSettings
+    from azure.core.exceptions import AzureError
+except ImportError:
+    BlobServiceClient = None
+    ContentSettings = None
+    AzureError = None
+
+import logging
+log = logging.getLogger('mapproxy.cache.azureblob')
+
+
+class AzureBlobConnectionError(Exception):
+    pass
+
+
+class AzureBlobCache(TileCacheBase):
+
+    def __init__(self, base_path, file_ext, directory_layout='tms', container_name='mapproxy',
+                 _concurrent_writer=4, _concurrent_reader=4, connection_string=None, coverage=None):
+        super(AzureBlobCache, self).__init__(coverage)
+        if BlobServiceClient is None:
+            raise ImportError("Azure Blob Cache requires 'azure-storage-blob' package")
+
+        self.lock_cache_id = 'azureblob-' + hashlib.md5(base_path.encode('utf-8')
+                                                        + container_name.encode('utf-8')).hexdigest()
+
+        self.connection_string = connection_string
+        self.container_name = container_name
+        self._container_client_cache = threading.local()
+
+        self.base_path = base_path
+        self.file_ext = file_ext
+        self._concurrent_writer = _concurrent_writer
+        self._concurrent_reader = _concurrent_reader
+        self._tile_location, _ = path.location_funcs(layout=directory_layout)
+
+    @property
+    def container_client(self):
+        if not getattr(self._container_client_cache, 'client', None):
+            container_client = BlobServiceClient.from_connection_string(self.connection_string) \
+                .get_container_client(self.container_name)
+            self._container_client_cache.client = container_client
+        return self._container_client_cache.client
+
+    def tile_key(self, tile):
+        return self._tile_location(tile, self.base_path, self.file_ext).lstrip('/')
+
+    def load_tile_metadata(self, tile, dimensions=None):
+        if tile.timestamp:
+            return
+        self.is_cached(tile, dimensions=dimensions)
+
+    @staticmethod
+    def _set_metadata(properties, tile):
+        tile.timestamp = calendar.timegm(properties.last_modified.timetuple())
+        tile.size = properties.size
+
+    def is_cached(self, tile, dimensions=None):
+        if tile.is_missing():
+            key = self.tile_key(tile)
+            blob = self.container_client.get_blob_client(key)
+            if not blob.exists():
+                return False
+            else:
+                self._set_metadata(blob.get_blob_properties(), tile)
+
+        return True
+
+    def load_tiles(self, tiles, with_metadata=True, dimensions=None):
+        p = async_.Pool(min(self._concurrent_reader, len(tiles)))
+        return all(p.map(self.load_tile, tiles))
+
+    def load_tile(self, tile, with_metadata=True, dimensions=None):
+        if not tile.cacheable:
+            return False
+
+        if not tile.is_missing():
+            return True
+
+        key = self.tile_key(tile)
+        log.debug('AzureBlob:load_tile, loading key: %s' % key)
+
+        try:
+            r = self.container_client.download_blob(key)
+            self._set_metadata(r.properties, tile)
+            tile.source = ImageSource(BytesIO(r.readall()))
+        except AzureError as e:
+            log.debug("AzureBlob:load_tile unable to load key: %s" % key, e)
+            tile.source = None
+            return False
+
+        log.debug("AzureBlob:load_tile loaded key: %s" % key)
+        return True
+
+    def remove_tile(self, tile, dimensions=None):
+        key = self.tile_key(tile)
+        log.debug('remove_tile, key: %s' % key)
+        self.container_client.delete_blob(key)
+
+    def store_tiles(self, tiles, dimensions=None):
+        p = async_.Pool(min(self._concurrent_writer, len(tiles)))
+        p.map(self.store_tile, tiles)
+
+    def store_tile(self, tile, dimensions=None):
+        if tile.stored:
+            return
+
+        key = self.tile_key(tile)
+        log.debug('AzureBlob: store_tile, key: %s' % key)
+
+        container_client = self.container_client
+        with tile_buffer(tile) as buf:
+            content_settings = ContentSettings(content_type='image/' + self.file_ext)
+            container_client.upload_blob(
+                name=key,
+                data=buf,
+                overwrite=True,
+                content_settings=content_settings)

mapproxy/cache/base.py

@@ -0,0 +1,120 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import time
+
+from contextlib import contextmanager
+
+from mapproxy.util.lock import FileLock, cleanup_lockdir, DummyLock
+
+
+class CacheBackendError(Exception):
+    pass
+
+
+@contextmanager
+def tile_buffer(tile):
+    data = tile.source.as_buffer(seekable=True)
+    data.seek(0)
+    yield data
+    tile.size = data.tell()
+    if not tile.timestamp:
+        tile.timestamp = time.time()
+    data.seek(0)
+    tile.stored = True
+
+
+class TileCacheBase(object):
+    """
+    Base implementation of a tile cache.
+    """
+
+    supports_timestamp = True
+    supports_dimensions = False
+
+    def __init__(self, coverage=None) -> None:
+        self.coverage = coverage
+
+    def load_tile(self, tile, with_metadata=False, dimensions=None):
+        raise NotImplementedError()
+
+    def load_tiles(self, tiles, with_metadata=False, dimensions=None):
+        all_succeed = True
+        for tile in tiles:
+            if not self.load_tile(tile, with_metadata=with_metadata, dimensions=dimensions):
+                all_succeed = False
+        return all_succeed
+
+    def store_tile(self, tile, dimensions=None):
+        raise NotImplementedError()
+
+    def store_tiles(self, tiles, dimensions=None):
+        all_succeed = True
+        for tile in tiles:
+            if not self.store_tile(tile, dimensions=dimensions):
+                all_succeed = False
+        return all_succeed
+
+    def remove_tile(self, tile, dimensions=None):
+        raise NotImplementedError()
+
+    def remove_tiles(self, tiles, dimensions=None):
+        for tile in tiles:
+            self.remove_tile(tile, dimensions=dimensions)
+
+    def is_cached(self, tile, dimensions=None):
+        """
+        Return ``True`` if the tile is cached.
+        """
+        raise NotImplementedError()
+
+    def load_tile_metadata(self, tile, dimensions=None):
+        """
+        Fill the metadata attributes of `tile`.
+        Sets ``.timestamp`` and ``.size``.
+        """
+        raise NotImplementedError()
+
+
+# whether we immediately remove lock files or not
+REMOVE_ON_UNLOCK = True
+if sys.platform == 'win32':
+    # windows does not handle this well
+    REMOVE_ON_UNLOCK = False
+
+
+class TileLocker(object):
+    def __init__(self, lock_dir, lock_timeout, lock_cache_id):
+        self.lock_dir = lock_dir
+        self.lock_timeout = lock_timeout
+        self.lock_cache_id = lock_cache_id
+
+    def lock_filename(self, tile):
+        return os.path.join(self.lock_dir, self.lock_cache_id + '-' +
+                            '-'.join(map(str, tile.coord)) + '.lck')
+
+    def lock(self, tile):
+        """
+        Returns a lock object for this tile.
+        """
+        if getattr(self, 'locking_disabled', False):
+            return DummyLock()
+        lock_filename = self.lock_filename(tile)
+        cleanup_lockdir(self.lock_dir, max_lock_time=self.lock_timeout + 10,
+                        force=False)
+        return FileLock(lock_filename, timeout=self.lock_timeout,
+                        remove_on_unlock=REMOVE_ON_UNLOCK)