MapProxy 1.16.1__py3-none-any.whl

mapproxy/cache/path.py

@@ -0,0 +1,250 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010-2016 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from mapproxy.compat import string_type
+from mapproxy.util.fs import ensure_directory
+from mapproxy.request.base import NoCaseMultiDict
+
+def location_funcs(layout):
+    if layout == 'tc':
+        return tile_location_tc, level_location
+    elif layout == 'mp':
+        return tile_location_mp, level_location
+    elif layout == 'tms':
+        return tile_location_tms, level_location
+    elif layout == 'reverse_tms':
+        return tile_location_reverse_tms, None
+    elif layout == 'quadkey':
+        return tile_location_quadkey, no_level_location
+    elif layout == 'arcgis':
+        return tile_location_arcgiscache, level_location_arcgiscache
+    else:
+        raise ValueError('unknown directory_layout "%s"' % layout)
+
+def level_location(level, cache_dir, dimensions=None):
+    """
+    Return the path where all tiles for `level` will be stored.
+
+    >>> os.path.abspath(level_location(2, '/tmp/cache')) == os.path.abspath('/tmp/cache/02')
+    True
+    """
+    dim_path = dimensions_part(dimensions)
+
+    if isinstance(level, string_type):
+        return os.path.join(cache_dir, dim_path, level)
+    else:
+        return os.path.join(cache_dir, dim_path, "%02d" % level)
+
+def dimensions_part(dimensions):
+    """
+    Return the subpath where all tiles for `dimensions` will be stored.
+    Dimensions prefixed with "dim_" are sorted after the predefined elevation and time dimensions
+    >>> dimensions_part({'time': '2020-08-25T00:00:00Z'})
+    'time-2020-08-25T00:00:00Z'
+    >>> dimensions_part({'time': '2020-08-25T00:00:00Z', 'dim_reference_time': '2020-08-25T00:00:00Z', 'dim_level': '700'})
+    'time-2020-08-25T00:00:00Z/dim_level-700/dim_reference_time-2020-08-25T00:00:00Z'
+
+    """
+    if dimensions:
+        dims = NoCaseMultiDict(dimensions)
+        predefined_dims, custom_dims = [], []
+        for dim in dims.keys():
+            (custom_dims if dim.startswith('dim_') else predefined_dims).append(dim)
+        dim_keys = sorted(predefined_dims) + sorted(custom_dims)
+        return os.path.join(*(map(lambda k: k + "-" + str(dims.get(k, 'default')), dim_keys)))
+    else:
+        return ""
+
+def level_part(level):
+    """
+    Return the path where all tiles for `level` will be stored.
+
+    >>> level_part(2)
+    '02'
+    >>> level_part('2')
+    '2'
+    """
+    if isinstance(level, string_type):
+        return level
+    else:
+        return "%02d" % level
+
+
+def tile_location_tc(tile, cache_dir, file_ext, create_dir=False, dimensions=None):
+    """
+    Return the location of the `tile`. Caches the result as ``location``
+    property of the `tile`.
+
+    :param tile: the tile object
+    :param create_dir: if True, create all necessary directories
+    :return: the full filename of the tile
+
+    >>> from mapproxy.cache.tile import Tile
+    >>> tile_location_tc(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/02/000/000/003/000/000/004.png'
+    """
+    if tile.location is None:
+        x, y, z = tile.coord
+        
+        parts = (cache_dir,
+                dimensions_part(dimensions),
+                level_part(z),
+                 "%03d" % int(x / 1000000),
+                 "%03d" % (int(x / 1000) % 1000),
+                 "%03d" % (int(x) % 1000),
+                 "%03d" % int(y / 1000000),
+                 "%03d" % (int(y / 1000) % 1000),
+                 "%03d.%s" % (int(y) % 1000, file_ext))
+        tile.location = os.path.join(*parts)
+    if create_dir:
+        ensure_directory(tile.location)
+    return tile.location
+
+def tile_location_mp(tile, cache_dir, file_ext, create_dir=False, dimensions=None):
+    """
+    Return the location of the `tile`. Caches the result as ``location``
+    property of the `tile`.
+
+    :param tile: the tile object
+    :param create_dir: if True, create all necessary directories
+    :return: the full filename of the tile
+
+    >>> from mapproxy.cache.tile import Tile
+    >>> tile_location_mp(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/02/0000/0003/0000/0004.png'
+    >>> tile_location_mp(Tile((12345678, 98765432, 22)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/22/1234/5678/9876/5432.png'
+    """
+    if tile.location is None:
+        x, y, z = tile.coord
+        parts = (cache_dir,
+                dimensions_part(dimensions),
+                level_part(z),
+                 "%04d" % int(x / 10000),
+                 "%04d" % (int(x) % 10000),
+                 "%04d" % int(y / 10000),
+                 "%04d.%s" % (int(y) % 10000, file_ext))
+        tile.location = os.path.join(*parts)
+    if create_dir:
+        ensure_directory(tile.location)
+    return tile.location
+
+def tile_location_tms(tile, cache_dir, file_ext, create_dir=False, dimensions=None):
+    """
+    Return the location of the `tile`. Caches the result as ``location``
+    property of the `tile`.
+
+    :param tile: the tile object
+    :param create_dir: if True, create all necessary directories
+    :return: the full filename of the tile
+
+    >>> from mapproxy.cache.tile import Tile
+    >>> tile_location_tms(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/2/3/4.png'
+    """
+    if tile.location is None:
+        x, y, z = tile.coord
+        tile.location = os.path.join(
+            cache_dir,dimensions_part(dimensions) ,level_part(str(z)),
+            str(x), str(y) + '.' + file_ext
+        )
+    if create_dir:
+        ensure_directory(tile.location)
+    return tile.location
+
+def tile_location_reverse_tms(tile, cache_dir, file_ext, create_dir=False, dimensions=None):
+    """
+    Return the location of the `tile`. Caches the result as ``location``
+    property of the `tile`.
+
+    :param tile: the tile object
+    :param create_dir: if True, create all necessary directories
+    :return: the full filename of the tile
+
+    >>> from mapproxy.cache.tile import Tile
+    >>> tile_location_reverse_tms(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/4/3/2.png'
+    """
+    if tile.location is None:
+        x, y, z = tile.coord
+        tile.location = os.path.join(
+            cache_dir,dimensions_part(dimensions),str(y), str(x), str(z) + '.' + file_ext
+        )
+    if create_dir:
+        ensure_directory(tile.location)
+    return tile.location
+
+def level_location_tms(level, cache_dir, dimensions=None):
+    return level_location(str(level), cache_dir=cache_dir)
+
+def tile_location_quadkey(tile, cache_dir, file_ext, create_dir=False, dimensions=None):
+    """
+    Return the location of the `tile`. Caches the result as ``location``
+    property of the `tile`.
+
+    :param tile: the tile object
+    :param create_dir: if True, create all necessary directories
+    :return: the full filename of the tile
+
+    >>> from mapproxy.cache.tile import Tile
+    >>> tile_location_quadkey(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/11.png'
+    """
+    if tile.location is None:
+        x, y, z = tile.coord
+        quadKey = ""
+        for i in range(z,0,-1):
+            digit = 0
+            mask = 1 << (i-1)
+            if (x & mask) != 0:
+                digit += 1
+            if (y & mask) != 0:
+                digit += 2
+            quadKey += str(digit)
+        tile.location = os.path.join(
+            cache_dir, quadKey + '.' + file_ext
+        )
+    if create_dir:
+        ensure_directory(tile.location)
+    return tile.location
+
+def no_level_location(level, cache_dir, dimensions=None):
+    # dummy for quadkey cache which stores all tiles in one directory
+    raise NotImplementedError('cache does not have any level location')
+
+def tile_location_arcgiscache(tile, cache_dir, file_ext, create_dir=False, dimensions=None):
+    """
+    Return the location of the `tile`. Caches the result as ``location``
+    property of the `tile`.
+
+    :param tile: the tile object
+    :param create_dir: if True, create all necessary directories
+    :return: the full filename of the tile
+
+    >>> from mapproxy.cache.tile import Tile
+    >>> tile_location_arcgiscache(Tile((1234567, 87654321, 9)), '/tmp/cache', 'png').replace('\\\\', '/')
+    '/tmp/cache/L09/R05397fb1/C0012d687.png'
+    """
+    if tile.location is None:
+        x, y, z = tile.coord
+        parts = (cache_dir, 'L%02d' % z, 'R%08x' % y, 'C%08x.%s' % (x, file_ext))
+        tile.location = os.path.join(*parts)
+    if create_dir:
+        ensure_directory(tile.location)
+    return tile.location
+
+def level_location_arcgiscache(z, cache_dir, dimensions=None):
+    return level_location('L%02d' % z, cache_dir=cache_dir, dimensions=dimensions)

mapproxy/cache/redis.py

@@ -0,0 +1,88 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2017 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import hashlib
+
+from mapproxy.image import ImageSource
+from mapproxy.cache.base import (
+    TileCacheBase,
+    tile_buffer,
+)
+from mapproxy.compat import BytesIO
+
+try:
+    import redis
+except ImportError:
+    redis = None
+
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class RedisCache(TileCacheBase):
+    def __init__(self, host, port, prefix, ttl=0, db=0):
+        if redis is None:
+            raise ImportError("Redis backend requires 'redis' package.")
+
+        self.prefix = prefix
+        self.lock_cache_id = 'redis-' + hashlib.md5((host + str(port) + prefix + str(db)).encode('utf-8')).hexdigest()
+        self.ttl = ttl
+        self.r = redis.StrictRedis(host=host, port=port, db=db)
+
+    def _key(self, tile):
+        x, y, z = tile.coord
+        return self.prefix + '-%d-%d-%d' % (z, x, y)
+
+    def is_cached(self, tile, dimensions=None):
+        if tile.coord is None or tile.source:
+            return True
+
+        return self.r.exists(self._key(tile))
+
+    def store_tile(self, tile, dimensions=None):
+        if tile.stored:
+            return True
+
+        key = self._key(tile)
+
+        with tile_buffer(tile) as buf:
+            data = buf.read()
+
+        r = self.r.set(key, data)
+        if self.ttl:
+            # use ms expire times for unit-tests
+            self.r.pexpire(key, int(self.ttl * 1000))
+        return r
+
+    def load_tile(self, tile, with_metadata=False, dimensions=None):
+        if tile.source or tile.coord is None:
+            return True
+        key = self._key(tile)
+        tile_data = self.r.get(key)
+        if tile_data:
+            tile.source = ImageSource(BytesIO(tile_data))
+            return True
+        return False
+
+    def remove_tile(self, tile, dimensions=None):
+        if tile.coord is None:
+            return True
+
+        key = self._key(tile)
+        self.r.delete(key)
+        return True

mapproxy/cache/renderd.py

@@ -0,0 +1,95 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2012, 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import hashlib
+
+try:
+    import json; json
+except ImportError:
+    json = None
+
+try:
+    import requests; requests
+except ImportError:
+    requests = None
+
+from mapproxy.client.log import log_request
+from mapproxy.cache.tile import TileCreator, Tile
+from mapproxy.source import SourceError
+from mapproxy.util.lock import LockTimeout
+
+def has_renderd_support():
+    if not json or not requests:
+        return False
+    return True
+
+class RenderdTileCreator(TileCreator):
+    def __init__(self, renderd_address, tile_mgr, dimensions=None, priority=100, tile_locker=None):
+        TileCreator.__init__(self, tile_mgr, dimensions)
+        self.tile_locker = tile_locker.lock or self.tile_mgr.lock
+        self.renderd_address = renderd_address
+        self.priority = priority
+
+    def _create_single_tile(self, tile):
+        with self.tile_locker(tile):
+            if not self.is_cached(tile):
+                self._create_renderd_tile(tile.coord)
+            self.cache.load_tile(tile)
+        return [tile]
+
+    def _create_meta_tile(self, meta_tile):
+        main_tile = Tile(meta_tile.main_tile_coord)
+        with self.tile_locker(main_tile):
+            if not all(self.is_cached(t) for t in meta_tile.tiles if t is not None):
+                self._create_renderd_tile(main_tile.coord)
+
+        tiles = [Tile(coord) for coord in meta_tile.tiles]
+        self.cache.load_tiles(tiles)
+        return tiles
+
+    def _create_renderd_tile(self, tile_coord):
+        start_time = time.time()
+        result = self._send_tile_request(self.tile_mgr.identifier, [tile_coord])
+        duration = time.time()-start_time
+
+        address = '%s:%s:%r' % (self.renderd_address,
+            self.tile_mgr.identifier, tile_coord)
+
+        if result['status'] == 'error':
+            log_request(address, 500, None, duration=duration, method='RENDERD')
+            raise SourceError("Error from renderd: %s" % result.get('error_message', 'unknown error from renderd'))
+        elif result['status'] == 'lock':
+            log_request(address, 503, None, duration=duration, method='RENDERD')
+            raise LockTimeout("Lock timeout from renderd: %s" % result.get('error_message', 'unknown lock timeout error from renderd'))
+
+        log_request(address, 200, None, duration=duration, method='RENDERD')
+
+    def _send_tile_request(self, cache_identifier, tile_coords):
+        identifier = hashlib.sha1(str((cache_identifier, tile_coords)).encode('ascii')).hexdigest()
+        message = {
+            'command': 'tile',
+            'id': identifier,
+            'tiles': tile_coords,
+            'cache_identifier': cache_identifier,
+            'priority': self.priority
+        }
+        try:
+            resp = requests.post(self.renderd_address, data=json.dumps(message))
+            return resp.json()
+        except ValueError:
+            raise SourceError("Error while communicating with renderd: invalid JSON")
+        except requests.RequestException as ex:
+            raise SourceError("Error while communicating with renderd: %s" % ex)

mapproxy/cache/riak.py

@@ -0,0 +1,202 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import threading
+import hashlib
+
+from io import BytesIO
+
+from mapproxy.image import ImageSource
+from mapproxy.cache.tile import Tile
+from mapproxy.cache.base import TileCacheBase, tile_buffer, CacheBackendError
+
+try:
+    import riak
+except ImportError:
+    riak = None
+except TypeError:
+    import warnings
+    warnings.warn("riak version not compatible with this Python version")
+    riak = None
+
+import logging
+log = logging.getLogger(__name__)
+
+class UnexpectedResponse(CacheBackendError):
+    pass
+
+class RiakCache(TileCacheBase):
+    def __init__(self, nodes, protocol, bucket, tile_grid, use_secondary_index=False, timeout=60):
+        if riak is None:
+            raise ImportError("Riak backend requires 'riak' package.")
+
+        self.nodes = nodes
+        self.protocol = protocol
+        self.lock_cache_id = 'riak-' + hashlib.md5(bucket.encode('utf-8')).hexdigest()
+        self.request_timeout = timeout * 1000
+        self.bucket_name = bucket
+        self.tile_grid = tile_grid
+        self.use_secondary_index = use_secondary_index
+        self._db_conn_cache = threading.local()
+
+    @property
+    def connection(self):
+        if not getattr(self._db_conn_cache, 'connection', None):
+            self._db_conn_cache.connection = riak.RiakClient(protocol=self.protocol, nodes=self.nodes)
+        return self._db_conn_cache.connection
+
+    @property
+    def bucket(self):
+        if not getattr(self._db_conn_cache, 'bucket', None):
+            self._db_conn_cache.bucket = self.connection.bucket(self.bucket_name)
+        return self._db_conn_cache.bucket
+
+    def _get_object(self, coord):
+        (x, y, z) = coord
+        key = '%(z)d_%(x)d_%(y)d' % locals()
+        obj = False
+        try:
+            obj = self.bucket.get(key, r=1, timeout=self.request_timeout)
+        except Exception as e:
+            log.warning('error while requesting %s: %s', key, e)
+
+        if not obj:
+            obj = self.bucket.new(key=key, data=None, content_type='application/octet-stream')
+        return obj
+
+    def _get_timestamp(self, obj):
+        metadata = obj.usermeta
+        timestamp = metadata.get('timestamp')
+        if timestamp != None:
+            return float(timestamp)
+
+        obj.usermeta = {'timestamp': '0'}
+        return 0.0
+
+    def is_cached(self, tile, dimensions=None):
+        return self.load_tile(tile, True)
+
+    def _store_bulk(self, tiles):
+        for tile in tiles:
+            res = self._get_object(tile.coord)
+            with tile_buffer(tile) as buf:
+                data = buf.read()
+            res.encoded_data = data
+            res.usermeta = {
+                'timestamp': str(tile.timestamp),
+                'size': str(tile.size),
+            }
+            if self.use_secondary_index:
+                x, y, z = tile.coord
+                res.add_index('tile_coord_bin', '%02d-%07d-%07d' % (z, x, y))
+
+            try:
+                res.store(w=1, dw=1, pw=1, return_body=False, timeout=self.request_timeout)
+            except riak.RiakError as ex:
+                log.warning('unable to store tile: %s', ex)
+                return False
+
+        return True
+
+    def store_tile(self, tile,dimensions=None):
+        if tile.stored:
+            return True
+
+        return self._store_bulk([tile])
+
+    def store_tiles(self, tiles,dimensions=None):
+        tiles = [t for t in tiles if not t.stored]
+        return self._store_bulk(tiles)
+
+    def load_tile_metadata(self, tile,dimensions=None):
+        if tile.timestamp:
+            return
+
+        # is_cached loads metadata
+        self.load_tile(tile, True)
+
+    def load_tile(self, tile, with_metadata=False, dimensions=None):
+        if tile.timestamp is None:
+            tile.timestamp = 0
+        if tile.source or tile.coord is None:
+            return True
+
+        res = self._get_object(tile.coord)
+        if res.exists:
+            tile_data = BytesIO(res.encoded_data)
+            tile.source = ImageSource(tile_data)
+            if with_metadata:
+                tile.timestamp = self._get_timestamp(res)
+                tile.size = len(res.encoded_data)
+            return True
+
+        return False
+
+    def remove_tile(self, tile):
+        if tile.coord is None:
+            return True
+
+        res = self._get_object(tile.coord)
+        if not res.exists:
+            # already removed
+            return True
+
+        try:
+            res.delete(w=1, r=1, dw=1, pw=1, timeout=self.request_timeout)
+        except riak.RiakError as ex:
+            log.warning('unable to remove tile: %s', ex)
+            return False
+        return True
+
+    def _fill_metadata_from_obj(self, obj, tile):
+        tile_md = obj.usermeta
+        timestamp = tile_md.get('timestamp')
+        if timestamp:
+            tile.timestamp = float(timestamp)
+
+    def _key_iterator(self, level):
+        """
+        Generator for all tile keys in `level`.
+        """
+        # index() returns a list of all keys so we check for tiles in
+        # batches of `chunk_size`*`chunk_size`.
+        grid_size = self.tile_grid.grid_sizes[level]
+        chunk_size = 256
+        for x in range(grid_size[0]/chunk_size):
+            start_x = x * chunk_size
+            end_x = start_x + chunk_size - 1
+            for y in range(grid_size[1]/chunk_size):
+                start_y = y * chunk_size
+                end_y = start_y + chunk_size - 1
+                query = self.bucket.get_index('tile_coord_bin',
+                    '%02d-%07d-%07d' % (level, start_x, start_y),
+                    '%02d-%07d-%07d' % (level, end_x, end_y))
+                for link in query.run():
+                    yield link.get_key()
+
+    def remove_tiles_for_level(self, level, before_timestamp=None):
+        bucket = self.bucket
+        client = self.connection
+        for key in self._key_iterator(level):
+            if before_timestamp:
+                obj = self.bucket.get(key, r=1)
+                dummy_tile = Tile((0, 0, 0))
+                self._fill_metadata_from_obj(obj, dummy_tile)
+                if dummy_tile.timestamp < before_timestamp:
+                    obj.delete()
+            else:
+                riak.RiakObject(client, bucket, key).delete()