MapProxy 2.1.0__py3-none-any.whl

mapproxy/test/unit/test_cache_geopackage.py

@@ -0,0 +1,256 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2016 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+
+import os
+import sqlite3
+import threading
+import time
+
+from io import BytesIO
+
+from mapproxy.cache.geopackage import GeopackageCache, GeopackageLevelCache
+from mapproxy.cache.tile import Tile
+from mapproxy.grid import tile_grid, TileGrid
+from mapproxy.image import ImageSource
+from mapproxy.layer import MapExtent
+from mapproxy.srs import SRS
+from mapproxy.test.helper import assert_files_in_dir
+from mapproxy.test.unit.test_cache_tile import TileCacheTestBase
+from mapproxy.util.coverage import coverage
+
+
+GLOBAL_WEBMERCATOR_EXTENT = MapExtent(
+    (-20037508.342789244, -20037508.342789244, 20037508.342789244, 20037508.342789244),
+    SRS(3857))
+
+
+class TestGeopackageCache(TileCacheTestBase):
+
+    always_loads_metadata = True
+
+    def setup_method(self):
+        TileCacheTestBase.setup_method(self)
+        self.gpkg_file = os.path.join(self.cache_dir, 'tmp.gpkg')
+        self.table_name = 'test_tiles'
+        self.cache = GeopackageCache(
+            self.gpkg_file,
+            tile_grid=tile_grid(3857, name='global-webmarcator'),
+            table_name=self.table_name,
+        )
+
+    def teardown_method(self):
+        if self.cache:
+            self.cache.cleanup()
+        TileCacheTestBase.teardown_method(self)
+
+    def test_new_geopackage(self):
+        assert os.path.exists(self.gpkg_file)
+
+        with sqlite3.connect(self.gpkg_file) as db:
+            cur = db.execute('''SELECT name FROM sqlite_master WHERE type='table' AND name=?''',
+                             (self.table_name,))
+            content = cur.fetchone()
+            assert content[0] == self.table_name
+
+        with sqlite3.connect(self.gpkg_file) as db:
+            cur = db.execute('''SELECT table_name, data_type FROM gpkg_contents WHERE table_name = ?''',
+                             (self.table_name,))
+            content = cur.fetchone()
+            assert content[0] == self.table_name
+            assert content[1] == 'tiles'
+
+        with sqlite3.connect(self.gpkg_file) as db:
+            cur = db.execute('''SELECT table_name FROM gpkg_tile_matrix WHERE table_name = ?''',
+                             (self.table_name,))
+            content = cur.fetchall()
+            assert len(content) == 20
+
+        with sqlite3.connect(self.gpkg_file) as db:
+            cur = db.execute('''SELECT table_name FROM gpkg_tile_matrix_set WHERE table_name = ?''',
+                             (self.table_name,))
+            content = cur.fetchone()
+            assert content[0] == self.table_name
+
+        assert self.cache.coverage is None
+        assert self.cache.bbox == GLOBAL_WEBMERCATOR_EXTENT.bbox
+
+    def test_load_empty_tileset(self):
+        assert self.cache.load_tiles([Tile(None)]) is True
+        assert self.cache.load_tiles([Tile(None), Tile(None), Tile(None)]) is True
+
+    def test_load_more_than_2000_tiles(self):
+        # prepare data
+        for i in range(0, 2010):
+            assert self.cache.store_tile(Tile((i, 0, 10),  ImageSource(BytesIO(b'foo'))))
+
+        tiles = [Tile((i, 0, 10)) for i in range(0, 2010)]
+        assert self.cache.load_tiles(tiles)
+
+    def test_timeouts(self):
+        self.cache._db_conn_cache.db = sqlite3.connect(self.cache.geopackage_file, timeout=0.05)
+
+        def block():
+            # block database by delaying the commit
+            db = sqlite3.connect(self.cache.geopackage_file)
+            cur = db.cursor()
+            stmt = "INSERT OR REPLACE INTO {0} (zoom_level, tile_column, tile_row, tile_data) " \
+                   "VALUES (?,?,?,?)".format(self.table_name)
+            cur.execute(stmt, (3, 1, 1, '1234'))
+            time.sleep(0.2)
+            db.commit()
+
+        try:
+            assert self.cache.store_tile(self.create_tile((0, 0, 1))) is True
+
+            t = threading.Thread(target=block)
+            t.start()
+            time.sleep(0.05)
+            assert self.cache.store_tile(self.create_tile((0, 0, 1))) is False
+        finally:
+            t.join()
+
+        assert self.cache.store_tile(self.create_tile((0, 0, 1))) is True
+
+
+class TestGeopackageCacheCoverage(TileCacheTestBase):
+    def setup_method(self):
+        TileCacheTestBase.setup_method(self)
+        self.gpkg_file = os.path.join(self.cache_dir, 'tmp.gpkg')
+        self.table_name = 'test_tiles'
+        self.cache = GeopackageCache(
+            self.gpkg_file,
+            tile_grid=tile_grid(4326, name='inspire-crs-84-quad'),
+            table_name=self.table_name,
+            coverage=coverage([20, 20, 30, 30], SRS(4326))
+        )
+
+    def teardown_method(self):
+        if self.cache:
+            self.cache.cleanup()
+        TileCacheTestBase.teardown_method(self)
+
+    def test_correct_coverage(self):
+        assert self.cache.bbox == [20, 20, 30, 30]
+
+
+class TestGeopackageLevelCache(TileCacheTestBase):
+
+    always_loads_metadata = True
+
+    def setup_method(self):
+        TileCacheTestBase.setup_method(self)
+        self.cache = GeopackageLevelCache(
+            self.cache_dir,
+            tile_grid=tile_grid(3857, name='global-webmarcator'),
+            table_name='test_tiles',
+        )
+
+    def teardown_method(self):
+        if self.cache:
+            self.cache.cleanup()
+        TileCacheTestBase.teardown_method(self)
+
+    def test_default_coverage(self):
+        assert self.cache.coverage is None
+
+    def test_level_files(self):
+        if os.path.exists(self.cache_dir):
+            assert_files_in_dir(self.cache_dir, [], glob='*.gpkg')
+
+        self.cache.store_tile(self.create_tile((0, 0, 1)))
+        assert_files_in_dir(self.cache_dir, ['1.gpkg'], glob='*.gpkg')
+
+        self.cache.store_tile(self.create_tile((0, 0, 5)))
+        assert_files_in_dir(self.cache_dir, ['1.gpkg', '5.gpkg'], glob='*.gpkg')
+
+    def test_remove_level_files(self):
+        self.cache.store_tile(self.create_tile((0, 0, 1)))
+        self.cache.store_tile(self.create_tile((0, 0, 2)))
+        assert_files_in_dir(self.cache_dir, ['1.gpkg', '2.gpkg'], glob='*.gpkg')
+
+        self.cache.remove_level_tiles_before(1, timestamp=0)
+        assert_files_in_dir(self.cache_dir, ['2.gpkg'], glob='*.gpkg')
+
+    def test_remove_level_tiles_before(self):
+        self.cache.store_tile(self.create_tile((0, 0, 1)))
+        self.cache.store_tile(self.create_tile((0, 0, 2)))
+
+        assert_files_in_dir(self.cache_dir, ['1.gpkg', '2.gpkg'], glob='*.gpkg')
+        assert self.cache.is_cached(Tile((0, 0, 1)))
+
+        self.cache.remove_level_tiles_before(1, timestamp=time.time() - 60)
+        assert self.cache.is_cached(Tile((0, 0, 1)))
+
+        self.cache.remove_level_tiles_before(1, timestamp=0)
+        assert not self.cache.is_cached(Tile((0, 0, 1)))
+
+        assert_files_in_dir(self.cache_dir, ['1.gpkg', '2.gpkg'], glob='*.gpkg')
+        assert self.cache.is_cached(Tile((0, 0, 2)))
+
+    def test_bulk_store_tiles_with_different_levels(self):
+        self.cache.store_tiles([
+            self.create_tile((0, 0, 1)),
+            self.create_tile((0, 0, 2)),
+            self.create_tile((1, 0, 2)),
+            self.create_tile((1, 0, 1)),
+        ])
+
+        assert_files_in_dir(self.cache_dir, ['1.gpkg', '2.gpkg'], glob='*.gpkg')
+        assert self.cache.is_cached(Tile((0, 0, 1)))
+        assert self.cache.is_cached(Tile((1, 0, 1)))
+        assert self.cache.is_cached(Tile((0, 0, 2)))
+        assert self.cache.is_cached(Tile((1, 0, 2)))
+
+
+class TestGeopackageCacheInitErrors(object):
+    table_name = 'cache'
+
+    def test_bad_config_geopackage_srs(self):
+        error_msg = None
+        gpkg_file = os.path.join(os.path.join(os.path.dirname(__file__),
+                                              'fixture'),
+                                 'cache.gpkg')
+        table_name = 'cache'
+        try:
+            GeopackageCache(gpkg_file, TileGrid(srs=4326), table_name)
+        except ValueError as ve:
+            error_msg = ve
+        assert "srs is improperly configured." in str(error_msg)
+
+    def test_bad_config_geopackage_tile(self):
+        error_msg = None
+        gpkg_file = os.path.join(os.path.join(os.path.dirname(__file__),
+                                              'fixture'),
+                                 'cache.gpkg')
+        table_name = 'cache'
+        try:
+            GeopackageCache(gpkg_file, TileGrid(srs=900913, tile_size=(512, 512)), table_name)
+        except ValueError as ve:
+            error_msg = ve
+        assert "tile_size is improperly configured." in str(error_msg)
+
+    def test_bad_config_geopackage_res(self):
+        error_msg = None
+        gpkg_file = os.path.join(os.path.join(os.path.dirname(__file__),
+                                              'fixture'),
+                                 'cache.gpkg')
+        table_name = 'cache'
+        try:
+            GeopackageCache(gpkg_file, TileGrid(srs=900913, res=[1000, 100, 10]), table_name)
+        except ValueError as ve:
+            error_msg = ve
+        assert "res is improperly configured." in str(error_msg)

mapproxy/test/unit/test_cache_redis.py

@@ -0,0 +1,123 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2017 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+import pytest
+
+try:
+    import redis
+except ImportError:
+    redis = None
+
+from mapproxy.cache.redis import RedisCache
+from mapproxy.cache.tile import Tile
+from mapproxy.test.unit.test_cache_tile import TileCacheTestBase
+
+
+@pytest.mark.skipif(not redis or not os.environ.get('MAPPROXY_TEST_REDIS'),
+                    reason="redis package and MAPPROXY_TEST_REDIS env required")
+class TestRedisCache(TileCacheTestBase):
+    always_loads_metadata = False
+
+    def setup_method(self):
+        redis_host = os.environ['MAPPROXY_TEST_REDIS']
+        self.host, self.port = redis_host.split(':')
+        if os.environ.get('MAPPROXY_TEST_REDIS_TLS'):
+            redis_host_tls = os.environ['MAPPROXY_TEST_REDIS_TLS']
+            self.tls_host, self.tls_port = redis_host_tls.split(':')
+        if os.environ.get('MAPPROXY_TEST_REDIS_AUTH'):
+            redis_host_tls = os.environ['MAPPROXY_TEST_REDIS_AUTH']
+            self.auth_host, self.auth_port = redis_host_tls.split(':')
+
+        TileCacheTestBase.setup_method(self)
+
+        self.cache = RedisCache(self.host, int(self.port), prefix='mapproxy-test', db=1)
+
+    def teardown_method(self):
+        for k in self.cache.r.keys('mapproxy-test-*'):
+            self.cache.r.delete(k)
+
+    def test_default_coverage(self):
+        assert self.cache.coverage is None
+
+    def test_expire(self):
+        cache = RedisCache(self.host, int(self.port), prefix='mapproxy-test', db=1, ttl=0)
+        t1 = self.create_tile(coord=(9382, 1234, 9))
+        assert cache.store_tile(t1)
+        time.sleep(0.1)
+        t2 = Tile(t1.coord)
+        assert cache.is_cached(t2)
+
+        cache = RedisCache(self.host, int(self.port), prefix='mapproxy-test', db=1, ttl=0.05)
+        t1 = self.create_tile(coord=(5382, 2234, 9))
+        assert cache.store_tile(t1)
+        time.sleep(0.1)
+        t2 = Tile(t1.coord)
+        assert not cache.is_cached(t2)
+
+    def test_double_remove(self):
+        tile = self.create_tile()
+        self.create_cached_tile(tile)
+        assert self.cache.remove_tile(tile)
+        assert self.cache.remove_tile(tile)
+
+    @pytest.mark.skipif(not redis or not os.environ.get('MAPPROXY_TEST_REDIS_TLS'),
+                        reason="MAPPROXY_TEST_REDIS_TLS is required")
+    def test_tls_authentication_enabled(self):
+        print(os.curdir)
+        ssl_certfile = 'mapproxy/test/unit/fixture/redis-client.crt'
+        ssl_keyfile = 'mapproxy/test/unit/fixture/redis-client.key'
+        ssl_ca_certs = 'mapproxy/test/unit/fixture/ca.crt'
+        cache = RedisCache(
+            self.tls_host, int(self.tls_port), prefix='mapproxy-test', db=1, ssl_certfile=ssl_certfile,
+            ssl_keyfile=ssl_keyfile, ssl_ca_certs=ssl_ca_certs)
+        assert cache.r.connection_pool.connection_kwargs['ssl_certfile'] == ssl_certfile
+        assert cache.r.connection_pool.connection_kwargs['ssl_keyfile'] == ssl_keyfile
+        assert cache.r.connection_pool.connection_kwargs['ssl_ca_certs'] == ssl_ca_certs
+        t1 = self.create_tile(coord=(5382, 3234, 9))
+        assert cache.store_tile(t1)
+        time.sleep(0.1)
+        t2 = Tile(t1.coord)
+        assert cache.is_cached(t2)
+
+    @pytest.mark.skipif(not redis or not os.environ.get('MAPPROXY_TEST_REDIS_TLS'),
+                        reason="MAPPROXY_TEST_REDIS_TLS is required")
+    def test_tls_authentication_disabled(self):
+        cache = RedisCache(self.tls_host, int(self.tls_port), prefix='mapproxy-test', db=1)
+        assert 'ssl_certfile' not in cache.r.connection_pool.connection_kwargs
+        assert 'ssl_keyfile' not in cache.r.connection_pool.connection_kwargs
+        assert 'ssl_ca_certs' not in cache.r.connection_pool.connection_kwargs
+        assert not cache.r.connection_pool.connection_kwargs.get('ssl', False)
+        t1 = self.create_tile(coord=(5382, 4234, 9))
+        assert not cache.store_tile(t1)
+        time.sleep(0.1)
+        t2 = Tile(t1.coord)
+        assert not cache.is_cached(t2)
+
+    @pytest.mark.skipif(not redis or not os.environ.get('MAPPROXY_TEST_REDIS_AUTH'),
+                        reason="MAPPROXY_TEST_REDIS_AUTH is required to test authentication")
+    def test_user_password_authentication(self):
+        username = 'test'
+        password = 'pw4test'
+        cache = RedisCache(
+            self.auth_host, int(self.auth_port), prefix='mapproxy-test', db=1, username=username, password=password)
+        assert cache.r.connection_pool.connection_kwargs['username'] == username
+        assert cache.r.connection_pool.connection_kwargs['password'] == password
+        t1 = self.create_tile(coord=(5382, 5234, 9))
+        assert cache.store_tile(t1)
+        t2 = Tile(t1.coord)
+        assert cache.is_cached(t2)

mapproxy/test/unit/test_cache_riak.py

@@ -0,0 +1,80 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import random
+from urllib.parse import urlparse
+
+import pytest
+
+from mapproxy.cache.riak import RiakCache
+from mapproxy.grid import tile_grid
+from mapproxy.test.image import create_tmp_image_buf
+from mapproxy.test.unit.test_cache_tile import TileCacheTestBase
+
+
+tile_image = create_tmp_image_buf((256, 256), color='blue')
+tile_image2 = create_tmp_image_buf((256, 256), color='red')
+
+
+@pytest.mark.skipif(sys.version_info > (3, 7), reason="riak is not compatible with this Python version")
+class RiakCacheTestBase(TileCacheTestBase):
+    always_loads_metadata = True
+
+    def setup_method(self):
+        url = os.environ[self.riak_url_env]
+        urlparts = urlparse(url)
+        protocol = urlparts.scheme.lower()
+        node = {'host': urlparts.hostname}
+        if ':' in urlparts.hostname:
+            if protocol == 'pbc':
+                node['pb_port'] = urlparts.port
+            if protocol in ('http', 'https'):
+                node['http_port'] = urlparts.port
+
+        db_name = 'mapproxy_test_%d' % random.randint(0, 100000)
+
+        TileCacheTestBase.setup_method(self)
+
+        self.cache = RiakCache([node], protocol, db_name, tile_grid=tile_grid(3857, name='global-webmarcator'))
+
+    def teardown_method(self):
+        import riak
+        bucket = self.cache.bucket
+        for k in bucket.get_keys():
+            riak.RiakObject(self.cache.connection, bucket, k).delete()
+        TileCacheTestBase.teardown_method(self)
+
+    def test_default_coverage(self):
+        assert self.cache.coverage is None
+
+    def test_double_remove(self):
+        tile = self.create_tile()
+        self.create_cached_tile(tile)
+        assert self.cache.remove_tile(tile)
+        assert self.cache.remove_tile(tile)
+
+
+@pytest.mark.skipif(not os.environ.get('MAPPROXY_TEST_RIAK_HTTP'),
+                    reason="MAPPROXY_TEST_RIAK_HTTP not set")
+class TestRiakCacheHTTP(RiakCacheTestBase):
+    riak_url_env = 'MAPPROXY_TEST_RIAK_HTTP'
+
+
+@pytest.mark.skipif(not os.environ.get('MAPPROXY_TEST_RIAK_PBC'),
+                    reason="MAPPROXY_TEST_RIAK_PBC not set")
+class TestRiakCachePBC(RiakCacheTestBase):
+    riak_url_env = 'MAPPROXY_TEST_RIAK_PBC'

mapproxy/test/unit/test_cache_s3.py

@@ -0,0 +1,93 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from mapproxy.layer import MapExtent
+from mapproxy.srs import SRS
+
+try:
+    import boto3
+    from moto import mock_s3
+except ImportError:
+    boto3 = None
+    mock_s3 = None
+
+from mapproxy.cache.s3 import S3Cache
+from mapproxy.test.unit.test_cache_tile import TileCacheTestBase
+
+
+GLOBAL_WEBMERCATOR_EXTENT = MapExtent(
+    (-20037508.342789244, -20037508.342789244, 20037508.342789244, 20037508.342789244),
+    SRS(3857))
+
+
+@pytest.mark.skipif(not mock_s3 or not boto3,
+                    reason="boto3 and moto required for S3 tests")
+class TestS3Cache(TileCacheTestBase):
+    always_loads_metadata = True
+    uses_utc = True
+
+    def setup_method(self):
+        TileCacheTestBase.setup_method(self)
+
+        self.mock = mock_s3()
+        self.mock.start()
+
+        self.bucket_name = "test"
+        dir_name = 'mapproxy'
+
+        boto3.client("s3").create_bucket(Bucket=self.bucket_name)
+
+        self.cache = S3Cache(dir_name,
+                             file_ext='png',
+                             directory_layout='tms',
+                             bucket_name=self.bucket_name,
+                             profile_name=None,
+                             _concurrent_writer=1,)  # moto is not thread safe
+
+    def teardown_method(self):
+        self.mock.stop()
+        TileCacheTestBase.teardown_method(self)
+
+    def test_default_coverage(self):
+        assert self.cache.coverage is None
+
+    @pytest.mark.parametrize('layout,tile_coord,key', [
+        ['mp', (12345, 67890,  2), 'mycache/webmercator/02/0001/2345/0006/7890.png'],
+        ['mp', (12345, 67890, 12), 'mycache/webmercator/12/0001/2345/0006/7890.png'],
+
+        ['tc', (12345, 67890,  2), 'mycache/webmercator/02/000/012/345/000/067/890.png'],
+        ['tc', (12345, 67890, 12), 'mycache/webmercator/12/000/012/345/000/067/890.png'],
+
+        ['tms', (12345, 67890,  2), 'mycache/webmercator/2/12345/67890.png'],
+        ['tms', (12345, 67890, 12), 'mycache/webmercator/12/12345/67890.png'],
+
+        ['quadkey', (0, 0, 0), 'mycache/webmercator/.png'],
+        ['quadkey', (0, 0, 1), 'mycache/webmercator/0.png'],
+        ['quadkey', (1, 1, 1), 'mycache/webmercator/3.png'],
+        ['quadkey', (12345, 67890, 12), 'mycache/webmercator/200200331021.png'],
+
+        ['arcgis', (1, 2, 3), 'mycache/webmercator/L03/R00000002/C00000001.png'],
+        ['arcgis', (9, 2, 3), 'mycache/webmercator/L03/R00000002/C00000009.png'],
+        ['arcgis', (10, 2, 3), 'mycache/webmercator/L03/R00000002/C0000000a.png'],
+        ['arcgis', (12345, 67890, 12), 'mycache/webmercator/L12/R00010932/C00003039.png'],
+    ])
+    def test_tile_key(self, layout, tile_coord, key):
+        cache = S3Cache('/mycache/webmercator', 'png', bucket_name=self.bucket_name, directory_layout=layout)
+        cache.store_tile(self.create_tile(tile_coord))
+
+        # raises, if key is missing
+        boto3.client("s3").head_object(Bucket=self.bucket_name, Key=key)