MapProxy 2.1.0__py3-none-any.whl

mapproxy/response.py

@@ -0,0 +1,237 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Service responses.
+"""
+
+import hashlib
+from mapproxy.util.times import format_httpdate, parse_httpdate, timestamp
+
+
+class Response(object):
+    charset = 'utf-8'
+    default_content_type = 'text/plain'
+    block_size = 1024 * 32
+
+    def __init__(self, response, status=None, content_type=None, mimetype=None):
+        self.response = response
+        if status is None:
+            status = 200
+        self.status = status
+        self._timestamp = None
+        self.headers = {}
+        if mimetype:
+            if mimetype.startswith('text/'):
+                content_type = mimetype + '; charset=' + self.charset
+            else:
+                content_type = mimetype
+        if content_type is None:
+            content_type = self.default_content_type
+        self.headers['Content-type'] = content_type
+
+        if content_type.startswith(('text/', 'application/')):
+            # Capability documents can be dependent on the value of a few X-headers.
+            # Tell this caching proxies via the Vary HTTP header. This also prevents
+            # malicious cache poisoning.
+            self.headers['Vary'] = 'X-Script-Name, X-Forwarded-Host, X-Forwarded-Proto'
+
+    def _status_set(self, status):
+        if isinstance(status, int):
+            status = status_code(status)
+        self._status = status
+
+    def _status_get(self):
+        return self._status
+
+    status = property(_status_get, _status_set)
+
+    def _last_modified_set(self, date):
+        if not date:
+            return
+        self._timestamp = timestamp(date)
+        self.headers['Last-modified'] = format_httpdate(self._timestamp)
+
+    def _last_modified_get(self):
+        return self.headers.get('Last-modified', None)
+
+    last_modified = property(_last_modified_get, _last_modified_set)
+
+    def _etag_set(self, value):
+        self.headers['ETag'] = value
+
+    def _etag_get(self):
+        return self.headers.get('ETag', None)
+
+    etag = property(_etag_get, _etag_set)
+
+    def cache_headers(self, timestamp=None, etag_data=None, max_age=None, no_cache=False):
+        """
+        Set cache-related headers.
+
+        :param timestamp: local timestamp of the last modification of the
+            response content
+        :param etag_data: list that will be used to build an ETag hash.
+            calls the str function on each item.
+        :param max_age: the maximum cache age in seconds
+        """
+        if etag_data:
+            hash_src = ''.join((str(x) for x in etag_data)).encode('ascii')
+            self.etag = hashlib.new('md5', hash_src, usedforsecurity=False).hexdigest()
+
+        if no_cache:
+            assert not timestamp and not max_age
+            self.headers['Cache-Control'] = 'no-cache, no-store'
+            self.headers['Pragma'] = 'no-cache'
+            self.headers['Expires'] = '-1'
+
+        self.last_modified = timestamp
+        if (timestamp or etag_data) and max_age is not None:
+            self.headers['Cache-control'] = 'public, max-age=%d, s-maxage=%d' % (max_age, max_age)
+
+    def make_conditional(self, req):
+        """
+        Make the response conditional to the HTTP headers in the CGI/WSGI `environ`.
+        Checks for ``If-none-match`` and ``If-modified-since`` headers and compares
+        to the etag and timestamp of this response. If the content was not modified
+        the repsonse will changed to HTTP 304 Not Modified.
+        """
+        if req is None:
+            return
+        environ = req.environ
+
+        not_modified = False
+
+        if self.etag == environ.get('HTTP_IF_NONE_MATCH', -1):
+            not_modified = True
+        elif self._timestamp is not None:
+            date = environ.get('HTTP_IF_MODIFIED_SINCE', None)
+            timestamp = parse_httpdate(date)
+            if timestamp is not None and self._timestamp <= timestamp:
+                not_modified = True
+
+        if not_modified:
+            self.status = 304
+            self.response = []
+            if 'Content-type' in self.headers:
+                del self.headers['Content-type']
+
+    @property
+    def content_length(self):
+        return int(self.headers.get('Content-length', 0))
+
+    @property
+    def content_type(self):
+        return self.headers['Content-type']
+
+    @property
+    def data(self):
+        if hasattr(self.response, 'read'):
+            return self.response.read()
+        else:
+            return b''.join(chunk.encode() for chunk in self.response)
+
+    @property
+    def fixed_headers(self):
+        headers = []
+        for key, value in self.headers.items():
+            if type(value) is not str:
+                # for str subclasses like ImageFormat
+                value = str(value)
+            headers.append((key, value))
+        return headers
+
+    def __call__(self, environ, start_response):
+        if hasattr(self.response, 'read'):
+            if ((not hasattr(self.response, 'ok_to_seek') or
+                self.response.ok_to_seek) and
+               (hasattr(self.response, 'seek') and
+                    hasattr(self.response, 'tell'))):
+                self.response.seek(0, 2)  # to EOF
+                self.headers['Content-length'] = str(self.response.tell())
+                self.response.seek(0)
+            if 'wsgi.file_wrapper' in environ:
+                resp_iter = environ['wsgi.file_wrapper'](self.response, self.block_size)
+            else:
+                resp_iter = iter(lambda: self.response.read(self.block_size), b'')
+        elif not self.response:
+            resp_iter = iter([])
+        elif isinstance(self.response, str):
+            self.response = self.response.encode(self.charset)
+            self.headers['Content-length'] = str(len(self.response))
+            resp_iter = iter([self.response])
+        elif isinstance(self.response, bytes):
+            self.headers['Content-length'] = str(len(self.response))
+            resp_iter = iter([self.response])
+        else:
+            resp_iter = self.response
+
+        start_response(self.status, self.fixed_headers)
+        return resp_iter
+
+    def iter_encode(self, chunks):
+        for chunk in chunks:
+            if isinstance(chunk, str):
+                chunk = chunk.encode(self.charset)
+            yield chunk
+
+
+# http://www.faqs.org/rfcs/rfc2616.html
+_status_codes = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    307: 'Temporary Redirect',
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Time-out',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Large',
+    415: 'Unsupported Media Type',
+    416: 'Requested range not satisfiable',
+    417: 'Expectation Failed',
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Time-out',
+    505: 'HTTP Version not supported',
+}
+
+
+def status_code(code):
+    return str(code) + ' ' + _status_codes[code]

mapproxy/script/conf/app.py

@@ -0,0 +1,222 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import codecs
+import sys
+import os
+import optparse
+import logging
+import textwrap
+import datetime
+import xml.etree.ElementTree
+import yaml
+
+from contextlib import contextmanager
+from io import BytesIO
+
+from .sources import sources
+from .layers import layers
+from .caches import caches
+from .seeds import seeds
+from .utils import update_config, MapProxyYAMLDumper, download_capabilities
+from .geopackage import get_geopackage_configuration_dict
+
+from mapproxy.config.loader import load_configuration
+from mapproxy.util.ext.wmsparse import parse_capabilities
+
+
+def setup_logging(level=logging.INFO):
+    mapproxy_log = logging.getLogger('mapproxy')
+    mapproxy_log.setLevel(level)
+
+    # do not init logging when stdout is captured
+    # eg. when running in tests
+    if isinstance(sys.stdout, BytesIO):
+        return
+
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter(
+        "[%(asctime)s] %(name)s - %(levelname)s - %(message)s")
+    ch.setFormatter(formatter)
+    mapproxy_log.addHandler(ch)
+
+
+def write_header(f, capabilities):
+    print('# MapProxy configuration automatically generated from:', file=f)
+    print('#   %s' % capabilities, file=f)
+    print('#', file=f)
+    print('# NOTE: The generated configuration can be highly inefficient,', file=f)
+    print('#       especially when multiple layers and caches are requested at once.', file=f)
+    print('#       Make sure you understand the generated configuration!', file=f)
+    print('#', file=f)
+    print('# Created on %s with:' % datetime.datetime.now(), file=f)
+    print(' \\\n'.join(textwrap.wrap(' '.join(sys.argv), initial_indent='# ', subsequent_indent='#    ')), file=f)
+    print('', file=f)
+
+
+@contextmanager
+def file_or_stdout(name):
+    if name == '-':
+        if hasattr(sys.stdout, 'buffer'):
+            yield codecs.getwriter('utf-8')(sys.stdout.buffer)
+        else:
+            yield codecs.getwriter('utf-8')(sys.stdout)
+    else:
+        with open(name, 'wb') as f:
+            yield codecs.getwriter('utf-8')(f)
+
+
+def config_command(args):
+    parser = optparse.OptionParser("usage: %prog autoconfig [options]")
+
+    parser.add_option('--capabilities',
+                      help="URL or filename of WMS 1.1.1/1.3.0 capabilities document")
+    parser.add_option('--geopackage',
+                      help="Filename of a geopackage file")
+    parser.add_option('--output', help="filename for created MapProxy config [default: -]", default="-")
+    parser.add_option('--output-seed', help="filename for created seeding config")
+
+    parser.add_option('--base', help='base config to include in created MapProxy config')
+
+    parser.add_option('--overwrite',
+                      help='YAML file with overwrites for the created MapProxy config')
+    parser.add_option('--overwrite-seed',
+                      help='YAML file with overwrites for the created seeding config')
+
+    parser.add_option('--force', default=False, action='store_true',
+                      help="overwrite existing files")
+
+    options, args = parser.parse_args(args)
+
+    if not (bool(options.capabilities) ^ bool(options.geopackage)):
+        parser.print_help()
+        print("\nERROR: --capabilities required or --geopackage required", file=sys.stderr)
+        return 2
+
+    if not options.output and not options.output_seed:
+        parser.print_help()
+        print("\nERROR: --output and/or --output-seed required", file=sys.stderr)
+        return 2
+
+    if not options.force:
+        if options.output and options.output != '-' and os.path.exists(options.output):
+            print("\nERROR: %s already exists, use --force to overwrite" % options.output, file=sys.stderr)
+            return 2
+        if options.output_seed and options.output_seed != '-' and os.path.exists(options.output_seed):
+            print("\nERROR: %s already exists, use --force to overwrite" % options.output_seed, file=sys.stderr)
+            return 2
+
+    log = logging.getLogger('mapproxy_conf_cmd')
+    log.addHandler(logging.StreamHandler())
+
+    setup_logging(logging.WARNING)
+
+    srs_grids = {}
+    if options.base:
+        base = load_configuration(options.base)
+        for name, grid_conf in base.grids.items():
+            if name.startswith('GLOBAL_'):
+                continue
+            srs_grids[grid_conf.tile_grid().srs.srs_code] = name
+
+    cap_doc = options.capabilities
+    gpkg = options.geopackage
+    if cap_doc:
+        if cap_doc.startswith(('http://', 'https://')):
+            cap_doc = download_capabilities(options.capabilities).read()
+        else:
+            cap_doc = open(cap_doc, 'rb').read()
+
+        try:
+            cap = parse_capabilities(BytesIO(cap_doc))
+        except (xml.etree.ElementTree.ParseError, ValueError) as ex:
+            print(ex, file=sys.stderr)
+            print(f"{cap_doc[:1000]} {'...' if len(cap_doc) > 1000 else ''}", file=sys.stderr)
+            return 3
+    elif gpkg:
+        if os.path.isfile(gpkg):
+            gpkg_dict = get_geopackage_configuration_dict(gpkg)
+        else:
+            print("ERROR: No file {} exists.".format(gpkg))
+            return 2
+
+    overwrite = None
+    if options.overwrite:
+        with open(options.overwrite, 'rb') as f:
+            overwrite = yaml.safe_load(f)
+
+    overwrite_seed = None
+    if options.overwrite_seed:
+        with open(options.overwrite_seed, 'rb') as f:
+            overwrite_seed = yaml.safe_load(f)
+
+    conf = {}
+
+    if options.base:
+        conf['base'] = os.path.abspath(options.base)
+    if cap_doc:
+        conf['services'] = {'wms': {'md': {'title': cap.metadata()['title']}}}
+    if gpkg:
+        conf['services'] = gpkg_dict['services']
+    if overwrite:
+        conf['services'] = update_config(conf['services'], overwrite.pop('service', {}))
+
+    if cap_doc:
+        conf['sources'] = sources(cap)
+    if overwrite:
+        conf['sources'] = update_config(conf['sources'], overwrite.pop('sources', {}))
+
+    if cap_doc:
+        conf['caches'] = caches(cap, conf['sources'], srs_grids=srs_grids)
+    if gpkg:
+        conf['caches'] = gpkg_dict['caches']
+    if overwrite:
+        conf['caches'] = update_config(conf['caches'], overwrite.pop('caches', {}))
+
+    if cap_doc:
+        conf['layers'] = layers(cap, conf['caches'])
+    if gpkg:
+        conf['layers'] = gpkg_dict['layers']
+    if overwrite:
+        conf['layers'] = update_config(conf['layers'], overwrite.pop('layers', {}))
+
+    if gpkg:
+        conf['grids'] = gpkg_dict['grids']
+    if overwrite:
+        conf = update_config(conf, overwrite)
+
+    seed_conf = {}
+    if cap_doc:
+        seed_conf['seeds'], seed_conf['cleanups'] = seeds(cap, conf['caches'])
+    if gpkg:
+        cap = None
+        seed_conf['seeds'], seed_conf['cleanups'] = seeds(cap, conf['caches'])
+    if overwrite_seed:
+        seed_conf = update_config(seed_conf, overwrite_seed)
+
+    if options.output:
+        with file_or_stdout(options.output) as f:
+            write_header(f, options.capabilities or options.geopackage)
+            yaml.dump(conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper)
+    if options.output_seed:
+        with file_or_stdout(options.output_seed) as f:
+            write_header(f, options.capabilities or options.geopackage)
+            yaml.dump(seed_conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper)
+
+    return 0

mapproxy/script/conf/caches.py

@@ -0,0 +1,44 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def caches(cap, sources, srs_grids):
+    caches = {}
+    for name, source in sources.items():
+        conf = for_source(name, source, srs_grids)
+        if not conf:
+            continue
+        caches[name[:-len('_wms')] + '_cache'] = conf
+
+    return caches
+
+
+def for_source(name, source, srs_grids):
+    cache = {
+        'sources': [name]
+    }
+
+    grids = []
+    for srs in source['supported_srs']:
+        if srs in srs_grids:
+            grids.append(srs_grids[srs])
+
+    if not grids:
+        return None
+
+    cache['grids'] = grids
+
+    return cache

mapproxy/script/conf/geopackage.py

@@ -0,0 +1,136 @@
+import yaml
+import sqlite3
+import os
+
+
+def conf_from_geopackage(geopackage_file, output_file=None):
+    conf = get_geopackage_configuration_dict(geopackage_file)
+    yaml.SafeDumper.add_representer(
+        type(None),
+        lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:null', '')
+    )
+
+    if output_file:
+        with open('data.yml', 'w') as outfile:
+            outfile.write(yaml.safe_dump(conf, default_flow_style=False))
+    else:
+        print(yaml.safe_dump(get_geopackage_configuration_dict(geopackage_file), default_flow_style=False))
+
+
+def get_gpkg_contents(geopackage_file, data_type='tiles'):
+    """
+    :param geopackage_file: Path to the geopackage file.
+    :param data_type: The type of layer to return tiles or features.
+    :return: One or more tuples with the table_name, min_x, min_y, max_x, max_y, srs_id for each layer in the
+        geopackage.
+    """
+    with sqlite3.connect(geopackage_file) as db:
+        cur = db.execute(
+            "SELECT table_name, data_type, identifier, description, last_change, min_x, min_y, max_x, "
+            "max_y, srs_id FROM gpkg_contents WHERE data_type = ?", (data_type,))
+    return cur.fetchall()
+
+
+def get_table_organization_coordsys_id(geopackage_file, srs_id):
+    """
+    :param geopackage_file: Path to the geopackage file.
+    :param srs_id: The srs_id which is the key value in the organization_coordsys_id.
+    :return: An integer representing the organization_coordsys_id as an EPSG code.
+    """
+    with sqlite3.connect(geopackage_file) as db:
+        cur = db.execute("SELECT organization_coordsys_id FROM gpkg_spatial_ref_sys WHERE srs_id = ?", (srs_id,))
+    results = cur.fetchone()
+    if results:
+        return results[0]
+
+
+def get_table_tile_matrix(geopackage_file, table_name):
+    """
+    :param geopackage_file: Path to the geopackage file.
+    :param table_name: The table_name associated with the tile_matrix data.
+    :return: A tuple of tuple containing zoom_level, matrix_width, matrix_height, tile_width, tile_height, pixel_x_size,
+    pixel_y_size for each zoom_level.
+    """
+
+    with sqlite3.connect(geopackage_file) as db:
+        cur = db.execute("SELECT zoom_level,"
+                         "matrix_width, "
+                         "matrix_height, "
+                         "tile_width, "
+                         "tile_height, "
+                         "pixel_x_size, "
+                         "pixel_y_size "
+                         "FROM gpkg_tile_matrix WHERE table_name = ?"
+                         "ORDER BY zoom_level", (table_name,))
+        return cur.fetchall()
+
+
+def get_estimated_tile_res_ratio(tile_matrix):
+    """
+
+    :param tile_matrix: A tuple of tuples representing the geopackage tile matrix (without the table name included).
+    :return: The rate at which the resolution increases between levels.
+    """
+    default_res_factor = 2
+    if len(tile_matrix) < 2:
+        return default_res_factor
+    layer = tile_matrix[0]
+    next_layer = tile_matrix[1]
+    return (layer[6]/next_layer[6])/(next_layer[0]-layer[0])
+
+
+def get_res_table(tile_matrix):
+    res_ratio = get_estimated_tile_res_ratio(tile_matrix)
+    res_table = []
+    if tile_matrix[0][0] == 0:
+        first_level_res = tile_matrix[0][5]
+    else:
+        first_level_res = tile_matrix[0][5]*(res_ratio**tile_matrix[0][0])
+    tile_matrix_set = {}
+    for level in tile_matrix:
+        tile_matrix_set[level[0]] = level
+    if not tile_matrix_set.get(0):
+        res_table += [first_level_res]
+    else:
+        res_table += [tile_matrix_set.get(0)[5]]
+    for level in range(1, 19):
+        res = tile_matrix_set.get(level)
+        if not res:
+            res_table += [first_level_res/(res_ratio**level)]
+        else:
+            res_table += [res[5]]
+    return res_table
+
+
+def get_geopackage_configuration_dict(geopackage_file):
+    gpkg_contents = get_gpkg_contents(geopackage_file, data_type='tiles')
+    conf = {'grids': {},
+            'caches': {},
+            'layers': [],
+            'services': {'demo': None,
+                         'tms': {'use_grid_names': True, 'origin': 'nw'},
+                         'kml': {'use_grid_names': True},
+                         'wmts': None,
+                         'wms': None}}
+
+    for gpkg_content in gpkg_contents:
+        table_name = str(gpkg_content[0])
+        tile_matrix = get_table_tile_matrix(geopackage_file, table_name)
+        srs = get_table_organization_coordsys_id(geopackage_file, gpkg_content[9])
+        if not tile_matrix or not srs:
+            continue
+        conf['grids']['{0}_{1}'.format(table_name, srs)] = {'srs': 'EPSG:{0}'.format(srs),
+                                                            'tile_size': [tile_matrix[0][3], tile_matrix[0][4]],
+                                                            'bbox': [gpkg_content[5],
+                                                                     gpkg_content[6],
+                                                                     gpkg_content[7],
+                                                                     gpkg_content[8]],
+                                                            'res': get_res_table(tile_matrix),
+                                                            'origin': 'nw'}
+        conf['caches']['{0}_cache'.format(table_name)] = {'sources': [],
+                                                          'grids': ['{0}_{1}'.format(table_name, srs)],
+                                                          'cache': {'type': 'geopackage',
+                                                                    'filename': os.path.abspath(geopackage_file),
+                                                                    'table_name': table_name}}
+        conf['layers'] += [{'name': table_name, 'title': table_name, 'sources': ['{0}_cache'.format(table_name)]}]
+    return conf