MapProxy 1.16.1__py3-none-any.whl

mapproxy/seed/util.py

@@ -0,0 +1,254 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010, 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function, division
+
+import os
+import sys
+import stat
+import math
+import time
+from datetime import datetime
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+from mapproxy.layer import map_extent_from_grid
+from mapproxy.util.fs import write_atomic
+
+import logging
+log = logging.getLogger(__name__)
+
+class bidict(dict):
+    """
+    Simplest bi-directional dictionary.
+    """
+    def __init__(self, iterator):
+        for key, val in iterator:
+            dict.__setitem__(self, key, val)
+            dict.__setitem__(self, val, key)
+
+class ProgressStore(object):
+    """
+    Reads and stores seed progresses to a file.
+    """
+    def __init__(self, filename=None, continue_seed=True):
+        self.filename = filename
+        if continue_seed:
+            self.status = self.load()
+        else:
+            self.status = {}
+
+    def load(self):
+        if not os.path.exists(self.filename):
+            pass
+        elif os.stat(self.filename).st_mode & stat.S_IWOTH:
+            log.error('progress file (%s) is world writable, ignoring file',
+                self.filename)
+        else:
+            with open(self.filename, 'rb') as f:
+                try:
+                    return pickle.load(f)
+                except (pickle.UnpicklingError, AttributeError,
+                    EOFError, ImportError, IndexError):
+                    log.error('unable to read progress file (%s), ignoring file',
+                        self.filename)
+
+        return {}
+
+    def write(self):
+        try:
+            write_atomic(self.filename, pickle.dumps(self.status))
+        except (IOError, OSError) as ex:
+            log.error('unable to write seed progress: %s', ex)
+
+    def remove(self):
+        self.status = {}
+        if os.path.exists(self.filename):
+            os.remove(self.filename)
+
+    def get(self, task_identifier):
+        return self.status.get(task_identifier, None)
+
+    def add(self, task_identifier, progress_identifier):
+        self.status[task_identifier] = progress_identifier
+
+class ProgressLog(object):
+    def __init__(self, out=None, silent=False, verbose=True, progress_store=None):
+        if not out:
+            out = sys.stdout
+        self.out = out
+        self._laststep = time.time()
+        self._lastprogress = 0
+
+        self.verbose = verbose
+        self.silent = silent
+        self.current_task_id = None
+        self.progress_store = progress_store
+
+    def log_message(self, msg):
+        self.out.write('[%s] %s\n' % (
+            timestamp(), msg,
+        ))
+        self.out.flush()
+
+    def log_step(self, progress):
+        if not self.verbose:
+            return
+        if (self._laststep + .5) < time.time():
+            # log progress at most every 500ms
+            self.out.write('[%s] %6.2f%%\t%-20s \r' % (
+                timestamp(), progress.progress*100, progress.progress_str,
+            ))
+            self.out.flush()
+            self._laststep = time.time()
+
+    def log_progress(self, progress, level, bbox, tiles):
+        progress_interval = 1
+        if not self.verbose:
+            progress_interval = 30
+
+        log_progess = False
+        if progress.progress == 1.0 or (self._lastprogress + progress_interval) < time.time():
+            self._lastprogress = time.time()
+            log_progess = True
+
+        if log_progess:
+            if self.progress_store and self.current_task_id:
+                self.progress_store.add(self.current_task_id,
+                    progress.current_progress_identifier())
+                self.progress_store.write()
+
+        if self.silent:
+            return
+
+        if log_progess:
+            self.out.write('[%s] %2s %6.2f%% %s (%d tiles)\n' % (
+                timestamp(), level, progress.progress*100,
+                format_bbox(bbox), tiles))
+            self.out.flush()
+
+
+def limit_sub_bbox(bbox, sub_bbox):
+    """
+    >>> limit_sub_bbox((0, 1, 10, 11), (-1, -1, 9, 8))
+    (0, 1, 9, 8)
+    >>> limit_sub_bbox((0, 0, 10, 10), (5, 2, 18, 18))
+    (5, 2, 10, 10)
+    """
+    minx = max(bbox[0], sub_bbox[0])
+    miny = max(bbox[1], sub_bbox[1])
+    maxx = min(bbox[2], sub_bbox[2])
+    maxy = min(bbox[3], sub_bbox[3])
+    return minx, miny, maxx, maxy
+
+def timestamp():
+    return datetime.now().strftime('%H:%M:%S')
+
+def format_bbox(bbox):
+    return ('%.5f, %.5f, %.5f, %.5f') % tuple(bbox)
+
+def status_symbol(i, total):
+    """
+    >>> status_symbol(0, 1)
+    '0'
+    >>> [status_symbol(i, 4) for i in range(5)]
+    ['.', 'o', 'O', '0', 'X']
+    >>> [status_symbol(i, 10) for i in range(11)]
+    ['.', '.', 'o', 'o', 'o', 'O', 'O', '0', '0', '0', 'X']
+    """
+    symbols = list(' .oO0')
+    i += 1
+    if 0 < i > total:
+        return 'X'
+    else:
+        return symbols[int(math.ceil(i/(total/4)))]
+
+class BackoffError(Exception):
+    pass
+
+def exp_backoff(func, args=(), kw={}, max_repeat=10, start_backoff_sec=2,
+        exceptions=(Exception,), ignore_exceptions=tuple(), max_backoff=60):
+    n = 0
+    while True:
+        try:
+            result = func(*args, **kw)
+        except ignore_exceptions:
+            time.sleep(0.01)
+        except exceptions as ex:
+            if n >= max_repeat:
+                print("An error occured. Giving up", file=sys.stderr)
+                raise BackoffError
+            wait_for = start_backoff_sec * 2**n
+            if wait_for > max_backoff:
+                wait_for = max_backoff
+            print("An error occured. Retry in %d seconds: %r. Retries left: %d" %
+                (wait_for, ex, (max_repeat - n)), file=sys.stderr)
+            time.sleep(wait_for)
+            n += 1
+        else:
+            return result
+
+def format_seed_task(task):
+    info = []
+    info.append('  %s:' % (task.md['name'], ))
+    if task.coverage is False:
+        info.append("    Empty coverage given for this task")
+        info.append("    Skipped")
+        return '\n'.join(info)
+
+    info.append("    Seeding cache '%s' with grid '%s' in %s" % (
+                 task.md['cache_name'], task.md['grid_name'], task.grid.srs.srs_code))
+    if task.coverage:
+        info.append('    Limited to coverage in: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
+    else:
+        info.append('   Complete grid: %s (EPSG:4326)' % (format_bbox(map_extent_from_grid(task.grid).llbbox), ))
+    info.append('    Levels: %s' % (task.levels, ))
+
+    if task.refresh_timestamp:
+        info.append('    Overwriting: tiles older than %s' %
+                    datetime.fromtimestamp(task.refresh_timestamp))
+    elif task.refresh_timestamp == 0:
+        info.append('    Overwriting: all tiles')
+    else:
+        info.append('    Overwriting: no tiles')
+
+    return '\n'.join(info)
+
+def format_cleanup_task(task):
+    info = []
+    info.append('  %s:' % (task.md['name'], ))
+    if task.coverage is False:
+        info.append("    Empty coverage given for this task")
+        info.append("    Skipped")
+        return '\n'.join(info)
+
+    info.append("    Cleaning up cache '%s' with grid '%s' in %s" % (
+                 task.md['cache_name'], task.md['grid_name'], task.grid.srs.srs_code))
+    if task.coverage:
+        info.append('    Limited to coverage in: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
+    else:
+        info.append('    Complete grid: %s (EPSG:4326)' % (format_bbox(map_extent_from_grid(task.grid).llbbox), ))
+    info.append('    Levels: %s' % (task.levels, ))
+
+    if task.remove_timestamp:
+        info.append('    Remove: tiles older than %s' %
+                    datetime.fromtimestamp(task.remove_timestamp))
+    else:
+        info.append('    Remove: all tiles')
+
+    return '\n'.join(info)

mapproxy/service/__init__.py

@@ -0,0 +1,14 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mapproxy/service/base.py

@@ -0,0 +1,46 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Service handler (WMS, TMS, etc.).
+"""
+from mapproxy.exception import RequestError
+
+class Server(object):
+    names = tuple()
+    request_parser = lambda x: None
+    request_methods = ()
+    
+    def handle(self, req):
+        try:
+            parsed_req = self.parse_request(req)
+            handler = getattr(self, parsed_req.request_handler_name)
+            return handler(parsed_req)
+        except RequestError as e:
+            return e.render()
+    
+    def parse_request(self, req):
+        return self.request_parser(req)
+
+    def decorate_img(self, image, service, layers, environ, query_extent):
+        """ Callback that allows the ImageSource associated with a response to
+            be modified before it is returned. The callback is passed the
+            ImageSource instance and must return a valid ImageSource """
+        if 'mapproxy.decorate_img' in environ:
+            image = environ['mapproxy.decorate_img'](
+                image, service, layers, environ=environ, query_extent=query_extent)
+        return image
+
+

mapproxy/service/demo.py

@@ -0,0 +1,356 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Demo service handler
+"""
+from __future__ import division
+
+import logging
+import re
+
+import os
+import pkg_resources
+import mimetypes
+from collections import defaultdict
+
+from mapproxy.config.config import base_config
+from mapproxy.compat import PY2
+from mapproxy.exception import RequestError
+from mapproxy.service.base import Server
+from mapproxy.response import Response
+from mapproxy.srs import SRS, get_epsg_num
+from mapproxy.layer import SRSConditional, CacheMapLayer, ResolutionConditional
+from mapproxy.source.wms import WMSSource
+from mapproxy.util.escape import escape_html
+
+if PY2:
+    import urllib2
+else:
+    from urllib import request as urllib2
+
+from mapproxy.template import template_loader, bunch
+env = {'bunch': bunch}
+get_template = template_loader(__name__, 'templates', namespace=env)
+
+# Used by plugins
+extra_demo_server_handlers = set()
+
+logger = logging.getLogger("demo")
+
+
+def register_extra_demo_server_handler(handler):
+    """ Method used by plugins to register a new handler for the demo service.
+        The handler passed to this method is invoked by the DemoServer.handle()
+        method when receiving an incoming request, and let a chance to the
+        handler to process it, if it is relevant to it.
+
+        :param handler: New handler for incoming requests
+        :type handler: function that takes 2 arguments (DemoServer instance and req) and
+                       returns a string with HTML content or None
+    """
+
+    extra_demo_server_handlers.add(handler)
+
+
+extra_substitution_handlers = set()
+
+
+def register_extra_demo_substitution_handler(handler):
+    """ Method used by plugins to register a new handler for doing substitutions
+        to the HTML template used by the demo service.
+        The handler passed to this method is invoked by the DemoServer._render_template()
+        method. The handler may modify the passed substitutions dictionary
+        argument. Keys of particular interest are 'extra_services_html_beginning'
+        and 'extra_services_html_end' to add HTML content before/after built-in
+        services.
+
+        :param handler: New handler for incoming requests
+        :type handler: function that takes 3 arguments(DemoServer instance, req and a substitutions dictionary argument).
+    """
+
+    extra_substitution_handlers.add(handler)
+
+
+def static_filename(name):
+    if base_config().template_dir:
+        return os.path.join(base_config().template_dir, name)
+    else:
+        return pkg_resources.resource_filename(__name__, os.path.join('templates', name))
+
+class DemoServer(Server):
+    names = ('demo',)
+
+    def __init__(self, layers, md, request_parser=None, tile_layers=None,
+                 srs=None, image_formats=None, services=None, restful_template=None):
+        Server.__init__(self)
+        self.layers = layers
+        self.tile_layers = tile_layers or {}
+        self.md = md
+        self.image_formats = image_formats
+        filter_image_format = []
+        for format in self.image_formats:
+            if 'image/jpeg' == format or 'image/png' == format:
+                filter_image_format.append(format)
+        self.image_formats = filter_image_format
+        self.srs = srs
+        self.services = services or []
+        self.restful_template = restful_template
+
+    @staticmethod
+    def get_capabilities_url(service_path, req):
+        if 'type' in req.args and req.args['type'] == 'external':
+            url = escape_html(req.script_url) + service_path
+        else:
+            url = req.server_script_url + service_path
+        return url
+
+    @staticmethod
+    def valid_url(url):
+        pattern = re.compile("^https?://")
+        if pattern.match(url):
+            return True
+        else:
+            logging.warn(f"A request was blocked that was trying to access a non-http resource: {url}")
+            return False
+
+    @staticmethod
+    def read_capabilities(url):
+        resp = urllib2.urlopen(url)
+        return escape_html(resp.read().decode())
+
+    def handle(self, req):
+        if req.path.startswith('/demo/static/'):
+            if '..' in req.path:
+                return Response('file not found', content_type='text/plain', status=404)
+            filename = req.path.lstrip('/')
+            filename = static_filename(filename)
+            if not os.path.isfile(filename):
+                return Response('file not found', content_type='text/plain', status=404)
+            type, encoding = mimetypes.guess_type(filename)
+            return Response(open(filename, 'rb'), content_type=type)
+
+        # we don't authorize the static files (css, js)
+        # since they are not confidential
+        try:
+            authorized = self.authorized_demo(req.environ)
+        except RequestError as ex:
+            return ex.render()
+        if not authorized:
+            return Response('forbidden', content_type='text/plain', status=403)
+
+        for handler in extra_demo_server_handlers:
+            demo = handler(self, req)
+            if demo is not None:
+                return Response(demo, content_type='text/html')
+
+        if 'wms_layer' in req.args:
+            demo = self._render_wms_template('demo/wms_demo.html', req)
+        elif 'tms_layer' in req.args:
+            demo = self._render_tms_template('demo/tms_demo.html', req)
+        elif 'wmts_layer' in req.args:
+            demo = self._render_wmts_template('demo/wmts_demo.html', req)
+        elif 'wms_capabilities' in req.args:
+            url = self.get_capabilities_url('/service?REQUEST=GetCapabilities&SERVICE=WMS', req)
+            if not self.valid_url(url):
+                return Response('bad request', content_type='text/plain', status=400)
+            capabilities = self.read_capabilities(url)
+            demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMS', url)
+        elif 'wmsc_capabilities' in req.args:
+            url = self.get_capabilities_url('/service?REQUEST=GetCapabilities&SERVICE=WMS&tiled=true', req)
+            if not self.valid_url(url):
+                return Response('bad request', content_type='text/plain', status=400)
+            capabilities = self.read_capabilities(url)
+            demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMS-C', url)
+        elif 'wmts_capabilities_kvp' in req.args:
+            url = self.get_capabilities_url('/service?REQUEST=GetCapabilities&SERVICE=WMTS', req)
+            if not self.valid_url(url):
+                return Response('bad request', content_type='text/plain', status=400)
+            capabilities = self.read_capabilities(url)
+            demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMTS', url)
+        elif 'wmts_capabilities' in req.args:
+            url = self.get_capabilities_url('/wmts/1.0.0/WMTSCapabilities.xml', req)
+            if not self.valid_url(url):
+                return Response('bad request', content_type='text/plain', status=400)
+            capabilities = self.read_capabilities(url)
+            demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMTS', url)
+        elif 'tms_capabilities' in req.args:
+            if 'layer' in req.args and 'srs' in req.args:
+                # prevent dir traversal (seems it's not possible with urllib2, but better safe than sorry)
+                layer = escape_html(req.args['layer'].replace('..', ''))
+                srs = escape_html(req.args['srs'].replace('..', ''))
+                service_path = f'/tms/1.0.0/{layer}/{srs}'
+            else:
+                service_path = '/tms/1.0.0/'
+            url = self.get_capabilities_url(service_path, req)
+            if not self.valid_url(url):
+                return Response('bad request', content_type='text/plain', status=400)
+            capabilities = self.read_capabilities(url)
+            demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'TMS', url)
+        elif req.path == '/demo/':
+            demo = self._render_template(req, 'demo/demo.html')
+        else:
+            resp = Response('', status=301)
+            resp.headers['Location'] = escape_html(req.script_url).rstrip('/') + '/demo/'
+            return resp
+        return Response(demo, content_type='text/html')
+
+    def layer_srs(self, layer):
+        """
+        Return a list tuples with title and name of all SRS for the layer.
+        The title of SRS that are native to the layer are suffixed with a '*'.
+        """
+        cached_srs = []
+        for map_layer in layer.map_layers:
+            # TODO unify map_layers interface
+            if isinstance(map_layer, SRSConditional):
+                for srs_key in map_layer.srs_map.keys():
+                    cached_srs.append(srs_key.srs_code)
+            elif isinstance(map_layer, CacheMapLayer):
+                cached_srs.append(map_layer.grid.srs.srs_code)
+            elif isinstance(map_layer, ResolutionConditional):
+                cached_srs.append(map_layer.srs.srs_code)
+            elif isinstance(map_layer, WMSSource):
+                if map_layer.supported_srs:
+                    for supported_srs in map_layer.supported_srs:
+                        cached_srs.append(supported_srs.srs_code)
+
+        uncached_srs = []
+
+        for srs_code in self.srs:
+            if srs_code not in cached_srs:
+                uncached_srs.append(srs_code)
+
+        def get_srs_key(srs):
+            epsg_num = get_epsg_num(srs)
+            return str(epsg_num if epsg_num else srs)
+
+        sorted_cached_srs = sorted(cached_srs, key=lambda srs: get_srs_key(srs))
+        sorted_uncached_srs = sorted(uncached_srs, key=lambda srs: get_srs_key(srs))
+        sorted_cached_srs = [(s + '*', s) for s in sorted_cached_srs]
+        sorted_uncached_srs = [(s, s) for s in sorted_uncached_srs]
+        return sorted_cached_srs + sorted_uncached_srs
+
+    def _render_template(self, req, template):
+        template = get_template(template, default_inherit="demo/static.html")
+        tms_tile_layers = defaultdict(list)
+        for layer in self.tile_layers:
+            name = self.tile_layers[layer].md.get('name')
+            tms_tile_layers[name].append(self.tile_layers[layer])
+        wmts_layers = tms_tile_layers.copy()
+
+        substitutions = dict(
+            extra_services_html_beginning='',
+            extra_services_html_end='',
+            layers=self.layers,
+            formats=self.image_formats,
+            srs=self.srs,
+            layer_srs=self.layer_srs,
+            tms_layers=tms_tile_layers,
+            wmts_layers=wmts_layers,
+            services=self.services
+        )
+
+        for add_substitution in extra_substitution_handlers:
+            add_substitution(self, req, substitutions)
+
+        return template.substitute(substitutions)
+
+    def _render_wms_template(self, template, req):
+        template = get_template(template, default_inherit="demo/static.html")
+        layer = self.layers[req.args['wms_layer']]
+        srs = escape_html(req.args['srs'])
+        bbox = layer.extent.bbox_for(SRS(srs))
+        width = bbox[2] - bbox[0]
+        height = bbox[3] - bbox[1]
+        min_res = max(width/256, height/256)
+        return template.substitute(layer=layer,
+                                   image_formats=self.image_formats,
+                                   format=escape_html(req.args['format']),
+                                   srs=srs,
+                                   layer_srs=self.layer_srs,
+                                   bbox=bbox,
+                                   res=min_res)
+
+    def _render_tms_template(self, template, req):
+        template = get_template(template, default_inherit="demo/static.html")
+        for layer in self.tile_layers.values():
+            if (layer.name == req.args['tms_layer'] and
+                    layer.grid.srs.srs_code == req.args['srs']):
+                tile_layer = layer
+                break
+
+        resolutions = tile_layer.grid.tile_sets
+        res = []
+        for level, resolution in resolutions:
+            res.append(resolution)
+
+        if tile_layer.grid.srs.is_latlong:
+            units = 'degree'
+        else:
+            units = 'm'
+
+        if tile_layer.grid.profile == 'local':
+            add_res_to_options = True
+        else:
+            add_res_to_options = False
+        return template.substitute(layer=tile_layer,
+                                   srs=escape_html(req.args['srs']),
+                                   format=escape_html(req.args['format']),
+                                   resolutions=res,
+                                   units=units,
+                                   add_res_to_options=add_res_to_options,
+                                   all_tile_layers=self.tile_layers)
+
+    def _render_wmts_template(self, template, req):
+        template = get_template(template, default_inherit="demo/static.html")
+        for layer in self.tile_layers.values():
+            if (layer.name == req.args['wmts_layer'] and
+                    layer.grid.srs.srs_code == req.args['srs']):
+                wmts_layer = layer
+                break
+
+        restful_url = self.restful_template.replace('{Layer}', wmts_layer.name, 1)
+        if '{Format}' in restful_url:
+            restful_url = restful_url.replace('{Format}', wmts_layer.format)
+
+        if wmts_layer.grid.srs.is_latlong:
+            units = 'degree'
+        else:
+            units = 'm'
+        return template.substitute(layer=wmts_layer,
+                                   matrix_set=wmts_layer.grid.name,
+                                   format=escape_html(req.args['format']),
+                                   srs=escape_html(req.args['srs']),
+                                   resolutions=wmts_layer.grid.resolutions,
+                                   units=units,
+                                   all_tile_layers=self.tile_layers,
+                                   restful_url=restful_url)
+
+    def _render_capabilities_template(self, template, xmlfile, service, url):
+        template = get_template(template, default_inherit="demo/static.html")
+        return template.substitute(capabilities = xmlfile,
+                                   service = service,
+                                   url = url)
+
+    def authorized_demo(self, environ):
+        if 'mapproxy.authorize' in environ:
+            result = environ['mapproxy.authorize']('demo', [], environ=environ)
+            if result['authorized'] == 'unauthenticated':
+                raise RequestError('unauthorized', status=401)
+            if result['authorized'] == 'full':
+                return True
+            return False
+        return True