MapProxy 2.1.0__py3-none-any.whl

mapproxy/script/util.py

@@ -0,0 +1,410 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import io
+import os
+import optparse
+import re
+import shutil
+import sys
+import textwrap
+import logging
+
+from mapproxy.config.loader import load_plugins
+from mapproxy.script.conf.app import config_command
+from mapproxy.script.defrag import defrag_command
+from mapproxy.script.export import export_command
+from mapproxy.script.grids import grids_command
+from mapproxy.script.scales import scales_command
+from mapproxy.script.wms_capabilities import wms_capabilities_command
+from mapproxy.version import version
+
+
+def setup_logging(level=logging.INFO, format=None):
+    mapproxy_log = logging.getLogger('mapproxy')
+    mapproxy_log.setLevel(level)
+
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setLevel(level)
+    if not format:
+        format = "[%(asctime)s] %(name)s - %(levelname)s - %(message)s"
+    formatter = logging.Formatter(format)
+    ch.setFormatter(formatter)
+    mapproxy_log.addHandler(ch)
+
+
+def serve_develop_command(args):
+    parser = optparse.OptionParser("usage: %prog serve-develop [options] mapproxy.yaml")
+    parser.add_option("-b", "--bind",
+                      dest="address", default='127.0.0.1:8080',
+                      help="Server socket [127.0.0.1:8080]. Use 0.0.0.0 for external access. :1234 to change port.")
+    parser.add_option("--debug", default=False, action='store_true',
+                      dest="debug",
+                      help="Enable debug mode")
+    options, args = parser.parse_args(args)
+
+    if len(args) != 2:
+        parser.print_help()
+        print("\nERROR: MapProxy configuration required.")
+        sys.exit(1)
+
+    mapproxy_conf = args[1]
+
+    host, port = parse_bind_address(options.address)
+
+    if options.debug and host not in ('localhost', '127.0.0.1'):
+        print(textwrap.dedent("""\
+        ################# WARNING! ##################
+        Running debug mode with non-localhost address
+        is a serious security vulnerability.
+        #############################################\
+        """))
+
+    if options.debug:
+        setup_logging(level=logging.DEBUG)
+    else:
+        setup_logging()
+    from mapproxy.wsgiapp import make_wsgi_app
+    from mapproxy.config.loader import ConfigurationError
+    from mapproxy.util.ext.serving import run_simple
+    try:
+        app = make_wsgi_app(mapproxy_conf, debug=options.debug)
+    except ConfigurationError:
+        sys.exit(2)
+
+    extra_files = app.config_files.keys()
+
+    if options.debug:
+        try:
+            from repoze.profile import ProfileMiddleware
+            app = ProfileMiddleware(
+                app,
+                log_filename='/tmp/mapproxy_profile.log',
+                discard_first_request=True,
+                flush_at_shutdown=True,
+                path='/__profile__',
+                unwind=False,
+            )
+            print('Installed profiler at /__profile__')
+        except ImportError:
+            pass
+
+    run_simple(host, port, app, use_reloader=True, processes=1,
+               threaded=True, passthrough_errors=True,
+               extra_files=extra_files)
+
+
+def serve_multiapp_develop_command(args):
+    parser = optparse.OptionParser("usage: %prog serve-multiapp-develop [options] projects/")
+    parser.add_option("-b", "--bind",
+                      dest="address", default='127.0.0.1:8080',
+                      help="Server socket [127.0.0.1:8080]")
+    parser.add_option("--debug", default=False, action='store_true',
+                      dest="debug",
+                      help="Enable debug mode")
+    options, args = parser.parse_args(args)
+
+    if len(args) != 2:
+        parser.print_help()
+        print("\nERROR: MapProxy projects directory required.")
+        sys.exit(1)
+
+    mapproxy_conf_dir = args[1]
+
+    host, port = parse_bind_address(options.address)
+
+    if options.debug and host not in ('localhost', '127.0.0.1'):
+        print(textwrap.dedent("""\
+        ################# WARNING! ##################
+        Running debug mode with non-localhost address
+        is a serious security vulnerability.
+        #############################################\
+        """))
+
+    setup_logging()
+    from mapproxy.multiapp import make_wsgi_app
+    from mapproxy.util.ext.serving import run_simple
+    app = make_wsgi_app(mapproxy_conf_dir, debug=options.debug)
+
+    run_simple(host, port, app, use_reloader=True, processes=1,
+               threaded=True, passthrough_errors=True)
+
+
+def parse_bind_address(address, default=('localhost', 8080)):
+    """
+    >>> parse_bind_address('80')
+    ('localhost', 80)
+    >>> parse_bind_address('0.0.0.0')
+    ('0.0.0.0', 8080)
+    >>> parse_bind_address('0.0.0.0:8081')
+    ('0.0.0.0', 8081)
+    """
+    if ':' in address:
+        host, port = address.split(':', 1)
+        port = int(port)
+    elif re.match(r'^\d+$', address):
+        host = default[0]
+        port = int(address)
+    else:
+        host = address
+        port = default[1]
+    return host, port
+
+
+def create_command(args):
+    cmd = CreateCommand(args)
+    cmd.run()
+
+
+class CreateCommand(object):
+    templates = {
+        'base-config': {},
+        'wsgi-app': {},
+        'log-ini': {},
+    }
+
+    def __init__(self, args):
+        parser = optparse.OptionParser("usage: %prog create [options] [destination]")
+        parser.add_option("-t", "--template", dest="template",
+                          help="Create a configuration from this template.")
+        parser.add_option("-l", "--list-templates", dest="list_templates",
+                          action="store_true", default=False,
+                          help="List all available configuration templates.")
+        parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
+                          help="Existing MapProxy configuration (required for some templates).")
+        parser.add_option("--force", dest="force", action="store_true",
+                          default=False, help="Force operation (e.g. overwrite existing files).")
+
+        self.options, self.args = parser.parse_args(args)
+        self.parser = parser
+
+    def log_error(self, msg, *args):
+        print('ERROR:', msg % args, file=sys.stderr)
+
+    def run(self):
+
+        if self.options.list_templates:
+            print_items(self.templates, title="Available templates")
+            sys.exit(1)
+        elif self.options.template:
+            if self.options.template not in self.templates:
+                self.log_error("unknown template " + self.options.template)
+                sys.exit(1)
+
+            if len(self.args) != 2:
+                self.log_error("template requires destination argument")
+                sys.exit(1)
+
+            sys.exit(
+                getattr(self, 'template_' + self.options.template.replace('-', '_'))()
+            )
+        else:
+            self.parser.print_help()
+            sys.exit(1)
+
+    @property
+    def mapproxy_conf(self):
+        if not self.options.mapproxy_conf:
+            self.parser.print_help()
+            self.log_error("template requires --mapproxy-conf option")
+            sys.exit(1)
+        return os.path.abspath(self.options.mapproxy_conf)
+
+    def template_dir(self):
+        import mapproxy.config_template
+        template_dir = os.path.join(
+            os.path.dirname(mapproxy.config_template.__file__),
+            'base_config')
+        return template_dir
+
+    def template_wsgi_app(self):
+        app_filename = self.args[1]
+        if '.' not in os.path.basename(app_filename):
+            app_filename += '.py'
+        mapproxy_conf = self.mapproxy_conf
+        if os.path.exists(app_filename) and not self.options.force:
+            self.log_error("%s already exists, use --force", app_filename)
+            return 1
+
+        print("writing MapProxy app to %s" % (app_filename, ))
+
+        template_dir = self.template_dir()
+        app_template = io.open(os.path.join(template_dir, 'config.wsgi'), encoding='utf-8').read()
+        with io.open(app_filename, 'w', encoding='utf-8') as f:
+            f.write(app_template % {'mapproxy_conf': mapproxy_conf,
+                                    'here': os.path.dirname(mapproxy_conf)})
+
+        return 0
+
+    def template_base_config(self):
+        outdir = self.args[1]
+        if not os.path.exists(outdir):
+            os.makedirs(outdir)
+
+        template_dir = self.template_dir()
+
+        for filename in ('mapproxy.yaml', 'seed.yaml',
+                         'full_example.yaml', 'full_seed_example.yaml'):
+            to = os.path.join(outdir, filename)
+            from_ = os.path.join(template_dir, filename)
+            if os.path.exists(to) and not self.options.force:
+                self.log_error("%s already exists, use --force", to)
+                return 1
+            print("writing %s" % (to, ))
+            shutil.copy(from_, to)
+
+        return 0
+
+    def template_log_ini(self):
+        log_filename = self.args[1]
+
+        if os.path.exists(log_filename) and not self.options.force:
+            self.log_error("%s already exists, use --force", log_filename)
+            return 1
+
+        template_dir = self.template_dir()
+        log_template = io.open(os.path.join(template_dir, 'log.ini'), encoding='utf-8').read()
+        with io.open(log_filename, 'w', encoding='utf-8') as f:
+            f.write(log_template)
+
+        return 0
+
+
+commands = {
+    'serve-develop': {
+        'func': serve_develop_command,
+        'help': 'Run MapProxy development server.'
+    },
+    'serve-multiapp-develop': {
+        'func': serve_multiapp_develop_command,
+        'help': 'Run MultiMapProxy development server.'
+    },
+    'create': {
+        'func': create_command,
+        'help': 'Create example configurations.'
+    },
+    'scales': {
+        'func': scales_command,
+        'help': 'Convert between scales and resolutions.'
+    },
+    'wms-capabilities': {
+        'func': wms_capabilities_command,
+        'help': 'Display WMS capabilites.',
+    },
+    'grids': {
+        'func': grids_command,
+        'help': 'Display detailed informations for configured grids.'
+    },
+    'export': {
+        'func': export_command,
+        'help': 'Export existing caches.'
+    },
+    'autoconfig': {
+        'func': config_command,
+        'help': 'Create config from WMS capabilities or a Geopackage file.'
+    },
+    'defrag-compact-cache': {
+        'func': defrag_command,
+        'help': 'De-fragmentate compact caches.'
+    }
+}
+
+
+def register_command(command_name, command_spec):
+    """ Method used by plugins to register a command.
+
+        :param command_name: Name of the service
+        :type command_name: str
+        :param command_spec: Definition of the command. Dictionary with a 'func' and 'help' member
+        :type command_spec: dict
+    """
+
+    commands[command_name] = command_spec
+
+
+class NonStrictOptionParser(optparse.OptionParser):
+    def _process_args(self, largs, rargs, values):
+        while rargs:
+            arg = rargs[0]
+            # We handle bare "--" explicitly, and bare "-" is handled by the
+            # standard arg handler since the short arg case ensures that the
+            # len of the opt string is greater than 1.
+            try:
+                if arg == "--":
+                    del rargs[0]
+                    return
+                elif arg[0:2] == "--":
+                    # process a single long option (possibly with value(s))
+                    self._process_long_opt(rargs, values)
+                elif arg[:1] == "-" and len(arg) > 1:
+                    # process a cluster of short options (possibly with
+                    # value(s) for the last one only)
+                    self._process_short_opts(rargs, values)
+                elif self.allow_interspersed_args:
+                    largs.append(arg)
+                    del rargs[0]
+                else:
+                    return
+            except optparse.BadOptionError:
+                largs.append(arg)
+
+
+def print_items(data, title='Commands'):
+    name_len = max(len(name) for name in data)
+
+    if title:
+        print('%s:' % (title, ), file=sys.stdout)
+    for name, item in data.items():
+        help = item.get('help', '')
+        name = ('%%-%ds' % name_len) % name
+        if help:
+            help = '  ' + help
+        print('  %s%s' % (name, help), file=sys.stdout)
+
+
+def main():
+
+    load_plugins()
+
+    parser = NonStrictOptionParser("usage: %prog COMMAND [options]",
+                                   add_help_option=False)
+    options, args = parser.parse_args()
+
+    if len(args) < 1 or args[0] in ('--help', '-h'):
+        parser.print_help()
+        print()
+        print_items(commands)
+        sys.exit(1)
+
+    if len(args) == 1 and args[0] == '--version':
+        print('MapProxy ' + version)
+        sys.exit(1)
+
+    command = args[0]
+    if command not in commands:
+        parser.print_help()
+        print()
+        print_items(commands)
+        print('\nERROR: unknown command %s' % (command,), file=sys.stdout)
+        sys.exit(1)
+
+    args = sys.argv[0:1] + sys.argv[2:]
+    commands[command]['func'](args)
+
+
+if __name__ == '__main__':
+    main()

mapproxy/script/wms_capabilities.py

@@ -0,0 +1,160 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import sys
+import optparse
+from io import BytesIO
+from urllib.parse import urlparse
+
+from xml import etree
+
+from mapproxy.client.http import open_url, HTTPClientError
+from mapproxy.request.base import BaseRequest, url_decode
+from mapproxy.util.ext import wmsparse
+
+
+class PrettyPrinter(object):
+    def __init__(self, indent=4, version='1.1.1'):
+        self.indent = indent
+        self.print_order = ['name', 'title', 'url', 'srs', 'llbbox', 'bbox']
+        self.marker = '- '
+        self.version = version
+
+    def print_line(self, indent, key, value=None, mark_first=False):
+        marker = ''
+        if value is None:
+            value = ''
+        if mark_first:
+            indent = indent - len(self.marker)
+            marker = self.marker
+        print(("%s%s%s: %s" % (' '*indent, marker, key, value)))
+
+    def _format_output(self, key, value, indent, mark_first=False):
+        if key == 'bbox':
+            self.print_line(indent, key)
+            for srs_code, bbox in value.items():
+                self.print_line(indent+self.indent, srs_code, value=bbox, mark_first=mark_first)
+        else:
+            if isinstance(value, set):
+                value = list(value)
+            self.print_line(indent, key, value=value, mark_first=mark_first)
+
+    def print_layers(self, capabilities, indent=None, root=False):
+        if root:
+            print("# Note: This is not a valid MapProxy configuration!")
+            print('Capabilities Document Version %s' % (self.version,))
+            print('Root-Layer:')
+            layer_list = capabilities.layers()['layers']
+        else:
+            layer_list = capabilities['layers']
+
+        indent = indent or self.indent
+        for layer in layer_list:
+            marked_first = False
+            # print ordered items first
+            for item in self.print_order:
+                if layer.get(item, False):
+                    if not marked_first:
+                        marked_first = True
+                        self._format_output(item, layer[item], indent, mark_first=marked_first)
+                    else:
+                        self._format_output(item, layer[item], indent)
+            # print remaining items except sublayers
+            for key, value in layer.items():
+                if key in self.print_order or key == 'layers':
+                    continue
+                self._format_output(key, value, indent)
+            # print the sublayers now
+            if layer.get('layers', False):
+                self.print_line(indent, 'layers')
+                self.print_layers(layer, indent=indent+self.indent)
+
+
+def log_error(msg, *args):
+    print(msg % args, file=sys.stderr)
+
+
+def wms_capapilities_url(url, version):
+    parsed_url = urlparse(url)
+    base_req = BaseRequest(
+        url=url.split('?', 1)[0],
+        param=url_decode(parsed_url.query),
+    )
+
+    base_req.params['service'] = 'WMS'
+    base_req.params['version'] = version
+    base_req.params['request'] = 'GetCapabilities'
+    return base_req.complete_url
+
+
+def parse_capabilities(fileobj, version='1.1.1'):
+    try:
+        return wmsparse.parse_capabilities(fileobj)
+    except ValueError as ex:
+        log_error('%s\n%s\n%s\n%s\nNot a capabilities document: %s',
+                  'Recieved document:', '-'*80, fileobj.getvalue(), '-'*80, ex.args[0])
+        sys.exit(1)
+    except etree.ElementTree.ParseError as ex:
+        log_error('%s\n%s\n%s\n%s\nCould not parse the document: %s',
+                  'Recieved document:', '-'*80, fileobj.getvalue(), '-'*80, ex.args[0])
+        sys.exit(1)
+
+
+def parse_capabilities_url(url, version='1.1.1'):
+    try:
+        capabilities_url = wms_capapilities_url(url, version)
+        capabilities_response = open_url(capabilities_url)
+    except HTTPClientError as ex:
+        log_error('ERROR: %s', ex.args[0])
+        sys.exit(1)
+
+    # after parsing capabilities_response will be empty, therefore cache it
+    capabilities = BytesIO(capabilities_response.read())
+    return parse_capabilities(capabilities, version=version)
+
+
+def wms_capabilities_command(args=None):
+    parser = optparse.OptionParser("%prog wms-capabilities [options] URL",
+                                   description="Read and parse WMS 1.1.1 capabilities and print out"
+                                   " information about each layer. It does _not_ return a valid"
+                                   " MapProxy configuration.")
+    parser.add_option("--host", dest="capabilities_url",
+                      help="WMS Capabilites URL")
+    parser.add_option(
+        "--version", dest="version", choices=['1.1.1', '1.3.0'], default='1.1.1',
+        help="Request GetCapabilities-document in version 1.1.1 or 1.3.0", metavar="<1.1.1 or 1.3.0>")
+
+    if args:
+        args = args[1:]  # remove script name
+
+    (options, args) = parser.parse_args(args)
+    if not options.capabilities_url:
+        if len(args) != 1:
+            parser.print_help()
+            sys.exit(2)
+        else:
+            options.capabilities_url = args[0]
+
+    try:
+        service = parse_capabilities_url(options.capabilities_url, version=options.version)
+
+        printer = PrettyPrinter(indent=4, version=options.version)
+        printer.print_layers(service, root=True)
+
+    except KeyError as ex:
+        log_error('XML-Element has no such attribute (%s).' % (ex.args[0],))
+        sys.exit(1)

mapproxy/seed/cachelock.py

@@ -0,0 +1,127 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2012 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import errno
+import os
+import sqlite3
+import time
+from contextlib import contextmanager
+
+
+class CacheLockedError(Exception):
+    pass
+
+
+class CacheLocker(object):
+    def __init__(self, lockfile, polltime=0.1):
+        self.lockfile = lockfile
+        self.polltime = polltime
+        self._initialize_lockfile()
+
+    def _initialize_lockfile(self):
+        db = sqlite3.connect(self.lockfile)
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS cache_locks (
+                cache_name TEXT NOT NULL,
+                created REAL NOT NULL,
+                pid INTEGER NUT NULL
+            );
+        """)
+        db.commit()
+        db.close()
+
+    @contextmanager
+    def _exclusive_db_cursor(self):
+        db = sqlite3.connect(self.lockfile, isolation_level="EXCLUSIVE")
+        db.row_factory = sqlite3.Row
+        cur = db.cursor()
+
+        try:
+            yield cur
+        finally:
+            db.commit()
+            db.close()
+
+    @contextmanager
+    def lock(self, cache_name, no_block=False):
+
+        pid = os.getpid()
+
+        while True:
+            with self._exclusive_db_cursor() as cur:
+                self._add_lock(cur, cache_name, pid)
+                if self._poll(cur, cache_name, pid):
+                    break
+                elif no_block:
+                    raise CacheLockedError()
+            time.sleep(self.polltime)
+
+        try:
+            yield
+        finally:
+            with self._exclusive_db_cursor() as cur:
+                self._remove_lock(cur, cache_name, pid)
+
+    def _poll(self, cur, cache_name, pid):
+        active_locks = False
+        cur.execute("SELECT * from cache_locks where cache_name = ? ORDER BY created", (cache_name, ))
+
+        for lock in cur:
+            if not active_locks and lock['cache_name'] == cache_name and lock['pid'] == pid:
+                # we are waiting and it is out turn
+                return True
+
+            if not is_running(lock['pid']):
+                self._remove_lock(cur, lock['cache_name'], lock['pid'])
+            else:
+                active_locks = True
+
+        return not active_locks
+
+    def _add_lock(self, cur, cache_name, pid):
+        cur.execute("SELECT count(*) from cache_locks WHERE cache_name = ? AND pid = ?", (cache_name, pid))
+        if cur.fetchone()[0] == 0:
+            cur.execute("INSERT INTO cache_locks (cache_name, pid, created) VALUES (?, ?, ?)",
+                        (cache_name, pid, time.time()))
+
+    def _remove_lock(self, cur, cache_name, pid):
+        cur.execute("DELETE FROM cache_locks WHERE cache_name = ?  AND pid = ?", (cache_name, pid))
+
+
+class DummyCacheLocker(object):
+    @contextmanager
+    def lock(self, cache_name, no_block=False):
+        yield
+
+
+def is_running(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError as err:
+        if err.errno == errno.ESRCH:
+            return False
+        elif err.errno == errno.EPERM:
+            return True
+        else:
+            raise err
+    else:
+        return True
+
+
+if __name__ == '__main__':
+    locker = CacheLocker('/tmp/cachelock_test')
+    with locker.lock('foo'):
+        pass