MapProxy 1.16.1__py3-none-any.whl

mapproxy/script/conf/utils.py

@@ -0,0 +1,143 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import copy
+from mapproxy.compat import iteritems
+
+__all__ = ['update_config', 'MapProxyYAMLDumper']
+
+def update_config(conf, overwrites):
+    wildcard_keys = []
+    for k, v in iteritems(overwrites):
+        if k == '__all__':
+            continue
+        if  k.startswith('___') or k.endswith('___'):
+            wildcard_keys.append(k)
+            continue
+
+        if k.endswith('__extend__'):
+            k = k[:-len('__extend__')]
+            if k not in conf:
+                conf[k] = v
+            elif isinstance(v, list):
+                conf[k].extend(v)
+            else:
+                raise ValueError('cannot extend non-list:', v)
+        elif k not in conf:
+            conf[k] = copy(v)
+        else:
+            if isinstance(conf[k], dict) and isinstance(v, dict):
+                conf[k] = update_config(conf[k], v)
+            else:
+                conf[k] = copy(v)
+
+    if '__all__' in overwrites:
+        v = overwrites['__all__']
+        for conf_k, conf_v in iteritems(conf):
+            if isinstance(conf_v, dict):
+                conf[conf_k] = update_config(conf_v, v)
+            else:
+                conf[conf_k] = v
+
+    if wildcard_keys:
+        for key in wildcard_keys:
+            v = overwrites[key]
+            if key.startswith('___'):
+                key = key[3:]
+                key_check = lambda x: x.endswith(key)
+            else:
+                key = key[:-3]
+                key_check = lambda x: x.startswith(key)
+            for conf_k, conf_v in iteritems(conf):
+                if not key_check(conf_k):
+                    continue
+                if isinstance(conf_v, dict):
+                    conf[conf_k] = update_config(conf_v, v)
+                else:
+                    conf[conf_k] = v
+
+    return conf
+
+
+from yaml.serializer import Serializer
+from yaml.nodes import ScalarNode, SequenceNode, MappingNode
+from yaml.emitter import Emitter
+from yaml.representer import SafeRepresenter
+from yaml.resolver import Resolver
+
+class _MixedFlowSortedSerializer(Serializer):
+    def serialize_node(self, node, parent, index):
+        # reset any anchors
+        if parent is None:
+            for k in self.anchors:
+                self.anchors[k] = None
+        self.serialized_nodes = {}
+
+        if isinstance(node, SequenceNode) and all(isinstance(item, ScalarNode) for item in node.value):
+            node.flow_style = True
+        elif isinstance(node, MappingNode):
+            node.value.sort(key=lambda x: x[0].value)
+        return Serializer.serialize_node(self, node, parent, index)
+
+class _EmptyNoneRepresenter(SafeRepresenter):
+    def represent_none(self, data):
+        return self.represent_scalar(u'tag:yaml.org,2002:null',
+                u'')
+_EmptyNoneRepresenter.add_representer(type(None), _EmptyNoneRepresenter.represent_none)
+
+class MapProxyYAMLDumper(Emitter, _MixedFlowSortedSerializer, _EmptyNoneRepresenter, Resolver):
+    """
+    YAML dumper that uses block style by default, except for
+    node-only sequences. Also sorts dicts by key, prevents `none`
+    for empty entries and prevents any anchors.
+    """
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=None):
+        Emitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width,
+                allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        _EmptyNoneRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
+from mapproxy.request.base import BaseRequest, url_decode
+from mapproxy.client.http import open_url
+from mapproxy.compat.modules import urlparse
+
+def wms_capapilities_url(url):
+    parsed_url = urlparse.urlparse(url)
+    base_req = BaseRequest(
+        url=url.split('?', 1)[0],
+        param=url_decode(parsed_url.query),
+    )
+
+    base_req.params['service'] = 'WMS'
+    if not base_req.params['version']:
+        base_req.params['version'] = '1.1.1'
+    base_req.params['request'] = 'GetCapabilities'
+    return base_req.complete_url
+
+def download_capabilities(url):
+    capabilities_url = wms_capapilities_url(url)
+    return open_url(capabilities_url)
+

mapproxy/script/defrag.py

@@ -0,0 +1,184 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2017 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import glob
+import optparse
+import os.path
+import re
+import sys
+from collections import OrderedDict
+
+from mapproxy.cache.compact import CompactCacheV1, CompactCacheV2
+from mapproxy.cache.tile import Tile
+from mapproxy.config import local_base_config
+from mapproxy.config.loader import load_configuration, ConfigurationError
+
+import logging
+log = logging.getLogger('mapproxy.defrag')
+
+def defrag_command(args=None):
+    parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf")
+    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
+        help="MapProxy configuration.")
+
+    parser.add_option("--min-percent", type=float, default=10.0,
+        help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)")
+
+    parser.add_option("--min-mb", type=float, default=1.0,
+        help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)")
+
+    parser.add_option("--dry-run", "-n", action="store_true",
+        help="Do not de-fragment, only print output")
+
+    parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...',
+        help="only defragment the named caches")
+
+
+    from mapproxy.script.util import setup_logging
+    import logging
+    setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s")
+
+    if args:
+        args = args[1:] # remove script name
+
+    (options, args) = parser.parse_args(args)
+    if not options.mapproxy_conf:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        proxy_configuration = load_configuration(options.mapproxy_conf)
+    except IOError as e:
+        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
+        sys.exit(2)
+    except ConfigurationError as e:
+        print(e, file=sys.stderr)
+        print('ERROR: invalid configuration (see above)', file=sys.stderr)
+        sys.exit(2)
+
+
+    with local_base_config(proxy_configuration.base_config):
+        available_caches = OrderedDict()
+        for name, cache_conf in proxy_configuration.caches.items():
+            for grid, extent, tile_mgr in cache_conf.caches():
+                if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)):
+                    available_caches.setdefault(name, []).append(tile_mgr.cache)
+
+        if options.cache_names:
+            defrag_caches = options.cache_names.split(',')
+            missing = set(defrag_caches).difference(available_caches.keys())
+            if missing:
+                print('unknown caches: %s' % (', '.join(missing), ))
+                print('available compact caches: %s' %
+                      (', '.join(available_caches.keys()), ))
+                sys.exit(1)
+        else:
+            defrag_caches = None
+
+        for name, caches in available_caches.items():
+            if defrag_caches and name not in defrag_caches:
+                continue
+            for cache in caches:
+                logger = DefragLog(name)
+                defrag_compact_cache(cache,
+                    min_percent=options.min_percent/100,
+                    min_bytes=options.min_mb*1024*1024,
+                    dry_run=options.dry_run,
+                    log_progress=logger,
+                )
+
+def bundle_offset(fname):
+    """
+    >>> bundle_offset("path/to/R0000C0000.bundle")
+    (0, 0)
+    >>> bundle_offset("path/to/R0380C1380.bundle")
+    (4992, 896)
+    """
+    match = re.search(r'R([A-F0-9]{4,})C([A-F0-9]{4,}).bundle$', fname, re.IGNORECASE)
+    if match:
+        r = int(match.group(1), 16)
+        c = int(match.group(2), 16)
+        return c, r
+
+class DefragLog(object):
+    def __init__(self, cache_name):
+        self.cache_name = cache_name
+    def log(self, fname, fragmentation, fragmentation_bytes, num, total, defrag):
+        msg = "%s: %3d/%d (%s) fragmentation is %.1f%% (%dkb)" % (
+            self.cache_name, num, total, fname, fragmentation, fragmentation_bytes/1024
+        )
+        if defrag:
+            msg += " - defragmenting"
+        else:
+            msg += " - skipping"
+        log.info(msg)
+
+def defrag_compact_cache(cache, min_percent=0.1, min_bytes=1024*1024, log_progress=None, dry_run=False):
+    bundles = glob.glob(os.path.join(cache.cache_dir, 'L??', 'R????C????.bundle'))
+
+    for i, bundle_file in enumerate(bundles):
+        offset = bundle_offset(bundle_file)
+        b = cache.bundle_class(bundle_file.rstrip('.bundle'), offset)
+        size, file_size = b.size()
+
+        defrag = 1 - float(size) / file_size
+        defrag_bytes = file_size - size
+
+
+        skip = False
+        if defrag < min_percent or defrag_bytes < min_bytes:
+            skip = True
+
+        if log_progress:
+            log_progress.log(
+                fname=bundle_file,
+                fragmentation=defrag * 100,
+                fragmentation_bytes=defrag_bytes,
+                num=i+1, total=len(bundles),
+                defrag=not skip,
+            )
+
+        if skip or dry_run:
+            continue
+
+        tmp_bundle = os.path.join(cache.cache_dir, 'tmp_defrag')
+        defb = cache.bundle_class(tmp_bundle, offset)
+        stored_tiles = False
+
+        for y in range(128):
+            tiles = [Tile((x, y, 0)) for x in range(128)]
+            b.load_tiles(tiles)
+            tiles = [t for t in tiles if t.source]
+            if tiles:
+                stored_tiles = True
+                defb.store_tiles(tiles)
+
+        # remove first
+        # - in case bundle is empty
+        # - windows does not support rename to existing files
+        if os.path.exists(bundle_file):
+            os.remove(bundle_file)
+        if os.path.exists(bundle_file[:-1] + 'x'):
+            os.remove(bundle_file[:-1] + 'x')
+
+        if stored_tiles:
+            os.rename(tmp_bundle + '.bundle', bundle_file)
+            if os.path.exists(tmp_bundle + '.bundlx'):
+                os.rename(tmp_bundle + '.bundlx', bundle_file[:-1] + 'x')
+            if os.path.exists(tmp_bundle + '.lck'):
+                os.unlink(tmp_bundle + '.lck')
+

mapproxy/script/export.py

@@ -0,0 +1,333 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2012 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function, division
+
+import os
+import re
+import shlex
+import sys
+import optparse
+
+import yaml
+
+from mapproxy.srs import SRS
+from mapproxy.config.coverage import load_coverage
+from mapproxy.config.loader import (
+    load_configuration, ConfigurationError,
+    CacheConfiguration, GridConfiguration,
+    ProxyConfiguration,
+)
+from mapproxy.util.coverage import  BBOXCoverage
+from mapproxy.seed.util import ProgressLog, format_bbox
+from mapproxy.seed.seeder import SeedTask, seed_task
+from mapproxy.config import spec as conf_spec
+from mapproxy.util.ext.dictspec.validator import validate, ValidationError
+
+
+def parse_levels(level_str):
+    """
+    >>> parse_levels('1,2,3,6')
+    [1, 2, 3, 6]
+    >>> parse_levels('1..6')
+    [1, 2, 3, 4, 5, 6]
+    >>> parse_levels('1..6, 8, 9, 13..14')
+    [1, 2, 3, 4, 5, 6, 8, 9, 13, 14]
+    """
+    levels = set()
+    for part in level_str.split(','):
+        part = part.strip()
+        if re.match(r'\d+..\d+', part):
+            from_level, to_level = part.split('..')
+            levels.update(list(range(int(from_level), int(to_level) + 1)))
+        else:
+            levels.add(int(part))
+
+    return sorted(levels)
+
+
+def resolve_source(source_name, conf):
+    """
+    Resolves the source with the given name.
+
+    >>> config = ProxyConfiguration({'sources': {'mysource': {'type': 'wms'}}})
+    >>> resolve_source('mysource', config)
+    (<mapproxy.config.loader.WMSSourceConfiguration object at 0x...>, False)
+
+    >>> config = ProxyConfiguration({'caches': {'mysource': {}}})
+    >>> resolve_source('mysource', config)
+    (<mapproxy.config.loader.CacheConfiguration object at 0x...>, True)
+
+    >>> config = ProxyConfiguration({'caches': {'mysource': {'type': 'foo'}}})
+    >>> resolve_source('nonexistingsource', config)
+    (None, None)
+
+    :param source_name: The name of the source to resolve.
+    :param conf: The mapproxy config to resolve from.
+    :rtype: (dict, bool) A tuple containing the resolved source and a boolean
+        indicating if the resolved source is a cache (True), or a source (False).
+    """
+    resolved_source = conf.sources.get(source_name)
+    if resolved_source is not None:
+        return resolved_source, False
+    resolved_source = conf.caches.get(source_name)
+    if resolved_source is not None:
+        return resolved_source, True
+    return None, None
+
+
+def parse_grid_definition(definition):
+    """
+    >>> sorted(parse_grid_definition("res=[10000,1000,100,10] srs=EPSG:4326 bbox=5,50,10,60").items())
+    [('bbox', '5,50,10,60'), ('res', [10000, 1000, 100, 10]), ('srs', 'EPSG:4326')]
+    """
+    args = shlex.split(definition)
+    grid_conf = {}
+    for arg in args:
+        key, value = arg.split('=')
+        value = yaml.safe_load(value)
+        grid_conf[key] = value
+
+    validate(conf_spec.grid_opts, grid_conf)
+    return grid_conf
+
+def supports_tiled_access(mgr):
+    if len(mgr.sources) == 1 and getattr(mgr.sources[0], 'supports_meta_tiles') == False:
+        return True
+    return False
+
+
+def format_export_task(task, custom_grid):
+    info = []
+    if custom_grid:
+        grid = "custom grid"
+    else:
+        grid = "grid '%s'" % task.md['grid_name']
+
+    info.append("Exporting cache '%s' to '%s' with %s in %s" % (
+                 task.md['cache_name'], task.md['dest'], grid, task.grid.srs.srs_code))
+    if task.coverage:
+        info.append('  Limited to: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
+    info.append('  Levels: %s' % (task.levels, ))
+
+    return '\n'.join(info)
+
+def export_command(args=None):
+    parser = optparse.OptionParser("%prog export [options] mapproxy_conf")
+    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
+        help="MapProxy configuration")
+
+    parser.add_option("-q", "--quiet",
+                      action="count", dest="quiet", default=0,
+                      help="reduce number of messages to stdout, repeat to disable progress output")
+
+    parser.add_option("--source", dest="source",
+        help="source to export (source or cache)")
+
+    parser.add_option("--grid",
+        help="grid for export. either the name of an existing grid or "
+        "the grid definition as a string")
+
+    parser.add_option("--dest",
+        help="destination of the export (directory or filename)")
+
+    parser.add_option("--type",
+        help="type of the export format")
+
+    parser.add_option("--levels",
+        help="levels to export: e.g 1,2,3 or 1..10")
+
+    parser.add_option("--fetch-missing-tiles", dest="fetch_missing_tiles",
+        action='store_true', default=False,
+        help="if missing tiles should be fetched from the sources")
+
+    parser.add_option("--force",
+        action='store_true', default=False,
+        help="overwrite/append to existing --dest files/directories")
+
+    parser.add_option("-n", "--dry-run",
+        action="store_true", default=False,
+        help="do not export, just print output")
+
+    parser.add_option("-c", "--concurrency", type="int",
+        dest="concurrency", default=1,
+        help="number of parallel export processes")
+
+    parser.add_option("--coverage",
+        help="the coverage for the export as a BBOX string, WKT file "
+        "or OGR datasource")
+    parser.add_option("--srs",
+        help="the SRS of the coverage")
+    parser.add_option("--where",
+        help="filter for OGR coverages")
+
+    from mapproxy.script.util import setup_logging
+    import logging
+    setup_logging(logging.WARN)
+
+    if args:
+        args = args[1:] # remove script name
+
+    (options, args) = parser.parse_args(args)
+
+    if not options.mapproxy_conf:
+        if len(args) != 1:
+            parser.print_help()
+            sys.exit(1)
+        else:
+            options.mapproxy_conf = args[0]
+
+    required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels']
+    for required in required_options:
+        if not getattr(options, required):
+            print('ERROR: missing required option --%s' % required.replace('_', '-'), file=sys.stderr)
+            parser.print_help()
+            sys.exit(1)
+
+    try:
+        conf = load_configuration(options.mapproxy_conf)
+    except IOError as e:
+        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
+        sys.exit(2)
+    except ConfigurationError as e:
+        print(e, file=sys.stderr)
+        print('ERROR: invalid configuration (see above)', file=sys.stderr)
+        sys.exit(2)
+
+
+    if '=' in options.grid:
+        try:
+            grid_conf = parse_grid_definition(options.grid)
+        except ValidationError as ex:
+            print('ERROR: invalid grid configuration', file=sys.stderr)
+            for error in ex.errors:
+                print(' ', error, file=sys.stderr)
+            sys.exit(2)
+        except ValueError:
+            print('ERROR: invalid grid configuration', file=sys.stderr)
+            sys.exit(2)
+        options.grid = 'tmp_mapproxy_export_grid'
+        grid_conf['name'] = options.grid
+        custom_grid = True
+        conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
+    else:
+        custom_grid = False
+
+    if os.path.exists(options.dest) and not options.force:
+        print('ERROR: destination exists, remove first or use --force', file=sys.stderr)
+        sys.exit(2)
+
+    cache_conf = {
+        'name': 'export',
+        'grids': [options.grid],
+        'sources': [options.source],
+    }
+
+    resolved_source, source_is_cache = resolve_source(options.source, conf)
+    if source_is_cache:
+        image_format = resolved_source.conf.get('format') or resolved_source.defaults.get('format')
+        if image_format is not None:
+            cache_conf['format'] = image_format
+        if image_format == 'mixed':
+            request_format = resolved_source.conf.get('request_format') or 'image/png'
+            cache_conf['request_format'] = request_format
+
+    if options.type == 'mbtile':
+        cache_conf['cache'] = {
+            'type': 'mbtiles',
+            'filename': options.dest,
+        }
+    elif options.type == 'sqlite':
+        cache_conf['cache'] = {
+            'type': 'sqlite',
+            'directory': options.dest,
+        }
+    elif options.type == 'geopackage':
+        cache_conf['cache'] = {
+            'type': 'geopackage',
+            'filename': options.dest,
+        }
+    elif options.type == 'compact-v1':
+        cache_conf['cache'] = {
+            'type': 'compact',
+            'version': 1,
+            'directory': options.dest,
+        }
+    elif options.type == 'compact-v2':
+        cache_conf['cache'] = {
+            'type': 'compact',
+            'version': 2,
+            'directory': options.dest,
+        }
+    elif options.type in ('tc', 'mapproxy'):
+        cache_conf['cache'] = {
+            'type': 'file',
+            'directory': options.dest,
+        }
+    elif options.type == 'arcgis':
+        cache_conf['cache'] = {
+            'type': 'file',
+            'directory_layout': 'arcgis',
+            'directory': options.dest,
+        }
+    elif options.type in ('tms', None): # default
+        cache_conf['cache'] = {
+            'type': 'file',
+            'directory_layout': 'tms',
+            'directory': options.dest,
+        }
+    else:
+        print('ERROR: unsupported --type %s' % (options.type, ), file=sys.stderr)
+        sys.exit(2)
+
+    if not options.fetch_missing_tiles:
+        for source in conf.sources.values():
+            source.conf['seed_only'] = True
+
+    tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]
+
+    levels = parse_levels(options.levels)
+    if levels[-1] >= tile_grid.levels:
+        print('ERROR: destination grid only has %d levels' % tile_grid.levels, file=sys.stderr)
+        sys.exit(2)
+
+    if options.srs:
+        srs = SRS(options.srs)
+    else:
+        srs = tile_grid.srs
+
+    if options.coverage:
+        seed_coverage = load_coverage(
+            {'datasource': options.coverage, 'srs': srs, 'where': options.where},
+            base_path=os.getcwd())
+    else:
+        seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)
+
+    if not supports_tiled_access(mgr):
+        print('WARN: grids are incompatible. needs to scale/reproject tiles for export.', file=sys.stderr)
+
+    md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest)
+    task = SeedTask(md, mgr, levels, 1, seed_coverage)
+
+    print(format_export_task(task, custom_grid=custom_grid))
+
+    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
+    try:
+        seed_task(task, progress_logger=logger, dry_run=options.dry_run,
+             concurrency=options.concurrency)
+    except KeyboardInterrupt:
+        print('stopping...', file=sys.stderr)
+        sys.exit(2)
+