MapProxy 2.1.0__py3-none-any.whl

mapproxy/script/conf/layers.py

@@ -0,0 +1,54 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def layers(cap, caches):
+    return [_layer(cap.layers(), caches)]
+
+
+def _layer(layer, caches):
+    name, conf = for_layer(layer, caches)
+    child_layers = []
+
+    for child_layer in layer['layers']:
+        child_layers.append(_layer(child_layer, caches))
+
+    if child_layers:
+        conf['layers'] = child_layers
+
+    return conf
+
+
+def for_layer(layer, caches):
+    conf = {
+        'title': layer['title'],
+    }
+
+    if layer['name']:
+        conf['name'] = layer['name']
+
+        if layer['name'] + '_cache' in caches:
+            conf['sources'] = [layer['name'] + '_cache']
+        else:
+            conf['sources'] = [layer['name'] + '_wms']
+
+    md = {}
+    if layer['abstract']:
+        md['abstract'] = layer['abstract']
+
+    if md:
+        conf['md'] = md
+
+    return layer['name'], conf

mapproxy/script/conf/seeds.py

@@ -0,0 +1,36 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def seeds(cap, caches):
+    seeds = {}
+    cleanups = {}
+
+    for cache_name, cache in caches.items():
+        for grid in cache['grids']:
+            seeds[cache_name + '_' + grid] = {
+                'caches': [cache_name],
+                'grids': [grid],
+            }
+            cleanups[cache_name + '_' + grid] = {
+                'caches': [cache_name],
+                'grids': [grid],
+                'remove_before': {
+                    'time': '1900-01-01T00:00:00',
+                }
+            }
+
+    return seeds, cleanups

mapproxy/script/conf/sources.py

@@ -0,0 +1,88 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mapproxy.srs import SRS
+
+import logging
+
+
+def sources(cap):
+    sources = {}
+    for layer in cap.layers_list():
+        name, conf = for_layer(cap, layer)
+        sources[name+'_wms'] = conf
+
+    return sources
+
+
+_checked_srs = {}
+
+
+def check_srs(srs):
+    if srs not in _checked_srs:
+        try:
+            SRS(srs)
+            _checked_srs[srs] = True
+        except Exception as ex:
+            logging.getLogger(__name__).warning('unable to initialize srs for %s: %s', srs, ex)
+            _checked_srs[srs] = False
+
+    return _checked_srs[srs]
+
+
+def for_layer(cap, layer):
+    source = {'type': 'wms'}
+
+    req = {
+        'url': layer['url'],
+        'layers': layer['name'],
+    }
+
+    if not layer['opaque']:
+        req['transparent'] = True
+
+    wms_opts = {}
+    if cap.version != '1.1.1':
+        wms_opts['version'] = cap.version
+    if layer['queryable']:
+        wms_opts['featureinfo'] = True
+    if layer['legend']:
+        wms_opts['legendurl'] = layer['legend']['url']
+    if wms_opts:
+        source['wms_opts'] = wms_opts
+
+    source['req'] = req
+
+    source['supported_srs'] = []
+    for srs in layer['srs']:
+        if check_srs(srs):
+            source['supported_srs'].append(srs)
+    source['supported_srs'].sort()
+
+    if layer['llbbox']:
+        source['coverage'] = {
+            'srs': 'EPSG:4326',
+            'bbox': layer['llbbox'],
+        }
+
+    res_hint = layer['res_hint']
+    if res_hint:
+        if res_hint[0]:
+            source['min_res'] = res_hint[0]
+        if res_hint[1]:
+            source['max_res'] = res_hint[1]
+
+    return layer['name'], source

mapproxy/script/conf/utils.py

@@ -0,0 +1,148 @@
+# -:- encoding: utf-8 -:-
+# This file is part of the MapProxy project.
+# Copyright (C) 2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mapproxy.client.http import open_url
+from mapproxy.request.base import BaseRequest, url_decode
+from yaml.resolver import Resolver
+from yaml.representer import SafeRepresenter
+from yaml.emitter import Emitter
+from yaml.nodes import ScalarNode, SequenceNode, MappingNode
+from yaml.serializer import Serializer
+from copy import copy
+from urllib.parse import urlparse
+
+
+__all__ = ['update_config', 'MapProxyYAMLDumper']
+
+
+def update_config(conf, overwrites):
+    wildcard_keys = []
+    for k, v in overwrites.items():
+        if k == '__all__':
+            continue
+        if k.startswith('___') or k.endswith('___'):
+            wildcard_keys.append(k)
+            continue
+
+        if k.endswith('__extend__'):
+            k = k[:-len('__extend__')]
+            if k not in conf:
+                conf[k] = v
+            elif isinstance(v, list):
+                conf[k].extend(v)
+            else:
+                raise ValueError('cannot extend non-list:', v)
+        elif k not in conf:
+            conf[k] = copy(v)
+        else:
+            if isinstance(conf[k], dict) and isinstance(v, dict):
+                conf[k] = update_config(conf[k], v)
+            else:
+                conf[k] = copy(v)
+
+    if '__all__' in overwrites:
+        v = overwrites['__all__']
+        for conf_k, conf_v in conf.items():
+            if isinstance(conf_v, dict):
+                conf[conf_k] = update_config(conf_v, v)
+            else:
+                conf[conf_k] = v
+
+    if wildcard_keys:
+        for key in wildcard_keys:
+            v = overwrites[key]
+            if key.startswith('___'):
+                key = key[3:]
+                def key_check(x): return x.endswith(key)
+            else:
+                key = key[:-3]
+                def key_check(x): return x.startswith(key)
+            for conf_k, conf_v in conf.items():
+                if not key_check(conf_k):
+                    continue
+                if isinstance(conf_v, dict):
+                    conf[conf_k] = update_config(conf_v, v)
+                else:
+                    conf[conf_k] = v
+
+    return conf
+
+
+class _MixedFlowSortedSerializer(Serializer):
+    def serialize_node(self, node, parent, index):
+        # reset any anchors
+        if parent is None:
+            for k in self.anchors:
+                self.anchors[k] = None
+        self.serialized_nodes = {}
+
+        if isinstance(node, SequenceNode) and all(isinstance(item, ScalarNode) for item in node.value):
+            node.flow_style = True
+        elif isinstance(node, MappingNode):
+            node.value.sort(key=lambda x: x[0].value)
+        return Serializer.serialize_node(self, node, parent, index)
+
+
+class _EmptyNoneRepresenter(SafeRepresenter):
+    def represent_none(self, data):
+        return self.represent_scalar(u'tag:yaml.org,2002:null',
+                                     u'')
+
+
+_EmptyNoneRepresenter.add_representer(type(None), _EmptyNoneRepresenter.represent_none)
+
+
+class MapProxyYAMLDumper(Emitter, _MixedFlowSortedSerializer, _EmptyNoneRepresenter, Resolver):
+    """
+    YAML dumper that uses block style by default, except for
+    node-only sequences. Also sorts dicts by key, prevents `none`
+    for empty entries and prevents any anchors.
+    """
+
+    def __init__(self, stream,
+                 default_style=None, default_flow_style=False,
+                 canonical=None, indent=None, width=None,
+                 allow_unicode=None, line_break=None,
+                 encoding=None, explicit_start=None, explicit_end=None,
+                 version=None, tags=None, sort_keys=None):
+        Emitter.__init__(self, stream, canonical=canonical,
+                         indent=indent, width=width,
+                         allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                            explicit_start=explicit_start, explicit_end=explicit_end,
+                            version=version, tags=tags)
+        _EmptyNoneRepresenter.__init__(self, default_style=default_style,
+                                       default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
+
+def wms_capapilities_url(url):
+    parsed_url = urlparse(url)
+    base_req = BaseRequest(
+        url=url.split('?', 1)[0],
+        param=url_decode(parsed_url.query),
+    )
+
+    base_req.params['service'] = 'WMS'
+    if not base_req.params['version']:
+        base_req.params['version'] = '1.1.1'
+    base_req.params['request'] = 'GetCapabilities'
+    return base_req.complete_url
+
+
+def download_capabilities(url):
+    capabilities_url = wms_capapilities_url(url)
+    return open_url(capabilities_url)

mapproxy/script/defrag.py

@@ -0,0 +1,187 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2017 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import glob
+import optparse
+import os.path
+import re
+import sys
+from collections import OrderedDict
+
+from mapproxy.cache.compact import CompactCacheV1, CompactCacheV2
+from mapproxy.cache.tile import Tile
+from mapproxy.config import local_base_config
+from mapproxy.config.loader import load_configuration, ConfigurationError
+
+import logging
+log = logging.getLogger('mapproxy.defrag')
+
+
+def defrag_command(args=None):
+    parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf")
+    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
+                      help="MapProxy configuration.")
+
+    parser.add_option(
+        "--min-percent", type=float, default=10.0,
+        help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)")
+
+    parser.add_option(
+        "--min-mb", type=float, default=1.0,
+        help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)")
+
+    parser.add_option("--dry-run", "-n", action="store_true",
+                      help="Do not de-fragment, only print output")
+
+    parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...',
+                      help="only defragment the named caches")
+
+    from mapproxy.script.util import setup_logging
+    import logging
+    setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s")
+
+    if args:
+        args = args[1:]  # remove script name
+
+    (options, args) = parser.parse_args(args)
+    if not options.mapproxy_conf:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        proxy_configuration = load_configuration(options.mapproxy_conf)
+    except IOError as e:
+        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
+        sys.exit(2)
+    except ConfigurationError as e:
+        print(e, file=sys.stderr)
+        print('ERROR: invalid configuration (see above)', file=sys.stderr)
+        sys.exit(2)
+
+    with local_base_config(proxy_configuration.base_config):
+        available_caches = OrderedDict()
+        for name, cache_conf in proxy_configuration.caches.items():
+            for grid, extent, tile_mgr in cache_conf.caches():
+                if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)):
+                    available_caches.setdefault(name, []).append(tile_mgr.cache)
+
+        if options.cache_names:
+            defrag_caches = options.cache_names.split(',')
+            missing = set(defrag_caches).difference(available_caches.keys())
+            if missing:
+                print('unknown caches: %s' % (', '.join(missing), ))
+                print('available compact caches: %s' %
+                      (', '.join(available_caches.keys()), ))
+                sys.exit(1)
+        else:
+            defrag_caches = None
+
+        for name, caches in available_caches.items():
+            if defrag_caches and name not in defrag_caches:
+                continue
+            for cache in caches:
+                logger = DefragLog(name)
+                defrag_compact_cache(cache,
+                                     min_percent=options.min_percent/100,
+                                     min_bytes=options.min_mb*1024*1024,
+                                     dry_run=options.dry_run,
+                                     log_progress=logger,
+                                     )
+
+
+def bundle_offset(fname):
+    """
+    >>> bundle_offset("path/to/R0000C0000.bundle")
+    (0, 0)
+    >>> bundle_offset("path/to/R0380C1380.bundle")
+    (4992, 896)
+    """
+    match = re.search(r'R([A-F0-9]{4,})C([A-F0-9]{4,}).bundle$', fname, re.IGNORECASE)
+    if match:
+        r = int(match.group(1), 16)
+        c = int(match.group(2), 16)
+        return c, r
+
+
+class DefragLog(object):
+    def __init__(self, cache_name):
+        self.cache_name = cache_name
+
+    def log(self, fname, fragmentation, fragmentation_bytes, num, total, defrag):
+        msg = "%s: %3d/%d (%s) fragmentation is %.1f%% (%dkb)" % (
+            self.cache_name, num, total, fname, fragmentation, fragmentation_bytes/1024
+        )
+        if defrag:
+            msg += " - defragmenting"
+        else:
+            msg += " - skipping"
+        log.info(msg)
+
+
+def defrag_compact_cache(cache, min_percent=0.1, min_bytes=1024*1024, log_progress=None, dry_run=False):
+    bundles = glob.glob(os.path.join(cache.cache_dir, 'L??', 'R????C????.bundle'))
+
+    for i, bundle_file in enumerate(bundles):
+        offset = bundle_offset(bundle_file)
+        b = cache.bundle_class(bundle_file.rstrip('.bundle'), offset)
+        size, file_size = b.size()
+
+        defrag = 1 - float(size) / file_size
+        defrag_bytes = file_size - size
+
+        skip = False
+        if defrag < min_percent or defrag_bytes < min_bytes:
+            skip = True
+
+        if log_progress:
+            log_progress.log(
+                fname=bundle_file,
+                fragmentation=defrag * 100,
+                fragmentation_bytes=defrag_bytes,
+                num=i+1, total=len(bundles),
+                defrag=not skip,
+            )
+
+        if skip or dry_run:
+            continue
+
+        tmp_bundle = os.path.join(cache.cache_dir, 'tmp_defrag')
+        defb = cache.bundle_class(tmp_bundle, offset)
+        stored_tiles = False
+
+        for y in range(128):
+            tiles = [Tile((x, y, 0)) for x in range(128)]
+            b.load_tiles(tiles)
+            tiles = [t for t in tiles if t.source]
+            if tiles:
+                stored_tiles = True
+                defb.store_tiles(tiles)
+
+        # remove first
+        # - in case bundle is empty
+        # - windows does not support rename to existing files
+        if os.path.exists(bundle_file):
+            os.remove(bundle_file)
+        if os.path.exists(bundle_file[:-1] + 'x'):
+            os.remove(bundle_file[:-1] + 'x')
+
+        if stored_tiles:
+            os.rename(tmp_bundle + '.bundle', bundle_file)
+            if os.path.exists(tmp_bundle + '.bundlx'):
+                os.rename(tmp_bundle + '.bundlx', bundle_file[:-1] + 'x')
+            if os.path.exists(tmp_bundle + '.lck'):
+                os.unlink(tmp_bundle + '.lck')