MapProxy 2.1.0__py3-none-any.whl

mapproxy/util/fs.py

@@ -0,0 +1,164 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010-2013 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+File system related utility functions.
+"""
+from __future__ import absolute_import
+import time
+import os
+import sys
+import random
+import errno
+import shutil
+
+
+def swap_dir(src_dir, dst_dir, keep_old=False, backup_ext='.tmp'):
+    """
+    Rename `src_dir` to `dst_dir`. The `dst_dir` is first renamed to
+    `dst_dir` + `backup_ext` to keep the interruption short.
+    Then the `src_dir` is renamed. If `keep_old` is False, the old content
+    of `dst_dir` will be removed.
+    """
+    tmp_dir = dst_dir + backup_ext
+    if os.path.exists(dst_dir):
+        os.rename(dst_dir, tmp_dir)
+
+    _force_rename_dir(src_dir, dst_dir)
+
+    if os.path.exists(tmp_dir) and not keep_old:
+        shutil.rmtree(tmp_dir)
+
+
+def _force_rename_dir(src_dir, dst_dir):
+    """
+    Rename `src_dir` to `dst_dir`. If `dst_dir` exists, it will be removed.
+    """
+    # someone might recreate the directory between rmtree and rename,
+    # so we try to remove it until we can rename our new directory
+    rename_tries = 0
+    while rename_tries < 10:
+        try:
+            os.rename(src_dir, dst_dir)
+        except OSError as ex:
+            if ex.errno == errno.ENOTEMPTY or ex.errno == errno.EEXIST:
+                if rename_tries > 0:
+                    time.sleep(2**rename_tries / 100.0)  # from 10ms to 5s
+                rename_tries += 1
+                shutil.rmtree(dst_dir)
+            else:
+                raise
+        else:
+            break  # on success
+
+
+def cleanup_directory(directory, before_timestamp, remove_empty_dirs=True,
+                      file_handler=None):
+    if file_handler is None:
+        if before_timestamp == 0 and remove_empty_dirs is True and os.path.exists(directory):
+            shutil.rmtree(directory, ignore_errors=True)
+            return
+
+        file_handler = os.remove
+
+    if os.path.exists(directory):
+        for dirpath, dirnames, filenames in os.walk(directory, topdown=False):
+            if not filenames:
+                if (remove_empty_dirs and not os.listdir(dirpath)
+                        and dirpath != directory):
+                    os.rmdir(dirpath)
+                continue
+            for filename in filenames:
+                filename = os.path.join(dirpath, filename)
+                try:
+                    if before_timestamp == 0:
+                        file_handler(filename)
+                    if os.lstat(filename).st_mtime < before_timestamp:
+                        file_handler(filename)
+                except OSError as ex:
+                    if ex.errno != errno.ENOENT:
+                        raise
+
+            if remove_empty_dirs:
+                remove_dir_if_emtpy(dirpath)
+
+        if remove_empty_dirs:
+            remove_dir_if_emtpy(directory)
+
+
+def remove_dir_if_emtpy(directory):
+    try:
+        os.rmdir(directory)
+    except OSError as ex:
+        if ex.errno != errno.ENOENT and ex.errno != errno.ENOTEMPTY:
+            raise
+
+
+def ensure_directory(file_name):
+    """
+    Create directory if it does not exist, else do nothing.
+    """
+    dir_name = os.path.dirname(file_name)
+    if not os.path.exists(dir_name):
+        try:
+            os.makedirs(dir_name)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+
+
+def write_atomic(filename, data):
+    """
+    write_atomic writes `data` to a random file in filename's directory
+    first and renames that file to the target filename afterwards.
+    Rename is atomic on all POSIX platforms.
+
+    Falls back to normal write on Windows.
+    """
+    if not sys.platform.startswith('win'):
+        # write to random filename to prevent concurrent writes in cases
+        # where file locking does not work (network fs)
+        path_tmp = filename + '.tmp-' + str(random.randint(0, 99999999))
+        try:
+            fd = os.open(path_tmp, os.O_EXCL | os.O_CREAT | os.O_WRONLY, 0o664)
+            with os.fdopen(fd, 'wb') as f:
+                f.write(data)
+            os.rename(path_tmp, filename)
+        except OSError as ex:
+            try:
+                os.unlink(path_tmp)
+            except OSError:
+                pass
+            raise ex
+    else:
+        fd = os.open(filename, os.O_CREAT | os.O_WRONLY, 0o664)
+        with os.fdopen(fd, 'wb') as f:
+            f.write(data)
+
+
+def find_exec(executable):
+    """
+    Search executable in PATH environment. Return path if found, None if not.
+    """
+    path = os.environ.get('PATH')
+    if not path:
+        return
+    for p in path.split(os.path.pathsep):
+        p = os.path.join(p, executable)
+        if os.path.exists(p):
+            return p
+        p += '.exe'
+        if os.path.exists(p):
+            return p

mapproxy/util/geom.py

@@ -0,0 +1,307 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+
+import os
+import json
+import codecs
+from functools import partial
+from contextlib import closing
+
+from mapproxy.grid import tile_grid
+
+import logging
+log_config = logging.getLogger('mapproxy.config.coverage')
+
+try:
+    import shapely.wkt
+    import shapely.geometry
+    import shapely.ops
+    import shapely.prepared
+    from shapely.errors import ShapelyError
+    geom_support = True
+except ImportError:
+    geom_support = False
+
+
+class GeometryError(Exception):
+    pass
+
+
+class EmptyGeometryError(Exception):
+    pass
+
+
+class CoverageReadError(Exception):
+    pass
+
+
+def require_geom_support():
+    if not geom_support:
+        raise ImportError('Shapely required for geometry support')
+
+
+def load_datasource(datasource, where=None):
+    """
+    Loads polygons from WKT text files or OGR datasources.
+
+    Returns a list of Shapely Polygons.
+    """
+    # check if it is a  wkt or geojson file
+    if os.path.exists(os.path.abspath(datasource)):
+        with open(os.path.abspath(datasource), 'rb') as fp:
+            data = fp.read(50)
+        if data.lower().lstrip().startswith((b'polygon', b'multipolygon')):
+            return load_polygons(datasource)
+        # only load geojson directly if we don't have a filter
+        if where is None and data and data.startswith(b'{'):
+            return load_geojson(datasource)
+    # otherwise pass to OGR
+    return load_ogr_datasource(datasource, where=where)
+
+
+def load_ogr_datasource(datasource, where=None):
+    """
+    Loads polygons from any OGR datasource.
+
+    Returns a list of Shapely Polygons.
+    """
+    from mapproxy.util.ogr import OGRShapeReader, OGRShapeReaderError
+
+    polygons = []
+    try:
+        with closing(OGRShapeReader(datasource)) as reader:
+            for wkt in reader.wkts(where):
+                if not isinstance(wkt, str):
+                    wkt = wkt.decode()
+                try:
+                    geom = shapely.wkt.loads(wkt)
+                except ShapelyError as ex:
+                    raise GeometryError(ex)
+                if geom.geom_type == 'Polygon':
+                    polygons.append(geom)
+                elif geom.geom_type == 'MultiPolygon':
+                    for p in geom.geoms:
+                        polygons.append(p)
+                else:
+                    log_config.warning('skipping %s geometry from %s: not a Polygon/MultiPolygon',
+                                       geom.geom_type, datasource)
+    except OGRShapeReaderError as ex:
+        raise CoverageReadError(ex)
+
+    return polygons
+
+
+def load_polygons(geom_files):
+    """
+    Loads WKT polygons from one or more text files.
+
+    Returns a list of Shapely Polygons.
+    """
+    polygons = []
+    if isinstance(geom_files, str):
+        geom_files = [geom_files]
+
+    for geom_file in geom_files:
+        # open with utf-8-sig encoding to get rid of UTF8 BOM from MS Notepad
+        with codecs.open(geom_file, encoding='utf-8-sig') as f:
+            polygons.extend(load_polygon_lines(f, source=geom_files))
+
+    return polygons
+
+
+def load_geojson(datasource):
+    with open(datasource) as f:
+        geojson = json.load(f)
+        t = geojson.get('type')
+        if not t:
+            raise CoverageReadError("not a GeoJSON")
+        geometries = []
+        if t == 'FeatureCollection':
+            for f in geojson.get('features'):
+                geom = f.get('geometry')
+                if geom:
+                    geometries.append(geom)
+        elif t == 'Feature':
+            if 'geometry' in geojson:
+                geometries.append(geojson['geometry'])
+        elif t in ('Polygon', 'MultiPolygon'):
+            geometries.append(geojson)
+        else:
+            log_config.warning('skipping feature of type %s from %s: not a Polygon/MultiPolygon',
+                               t, datasource)
+
+    polygons = []
+    for geom in geometries:
+        geom = shapely.geometry.shape(geom)
+        if geom.geom_type == 'Polygon':
+            polygons.append(geom)
+        elif geom.geom_type == 'MultiPolygon':
+            for p in geom.geoms:
+                polygons.append(p)
+        else:
+            log_config.warning('ignoring non-polygon geometry (%s) from %s',
+                               geom.geom_type, datasource)
+
+    return polygons
+
+
+def load_polygon_lines(line_iter, source='<string>'):
+    polygons = []
+    for line in line_iter:
+        if not line.strip():
+            continue
+        geom = shapely.wkt.loads(line)
+        if geom.geom_type == 'Polygon':
+            polygons.append(geom)
+        elif geom.geom_type == 'MultiPolygon':
+            for p in geom.geoms:
+                polygons.append(p)
+        else:
+            log_config.warning('ignoring non-polygon geometry (%s) from %s',
+                               geom.geom_type, source)
+
+    return polygons
+
+
+def build_multipolygon(polygons, simplify=False):
+    if not polygons:
+        raise EmptyGeometryError('no polygons')
+
+    if len(polygons) == 1:
+        geom = polygons[0]
+        if simplify:
+            geom = simplify_geom(geom)
+        return geom.bounds, geom
+
+    if simplify:
+        polygons = [simplify_geom(g) for g in polygons]
+
+    # eliminate any self-overlaps
+    mp = shapely.ops.unary_union(polygons)
+
+    return mp.bounds, mp
+
+
+def simplify_geom(geom):
+    bounds = geom.bounds
+    if not bounds:
+        raise EmptyGeometryError('Empty geometry given')
+    w, h = bounds[2] - bounds[0], bounds[3] - bounds[1]
+    tolerance = min((w/1e6, h/1e6))
+    geom = geom.simplify(tolerance, preserve_topology=True)
+    if not geom.is_valid:
+        geom = geom.buffer(0)
+    return geom
+
+
+def bbox_polygon(bbox):
+    """
+    Create Polygon that covers the given bbox.
+    """
+    return shapely.geometry.Polygon((
+        (bbox[0], bbox[1]),
+        (bbox[2], bbox[1]),
+        (bbox[2], bbox[3]),
+        (bbox[0], bbox[3]),
+    ))
+
+
+def transform_geometry(from_srs, to_srs, geometry):
+    transf = partial(transform_xy, from_srs, to_srs)
+
+    if geometry.geom_type == 'Polygon':
+        result = transform_polygon(transf, geometry)
+    elif geometry.geom_type == 'MultiPolygon':
+        result = transform_multipolygon(transf, geometry)
+    else:
+        raise ValueError('cannot transform %s' % geometry.geom_type)
+
+    if not result.is_valid:
+        result = result.buffer(0)
+    return result
+
+
+def transform_polygon(transf, polygon):
+    ext = transf(polygon.exterior.xy)
+    ints = [transf(ring.xy) for ring in polygon.interiors]
+    return shapely.geometry.Polygon(ext, ints)
+
+
+def transform_multipolygon(transf, multipolygon):
+    transformed_polygons = []
+    for polygon in multipolygon.geoms:
+        transformed_polygons.append(transform_polygon(transf, polygon))
+    return shapely.geometry.MultiPolygon(transformed_polygons)
+
+
+def transform_xy(from_srs, to_srs, xy):
+    return list(from_srs.transform_to(to_srs, list(zip(*xy))))
+
+
+def flatten_to_polygons(geometry):
+    """
+    Return a list of all polygons of this (multi)`geometry`.
+    """
+    if geometry.geom_type == 'Polygon':
+        return [geometry]
+
+    if geometry.geom_type == 'MultiPolygon':
+        return list(geometry.geoms)
+
+    if hasattr(geometry, 'geoms'):
+        # GeometryCollection or MultiLineString? return list of all polygons
+        geoms = []
+        for part in geometry.geoms:
+            if part.type == 'Polygon':
+                geoms.append(part)
+
+        if geoms:
+            return geoms
+
+    return []
+
+
+def load_expire_tiles(expire_dir, grid=None):
+    if grid is None:
+        grid = tile_grid(3857, origin='nw')
+    tiles = set()
+
+    def parse(filename):
+        with open(filename) as f:
+            try:
+                for line in f:
+                    if not line:
+                        continue
+                    tile = tuple(map(int, line.split('/')))
+                    tiles.add(tile)
+            except Exception:
+                log_config.warning('found error in %s, skipping rest of file', filename)
+
+    if os.path.isdir(expire_dir):
+        for root, dirs, files in os.walk(expire_dir):
+            for name in files:
+                filename = os.path.join(root, name)
+                parse(filename)
+    else:
+        parse(expire_dir)
+
+    boxes = []
+    for tile in tiles:
+        z, x, y = tile
+        boxes.append(shapely.geometry.box(*grid.tile_bbox((x, y, z))))
+
+    return boxes

mapproxy/util/lib.py

@@ -0,0 +1,117 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+ctypes utilities.
+"""
+from __future__ import print_function
+
+import sys
+import os
+
+from ctypes import CDLL
+from ctypes.util import find_library as _find_library
+
+
+default_locations = dict(
+    darwin=dict(
+        paths=['/opt/local/lib'],
+        exts=['.dylib'],
+    ),
+    win32=dict(
+        paths=[os.path.dirname(os.__file__) + '/../../../DLLs'],
+        exts=['.dll']
+    ),
+    other=dict(
+        paths=[],  # MAPPROXY_LIB_PATH will add paths here
+        exts=['.so']
+    ),
+)
+
+additional_lib_path = os.environ.get('MAPPROXY_LIB_PATH')
+if additional_lib_path:
+    additional_lib_path = additional_lib_path.split(os.pathsep)
+    additional_lib_path.reverse()
+    for locs in default_locations.values():
+        for path in additional_lib_path:
+            locs['paths'].insert(0, path)
+
+
+def load_library(lib_names, locations_conf=default_locations):
+    """
+    Load the `lib_name` library with ctypes.
+    If ctypes.util.find_library does not find the library,
+    different path and filename extensions will be tried.
+
+    Retruns the loaded library or None.
+    """
+    if isinstance(lib_names, str):
+        lib_names = [lib_names]
+
+    for lib_name in lib_names:
+        lib = load_library_(lib_name, locations_conf)
+        if lib is not None:
+            return lib
+
+
+def load_library_(lib_name, locations_conf=default_locations):
+    lib_path = find_library(lib_name)
+
+    if lib_path:
+        return CDLL(lib_path)
+
+    if sys.platform in locations_conf:
+        paths = locations_conf[sys.platform]['paths']
+        exts = locations_conf[sys.platform]['exts']
+        lib_path = find_library(lib_name, paths, exts)
+    else:
+        paths = locations_conf['other']['paths']
+        exts = locations_conf['other']['exts']
+        lib_path = find_library(lib_name, paths, exts)
+
+    if lib_path:
+        return CDLL(lib_path)
+
+
+def find_library(lib_name, paths=None, exts=None):
+    """
+    Search for library in all permutations of `paths` and `exts`.
+    If nothing is found None is returned.
+    """
+    if not paths or not exts:
+        lib = None
+        try:
+            lib = _find_library(lib_name)
+        except FileNotFoundError:
+            pass
+        if lib is None and lib_name.startswith('lib'):
+            try:
+                lib = _find_library(lib_name[3:])
+            except FileNotFoundError:
+                pass
+        return lib
+
+    for lib_name in [lib_name] + ([lib_name[3:]] if lib_name.startswith('lib') else []):
+        for path in paths:
+            for ext in exts:
+                lib_path = os.path.join(path, lib_name + ext)
+                if os.path.exists(lib_path):
+                    return lib_path
+
+    return None
+
+
+if __name__ == '__main__':
+    print(load_library(sys.argv[1]))