MapProxy 1.16.1__py3-none-any.whl

mapproxy/seed/cleanup.py

@@ -0,0 +1,187 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2010 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import os
+from mapproxy.compat.itertools import izip_longest
+from mapproxy.seed.util import format_cleanup_task
+from mapproxy.util.fs import cleanup_directory
+from mapproxy.seed.seeder import (
+    TileWorkerPool, TileWalker, TileCleanupWorker,
+    SeedProgress,
+)
+from mapproxy.seed.util import ProgressLog
+
+def cleanup(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0,
+               verbose=True, progress_logger=None):
+    for task in tasks:
+        print(format_cleanup_task(task))
+
+        if task.coverage is False:
+            continue
+
+        # seed_progress for tilewalker cleanup
+        seed_progress = None
+        # cleanup_progress for os.walk based cleanup
+        cleanup_progress = None
+        if progress_logger and progress_logger.progress_store:
+            progress_logger.current_task_id = task.id
+            start_progress = progress_logger.progress_store.get(task.id)
+            seed_progress = SeedProgress(old_progress_identifier=start_progress)
+            cleanup_progress = DirectoryCleanupProgress(old_dir=start_progress)
+
+        if task.complete_extent:
+            if callable(getattr(task.tile_manager.cache, 'level_location', None)):
+                simple_cleanup(task, dry_run=dry_run, progress_logger=progress_logger,
+                    cleanup_progress=cleanup_progress)
+                task.tile_manager.cleanup()
+                continue
+            elif callable(getattr(task.tile_manager.cache, 'remove_level_tiles_before', None)):
+                cache_cleanup(task, dry_run=dry_run, progress_logger=progress_logger)
+                task.tile_manager.cleanup()
+                continue
+
+        tilewalker_cleanup(task, dry_run=dry_run, concurrency=concurrency,
+                         skip_geoms_for_last_levels=skip_geoms_for_last_levels,
+                         progress_logger=progress_logger,
+                         seed_progress=seed_progress,
+        )
+        task.tile_manager.cleanup()
+
+
+def simple_cleanup(task, dry_run, progress_logger=None, cleanup_progress=None):
+    """
+    Cleanup cache level on file system level.
+    """
+
+    for level in task.levels:
+        level_dir = task.tile_manager.cache.level_location(level)
+        if dry_run:
+            def file_handler(filename):
+                print('removing ' + filename)
+        else:
+            file_handler = None
+        if progress_logger:
+            progress_logger.log_message('removing old tiles in ' + normpath(level_dir))
+            if progress_logger.progress_store:
+                cleanup_progress.step_dir(level_dir)
+                if cleanup_progress.already_processed():
+                    continue
+                progress_logger.progress_store.add(
+                    task.id,
+                    cleanup_progress.current_progress_identifier(),
+                )
+                progress_logger.progress_store.write()
+
+        cleanup_directory(level_dir, task.remove_timestamp,
+            file_handler=file_handler, remove_empty_dirs=True)
+
+def cache_cleanup(task, dry_run, progress_logger=None):
+    for level in task.levels:
+        if progress_logger:
+            progress_logger.log_message('removing old tiles for level %s' % level)
+        if not dry_run:
+            task.tile_manager.cache.remove_level_tiles_before(level, task.remove_timestamp)
+            task.tile_manager.cleanup()
+
+def normpath(path):
+    # relpath doesn't support UNC
+    if path.startswith('\\'):
+        return path
+
+    path = os.path.relpath(path)
+
+    if path.startswith('../../'):
+        path = os.path.abspath(path)
+    return path
+
+def tilewalker_cleanup(task, dry_run, concurrency, skip_geoms_for_last_levels,
+    progress_logger=None, seed_progress=None):
+    """
+    Cleanup tiles with tile traversal.
+    """
+    task.tile_manager._expire_timestamp = task.remove_timestamp
+    task.tile_manager.minimize_meta_requests = False
+    tile_worker_pool = TileWorkerPool(task, TileCleanupWorker, progress_logger=progress_logger,
+                                      dry_run=dry_run, size=concurrency)
+    tile_walker = TileWalker(task, tile_worker_pool, handle_stale=True,
+                             work_on_metatiles=False, progress_logger=progress_logger,
+                             skip_geoms_for_last_levels=skip_geoms_for_last_levels,
+                             seed_progress=seed_progress)
+    try:
+        tile_walker.walk()
+    except KeyboardInterrupt:
+        tile_worker_pool.stop(force=True)
+        raise
+    finally:
+        tile_worker_pool.stop()
+
+
+class DirectoryCleanupProgress(object):
+    def __init__(self, old_dir=None):
+        self.old_dir = old_dir
+        self.current_dir = None
+
+    def step_dir(self, dir):
+        self.current_dir = dir
+
+    def already_processed(self):
+        return self.can_skip(self.old_dir, self.current_dir)
+
+    def current_progress_identifier(self):
+        if self.already_processed() or self.current_dir is None:
+            return self.old_dir
+        return self.current_dir
+
+    @staticmethod
+    def can_skip(old_dir, current_dir):
+        """
+        Return True if the `current_dir` is before `old_dir` when compared
+        lexicographic.
+
+        >>> DirectoryCleanupProgress.can_skip(None, '/00')
+        False
+        >>> DirectoryCleanupProgress.can_skip(None, '/00/000/000')
+        False
+
+        >>> DirectoryCleanupProgress.can_skip('/01/000/001', '/00')
+        True
+        >>> DirectoryCleanupProgress.can_skip('/01/000/001', '/01/000/000')
+        True
+        >>> DirectoryCleanupProgress.can_skip('/01/000/001', '/01/000/000/000')
+        True
+        >>> DirectoryCleanupProgress.can_skip('/01/000/001', '/01/000/001')
+        False
+        >>> DirectoryCleanupProgress.can_skip('/01/000/001', '/01/000/001/000')
+        False
+        """
+        if old_dir is None:
+            return False
+        if current_dir is None:
+            return False
+        for old, current in izip_longest(old_dir.split(os.path.sep), current_dir.split(os.path.sep), fillvalue=None):
+            if old is None:
+                return False
+            if current is None:
+                return False
+            if old < current:
+                return False
+            if old > current:
+                return True
+        return False
+
+    def running(self):
+        return True

mapproxy/seed/config.py

@@ -0,0 +1,469 @@
+# This file is part of the MapProxy project.
+# Copyright (C) 2011 Omniscale <http://omniscale.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import os
+import sys
+import time
+import operator
+from functools import reduce
+
+from mapproxy.cache.dummy import DummyCache
+from mapproxy.compat import iteritems, itervalues, iterkeys
+from mapproxy.config import abspath
+from mapproxy.config.loader import ConfigurationError
+from mapproxy.config.coverage import load_coverage
+from mapproxy.srs import SRS, TransformationError
+from mapproxy.util.py import memoize
+from mapproxy.util.times import timestamp_from_isodate, timestamp_before
+from mapproxy.util.coverage import MultiCoverage, BBOXCoverage, GeomCoverage
+from mapproxy.util.geom import GeometryError, EmptyGeometryError, CoverageReadError
+from mapproxy.util.yaml import load_yaml_file, YAMLError
+from mapproxy.seed.util import bidict
+from mapproxy.seed.seeder import SeedTask, CleanupTask
+from mapproxy.seed.spec import validate_seed_conf
+
+class SeedConfigurationError(ConfigurationError):
+    pass
+
+class EmptyCoverageError(Exception):
+    pass
+
+
+import logging
+log = logging.getLogger('mapproxy.seed.config')
+
+def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
+    try:
+        conf = load_yaml_file(seed_conf_filename)
+    except YAMLError as ex:
+        raise SeedConfigurationError(ex)
+
+    if 'views' in conf:
+        # TODO: deprecate old config
+        seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
+    else:
+        errors, informal_only = validate_seed_conf(conf)
+        for error in errors:
+            log.warning(error)
+        if not informal_only:
+            raise SeedConfigurationError('invalid configuration')
+        seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
+    return seed_conf
+
+class LegacySeedingConfiguration(object):
+    """
+    Read old seed.yaml configuration (with seed and views).
+    """
+    def __init__(self, seed_conf, mapproxy_conf):
+        self.conf = seed_conf
+        self.mapproxy_conf = mapproxy_conf
+        self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in iteritems(self.mapproxy_conf.grids))
+        self.seed_tasks = []
+        self.cleanup_tasks = []
+        self._init_tasks()
+
+    def _init_tasks(self):
+        for cache_name, options in iteritems(self.conf['seeds']):
+            remove_before = None
+            if 'remove_before' in options:
+                remove_before = before_timestamp_from_options(options['remove_before'])
+            try:
+                caches = self.mapproxy_conf.caches[cache_name].caches()
+            except KeyError:
+                print('error: cache %s not found. available caches: %s' % (
+                    cache_name, ','.join(self.mapproxy_conf.caches.keys())), file=sys.stderr)
+                return
+            caches = dict((grid, tile_mgr) for grid, extent, tile_mgr in caches)
+            for view in options['views']:
+                view_conf = self.conf['views'][view]
+                coverage = load_coverage(view_conf)
+
+                cache_srs = view_conf.get('srs', None)
+                if cache_srs is not None:
+                    cache_srs = [SRS(s) for s in cache_srs]
+
+                level = view_conf.get('level', None)
+                assert len(level) == 2
+
+                for grid, tile_mgr in iteritems(caches):
+                    if cache_srs and grid.srs not in cache_srs: continue
+                    md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
+                    levels = list(range(level[0], level[1]+1))
+                    if coverage:
+                        if isinstance(coverage, GeomCoverage) and coverage.geom.is_empty:
+                            continue
+                        seed_coverage = coverage.transform_to(grid.srs)
+                    else:
+                        seed_coverage = BBOXCoverage(grid.bbox, grid.srs)
+
+                    self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))
+
+                    if remove_before:
+                        levels = list(range(grid.levels))
+                        complete_extent = bool(coverage)
+                        self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
+                            seed_coverage, complete_extent=complete_extent))
+
+    def seed_tasks_names(self):
+        return self.conf['seeds'].keys()
+
+    def cleanup_tasks_names(self):
+        return self.conf['seeds'].keys()
+
+    def seeds(self, names=None):
+        if names is None:
+            return self.seed_tasks
+        else:
+            return [t for t in self.seed_tasks if t.md['name'] in names]
+
+    def cleanups(self, names=None):
+        if names is None:
+            return self.cleanup_tasks
+        else:
+            return [t for t in self.cleanup_tasks if t.md['name'] in names]
+
+class SeedingConfiguration(object):
+    def __init__(self, seed_conf, mapproxy_conf):
+        self.conf = seed_conf
+        self.mapproxy_conf = mapproxy_conf
+        self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in iteritems(self.mapproxy_conf.grids))
+
+    @memoize
+    def coverage(self, name):
+        coverage_conf = (self.conf.get('coverages') or {}).get(name)
+        if coverage_conf is None:
+            raise SeedConfigurationError('coverage %s not found. available coverages: %s' % (
+                name, ','.join((self.conf.get('coverages') or {}).keys())))
+
+        try:
+            coverage = load_coverage(coverage_conf)
+        except CoverageReadError as ex:
+            raise SeedConfigurationError("can't load coverage '%s'. %s" % (name, ex))
+        except GeometryError as ex:
+            raise SeedConfigurationError("invalid geometry in coverage '%s'. %s" % (name, ex))
+        except EmptyGeometryError as ex:
+            raise EmptyCoverageError("coverage '%s' contains no geometries. %s" % (name, ex))
+
+        # without extend we have an empty coverage
+        if not coverage.extent.llbbox:
+            raise EmptyCoverageError("coverage '%s' contains no geometries." % name)
+        return coverage
+
+    def cache(self, cache_name):
+        cache = {}
+        if cache_name not in self.mapproxy_conf.caches:
+            raise SeedConfigurationError('cache %s not found. available caches: %s' % (
+                cache_name, ','.join(self.mapproxy_conf.caches.keys())))
+        for tile_grid, extent, tile_mgr in self.mapproxy_conf.caches[cache_name].caches():
+            if isinstance(tile_mgr.cache, DummyCache):
+                raise SeedConfigurationError('can\'t seed cache %s (disable_storage: true)' %
+                    cache_name)
+            grid_name = self.grids[tile_grid]
+            cache[grid_name] = tile_mgr
+        return cache
+
+    def seed_tasks_names(self):
+        seeds = self.conf.get('seeds') or {}
+        if seeds:
+            return seeds.keys()
+        return []
+
+    def cleanup_tasks_names(self):
+        cleanups = self.conf.get('cleanups') or {}
+        if cleanups:
+            return cleanups.keys()
+        return []
+
+    def seeds(self, names=None):
+        """
+        Return seed tasks.
+        """
+        tasks = []
+        if names is None:
+            names = (self.conf.get('seeds') or {}).keys()
+
+        for seed_name in names:
+            seed_conf = self.conf['seeds'][seed_name]
+            seed_conf = SeedConfiguration(seed_name, seed_conf, self)
+            for task in seed_conf.seed_tasks():
+                tasks.append(task)
+        return tasks
+
+    def cleanups(self, names=None):
+        """
+        Return cleanup tasks.
+        """
+        tasks = []
+        if names is None:
+            names = (self.conf.get('cleanups') or {}).keys()
+
+        for cleanup_name in names:
+            cleanup_conf = self.conf['cleanups'][cleanup_name]
+            cleanup_conf = CleanupConfiguration(cleanup_name, cleanup_conf, self)
+            for task in cleanup_conf.cleanup_tasks():
+                tasks.append(task)
+        return tasks
+
+
+class ConfigurationBase(object):
+    def __init__(self, name, conf, seeding_conf):
+        self.name = name
+        self.conf = conf
+        self.seeding_conf = seeding_conf
+
+        self.coverage = self._coverages()
+        self.caches = self._caches()
+        self.grids = self._grids(self.caches)
+        self.levels = levels_from_options(conf)
+
+    def _coverages(self):
+        coverage = None
+        if 'coverages' in self.conf:
+            try:
+                coverages = [self.seeding_conf.coverage(c) for c in self.conf.get('coverages', {})]
+            except EmptyCoverageError:
+                return False
+            if len(coverages) == 1:
+                coverage = coverages[0]
+            else:
+                coverage = MultiCoverage(coverages)
+        return coverage
+
+    def _grids(self, caches):
+        grids = []
+
+        if 'grids' in self.conf:
+            # grids available for all caches
+            available_grids = reduce(operator.and_, (set(cache) for cache in caches.values()))
+            for grid_name in self.conf['grids']:
+                if grid_name not in available_grids:
+                    raise SeedConfigurationError('%s not defined for caches' % grid_name)
+                grids.append(grid_name)
+        else:
+            # check that all caches have the same grids configured
+            last = []
+            for cache_grids in (set(iterkeys(cache)) for cache in itervalues(caches)):
+                if not last:
+                    last = cache_grids
+                else:
+                    if last != cache_grids:
+                        raise SeedConfigurationError('caches in same seed task require identical grids')
+            grids = list(last or [])
+        return grids
+
+    def _caches(self):
+        """
+        Returns a dictionary with all caches for this seed.
+
+        e.g.: {'seed1': {'grid1': tilemanager1, 'grid2': tilemanager2}}
+        """
+        caches = {}
+        for cache_name in self.conf.get('caches', []):
+            caches[cache_name] = self.seeding_conf.cache(cache_name)
+        return caches
+
+
+class SeedConfiguration(ConfigurationBase):
+    def __init__(self, name, conf, seeding_conf):
+        ConfigurationBase.__init__(self, name, conf, seeding_conf)
+
+        self.refresh_timestamp = None
+        if 'refresh_before' in self.conf:
+            self.refresh_timestamp = before_timestamp_from_options(self.conf['refresh_before'])
+
+    def seed_tasks(self):
+        for grid_name in self.grids:
+            for cache_name, cache in iteritems(self.caches):
+                tile_manager = cache[grid_name]
+                grid = self.seeding_conf.grids[grid_name]
+                if self.coverage is False:
+                    coverage = False
+                elif self.coverage:
+                    coverage = self.coverage.transform_to(grid.srs)
+                else:
+                    coverage = BBOXCoverage(grid.bbox, grid.srs)
+
+                try:
+                    if coverage is not False:
+                        coverage.extent.llbbox
+                except TransformationError:
+                    raise SeedConfigurationError('%s: coverage transformation error' % self.name)
+
+                if self.levels:
+                    levels = self.levels.for_grid(grid)
+                else:
+                    levels = list(range(0, grid.levels))
+
+                if not tile_manager.cache.supports_timestamp:
+                    if self.refresh_timestamp:
+                        # remove everything
+                        self.refresh_timestamp = 0
+
+                md = dict(name=self.name, cache_name=cache_name, grid_name=grid_name)
+
+                if tile_manager.rescale_tiles:
+                    if tile_manager.rescale_tiles > 0:
+                        levels = levels[::-1]
+                    for l in levels:
+                        yield SeedTask(md, tile_manager, [l], self.refresh_timestamp, coverage)
+                else:
+                    yield SeedTask(md, tile_manager, levels, self.refresh_timestamp, coverage)
+
+class CleanupConfiguration(ConfigurationBase):
+    def __init__(self, name, conf, seeding_conf):
+        ConfigurationBase.__init__(self, name, conf, seeding_conf)
+        self.init_time = time.time()
+
+        if self.conf.get('remove_all') == True:
+            self.remove_timestamp = 0
+        elif 'remove_before' in self.conf:
+            self.remove_timestamp = before_timestamp_from_options(self.conf['remove_before'])
+        else:
+            # use now as remove_before date. this should not remove
+            # fresh seeded tiles, since this should be configured before seeding
+            self.remove_timestamp = self.init_time
+
+    def cleanup_tasks(self):
+        for grid_name in self.grids:
+            for cache_name, cache in iteritems(self.caches):
+                tile_manager = cache[grid_name]
+                grid = self.seeding_conf.grids[grid_name]
+                if self.coverage is False:
+                    coverage = False
+                    complete_extent = False
+                elif self.coverage:
+                    coverage = self.coverage.transform_to(grid.srs)
+                    complete_extent = False
+                else:
+                    coverage = BBOXCoverage(grid.bbox, grid.srs)
+                    complete_extent = True
+
+                try:
+                    if coverage is not False:
+                        coverage.extent.llbbox
+                except TransformationError:
+                    raise SeedConfigurationError('%s: coverage transformation error' % self.name)
+
+                if self.levels:
+                    levels = self.levels.for_grid(grid)
+                else:
+                    levels = list(range(0, grid.levels))
+
+                if not tile_manager.cache.supports_timestamp:
+                    # for caches without timestamp support (like MBTiles)
+                    if self.remove_timestamp is self.init_time or self.remove_timestamp == 0:
+                        # remove everything
+                        self.remove_timestamp = 0
+                    else:
+                        raise SeedConfigurationError("cleanup does not support remove_before for '%s'"
+                            " because cache '%s' does not support timestamps" % (self.name, cache_name))
+                md = dict(name=self.name, cache_name=cache_name, grid_name=grid_name)
+                yield CleanupTask(md, tile_manager, levels, self.remove_timestamp,
+                    coverage=coverage, complete_extent=complete_extent)
+
+
+def levels_from_options(conf):
+    levels = conf.get('levels')
+    if levels:
+        if isinstance(levels, list):
+            return LevelsList(levels)
+        from_level = levels.get('from')
+        to_level = levels.get('to')
+        return LevelsRange((from_level, to_level))
+    resolutions = conf.get('resolutions')
+    if resolutions:
+        if isinstance(resolutions, list):
+            return LevelsResolutionList(resolutions)
+        from_res = resolutions.get('from')
+        to_res = resolutions.get('to')
+        return LevelsResolutionRange((from_res, to_res))
+    return None
+
+def before_timestamp_from_options(conf):
+    """
+    >>> import time
+    >>> t = before_timestamp_from_options({'hours': 4})
+    >>> time.time() - t - 4 * 60 * 60 < 1
+    True
+    """
+    if 'time' in conf:
+        try:
+            return timestamp_from_isodate(conf['time'])
+        except ValueError:
+            raise SeedConfigurationError(
+                "can't parse time '%s'. should be ISO time string" % (conf["time"], ))
+    if 'mtime' in conf:
+        datasource = abspath(conf['mtime'])
+        try:
+            return os.path.getmtime(datasource)
+        except OSError as ex:
+            raise SeedConfigurationError(
+                "can't parse last modified time from file '%s'." % (datasource, ), ex)
+    deltas = {}
+    for delta_type in ('weeks', 'days', 'hours', 'minutes', 'seconds'):
+        deltas[delta_type] = conf.get(delta_type, 0)
+    return timestamp_before(**deltas)
+
+
+class LevelsList(object):
+    def __init__(self, levels=None):
+        self.levels = levels
+
+    def for_grid(self, grid):
+        uniqe_valid_levels = set(l for l in self.levels if 0 <= l <= (grid.levels-1))
+        return sorted(uniqe_valid_levels)
+
+class LevelsRange(object):
+    def __init__(self, level_range=None):
+        self.level_range = level_range
+
+    def for_grid(self, grid):
+        start, stop = self.level_range
+        if start is None:
+            start = 0
+        if stop is None:
+            stop = 999
+
+        stop = min(stop, grid.levels-1)
+
+        return list(range(start, stop+1))
+
+class LevelsResolutionRange(object):
+    def __init__(self, res_range=None):
+        self.res_range = res_range
+    def for_grid(self, grid):
+        start, stop = self.res_range
+        if start is None:
+            start = 0
+        else:
+            start = grid.closest_level(start)
+
+        if stop is None:
+            stop = grid.levels-1
+        else:
+            stop = grid.closest_level(stop)
+
+        return list(range(start, stop+1))
+
+class LevelsResolutionList(object):
+    def __init__(self, resolutions=None):
+        self.resolutions = resolutions
+
+    def for_grid(self, grid):
+        levels = set(grid.closest_level(res) for res in self.resolutions)
+        return sorted(levels)
+