r5py 1.1.0__py3-none-any.whl

r5py/util/classpath.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+"""Make sure R5 is in the class path, download it if not."""
+
+import hashlib
+import pathlib
+import string
+import urllib.parse
+import warnings
+
+import requests
+
+from .config import Config
+from .exceptions import UnexpectedClasspathSchema
+from .validating_requests_session import ValidatingRequestsSession
+from .warnings import R5pyWarning
+
+# update these to use a newer R5 version if no R5 available locally
+R5_JAR_URL = (
+    "https://github.com/r5py/r5/releases/download/v7.5-r5py/r5-v7.5-r5py-all.jar"
+)
+R5_JAR_SHA256 = "70c89e57f9f98dd96c9d2be5fdc00d53a133893b2620e0d8d05e3e602f34e4d5"
+# ---
+
+
+__all__ = ["R5_CLASSPATH"]
+
+
+config = Config()
+
+config.argparser.add(
+    "-r",
+    "--r5-classpath",
+    help="R5’s class path, can point to r5-all.jar",
+    default="",
+)
+
+
+def find_r5_classpath(arguments):
+    r5_classpath = None
+
+    if arguments.r5_classpath:
+        schema, *_ = urllib.parse.urlparse(arguments.r5_classpath)
+
+        # fmt: off
+        if (
+            schema in ("file", "")
+            or (len(schema) == 1 and schema in string.ascii_letters)
+            # windows drive letter
+        ):
+            # fmt: on
+            if pathlib.Path(arguments.r5_classpath).exists():
+                r5_classpath = arguments.r5_classpath
+
+        elif schema in ("https", "http"):
+            r5_classpath = config.CACHE_DIR / pathlib.Path(arguments.r5_classpath).name
+            with requests.get(arguments.r5_classpath) as response:
+                r5_classpath.write_bytes(response.content)
+            r5_classpath = str(r5_classpath)
+
+        else:
+            raise UnexpectedClasspathSchema(
+                f"Could not parse `r5_classpath`: "
+                f"schema {schema}:// is not supported"
+            )
+
+    if r5_classpath is None:
+        r5_classpath = str(config.CACHE_DIR / pathlib.Path(R5_JAR_URL).name)
+        try:
+            with open(r5_classpath, "rb") as jar:
+                assert hashlib.sha256(jar.read()).hexdigest() == R5_JAR_SHA256
+        except (AssertionError, FileNotFoundError):
+            if arguments.verbose:
+                warnings.warn(
+                    "Could not find R5 jar, trying to download it from upstream",
+                    R5pyWarning,
+                    stacklevel=1,
+                )
+            with (
+                ValidatingRequestsSession() as session,
+                session.get(R5_JAR_URL, R5_JAR_SHA256) as response,
+                open(r5_classpath, "wb") as jar,
+            ):
+                jar.write(response.content)
+            if arguments.verbose:
+                warnings.warn(
+                    f"Successfully downloaded {pathlib.Path(R5_JAR_URL).name}",
+                    R5pyWarning,
+                    stacklevel=1,
+                )
+
+    return r5_classpath
+
+
+R5_CLASSPATH = find_r5_classpath(config.arguments)

r5py/util/config.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+
+"""Handle configuration options and command line options."""
+
+import datetime
+import functools
+import importlib.resources
+import os
+import pathlib
+import sys
+import tempfile
+
+import configargparse
+
+__all__ = ["Config"]
+
+
+PACKAGE = __package__.split(".")[0]
+CONFIG_FILE_TEMPLATE = importlib.resources.files(f"{PACKAGE}.util").joinpath(
+    f"{PACKAGE}.yml.template"
+)
+CACHE_MAX_AGE = datetime.timedelta(weeks=2)
+
+if "HOME" not in os.environ:  # e.g., testing environment or container
+    os.environ["HOME"] = "."
+
+
+class Config:
+    """Load configuration from config files or command line arguments."""
+
+    _instance = None  # stores singleton instance
+
+    def __init__(self):
+        """Load configuration from config files or command line arguments."""
+        self.argparser.add(
+            "-v",
+            "--verbose",
+            help="Enable verbose output from R5 and r5py",
+            action="store_true",
+        )
+
+    def __new__(cls):
+        """Load configuration from config files or command line arguments."""
+        if cls._instance is None:
+            cls._instance = super(Config, cls).__new__(cls)
+        return cls._instance
+
+    @property
+    def arguments(self):
+        """
+        Arguments passed from command line or config file.
+
+        Ignores `--help`: can be used while not all modules have added arguments.
+        """
+        return self.get_arguments(ignore_help_args=True)
+
+    @property
+    def argparser(self):
+        """Return a singleton instance of a `configargparse.ArgumentParser`."""
+        try:
+            argparser = configargparse.get_argument_parser(
+                prog=PACKAGE,
+                description=sys.modules[PACKAGE].__doc__,
+                default_config_files=self.CONFIG_FILES,
+            )
+        except ValueError:  # has been instantiated, already
+            argparser = configargparse.get_argument_parser()
+        return argparser
+
+    @functools.cached_property
+    def CACHE_DIR(self):
+        """Save persistent cache files into this directory."""
+        cache_dir = (
+            pathlib.Path(
+                os.environ.get("LOCALAPPDATA")
+                or os.environ.get("XDG_CACHE_HOME")
+                or (pathlib.Path(os.environ.get("HOME")) / ".cache")
+            )
+            / PACKAGE
+        )
+        cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # clean old files to keep cache dir from growing too much
+        cache_treshold = (datetime.datetime.now() - CACHE_MAX_AGE).timestamp()
+        for cached_file in cache_dir.glob("**/*"):
+            try:
+                *_, atime, mtime, _ = cached_file.stat()
+                assert max(atime, mtime) > cache_treshold
+            except (
+                AssertionError,  # expired
+                FileNotFoundError,  # broken symlink
+                PermissionError,
+            ):
+                try:
+                    cached_file.unlink()
+                except (
+                    IsADirectoryError,  # only available on Linux kernels
+                    PermissionError,  # what’s raised instead on Win and MacOs
+                ):
+                    pass
+
+        return cache_dir
+
+    @functools.cached_property
+    def CONFIG_FILES(self):
+        """List locations of potential configuration files."""
+        config_files = [
+            pathlib.Path(f"/etc/{PACKAGE}.yml"),
+            pathlib.Path(
+                os.environ.get("APPDATA")
+                or os.environ.get("XDG_CONFIG_HOME")
+                or (pathlib.Path(os.environ.get("HOME")) / ".config")
+            )
+            / f"{PACKAGE}.yml",
+        ]
+
+        # write a template configuration file to possible locations
+        for config_file in config_files:
+            self._copy_config_file_template(config_file)
+
+        # argparse does not understand pathlib.Path
+        config_files = [str(config_file) for config_file in config_files]
+
+        return config_files
+
+    @staticmethod
+    def _copy_config_file_template(destination_path):
+        """
+        Try to copy a configuration file template.
+
+        Arguments:
+        ----------
+        destination_path : pathlib.Path
+            Where could/should a configuration file exist?
+        """
+        if not destination_path.exists():
+            try:
+                destination_path.parent.mkdir(parents=True, exist_ok=True)
+
+                with importlib.resources.as_file(CONFIG_FILE_TEMPLATE) as template:
+                    destination_path.write_text(template.read_text())
+
+            except (
+                FileNotFoundError,
+                FileExistsError,
+                IsADirectoryError,
+                PermissionError,
+            ):
+                pass
+
+    def get_arguments(self, ignore_help_args=False):
+        """Parse arguments passed from command line or config file."""
+        return self.argparser.parse_known_args(ignore_help_args=ignore_help_args)[0]
+
+    @functools.cached_property
+    def TEMP_DIR(self):
+        """
+        Save temporary files to this directory.
+
+        read-only property,
+        use command-line option `--temporary-directory` to change.
+        """
+        parent_dir = self.arguments.temporary_directory
+        temp_dir = pathlib.Path(tempfile.mkdtemp(prefix=self.PACKAGE, dir=parent_dir))
+        return temp_dir
+
+    PACKAGE = PACKAGE
+
+
+Config().argparser.add(
+    "-t",
+    "--temporary-directory",
+    help="Directory for temporary files, overrides system default",
+    default=None,
+    type=pathlib.Path,
+)

r5py/util/contains_gtfs_data.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+"""Check whether a file contains a GTFS data set."""
+
+import zipfile
+
+__all__ = ["contains_gtfs_data"]
+
+
+# https://developers.google.com/transit/gtfs/reference#field_definitions
+GTFS_REQUIRED_FILES = (
+    "agency.txt",
+    "stops.txt",
+    "routes.txt",
+    "trips.txt",
+    "stop_times.txt",
+)
+
+
+def contains_gtfs_data(path):
+    """
+    Check whether the file in `path` contains a GTFS dataset.
+
+    This is a rather heuristic approach: it tries to open the file
+    as a ZIP archive, and confirm the presence of the files required
+    by the GTFS standard reference.
+
+    Arguments:
+    ----------
+    path : str | pathlib.Path | file-like
+        The file to check. Should be opened in binary mode, if passed as a
+        file-like object.
+
+    Returns:
+    --------
+    bool : `True` if `path` likely contains a GTFS dataset, `False` if not.
+    """
+    try:
+        archive = zipfile.ZipFile(path)
+        assert all(
+            gtfs_field in archive.namelist() for gtfs_field in GTFS_REQUIRED_FILES
+        )
+        contains_gtfs_data = True
+    except (AssertionError, FileNotFoundError, zipfile.BadZipFile):
+        contains_gtfs_data = False
+    return contains_gtfs_data

r5py/util/data_validation.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+"""Utility functions for validating input data."""
+
+from .exceptions import NoIDColumnError, NonUniqueIDError, NoCrsError
+
+__all__ = ["check_od_data_set"]
+
+
+def check_od_data_set(od_data_set):
+    """Check whether an origin/destination data set fulfils certain minimum criteria.
+
+    Checks whether `od_data_set` has an `id` column with unique values, and a coordinate
+    reference system defined. Raises a `r5py.util.exceptions.NoIDColumnError`, a
+    `r5py.util.exceptions.NonUniqueIDError`, or a `r5py.util.exceptions.NoCrsError`
+    otherwise.
+
+    Arguments
+    ---------
+        od_data_set : geopandas.GeoDataFrame
+            The origin/destination data set to check.
+    """
+    if "id" not in od_data_set.columns:
+        raise NoIDColumnError("Data set must contain an 'id' column.")
+    if not od_data_set.id.is_unique:
+        raise NonUniqueIDError("Id values must be unique.")
+    if od_data_set.crs is None:
+        raise NoCrsError("Data set has to have a coordinate reference system defined.")

r5py/util/environment.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+
+"""Normalise some environment variables that might not always get set."""
+
+import os
+import pathlib
+
+# if a readthedocs runner uses a conda environment, it fails to
+# properly initialise the JAVA_HOME and PROJ_LIB environment variables
+#
+# this might happen on other installation, so let’s keep this as general
+# as possible.
+#
+# As readthedocs also does not export CONDA_PREFIX, we first reconstruct
+# it from CONDA_ENVS_PATH and CONDA_DEFAULT_ENV
+if (
+    "CONDA_PREFIX" not in os.environ
+    and "CONDA_DEFAULT_ENV" in os.environ
+    and "CONDA_ENVS_PATH" in os.environ
+):  # pragma: no cover
+    os.environ["CONDA_PREFIX"] = str(
+        pathlib.Path(os.environ["CONDA_ENVS_PATH"]) / os.environ["CONDA_DEFAULT_ENV"]
+    )
+if "JAVA_HOME" not in os.environ and "CONDA_PREFIX" in os.environ:  # pragma: no cover
+    os.environ["JAVA_HOME"] = str(
+        pathlib.Path(os.environ["CONDA_PREFIX"]) / "lib" / "jvm"
+    )
+if "PROJ_LIB" not in os.environ and "CONDA_PREFIX" in os.environ:  # pragma: no cover
+    os.environ["PROJ_LIB"] = str(
+        pathlib.Path(os.environ["CONDA_PREFIX"]) / "share" / "proj"
+    )

r5py/util/exceptions.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+"""R5py-specific exceptions."""
+
+import requests
+
+
+# base class for r5py-related exceptions
+class R5pyError(Exception):
+    """Generic base exception for r5py errors."""
+
+
+# more specific exceptions
+class GtfsFileError(R5pyError):
+    """GTFS file contained errors."""
+
+
+class ChecksumFailed(requests.RequestException, R5pyError):
+    """Requested resource did not pass checksum test."""
+
+
+class MissingColumnError(ValueError, R5pyError):
+    """An input data set is missing a required column."""
+
+
+class NoCrsError(ValueError, R5pyError):
+    """An input data set’s geometry column does not have a reference system defined."""
+
+
+class NonUniqueIDError(ValueError, R5pyError):
+    """An input data set’s `id` column has non-unique values."""
+
+
+class NoIDColumnError(MissingColumnError):
+    """An input data set does not have a required `id` column."""
+
+
+class UnexpectedClasspathSchema(ValueError, R5pyError):
+    """A classpath was supplied as an URI, but could not be parsed."""
+
+
+class UnexpectedCrsError(ValueError, R5pyError):
+    """A geometry is in an unexpected reference system."""

r5py/util/file_digest.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+"""Create a hash sum of a file."""
+
+import hashlib
+import pathlib
+
+__all__ = ["FileDigest"]
+
+
+BUFFER_SIZE = 64 * 1024
+
+
+class FileDigest(str):
+    """Create a hash sum of a file."""
+
+    def __new__(cls, input_file, digest="blake2s"):
+        """
+        Create a hash sum of a file.
+
+        Arguments
+        ---------
+        input_file : pathlib.Path | str
+            for which file to compute a hash digest
+        digest : str | func
+            name of hash algorithm (s.
+            https://docs.python.org/3/library/hashlib.html) or function that
+            returns a hash sum
+        """
+        input_file = pathlib.Path(input_file)
+        try:
+            with input_file.open("rb") as f:
+                hashdigest = hashlib.file_digest(f, digest)
+        except AttributeError:  # Python<=3.10
+            hashdigest = hashlib.new(digest)
+            with input_file.open("rb") as f:
+                while data := f.read(BUFFER_SIZE):
+                    hashdigest.update(data)
+
+        return hashdigest.hexdigest()

r5py/util/good_enough_equidistant_crs.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+
+"""Find the most appropriate equidistant (UTM) reference system for an extent."""
+
+import pyproj
+import shapely
+
+from .exceptions import UnexpectedCrsError
+
+FALLBACK_CRS = 3857
+DATUM_NAME = "WGS 84"
+VERY_SMALL_BUFFER_SIZE = 0.001
+
+
+class GoodEnoughEquidistantCrs(pyproj.CRS):
+    """
+    Find the most appropriate UTM reference system for the current extent.
+
+    (We need this to be able to calculate lengths in meters.
+    Results don’t have to be perfect, so also the neighbouring UTM grid will do.)
+    """
+
+    def __new__(cls, extent):
+        """
+        Find the most appropriate UTM reference system for the current extent.
+
+        (We need this to be able to calculate lengths in meters.
+        Results don’t have to be perfect, so also the neighbouring UTM grid will do.)
+
+        Arguments
+        ---------
+        extent: shapely.Geometry
+            The geographical extent for which to find an equidistant reference
+            system, in `EPSG:4326`
+        """
+        if GoodEnoughEquidistantCrs._is_plausible_in_epsg4326(extent):
+            # default CRS in case we do not find any better match
+            crs = pyproj.CRS.from_epsg(FALLBACK_CRS)
+
+            # buffer extent (so everything is a polygon)
+            extent = extent.buffer(VERY_SMALL_BUFFER_SIZE)
+
+            crsinfo = pyproj.database.query_utm_crs_info(
+                datum_name=DATUM_NAME,
+                area_of_interest=pyproj.aoi.AreaOfInterest(*extent.bounds),
+            )
+            for candidate_crs in crsinfo:
+                area_of_use = shapely.box(*candidate_crs.area_of_use.bounds)
+                coverage = shapely.intersection(extent, area_of_use).area / extent.area
+
+                if coverage > 0.5:
+                    # more than half of extent covered by crs’ area of use
+                    # -> good enough
+                    crs = pyproj.CRS.from_authority(
+                        candidate_crs.auth_name, candidate_crs.code
+                    )
+                    break
+
+            return crs
+
+        else:
+            raise UnexpectedCrsError("`extent` does not seem to be in `EPSG:4326`")
+
+    @staticmethod
+    def _is_plausible_in_epsg4326(geometry):
+        try:
+            minx, miny, maxx, maxy = geometry.bounds
+            assert -180 <= minx <= maxx <= 180
+            assert -90 <= miny <= maxy <= 90
+            return True
+        except AssertionError:
+            return False

r5py/util/jvm.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+"""Set up a JVM and import basic java classes."""
+
+import os
+import pathlib
+import shutil
+import sys
+
+import jpype
+import jpype.imports
+
+from .classpath import R5_CLASSPATH
+from .config import Config
+from .memory_footprint import MAX_JVM_MEMORY
+
+__all__ = ["start_jvm"]
+
+
+def start_jvm():
+    """
+    Start a Java Virtual Machine (JVM) if none is running already.
+
+    Takes into account the `--max-memory` and `--verbose` command
+    line and configuration options.
+    """
+    if not jpype.isJVMStarted():
+        # preload signal handling; this, among other things, prevents some of
+        # the warning messages we have been seeing
+        # (cf. https://stackoverflow.com/q/15790403 and
+        #  https://docs.oracle.com/en/java/javase/19/vm/signal-chaining.html )
+        JVM_PATH = pathlib.Path(jpype.getDefaultJVMPath()).resolve()
+        if sys.platform == "linux":
+            try:
+                LIBJSIG = next(JVM_PATH.parent.glob("**/libjsig.so"))
+                os.environ["LD_PRELOAD"] = str(LIBJSIG)
+            except StopIteration:  # pragma: no cover
+                pass  # don’t fail completely if libjsig not found
+        elif sys.platform == "darwin":
+            try:
+                LIBJSIG = next(JVM_PATH.parent.glob("**/libjsig.dylib"))
+                os.environ["DYLD_INSERT_LIBRARIES"] = str(LIBJSIG)
+            except StopIteration:  # pragma: no cover
+                pass  # don’t fail completely if libjsig not found
+
+        TEMP_DIR = Config().TEMP_DIR
+
+        jpype.startJVM(
+            f"-Xmx{MAX_JVM_MEMORY:d}",
+            "-XX:+RestoreMXCSROnJNICalls",  # https://github.com/r5py/r5py/issues/485
+            "-Xcheck:jni",
+            "-Xrs",  # https://stackoverflow.com/q/34951812
+            "-Duser.language=en",  # Set a default locale, …
+            "-Duser.country=US",  # … as R5 formats numeric return …
+            "-Duser.variant=",  # … values as a localised string
+            f"-Djava.io.tmpdir={TEMP_DIR}",
+            "--enable-native-access=ALL-UNNAMED",
+            classpath=[R5_CLASSPATH],
+            interrupt=True,
+        )
+
+        # Add shutdown hook that cleans up the temporary directory
+        @jpype.JImplements("java.lang.Runnable")
+        class ShutdownHookToCleanUpTempDir:
+            @jpype.JOverride
+            def run(self):  # pragma: no cover
+                shutil.rmtree(TEMP_DIR)
+
+        import java.lang
+
+        java.lang.Runtime.getRuntime().addShutdownHook(
+            java.lang.Thread(ShutdownHookToCleanUpTempDir())
+        )
+
+        if not Config().arguments.verbose:
+            import ch.qos.logback.classic
+            import java.io
+            import java.lang
+            import org.slf4j.LoggerFactory
+
+            logger_context = org.slf4j.LoggerFactory.getILoggerFactory()
+            for log_target in (
+                "com.conveyal.gtfs",
+                "com.conveyal.osmlib",
+                "com.conveyal.r5",
+                "com.conveyal.r5.profile.ExecutionTimer",
+                "com.conveyal.r5.profile.FastRaptorWorker",
+                "graphql.GraphQL",
+                "org.eclipse.jetty",
+                "org.hsqldb.persist.Logger" "org.mongodb.driver.connection",
+            ):
+                logger_context.getLogger(log_target).setLevel(
+                    ch.qos.logback.classic.Level.valueOf("OFF")
+                )
+
+            if sys.platform == "win32":  # Windows
+                null_stream = java.io.PrintStream("NUL")
+            else:
+                null_stream = java.io.PrintStream("/dev/null")
+            java.lang.System.setErr(null_stream)
+            java.lang.System.setOut(null_stream)
+
+
+# The JVM should be started before we attempt to import any Java package.
+# If we run `start_jvm()` before another `import` statement, linting our
+# code would result in many E402 (‘Module level import not at top of file’)
+# warnings, if the JVM would start implicitely when `__file__` is imported,
+# we would end up with F401 (‘Module imported but unused’) warnings.
+
+# This below is a middle way: We don’t start the JVM right away, only
+# when `start_jvm()` is called. However, if we attempt to import a
+# Java package (or, more precisely, a package that’s likely to be a
+# Java package), the `import` statement would trigger `start_jvm()`
+
+# see:
+# https://github.com/jpype-project/jpype/blob/master/jpype/imports.py#L146
+
+
+class _JImportLoaderThatStartsTheJvm(jpype.imports._JImportLoader):
+    """Find Java packages for import statements, start JVM before that."""
+
+    def find_spec(self, name, path, target=None):
+        # we got this far in `sys.meta_path` (no other finder/loader
+        # knew about the package we try to load), and naturally, we’re
+        # towards the end of that list.
+
+        # Let’s assume the requested packages is a Java package,
+        # and start the JVM
+        start_jvm()
+
+        # then go the standard jpype way:
+        return super().find_spec(name, path, target)
+
+
+# replace jpype’s _JImportLoader with our own:
+for i, finder in enumerate(sys.meta_path):
+    if isinstance(finder, jpype.imports._JImportLoader):
+        sys.meta_path[i] = _JImportLoaderThatStartsTheJvm()