nefino-geosync 0.1.0__py3-none-any.whl

nefino_geosync/access_rule_filter.py

@@ -0,0 +1,11 @@
+
+class AccessRuleFilter:
+    def __init__(self, access_rules):
+        self.access_rules = access_rules
+
+    def check(self, place, cluster):
+        for access_rule in self.access_rules:
+            if place in access_rule.places:
+                if access_rule.all_clusters_enabled or cluster in access_rule.clusters:
+                    return True
+        return False

nefino_geosync/api_client.py

@@ -0,0 +1,87 @@
+"""This module handles the API client for the Nefino API.
+If you want to use the Nefino API for something other than fetching the latest geodata,
+you can use this client to interact with the API directly."""
+
+from typing import Any, Dict, List 
+from .schema import GeoAnalysisInput, PlaceTypeGeo, schema
+from sgqlc.endpoint.http import HTTPEndpoint
+from sgqlc.operation import Operation
+from .config import Config
+
+def get_client(api_host: str="https://api.nefino.li") -> HTTPEndpoint:
+    """Returns an HTTP client for the Nefino API."""
+    headers = {'Authorization': Config.singleton().api_key}
+    return HTTPEndpoint(f'{api_host}/external', headers)
+
+def general_availability_operation() -> Operation:
+    """Returns the general availability of layers and access permissions from Nefino API."""
+    operation = Operation(schema.Query)
+    analysis_areas = operation.allowed_analysis_areas()
+    analysis_areas.all_areas_enabled()
+    analysis_areas.enabled_states().place_id()
+
+    access_rules = operation.access_rules()
+    access_rules.all_clusters_enabled()
+    access_rules.clusters()
+    access_rules.places()
+
+    clusters = operation.clusters()
+    clusters.name()
+    clusters.has_access()
+    layers = clusters.layers()
+    layers.name()
+    layers.last_update()
+    layers.is_regional()
+    layers.pre_buffer()
+    return operation
+
+# any is the most specific type we can write for the results from the availability query
+# this is a limitation of sgqlc types
+# GitHub issue: https://github.com/profusion/sgqlc/issues/129
+GeneralAvailabilityResult = Any
+LocalAvailabilityResult = Any
+
+
+def local_availability_operation(availability_result: GeneralAvailabilityResult) -> Operation:
+    """Builds an operation to determine location-specific details of all layers."""
+    operation = Operation(schema.Query)
+    for state in build_states_list(availability_result):
+        regional_layers = operation.regional_layers(
+            # if you request the same field multiple times with different arguments,
+            # you need to give each copy a unique alias
+            __alias__=f'regionalLayers_{state}',
+            place_id=state, place_type=PlaceTypeGeo('FEDERAL_STATE_GEO'))
+        regional_layers.name()
+        regional_layers.last_update()
+    return operation
+
+def build_states_list(availability_result: GeneralAvailabilityResult) -> List[str]:
+    """Returns a list of states from the availability result."""
+    if availability_result.allowed_analysis_areas is None:
+        return []
+    if availability_result.allowed_analysis_areas.all_areas_enabled:
+        # DE1 to DEG are the place_ids for the German states (EU scheme)
+        return [f'DE{i}' for i in list("123456789ABCDEFG")]
+    return [state.place_id for state in 
+            availability_result.allowed_analysis_areas.enabled_states]
+
+def start_analyses_operation(inputs: Dict[str, GeoAnalysisInput]) -> Operation:
+    """Builds an operation to start analyses with the given inputs."""
+    operation = Operation(schema.Mutation)
+    for state, input_data in inputs.items():
+        start_analysis = operation.start_analysis(inputs=input_data,
+                                                  __alias__=f'startAnalysis_{state}')
+        start_analysis.pk()
+        start_analysis.status()
+        start_analysis.url()
+    return operation
+
+def get_analyses_operation() -> Operation:
+    """Builds an operation to get all analyses."""
+    operation = Operation(schema.Query)
+    analyses = operation.analysis_metadata()
+    analyses.pk()
+    analyses.status()
+    analyses.url()
+    analyses.started_at()
+    return operation

nefino_geosync/compose_requests.py

@@ -0,0 +1,68 @@
+from typing import Dict, List, Set 
+from .schema import CoordinateInput, GeoAnalysisInput, GeoAnalysisLayerInput, GeoAnalysisObjectInput, GeoAnalysisOutputFormatInput, GeoAnalysisRequestInput, GeoAnalysisScopeInput, ScopeType
+from .api_client import GeneralAvailabilityResult, LocalAvailabilityResult, build_states_list
+from .journal import Journal
+from .config import Config
+from .access_rule_filter import AccessRuleFilter
+
+# Place analyses require a dummy coordinate. It will be ignored in calculations.
+DUMMY_COORDINATE = CoordinateInput(lon=9.0, lat=52.0)
+# The API requires input of combining operations, even if they are not used.
+DUMMY_OPERATIONS = []
+
+def compose_complete_requests(general_availability: GeneralAvailabilityResult,
+                              local_availability: LocalAvailabilityResult
+                              ) -> Dict[str, GeoAnalysisInput]:
+    """Use fetched data to build the complete requests for all available layers."""
+    available_states = build_states_list(general_availability)
+    requests_as_tuples = [(state, compose_single_request(state, general_availability, local_availability)) 
+                          for state in available_states]
+    return {state: request for (state, request) in requests_as_tuples 
+            if request is not None}
+
+def compose_layer_inputs(layers: list, local_layers: Set[str], state: str) -> List[GeoAnalysisLayerInput]:
+    """Build a list of layer inputs from output lists."""
+    journal = Journal.singleton()
+    return [GeoAnalysisLayerInput(layer_name=layer['name'], 
+                                  buffer_m=[layer['pre_buffer']]) 
+            for layer in layers 
+            if ((not layer.is_regional) or (layer.name in local_layers)) 
+            and journal.is_newer_than_saved(layer.name, state, layer.last_update)]
+
+def compose_single_request(state: str,
+                           general_availability: GeneralAvailabilityResult,
+                           local_availability: LocalAvailabilityResult
+                           ) -> GeoAnalysisInput:
+    """Build a single request for a given state."""
+    config = Config.singleton()
+    rules = AccessRuleFilter(general_availability.access_rules)
+    # specify the data we want to add to the analysis
+    state_local_layers = {layer.name for layer in 
+                          local_availability[f'regionalLayers_{state}']}
+
+    for skip_layer in config.skip_layers:
+        state_local_layers.discard(skip_layer)
+
+    requests_as_tuples = [(cluster, compose_layer_inputs(cluster.layers, state_local_layers, state)) 
+                          for cluster in general_availability.clusters 
+                          if cluster.has_access and rules.check(state, cluster.name)]
+
+    requests =[GeoAnalysisRequestInput(cluster_name=cluster.name, layers=layers)
+               for (cluster, layers) in requests_as_tuples if len(layers) > 0]
+
+    if len(requests) == 0:
+        return None
+    # Specify the output format
+    # TODO: this should be configurable
+    output = GeoAnalysisOutputFormatInput(template_name='default', 
+                                          type=config.output_format,
+                                          crs=config.crs)
+    # specify where the analysis should be done
+    scope = GeoAnalysisScopeInput(place=state, type=ScopeType('FEDERAL_STATE'))
+    # put everything together into a specification for an analysis
+    spec = GeoAnalysisObjectInput(coordinate=DUMMY_COORDINATE, 
+                                  output=output, 
+                                  scope=scope, 
+                                  requests=requests, 
+                                  operations=DUMMY_OPERATIONS)
+    return GeoAnalysisInput(name=f'sync_{state}', specs=[spec])

nefino_geosync/config.py

@@ -0,0 +1,82 @@
+import json
+import os
+from typing import List
+
+import questionary
+from .schema import CRSType, OutputObjectType
+from .storage import get_app_directory
+
+
+class Config:
+    """This class handles storing and retrieving user preferences."""
+    # This is a singleton class. There should only be one instance of Config.
+    _instance = None
+
+    @classmethod
+    def singleton(cls):
+        """Returns the singleton instance of Journal."""
+        if not cls._instance:
+            cls._instance = Config()
+        return cls._instance
+    
+    @property
+    def _config_file_path(self) -> str:
+        """Returns the path to the config file."""
+        return os.path.join(get_app_directory(), "config.json")
+
+    def __init__(self):
+        if Config._instance:
+            raise Exception("Config is a singleton class. Use Config.singleton() to get the instance.")
+        self.already_prompted = False
+        if not os.path.exists(self._config_file_path):
+            self.run_config_prompts(missing_config=True)
+        else:
+            with open(self._config_file_path, "r") as f:
+                config = json.load(f)
+                self.output_path: str = config['output_path']
+                self.output_format: OutputObjectType = OutputObjectType(
+                    config['output_format'])
+                self.crs: CRSType = CRSType(config['crs'])
+                self.skip_layers: List[str] = config['skip_layers']
+                self.api_key: str = config['api_key']
+    
+    def save(self):
+        """Saves the config to a file."""
+        with open(self._config_file_path, "w") as f:
+            json.dump({
+                'output_path': self.output_path,
+                'output_format': self.output_format,
+                'skip_layers': self.skip_layers,
+                'api_key': self.api_key,
+                'crs': self.crs
+            }, f)
+
+    def run_config_prompts(self, missing_config: bool = False):
+        """Runs the configuration wizard."""
+        self.output_path = questionary.text(
+            "Where do you want to collect downloaded geodata files?",
+            default=os.path.join(get_app_directory(), "newestData") \
+                if missing_config else self.output_path).ask()
+        self.output_format = OutputObjectType(
+            questionary.select(
+                "What format do you want to use for the output files?",
+                instruction="Changing this value after first run will require wiping the downloaded data.",
+                choices=['GPKG', 'SHP'],
+                default='GPKG' if missing_config else self.output_format
+                ).ask())
+        self.crs = CRSType(
+            questionary.select(
+                "What coordinate reference system do you want to use?",
+                choices=[crs for crs in CRSType],
+                default='EPSG_4326' if missing_config else self.crs
+            ).ask())
+        self.api_key = questionary.text(
+            "Enter your API key:",
+            default="" if missing_config else self.api_key).ask()
+        skip_layer_string = questionary.text(
+            "Enter the names of any layers you want to skip downloading, separated by commas.",
+            instruction="Layer names can be found on https://docs.nefino.li/geo.",
+            default="" if missing_config else ",".join(self.skip_layers)).ask()
+        self.skip_layers = [] if skip_layer_string == "" else skip_layer_string.split(",")
+        self.save()
+        self.already_prompted = True

nefino_geosync/download_analysis.py

@@ -0,0 +1,70 @@
+import os
+import re
+from shutil import move, rmtree
+from urllib.request import urlretrieve
+import zipfile
+from .get_downloadable_analyses import AnalysisResult
+from .journal import Journal
+from .config import Config
+from .storage import get_download_directory 
+from datetime import datetime
+
+def download_analysis(analysis: AnalysisResult) -> None:
+    """Downloads the analysis to the local machine."""
+    journal = Journal.singleton()
+    download_dir = get_download_directory(analysis.pk)
+    download_file = os.path.join(download_dir, "download.zip")
+    if not os.path.exists(download_file):
+        urlretrieve(analysis.url.replace(" ", "%20"), download_file)
+    with zipfile.ZipFile(download_file, 'r') as zip_ref:
+        zip_ref.extractall(download_dir)
+    zip_root = get_zip_root(download_dir)
+    unpack_items(zip_root, analysis.pk, analysis.started_at)
+    journal.record_analysis_synced(analysis.pk)
+
+
+def get_zip_root(download_dir: str) -> str:
+    """Returns the root directory of the extracted zip file."""
+    # earlier we had a heavily nested structure
+    return download_dir
+
+
+FILE_NAME_PATTERN = re.compile(r"(?P<layer>^.*?)(?P<buffer>__[0-9]+m)?(?P<ext>\..{3,4}$)")
+
+def unpack_items(zip_root: str, pk: str, started_at: datetime) -> None:
+    """Unpacks the layers from the zip file."""
+    journal = Journal.singleton()
+    config = Config.singleton()
+    if pk not in journal.analysis_states:
+        print(f"Analysis {pk} not found in journal; skipping download")
+        return
+    state = journal.get_state_for_analysis(pk)
+    for cluster in (f for f in os.listdir(zip_root) 
+                    if f != 'analysis_area'
+                    and os.path.isdir(os.path.join(zip_root, f))):
+        cluster_dir = os.path.join(zip_root, cluster)
+        layers = set()
+        for file in os.listdir(cluster_dir):
+            if journal.is_newer_than_saved(file, state, started_at):
+                output_dir = os.path.join(config.output_path, state)
+                if not os.path.exists(output_dir):
+                    os.makedirs(output_dir)
+                file_path = os.path.join(cluster_dir, file)
+                match = re.match(FILE_NAME_PATTERN, file)
+                layer, ext = (match.group("layer"), match.group("ext"))
+                # remove any existing files for the same layer
+                # this is important to avoid confusion if the pre-buffer changes
+                for matching_file in (f for f in os.listdir(output_dir)
+                                      if f.startswith(layer)):
+                    output_match = re.match(FILE_NAME_PATTERN, matching_file)
+                    # only remove files that match the layer and extension
+                    # otherwise, only the last extension to be unpacked would survive
+                    # also, we are double-checking the layer name here in case we have
+                    # a layer name which starts with a different layer's name
+                    if output_match.group("layer") == layer and \
+                          output_match.group("ext") == ext:
+                        os.remove(os.path.join(output_dir, matching_file))
+                move(file_path, output_dir)
+                layers.add(layer)
+        journal.record_layers_unpacked(layers, state, started_at)
+    rmtree(zip_root)

nefino_geosync/download_completed_analyses.py

@@ -0,0 +1,19 @@
+from .journal import Journal
+from .get_downloadable_analyses import get_downloadable_analyses
+from .download_analysis import download_analysis
+from sgqlc.endpoint.http import HTTPEndpoint
+from .parse_args import parse_args
+
+def download_completed_analyses(client: HTTPEndpoint) -> None:
+    """Downloads the analyses that have been completed."""
+    journal = Journal.singleton()
+    args = parse_args()
+    for analysis in get_downloadable_analyses(client):
+        if not analysis.pk in journal.synced_analyses:
+            if analysis.pk in journal.analysis_states:
+                download_analysis(analysis)
+                print(f"Downloaded analysis {analysis.pk}")
+            else:
+                print(f"Analysis {analysis.pk} missing metadata; skipping download")
+        elif args.verbose:
+            print(f"Analysis {analysis.pk} already downloaded")

nefino_geosync/get_downloadable_analyses.py

@@ -0,0 +1,45 @@
+from time import sleep
+from typing import Generator, Protocol
+from .api_client import get_analyses_operation
+from .schema import DateTime, Status
+from .graphql_errors import check_errors
+from sgqlc.endpoint.http import HTTPEndpoint
+from .parse_args import parse_args
+
+# Let's give a quick description of what we want to be fetching.
+# This does depend on what get_analysis_operation() actually does.
+class AnalysisResult(Protocol):
+    status: Status
+    pk: str
+    url: str
+    started_at: DateTime
+
+
+def get_downloadable_analyses(client: HTTPEndpoint) -> Generator[AnalysisResult, None, None]:
+    """Yields analyses that are available for download.
+    Polls for more analyses and yields them until no more are available.
+    """
+    verbose = parse_args().verbose
+    op = get_analyses_operation()
+    reported_pks = set()
+    print("Checking for analyses to download...")
+    while True:
+        data = client(op)
+        check_errors(data)
+        analyses = op + data
+        found_outstanding_analysis = False
+
+        for analysis in analyses.analysis_metadata:
+            if analysis.status == Status("PENDING") or analysis.status == Status("RUNNING"):
+                if verbose:
+                    print(f"Analysis {analysis.pk} is still pending or running.")
+                found_outstanding_analysis = True
+            if analysis.status == Status("SUCCESS") and analysis.pk not in reported_pks:
+                reported_pks.add(analysis.pk)
+                yield analysis
+
+        if not found_outstanding_analysis:
+            break
+        if verbose:
+            print("Waiting for more analyses to finish...")
+        sleep(10)

nefino_geosync/graphql_errors.py

@@ -0,0 +1,40 @@
+from prompt_toolkit import print_formatted_text
+from prompt_toolkit.formatted_text import HTML
+from .parse_args import parse_args
+import json
+import html
+
+def check_errors(data: dict) -> None:
+    """Check for errors in a GraphQL response."""
+    args = parse_args()
+    if 'errors' in data:
+        if args.verbose:
+            pp("<b>GraphQL operation with errors:</b> " + html.escape(json.dumps(data, indent=4)))
+
+        if is_token_invalid(data):
+            pp('<b fg="red">ERROR:</b> Invalid token. Please run <b>nefino-geosync --configure</b> and double-check your API key.')
+        else:
+            if not args.verbose:
+                try:
+                    pp("<b>Received GraphQL error from server:</b> " + html.escape(json.dumps(data['errors'], indent=4)))
+                except Exception as e:
+                    print(e)
+                    print(data["errors"])
+                pp("""<b fg="red">ERROR:</b> A GraphQL error occurred. Run with <b>--verbose</b> to see more information.
+Exiting due to the above error.""")
+            if args.verbose:
+                pp('<b fg="red">ERROR:</b> A GraphQL error occurred. Exiting due to the above error.')
+
+        exit(1)
+
+def pp(to_print: str):
+    print_formatted_text(HTML(to_print))
+
+def is_token_invalid(data: dict) -> bool:
+    """Check if the token is invalid."""
+    try:
+        if data['errors'][0]['extensions']['nefino_type'] == "AuthTokenInvalid":
+            return True
+    except KeyError:
+        return False
+    return False

nefino_geosync/journal.py

@@ -0,0 +1,120 @@
+import json
+import os
+import re
+from datetime import datetime
+from typing import Dict, Set
+from .storage import get_app_directory
+
+class Journal:
+    """Handles metadata about analyses for efficient downloading."""
+
+    # This is a singleton class. There should only be one instance of Journal.
+    _instance = None
+
+    @classmethod
+    def singleton(cls):
+        """Returns the singleton instance of Journal."""
+        if not cls._instance:
+            cls._instance = Journal()
+        return cls._instance
+
+    def __init__(self):
+        if Journal._instance:
+            raise Exception("Journal is a singleton class. Use Journal.singleton() to get the instance.")
+        # Mapping from analysis pk to the state where the analysis was started
+        self.analysis_states: Dict[str, str] = dict()
+
+        # Mapping from layer name to where it was last updated and when
+        self.layer_last_updates: Dict[str, Dict[str, datetime]] = dict()
+
+        # Record which analyses have been successfully started
+        self.synced_analyses: Set[str] = set()
+
+        self.load_analysis_states()
+        self.load_layer_last_updates()
+        self.load_synced_analyses()
+
+    def save_analysis_states(self):
+        """Saves the analysis states to a file."""
+        with open(os.path.join(get_app_directory(), "analysis_states.json"), "w") as f:
+            json.dump(self.analysis_states, f)
+    
+    def load_analysis_states(self):
+        """Loads the analysis states from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), "analysis_states.json"), "r") as f:
+                self.analysis_states = json.load(f)
+        except FileNotFoundError:
+            # we already have an empty dictionary as the field value
+            print("No saved analysis states found.")
+    
+    def save_layer_last_updates(self):
+        """Saves the layer last updates to a file."""
+        with open(os.path.join(get_app_directory(), "layer_last_updates.json"), "w") as f:
+            json.dump(self.layer_last_updates, f, default=lambda x: x.isoformat())
+
+    def load_layer_last_updates(self):
+        """Loads the layer last updates from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), "layer_last_updates.json"), "r") as f:
+                self.layer_last_updates = json.load(f)
+                for cluster in self.layer_last_updates.values():
+                    for state, timestamp in cluster.items():
+                        cluster[state] = datetime.fromisoformat(timestamp) if timestamp else None
+        except FileNotFoundError:
+            # we already have an empty dictionary as the field value
+            print("No saved layer last updates found.")
+    
+    def save_synced_analyses(self):
+        """Saves the list of processed analyses to a file."""
+        with open(os.path.join(get_app_directory(), "synced_analyses.json"), "w") as f:
+            json.dump(list(self.synced_analyses), f)
+    
+    def load_synced_analyses(self):
+        """Loads the list of processed analyses from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), "synced_analyses.json"), "r") as f:
+                self.synced_analyses = set(json.load(f))
+        except FileNotFoundError:
+            # we already have an empty set as the field value
+            print("No saved downloaded analyses found.")
+
+    def record_analyses_requested(self, start_analyses_result):
+        """Records the analyses that have been started, and where they were started."""
+        pattern = r"^startAnalysis_(?P<state>DE[1-9A-G])$"
+        for alias, analysis_metadata in start_analyses_result.__dict__.items(): 
+            match = re.match(pattern, alias)
+            if not match:
+                continue
+            state = match.group("state")
+            # record where the analysis was started
+            self.analysis_states[analysis_metadata.pk] = state
+        self.save_analysis_states()
+    
+    def record_layers_unpacked(self, layers: Set[str], state: str, started_at: datetime):
+        """Records the layers that have been unpacked, and when they were last updated."""
+        print(f"Recording layers {layers} as unpacked for state {state}")
+        for layer in layers:
+            if layer not in self.layer_last_updates:
+                self.layer_last_updates[layer] = dict()
+            self.layer_last_updates[layer][state] = started_at
+        self.save_layer_last_updates()
+    
+    def get_state_for_analysis(self, pk: str) -> str:
+        """Returns the state where the analysis was started."""
+        return self.analysis_states[pk]
+    
+    def is_newer_than_saved(self, layer: str, state: str, timestamp: datetime) -> bool:
+        """Checks if the layer needs to be unpacked."""
+        if layer not in self.layer_last_updates:
+            return True
+        if state not in self.layer_last_updates[layer]:
+            return True
+        if not self.layer_last_updates[layer][state]:
+            return True
+        return self.layer_last_updates[layer][state] < timestamp
+    
+    def record_analysis_synced(self, pk: str):
+        """Records that the analysis has been downloaded and unpacked."""
+        self.synced_analyses.add(pk)
+        self.save_synced_analyses()

nefino_geosync/parse_args.py

@@ -0,0 +1,15 @@
+import argparse
+
+def parse_args(cached=[]):
+    if len(cached) > 0:
+        return cached[0]
+    parser = argparse.ArgumentParser(
+        prog="Nefino GeoSync",
+        description='Download available geodata from the Nefino API.',
+        epilog='If you have further questions please reach out to us! The maintainers for this tool can be found on https://github.com/nefino/geosync-py.')
+    parser.add_argument('-c', '--configure', action='store_true', help='Edit your existing configuration. The first-run wizard will be shown again, with your existing configuration pre-filled.')
+    parser.add_argument('-r', '--resume', action='store_true', help='Resume checking for completed analyses and downloading them. This will skip the analysis start step.')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Print more information to the console.')
+    args = parser.parse_args()
+    cached.append(args)
+    return args

nefino_geosync/run.py

@@ -0,0 +1,28 @@
+"""This is the main entry point of the application."""
+import os
+from .api_client import get_client
+from .start_analyses import start_analyses
+from .download_completed_analyses import download_completed_analyses
+from .config import Config
+from .parse_args import parse_args
+
+def main():
+    args = parse_args()
+
+    if args.configure:
+        config = Config.singleton()
+        # if you are running with --configure on the first run (you don't need to)
+        # you will be prompted to configure the app by the config singleton init.
+        # In that case, don't prompt the user again.
+        if not config.already_prompted:
+            config.run_config_prompts()
+
+    client = get_client(api_host=os.getenv("NEFINO_API_HOST", default="https://api.nefino.li"))
+
+    if not args.resume:
+        start_analyses(client)
+    else:
+        download_completed_analyses(client)
+
+if __name__ == "__main__":
+    main()