nefino-geosync 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

nefino_geosync/api_client.py

@@ -1,18 +1,21 @@
 """This module handles the API client for the Nefino API.
 If you want to use the Nefino API for something other than fetching the latest geodata,
-you can use this client to interact with the API directly."""
+you can use this client to interact with the API directly.
+"""
 
-from typing import Any, Dict, List 
+from .config import Config
 from .schema import GeoAnalysisInput, PlaceTypeGeo, schema
 from sgqlc.endpoint.http import HTTPEndpoint
 from sgqlc.operation import Operation
-from .config import Config
+from typing import Any, Dict, List
 
-def get_client(api_host: str="https://api.nefino.li") -> HTTPEndpoint:
+
+def get_client(api_host: str = 'https://api.nefino.li') -> HTTPEndpoint:
     """Returns an HTTP client for the Nefino API."""
     headers = {'Authorization': Config.singleton().api_key}
     return HTTPEndpoint(f'{api_host}/external', headers)
 
+
 def general_availability_operation() -> Operation:
     """Returns the general availability of layers and access permissions from Nefino API."""
     operation = Operation(schema.Query)
@@ -35,6 +38,7 @@ def general_availability_operation() -> Operation:
     layers.pre_buffer()
     return operation
 
+
 # any is the most specific type we can write for the results from the availability query
 # this is a limitation of sgqlc types
 # GitHub issue: https://github.com/profusion/sgqlc/issues/129
@@ -42,7 +46,9 @@ GeneralAvailabilityResult = Any
 LocalAvailabilityResult = Any
 
 
-def local_availability_operation(availability_result: GeneralAvailabilityResult) -> Operation:
+def local_availability_operation(
+    availability_result: GeneralAvailabilityResult,
+) -> Operation:
     """Builds an operation to determine location-specific details of all layers."""
     operation = Operation(schema.Query)
     for state in build_states_list(availability_result):
@@ -50,32 +56,35 @@ def local_availability_operation(availability_result: GeneralAvailabilityResult)
             # if you request the same field multiple times with different arguments,
             # you need to give each copy a unique alias
             __alias__=f'regionalLayers_{state}',
-            place_id=state, place_type=PlaceTypeGeo('FEDERAL_STATE_GEO'))
+            place_id=state,
+            place_type=PlaceTypeGeo('FEDERAL_STATE_GEO'),
+        )
         regional_layers.name()
         regional_layers.last_update()
     return operation
 
+
 def build_states_list(availability_result: GeneralAvailabilityResult) -> List[str]:
     """Returns a list of states from the availability result."""
     if availability_result.allowed_analysis_areas is None:
         return []
     if availability_result.allowed_analysis_areas.all_areas_enabled:
         # DE1 to DEG are the place_ids for the German states (EU scheme)
-        return [f'DE{i}' for i in list("123456789ABCDEFG")]
-    return [state.place_id for state in 
-            availability_result.allowed_analysis_areas.enabled_states]
+        return [f'DE{i}' for i in list('123456789ABCDEFG')]
+    return [state.place_id for state in availability_result.allowed_analysis_areas.enabled_states]
+
 
 def start_analyses_operation(inputs: Dict[str, GeoAnalysisInput]) -> Operation:
     """Builds an operation to start analyses with the given inputs."""
     operation = Operation(schema.Mutation)
     for state, input_data in inputs.items():
-        start_analysis = operation.start_analysis(inputs=input_data,
-                                                  __alias__=f'startAnalysis_{state}')
+        start_analysis = operation.start_analysis(inputs=input_data, __alias__=f'startAnalysis_{state}')
         start_analysis.pk()
         start_analysis.status()
         start_analysis.url()
     return operation
 
+
 def get_analyses_operation() -> Operation:
     """Builds an operation to get all analyses."""
     operation = Operation(schema.Query)
@@ -84,4 +93,27 @@ def get_analyses_operation() -> Operation:
     analyses.status()
     analyses.url()
     analyses.started_at()
-    return operation
+    return operation
+
+
+def layer_changelog_operation(timestamp_start: str = None) -> Operation:
+    """Builds an operation to get layer changelog entries."""
+    operation = Operation(schema.Query)
+
+    # Build the input object for the changelog query
+    changelog_input = {}
+    if timestamp_start:
+        changelog_input['timestampStart'] = timestamp_start
+
+    changelog = operation.layer_changelog(inputs=changelog_input)
+    changelog.layer_name()
+    changelog.timestamp()
+    changelog.action()
+    changelog.changed_fields()
+    changelog.attributes()
+    changelog.layer_id()
+    changelog.last_update()
+    changelog.cluster_name()
+    changelog.cluster_id()
+
+    return operation

nefino_geosync/compose_requests.py

@@ -1,68 +1,134 @@
-from typing import Dict, List, Set 
-from .schema import CoordinateInput, GeoAnalysisInput, GeoAnalysisLayerInput, GeoAnalysisObjectInput, GeoAnalysisOutputFormatInput, GeoAnalysisRequestInput, GeoAnalysisScopeInput, ScopeType
-from .api_client import GeneralAvailabilityResult, LocalAvailabilityResult, build_states_list
-from .journal import Journal
-from .config import Config
 from .access_rule_filter import AccessRuleFilter
+from .api_client import (
+    GeneralAvailabilityResult,
+    LocalAvailabilityResult,
+    build_states_list,
+)
+from .config import Config
+from .journal import Journal
+from .layer_changelog import LayerChangelogResult, layer_has_relevant_changes_in_changelog
+from .parse_args import parse_args
+from .schema import (
+    CoordinateInput,
+    GeoAnalysisInput,
+    GeoAnalysisLayerInput,
+    GeoAnalysisObjectInput,
+    GeoAnalysisOutputFormatInput,
+    GeoAnalysisRequestInput,
+    GeoAnalysisScopeInput,
+    ScopeType,
+)
+from typing import Dict, List, Set
 
 # Place analyses require a dummy coordinate. It will be ignored in calculations.
 DUMMY_COORDINATE = CoordinateInput(lon=9.0, lat=52.0)
 # The API requires input of combining operations, even if they are not used.
 DUMMY_OPERATIONS = []
 
-def compose_complete_requests(general_availability: GeneralAvailabilityResult,
-                              local_availability: LocalAvailabilityResult
-                              ) -> Dict[str, GeoAnalysisInput]:
+
+def compose_complete_requests(
+    general_availability: GeneralAvailabilityResult,
+    local_availability: LocalAvailabilityResult,
+    changelog_result: LayerChangelogResult = None,
+) -> Dict[str, GeoAnalysisInput]:
     """Use fetched data to build the complete requests for all available layers."""
     available_states = build_states_list(general_availability)
-    requests_as_tuples = [(state, compose_single_request(state, general_availability, local_availability)) 
-                          for state in available_states]
-    return {state: request for (state, request) in requests_as_tuples 
-            if request is not None}
 
-def compose_layer_inputs(layers: list, local_layers: Set[str], state: str) -> List[GeoAnalysisLayerInput]:
+    # Log the list of available federal states
+    if available_states:
+        print(f'📍 Checking {len(available_states)} available federal state(s): {", ".join(sorted(available_states))}')
+    else:
+        print('⚠️ No federal states available for your account')
+        return {}
+
+    requests_as_tuples = [
+        (state, compose_single_request(state, general_availability, local_availability, changelog_result))
+        for state in available_states
+    ]
+
+    # Filter out None requests and notify user about up-to-date states
+    result = {}
+    for state, request in requests_as_tuples:
+        if request is not None:
+            result[state] = request
+        else:
+            print(f'✅ {state} is up-to-date')
+
+    return result
+
+
+def compose_layer_inputs(
+    layers: list, local_layers: Set[str], state: str, cluster_name: str, changelog_result: LayerChangelogResult = None
+) -> List[GeoAnalysisLayerInput]:
     """Build a list of layer inputs from output lists."""
+    args = parse_args()
     journal = Journal.singleton()
-    return [GeoAnalysisLayerInput(layer_name=layer['name'], 
-                                  buffer_m=[layer['pre_buffer']]) 
-            for layer in layers 
-            if ((not layer.is_regional) or (layer.name in local_layers)) 
-            and journal.is_newer_than_saved(layer.name, state, layer.last_update)]
-
-def compose_single_request(state: str,
-                           general_availability: GeneralAvailabilityResult,
-                           local_availability: LocalAvailabilityResult
-                           ) -> GeoAnalysisInput:
+    updated_layers = []
+
+    print(f'  🔍 Checking layers in cluster {cluster_name} for {state}...')
+
+    for layer in layers:
+        # Check if layer should be processed
+        is_available = (not layer.is_regional) or (layer.name in local_layers)
+        needs_update = journal.is_newer_than_saved(layer.name, state, layer.last_update)
+        has_relevant_changes = layer_has_relevant_changes_in_changelog(changelog_result, layer.name, cluster_name)
+
+        if is_available and (needs_update or has_relevant_changes):
+            updated_layers.append(layer)
+            if args.verbose:
+                reason = 'last update' if needs_update else 'relevant changes'
+                print(f'    📄 {layer.name} needs update ({reason}: {layer.last_update})')
+
+    if updated_layers:
+        print(f'    ⚡ Found {len(updated_layers)} in cluster {cluster_name} layers to update for {state}')
+    else:
+        print(f'    ✅ All layers are up-to-date in cluster {cluster_name} for {state}')
+
+    return [GeoAnalysisLayerInput(layer_name=layer.name, buffer_m=[layer.pre_buffer]) for layer in updated_layers]
+
+
+def compose_single_request(
+    state: str,
+    general_availability: GeneralAvailabilityResult,
+    local_availability: LocalAvailabilityResult,
+    changelog_result: LayerChangelogResult = None,
+) -> GeoAnalysisInput:
     """Build a single request for a given state."""
+    print(f'🔍 Checking layers for {state}...')
+
     config = Config.singleton()
     rules = AccessRuleFilter(general_availability.access_rules)
     # specify the data we want to add to the analysis
-    state_local_layers = {layer.name for layer in 
-                          local_availability[f'regionalLayers_{state}']}
+    state_local_layers = {layer.name for layer in local_availability[f'regionalLayers_{state}']}
 
     for skip_layer in config.skip_layers:
         state_local_layers.discard(skip_layer)
 
-    requests_as_tuples = [(cluster, compose_layer_inputs(cluster.layers, state_local_layers, state)) 
-                          for cluster in general_availability.clusters 
-                          if cluster.has_access and rules.check(state, cluster.name)]
+    requests_as_tuples = [
+        (cluster, compose_layer_inputs(cluster.layers, state_local_layers, state, cluster.name, changelog_result))
+        for cluster in general_availability.clusters
+        if cluster.has_access and rules.check(state, cluster.name)
+    ]
 
-    requests =[GeoAnalysisRequestInput(cluster_name=cluster.name, layers=layers)
-               for (cluster, layers) in requests_as_tuples if len(layers) > 0]
+    requests = [
+        GeoAnalysisRequestInput(cluster_name=cluster.name, layers=layers)
+        for (cluster, layers) in requests_as_tuples
+        if len(layers) > 0
+    ]
 
     if len(requests) == 0:
         return None
     # Specify the output format
     # TODO: this should be configurable
-    output = GeoAnalysisOutputFormatInput(template_name='default', 
-                                          type=config.output_format,
-                                          crs=config.crs)
+    output = GeoAnalysisOutputFormatInput(template_name='default', type=config.output_format, crs=config.crs)
     # specify where the analysis should be done
     scope = GeoAnalysisScopeInput(place=state, type=ScopeType('FEDERAL_STATE'))
     # put everything together into a specification for an analysis
-    spec = GeoAnalysisObjectInput(coordinate=DUMMY_COORDINATE, 
-                                  output=output, 
-                                  scope=scope, 
-                                  requests=requests, 
-                                  operations=DUMMY_OPERATIONS)
-    return GeoAnalysisInput(name=f'sync_{state}', specs=spec)
+    spec = GeoAnalysisObjectInput(
+        coordinate=DUMMY_COORDINATE,
+        output=output,
+        scope=scope,
+        requests=requests,
+        operations=DUMMY_OPERATIONS,
+    )
+    return GeoAnalysisInput(name=f'sync_{state}', specs=spec)

nefino_geosync/download_analysis.py

@@ -14,12 +14,12 @@ def download_analysis(analysis: AnalysisResult) -> None:
     """Downloads the analysis to the local machine."""
     journal = Journal.singleton()
     download_dir = get_download_directory(analysis.pk)
-    download_file = os.path.join(download_dir, "download.zip")
+    download_file = os.path.join(download_dir, 'download.zip')
     if os.path.exists(download_file):
         # remove any failed download
         os.remove(download_file)
-    urlretrieve(analysis.url.replace(" ", "%20"), download_file)
-    with zipfile.ZipFile(download_file, "r") as zip_ref:
+    urlretrieve(analysis.url.replace(' ', '%20'), download_file)
+    with zipfile.ZipFile(download_file, 'r') as zip_ref:
         zip_ref.extractall(download_dir)
     zip_root = get_zip_root(download_dir)
     unpack_items(zip_root, analysis.pk, analysis.started_at)
@@ -32,9 +32,7 @@ def get_zip_root(download_dir: str) -> str:
     return download_dir
 
 
-FILE_NAME_PATTERN = re.compile(
-    r"(?P<layer>^.*?)(?P<buffer>__[0-9]+m)?(?P<ext>\..{3,4}$)"
-)
+FILE_NAME_PATTERN = re.compile(r'(?P<layer>^.*?)(?P<buffer>__[0-9]+m)?(?P<ext>\..{3,4}$)')
 
 
 def unpack_items(zip_root: str, pk: str, started_at: datetime) -> None:
@@ -50,7 +48,7 @@ def unpack_items(zip_root: str, pk: str, started_at: datetime) -> None:
     config = Config.singleton()
 
     if pk not in journal.analysis_states:
-        print(f"Analysis {pk} not found in journal; skipping download")
+        print(f'Analysis {pk} not found in journal; skipping download')
         return
 
     state = journal.get_state_for_analysis(pk)
@@ -58,12 +56,9 @@ def unpack_items(zip_root: str, pk: str, started_at: datetime) -> None:
 
     # Iterate through cluster folders inside the analysis subfolder
     for cluster in (
-        f
-        for f in os.listdir(base_path)
-        if f != "analysis_area" and os.path.isdir(os.path.join(base_path, f))
+        f for f in os.listdir(base_path) if f != 'analysis_area' and os.path.isdir(os.path.join(base_path, f))
     ):
         cluster_dir = os.path.join(base_path, cluster)
-        layers = set()
 
         for file in os.listdir(cluster_dir):
             if journal.is_newer_than_saved(file, state, started_at):
@@ -73,29 +68,27 @@ def unpack_items(zip_root: str, pk: str, started_at: datetime) -> None:
 
                 file_path = os.path.join(cluster_dir, file)
                 match = re.match(FILE_NAME_PATTERN, file)
-                layer, ext = (match.group("layer"), match.group("ext"))
+                layer, ext = (match.group('layer'), match.group('ext'))
 
                 # Remove any existing files for the same layer
                 # this is important to avoid confusion if the pre-buffer changes
-                for matching_file in (
-                    f for f in os.listdir(output_dir) if f.startswith(layer)
-                ):
+                for matching_file in (f for f in os.listdir(output_dir) if f.startswith(layer)):
                     output_match = re.match(FILE_NAME_PATTERN, matching_file)
                     # only remove files that match the layer and extension
                     # otherwise, only the last extension to be unpacked would survive
                     # also, we are double-checking the layer name here in case we have
                     # a layer name which starts with a different layer's name
-                    if (
-                        output_match.group("layer") == layer
-                        and output_match.group("ext") == ext
-                    ):
+                    if output_match.group('layer') == layer and output_match.group('ext') == ext:
                         os.remove(os.path.join(output_dir, matching_file))
 
                 move(file_path, output_dir)
-                layers.add(layer)
 
-        journal.record_layers_unpacked(layers, state, started_at)
+    # Update the journal to mark layers as updated. We might have empty layers so we do set all requested layers as
+    # updated.
+    layers_to_mark_updated = journal.analysis_requested_layers[pk]
+    print(f'Recording {len(layers_to_mark_updated)} requested layers as updated for state {state}')
 
+    journal.record_layers_unpacked(layers_to_mark_updated, state, started_at)
     rmtree(zip_root)
 
 
@@ -116,11 +109,9 @@ def get_base_path(zip_root: str) -> str:
     Returns:
         str: Path to the directory containing the cluster folders and analysis_summary.xlsx
     """
-    if "analysis_summary.xlsx" in os.listdir(zip_root):
+    if 'analysis_summary.xlsx' in os.listdir(zip_root):
         # Old structure - use zip_root
         return zip_root
     # Get the analysis subfolder name (first and only directory in zip_root)
-    analysis_subfolder = next(
-        f for f in os.listdir(zip_root) if os.path.isdir(os.path.join(zip_root, f))
-    )
+    analysis_subfolder = next(f for f in os.listdir(zip_root) if os.path.isdir(os.path.join(zip_root, f)))
     return os.path.join(zip_root, analysis_subfolder)

nefino_geosync/download_completed_analyses.py

@@ -1,19 +1,18 @@
-from .journal import Journal
-from .get_downloadable_analyses import get_downloadable_analyses
 from .download_analysis import download_analysis
-from sgqlc.endpoint.http import HTTPEndpoint
+from .get_downloadable_analyses import get_downloadable_analyses
+from .journal import Journal
 from .parse_args import parse_args
+from sgqlc.endpoint.http import HTTPEndpoint
+
 
 def download_completed_analyses(client: HTTPEndpoint) -> None:
     """Downloads the analyses that have been completed."""
     journal = Journal.singleton()
     args = parse_args()
     for analysis in get_downloadable_analyses(client):
-        if not analysis.pk in journal.synced_analyses:
+        if analysis.pk not in journal.synced_analyses:
             if analysis.pk in journal.analysis_states:
                 download_analysis(analysis)
-                print(f"Downloaded analysis {analysis.pk}")
-            else:
-                print(f"Analysis {analysis.pk} missing metadata; skipping download")
+                print(f'Downloaded analysis {analysis.pk}')
         elif args.verbose:
-            print(f"Analysis {analysis.pk} already downloaded")
+            print(f'Analysis {analysis.pk} already downloaded')

nefino_geosync/get_downloadable_analyses.py

@@ -1,10 +1,11 @@
-from time import sleep
-from typing import Generator, Protocol
 from .api_client import get_analyses_operation
-from .schema import DateTime, Status
 from .graphql_errors import check_errors
-from sgqlc.endpoint.http import HTTPEndpoint
 from .parse_args import parse_args
+from .schema import DateTime, Status
+from sgqlc.endpoint.http import HTTPEndpoint
+from time import sleep
+from typing import Generator, Protocol
+
 
 # Let's give a quick description of what we want to be fetching.
 # This does depend on what get_analysis_operation() actually does.
@@ -15,31 +16,33 @@ class AnalysisResult(Protocol):
     started_at: DateTime
 
 
-def get_downloadable_analyses(client: HTTPEndpoint) -> Generator[AnalysisResult, None, None]:
+def get_downloadable_analyses(
+    client: HTTPEndpoint,
+) -> Generator[AnalysisResult, None, None]:
     """Yields analyses that are available for download.
     Polls for more analyses and yields them until no more are available.
     """
     verbose = parse_args().verbose
     op = get_analyses_operation()
     reported_pks = set()
-    print("Checking for analyses to download...")
+    print('Checking for analyses to download...')
     while True:
         data = client(op)
-        check_errors(data)
+        check_errors(data, 'Failed to fetch analysis status')
         analyses = op + data
         found_outstanding_analysis = False
 
         for analysis in analyses.analysis_metadata:
-            if analysis.status == Status("PENDING") or analysis.status == Status("RUNNING"):
+            if analysis.status == Status('PENDING') or analysis.status == Status('RUNNING'):
                 if verbose:
-                    print(f"Analysis {analysis.pk} is still pending or running.")
+                    print(f'Analysis {analysis.pk} is still pending or running.')
                 found_outstanding_analysis = True
-            if analysis.status == Status("SUCCESS") and analysis.pk not in reported_pks:
+            if analysis.status == Status('SUCCESS') and analysis.pk not in reported_pks:
                 reported_pks.add(analysis.pk)
                 yield analysis
 
         if not found_outstanding_analysis:
             break
         if verbose:
-            print("Waiting for more analyses to finish...")
+            print('Waiting for more analyses to finish...')
         sleep(10)

nefino_geosync/graphql_errors.py

@@ -1,40 +1,73 @@
+import html
+import json
+import re
+import sys
+from .parse_args import parse_args
+from datetime import datetime
 from prompt_toolkit import print_formatted_text
 from prompt_toolkit.formatted_text import HTML
-from .parse_args import parse_args
-import json
-import html
 
-def check_errors(data: dict) -> None:
+
+def check_errors(data: dict, context: str = None) -> None:
     """Check for errors in a GraphQL response."""
     args = parse_args()
     if 'errors' in data:
         if args.verbose:
-            pp("<b>GraphQL operation with errors:</b> " + html.escape(json.dumps(data, indent=4)))
+            pp('<b>GraphQL operation with errors:</b> ' + html.escape(json.dumps(data, indent=4)))
 
         if is_token_invalid(data):
-            pp('<b fg="red">ERROR:</b> Invalid token. Please run <b>nefino-geosync --configure</b> and double-check your API key.')
+            pp(
+                '<b fg="red">ERROR:</b> Invalid token. Please run <b>nefino-geosync --configure</b> and double-check your API key.'
+            )
         else:
             if not args.verbose:
                 try:
-                    pp("<b>Received GraphQL error from server:</b> " + html.escape(json.dumps(data['errors'], indent=4)))
+                    pp(
+                        '<b>Received GraphQL error from server:</b> '
+                        + html.escape(json.dumps(data['errors'], indent=4))
+                    )
                 except Exception as e:
                     print(e)
-                    print(data["errors"])
+                    print(data['errors'])
+
+            # Add context information if provided
+            if context:
+                pp(f'<b fg="red">Context:</b> {context}')
+
+            if not args.verbose:
                 pp("""<b fg="red">ERROR:</b> A GraphQL error occurred. Run with <b>--verbose</b> to see more information.
+If this error persists, please contact Nefino support: https://www.nefino.de/kontakt
 Exiting due to the above error.""")
-            if args.verbose:
-                pp('<b fg="red">ERROR:</b> A GraphQL error occurred. Exiting due to the above error.')
+            else:
+                pp('<b fg="red">ERROR:</b> A GraphQL error occurred.')
+                pp(
+                    '<b fg="red">If this error persists, please contact Nefino support: https://www.nefino.de/kontakt</b>'
+                )
+                pp('<b fg="red">Exiting due to the above error.</b>')
+
+        sys.exit(1)
 
-        exit(1)
 
-def pp(to_print: str):
+def pp(to_print: str) -> None:
+    # Display formatted text in console
     print_formatted_text(HTML(to_print))
 
+    # For logging: check if stdout has been replaced by TeeStream
+    # If so, write plain text directly to the log file to avoid duplication
+    if hasattr(sys.stdout, 'log_file'):
+        # Remove HTML tags for plain text logging
+        plain_text = re.sub(r'<[^>]+>', '', to_print)
+
+        timestamp = datetime.now().strftime('%H:%M:%S')
+        sys.stdout.log_file.write(f'[{timestamp}] [STDOUT] {plain_text}\n')
+        sys.stdout.log_file.flush()
+
+
 def is_token_invalid(data: dict) -> bool:
     """Check if the token is invalid."""
     try:
-        if data['errors'][0]['extensions']['nefino_type'] == "AuthTokenInvalid":
+        if data['errors'][0]['extensions']['nefino_type'] == 'AuthTokenInvalid':
             return True
     except KeyError:
         return False
-    return False
+    return False