nefino-geosync 0.2.3__py3-none-any.whl

nefino_geosync/journal.py

@@ -0,0 +1,199 @@
+import json
+import os
+import re
+from .storage import get_app_directory
+from datetime import datetime, timezone
+from typing import Dict, Set
+
+
+class Journal:
+    """Handles metadata about analyses for efficient downloading."""
+
+    # This is a singleton class. There should only be one instance of Journal.
+    _instance = None
+
+    @classmethod
+    def singleton(cls):
+        """Returns the singleton instance of Journal."""
+        if not cls._instance:
+            cls._instance = Journal()
+        return cls._instance
+
+    def __init__(self) -> None:
+        if Journal._instance:
+            raise Exception('Journal is a singleton class. Use Journal.singleton() to get the instance.')
+        # Mapping from analysis pk to the state where the analysis was started
+        self.analysis_states: Dict[str, str] = dict()
+
+        # Mapping from analysis pk to the layers that were requested
+        self.analysis_requested_layers: Dict[str, Set[str]] = dict()
+
+        # Mapping from layer name to where it was last updated and when
+        self.layer_last_updates: Dict[str, Dict[str, datetime]] = dict()
+
+        # Record which analyses have been successfully started
+        self.synced_analyses: Set[str] = set()
+
+        # Track the last successful geosync run for changelog queries
+        self.last_geosync_run: datetime = None
+
+        self.load_analysis_states()
+        self.load_analysis_requested_layers()
+        self.load_layer_last_updates()
+        self.load_synced_analyses()
+        self.load_last_geosync_run()
+
+    def save_analysis_states(self) -> None:
+        """Saves the analysis states to a file."""
+        with open(os.path.join(get_app_directory(), 'analysis_states.json'), 'w') as f:
+            json.dump(self.analysis_states, f)
+
+    def load_analysis_states(self) -> None:
+        """Loads the analysis states from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), 'analysis_states.json'), 'r') as f:
+                self.analysis_states = json.load(f)
+        except FileNotFoundError:
+            # we already have an empty dictionary as the field value
+            print('No saved analysis states found.')
+
+    def save_analysis_requested_layers(self) -> None:
+        """Saves the analysis requested layers to a file."""
+        # Convert sets to lists for JSON serialization
+        serializable_data = {pk: list(layers) for pk, layers in self.analysis_requested_layers.items()}
+        with open(os.path.join(get_app_directory(), 'analysis_requested_layers.json'), 'w') as f:
+            json.dump(serializable_data, f)
+
+    def load_analysis_requested_layers(self) -> None:
+        """Loads the analysis requested layers from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), 'analysis_requested_layers.json'), 'r') as f:
+                data = json.load(f)
+                # Convert lists back to sets
+                self.analysis_requested_layers = {pk: set(layers) for pk, layers in data.items()}
+        except FileNotFoundError:
+            # we already have an empty dictionary as the field value
+            print('No saved analysis requested layers found.')
+
+    def save_layer_last_updates(self) -> None:
+        """Saves the layer last updates to a file."""
+        with open(os.path.join(get_app_directory(), 'layer_last_updates.json'), 'w') as f:
+            json.dump(self.layer_last_updates, f, default=lambda x: x.isoformat())
+
+    def load_layer_last_updates(self) -> None:
+        """Loads the layer last updates from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), 'layer_last_updates.json'), 'r') as f:
+                self.layer_last_updates = json.load(f)
+                for cluster in self.layer_last_updates.values():
+                    for state, timestamp in cluster.items():
+                        cluster[state] = datetime.fromisoformat(timestamp) if timestamp else None
+        except FileNotFoundError:
+            # we already have an empty dictionary as the field value
+            print('No saved layer last updates found.')
+
+    def save_synced_analyses(self) -> None:
+        """Saves the list of processed analyses to a file."""
+        with open(os.path.join(get_app_directory(), 'synced_analyses.json'), 'w') as f:
+            json.dump(list(self.synced_analyses), f)
+
+    def load_synced_analyses(self) -> None:
+        """Loads the list of processed analyses from a file."""
+        try:
+            with open(os.path.join(get_app_directory(), 'synced_analyses.json'), 'r') as f:
+                self.synced_analyses = set(json.load(f))
+        except FileNotFoundError:
+            # we already have an empty set as the field value
+            print('No saved downloaded analyses found.')
+
+    def save_last_geosync_run(self) -> None:
+        """Saves the timestamp of the last successful geosync run."""
+        with open(os.path.join(get_app_directory(), 'last_geosync_run.json'), 'w') as f:
+            json.dump(self.last_geosync_run.isoformat() if self.last_geosync_run else None, f)
+
+    def load_last_geosync_run(self) -> None:
+        """Loads the timestamp of the last successful geosync run."""
+        try:
+            with open(os.path.join(get_app_directory(), 'last_geosync_run.json'), 'r') as f:
+                timestamp_str = json.load(f)
+                self.last_geosync_run = datetime.fromisoformat(timestamp_str) if timestamp_str else None
+        except FileNotFoundError:
+            # we already have None as the field value
+            print('No saved last geosync run timestamp found.')
+
+    def record_successful_geosync_run(self, start_time: datetime) -> None:
+        """Records the current time as the last successful geosync run."""
+        self.last_geosync_run = start_time
+        self.save_last_geosync_run()
+
+    def record_analyses_requested(self, start_analyses_result, analysis_inputs) -> None:
+        """Records the analyses that have been started, where they were started, and which layers were requested."""
+        pattern = r'^startAnalysis_(?P<state>DE[1-9A-G])$'
+        for alias, analysis_metadata in start_analyses_result.__dict__.items():
+            match = re.match(pattern, alias)
+            if not match:
+                continue
+            state = match.group('state')
+            # record where the analysis was started
+            self.analysis_states[analysis_metadata.pk] = state
+            # record which layers were requested
+            requested_layers = set()
+            for request in analysis_inputs[state].specs.requests:
+                for layer in request.layers:
+                    requested_layers.add(layer.layer_name)
+            self.analysis_requested_layers[analysis_metadata.pk] = requested_layers
+        self.save_analysis_states()
+        self.save_analysis_requested_layers()
+
+    def clear_analysis_requested_layers(self) -> None:
+        """Clears all analysis requested layers at the start of a new run."""
+        if self.analysis_requested_layers:
+            print(
+                f"Clearing {len(self.analysis_requested_layers)} old analysis metadata entries from previous runs"
+            )
+            self.analysis_requested_layers.clear()
+            self.save_analysis_requested_layers()
+
+    def record_layers_unpacked(self, layers: Set[str], state: str, started_at: datetime) -> None:
+        """Records the layers that have been unpacked, and when they were last updated."""
+        print(f'Recording layers {layers} as unpacked for state {state}')
+
+        for layer in layers:
+            if layer not in self.layer_last_updates:
+                self.layer_last_updates[layer] = dict()
+            self.layer_last_updates[layer][state] = started_at
+        self.save_layer_last_updates()
+
+    def get_state_for_analysis(self, pk: str) -> str:
+        """Returns the state where the analysis was started."""
+        return self.analysis_states[pk]
+
+    def is_newer_than_saved(self, layer: str, state: str, timestamp: datetime) -> bool:
+        """Checks if the layer needs to be unpacked."""
+        if layer not in self.layer_last_updates:
+            return True
+        if state not in self.layer_last_updates[layer]:
+            return True
+        if not self.layer_last_updates[layer][state]:
+            return True
+
+        saved_timestamp = self.layer_last_updates[layer][state]
+
+        # Handle timezone comparison issues by making both timezone-aware
+        if saved_timestamp.tzinfo is None and timestamp.tzinfo is not None:
+            # Assume saved timestamp is UTC if it has no timezone info
+            saved_timestamp = saved_timestamp.replace(tzinfo=timezone.utc)
+        elif saved_timestamp.tzinfo is not None and timestamp.tzinfo is None:
+            # Make the API timestamp timezone-aware (assume UTC)
+            timestamp = timestamp.replace(tzinfo=timezone.utc)
+
+        return saved_timestamp < timestamp
+
+    def record_analysis_synced(self, pk: str) -> None:
+        """Records that the analysis has been downloaded and unpacked."""
+        self.synced_analyses.add(pk)
+        self.save_synced_analyses()
+        # Clean up the requested layers for this analysis to prevent unbounded growth
+        if pk in self.analysis_requested_layers:
+            del self.analysis_requested_layers[pk]
+            self.save_analysis_requested_layers()

nefino_geosync/layer_changelog.py

@@ -0,0 +1,240 @@
+"""Module for querying and logging layer changelog information."""
+
+import csv
+import os
+from .api_client import general_availability_operation, layer_changelog_operation
+from .config import Config
+from .graphql_errors import check_errors
+from .journal import Journal
+from .schema import LayerChangelogEntry
+from datetime import datetime, timezone
+from nefino_geosync.access_rule_filter import AccessRuleFilter
+from sgqlc.endpoint.http import HTTPEndpoint
+from typing import Any
+
+LayerChangelogResult = Any
+
+
+def query_layer_changelog(client: HTTPEndpoint, timestamp_start: str = None) -> LayerChangelogResult:
+    """Queries the layer changelog from the GraphQL API."""
+    changelog_op = layer_changelog_operation(timestamp_start)
+    changelog_data = client(changelog_op)
+    check_errors(changelog_data, 'Failed to fetch layer changelog')
+    return changelog_op + changelog_data
+
+
+def record_layer_changes_since_last_run(client: HTTPEndpoint) -> LayerChangelogResult:
+    """Logs all layer changes since the last successful geosync run and returns the changelog data."""
+    journal = Journal.singleton()
+
+    # Get the timestamp of the last successful run
+    timestamp_start = None
+    if journal.last_geosync_run:
+        # Format timestamp as required: 2025-09-19T10:30:20.383210+00:00
+        if journal.last_geosync_run.tzinfo is None:
+            # If timezone-naive, assume UTC
+
+            aware_timestamp = journal.last_geosync_run.replace(tzinfo=timezone.utc)
+        else:
+            aware_timestamp = journal.last_geosync_run
+
+        timestamp_start = aware_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
+        # Ensure the timezone format includes the colon (e.g., +00:00 not +0000)
+        if len(timestamp_start) >= 4 and timestamp_start[-4:].isdigit():
+            timestamp_start = timestamp_start[:-2] + ':' + timestamp_start[-2:]
+
+        print(f'Checking for layer changes since last geosync run: {timestamp_start}')
+    else:
+        print('No previous geosync run found, skipping changelog check')
+        return None
+
+    # Get available clusters to filter changelog results
+    try:
+        # First get general availability to determine accessible clusters
+        general_op = general_availability_operation()
+        general_data = client(general_op)
+        check_errors(general_data, 'Failed to fetch general availability for changelog filtering')
+        general_availability = general_op + general_data
+
+        # Use AccessRuleFilter to determine accessible clusters
+        rules = AccessRuleFilter(general_availability.access_rules)
+
+        # Get all places from access rules to check against
+        all_places = set()
+        for rule in general_availability.access_rules:
+            all_places.update(rule.places)
+
+        accessible_clusters = {
+            cluster.name
+            for cluster in general_availability.clusters
+            if cluster.has_access and any(rules.check(place, cluster.name) for place in all_places)
+        }
+        print(f'Accessible clusters: {accessible_clusters}')
+
+    except Exception as e:
+        print(f'Failed to fetch accessible clusters, showing all changelog entries: {e}')
+        accessible_clusters = set()  # Empty set means show all
+
+    # Query the changelog
+    try:
+        changelog_result = query_layer_changelog(client, timestamp_start)
+        log_changelog_entries(changelog_result, accessible_clusters)
+        return changelog_result
+    except Exception as e:
+        print(f'Failed to retrieve layer changelog: {e}')
+        return None
+
+
+def log_changelog_entries(changelog_result: LayerChangelogResult, accessible_clusters: set = None) -> None:
+    """Logs changelog entries, focusing on relevant changes and filtering by accessible clusters."""
+    if not hasattr(changelog_result, 'layer_changelog'):
+        print('No layer changelog data received')
+        return
+
+    changelog_entries = changelog_result.layer_changelog
+    if not changelog_entries:
+        print('✅ No layer changes detected for accessible clusters since last run')
+        return
+
+    # Filter entries by accessible clusters and collect those with relevant changes
+    relevant_entries = []
+    for entry in changelog_entries:
+        # Skip if not in accessible clusters
+        cluster_name = getattr(entry, 'cluster_name', None)
+        if accessible_clusters and (not cluster_name or cluster_name not in accessible_clusters):
+            continue
+
+        # Check for relevant changes
+        relevant_changes = _get_relevant_changes(entry)
+        if relevant_changes:
+            relevant_entries.append((entry, relevant_changes))
+
+    # If no relevant changes found, show success message
+    if not relevant_entries:
+        print('✅ No layer changes detected for accessible clusters since last run')
+        return
+
+    print(f'📋 Found {len(relevant_entries)} layer change(s) for accessible clusters since last run:')
+
+    for entry, relevant_changes in relevant_entries:
+        _log_entry_details(entry, relevant_changes)
+
+    # Save to CSV (use original filtered entries for CSV)
+    filtered_entries = [entry for entry, _ in relevant_entries]
+    save_changelog_to_csv(filtered_entries)
+
+
+def _get_relevant_changes(entry: LayerChangelogEntry) -> list:
+    """Extract relevant changes from a changelog entry."""
+    relevant_changes = []
+    if hasattr(entry, 'changed_fields') and entry.changed_fields:
+        for field in entry.changed_fields:
+            if field in ['attributes', 'layer_name', 'cluster_name']:
+                relevant_changes.append(field)
+    return relevant_changes
+
+
+def _log_entry_details(entry: LayerChangelogEntry, relevant_changes: list) -> None:
+    """Log details for a single changelog entry."""
+    layer_name = getattr(entry, 'layer_name', 'Unknown')
+    cluster_name = getattr(entry, 'cluster_name', 'Unknown')
+    action = getattr(entry, 'action', 'Unknown')
+    timestamp = getattr(entry, 'timestamp', 'Unknown')
+
+    print(f"  📦 Layer '{layer_name}' (cluster: {cluster_name})")
+    print(f'      Action: {action}')
+    print(f'      Changed fields: {", ".join(relevant_changes)}')
+    print(f'      Timestamp: {timestamp}')
+
+    # If attributes changed, log the attributes
+    if 'attributes' in relevant_changes and hasattr(entry, 'attributes') and entry.attributes:
+        print(f'      New attributes: {", ".join(entry.attributes)}')
+
+    print('')  # Empty line for readability
+
+
+def save_changelog_to_csv(filtered_entries: list) -> None:
+    """Saves changelog entries to a CSV file in the output directory."""
+    if not filtered_entries:
+        return
+
+    config = Config.singleton()
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    csv_filename = f'layer_changelog_{timestamp}.csv'
+    csv_path = os.path.join(config.output_path, csv_filename)
+
+    # Ensure output directory exists
+    os.makedirs(config.output_path, exist_ok=True)
+
+    # Define CSV headers
+    headers = ['timestamp', 'layer_name', 'cluster_name', 'action', 'changed_fields', 'attributes']
+
+    try:
+        with open(csv_path, 'w', newline='', encoding='utf-8') as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=headers)
+            writer.writeheader()
+
+            for entry in filtered_entries:
+                # Filter for relevant changes
+                relevant_changes = []
+                if hasattr(entry, 'changed_fields') and entry.changed_fields:
+                    for field in entry.changed_fields:
+                        if field in ['attributes', 'layer_name', 'cluster_name']:
+                            relevant_changes.append(field)
+
+                if not relevant_changes:
+                    continue  # Skip entries without relevant changes
+
+                # Extract entry data
+                layer_name = getattr(entry, 'layer_name', 'Unknown')
+                cluster_name = getattr(entry, 'cluster_name', 'Unknown')
+                action = getattr(entry, 'action', 'Unknown')
+                timestamp_str = getattr(entry, 'timestamp', 'Unknown')
+                attributes = ''
+                if hasattr(entry, 'attributes') and entry.attributes:
+                    attributes = ', '.join(entry.attributes)
+
+                writer.writerow(
+                    {
+                        'timestamp': timestamp_str,
+                        'layer_name': layer_name,
+                        'cluster_name': cluster_name,
+                        'action': action,
+                        'changed_fields': ', '.join(relevant_changes),
+                        'attributes': attributes,
+                    }
+                )
+
+        print(f'📊 Changelog saved to CSV: {csv_path}')
+    except Exception as e:
+        print(f'⚠️  Failed to save changelog to CSV: {e}')
+
+
+def record_successful_geosync_completion(start_time: datetime) -> None:
+    """Records that a geosync run completed successfully."""
+    journal = Journal.singleton()
+    journal.record_successful_geosync_run(start_time)
+    print('✅ Geosync completed successfully')
+
+
+def layer_has_relevant_changes_in_changelog(
+    changelog_result: LayerChangelogResult, layer_name: str, cluster_name: str
+) -> bool:
+    """Check if a layer has relevant changes in the provided changelog data."""
+    if not changelog_result or not hasattr(changelog_result, 'layer_changelog'):
+        return False
+
+    if not changelog_result.layer_changelog:
+        return False
+
+    # Check if this specific layer has relevant changes
+    for entry in changelog_result.layer_changelog:
+        entry_layer_name = getattr(entry, 'layer_name', None)
+        entry_cluster_name = getattr(entry, 'cluster_name', None)
+
+        if entry_layer_name == layer_name and entry_cluster_name == cluster_name:
+            relevant_changes = _get_relevant_changes(entry)
+            if relevant_changes:
+                return True
+
+    return False

nefino_geosync/parse_args.py

@@ -0,0 +1,15 @@
+import argparse
+
+def parse_args(cached=[]):
+    if len(cached) > 0:
+        return cached[0]
+    parser = argparse.ArgumentParser(
+        prog="Nefino GeoSync",
+        description='Download available geodata from the Nefino API.',
+        epilog='If you have further questions please reach out to us! The maintainers for this tool can be found on https://github.com/nefino/geosync-py.')
+    parser.add_argument('-c', '--configure', action='store_true', help='Edit your existing configuration. The first-run wizard will be shown again, with your existing configuration pre-filled.')
+    parser.add_argument('-r', '--resume', action='store_true', help='Resume checking for completed analyses and downloading them. This will skip the analysis start step.')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Print more information to the console.')
+    args = parser.parse_args()
+    cached.append(args)
+    return args

nefino_geosync/run.py

@@ -0,0 +1,59 @@
+"""This is the main entry point of the application."""
+
+import atexit
+import os
+from .api_client import get_client
+from .config import Config
+from .download_completed_analyses import download_completed_analyses
+from .layer_changelog import (
+    record_layer_changes_since_last_run,
+    record_successful_geosync_completion,
+)
+from .parse_args import parse_args
+from .session_logger import start_session_logging, stop_session_logging
+from .start_analyses import start_analyses
+from datetime import UTC, datetime
+
+
+def main() -> None:
+    # Start session-wide logging
+    print('Starting Nefino GeoSync...')
+    start_session_logging()
+    start_time = datetime.now(tz=UTC)
+    # Ensure logging stops when the program exits
+    atexit.register(stop_session_logging)
+
+    try:
+        args = parse_args()
+
+        if args.configure:
+            config = Config.singleton()
+            # if you are running with --configure on the first run (you don't need to)
+            # you will be prompted to configure the app by the config singleton init.
+            # In that case, don't prompt the user again.
+            if not config.already_prompted:
+                config.run_config_prompts()
+
+        client = get_client(api_host=os.getenv('NEFINO_API_HOST', default='https://api.nefino.li'))
+
+        # Check for layer changes since last run before starting new analyses
+        changelog_result = record_layer_changes_since_last_run(client)
+
+        if not args.resume:
+            start_analyses(client, changelog_result)
+        else:
+            download_completed_analyses(client)
+
+        # Record successful completion
+        record_successful_geosync_completion(start_time)
+
+    except Exception as e:
+        print(f'Fatal error: {e}')
+        raise
+    finally:
+        # Ensure logging stops even if there's an error
+        stop_session_logging()
+
+
+if __name__ == '__main__':
+    main()