nefino-geosync 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

nefino_geosync/journal.py

@@ -1,9 +1,10 @@
 import json
 import os
 import re
-from datetime import datetime
-from typing import Dict, Set
 from .storage import get_app_directory
+from datetime import datetime, timezone
+from typing import Dict, Set
+
 
 class Journal:
     """Handles metadata about analyses for efficient downloading."""
@@ -18,92 +19,126 @@ class Journal:
             cls._instance = Journal()
         return cls._instance
 
-    def __init__(self):
+    def __init__(self) -> None:
         if Journal._instance:
-            raise Exception("Journal is a singleton class. Use Journal.singleton() to get the instance.")
+            raise Exception('Journal is a singleton class. Use Journal.singleton() to get the instance.')
         # Mapping from analysis pk to the state where the analysis was started
         self.analysis_states: Dict[str, str] = dict()
 
+        # Mapping from analysis pk to the layers that were requested
+        self.analysis_requested_layers: Dict[str, Set[str]] = dict()
+
         # Mapping from layer name to where it was last updated and when
         self.layer_last_updates: Dict[str, Dict[str, datetime]] = dict()
 
         # Record which analyses have been successfully started
         self.synced_analyses: Set[str] = set()
 
+        # Track the last successful geosync run for changelog queries
+        self.last_geosync_run: datetime = None
+
         self.load_analysis_states()
         self.load_layer_last_updates()
         self.load_synced_analyses()
+        self.load_last_geosync_run()
 
-    def save_analysis_states(self):
+    def save_analysis_states(self) -> None:
         """Saves the analysis states to a file."""
-        with open(os.path.join(get_app_directory(), "analysis_states.json"), "w") as f:
+        with open(os.path.join(get_app_directory(), 'analysis_states.json'), 'w') as f:
             json.dump(self.analysis_states, f)
-    
-    def load_analysis_states(self):
+
+    def load_analysis_states(self) -> None:
         """Loads the analysis states from a file."""
         try:
-            with open(os.path.join(get_app_directory(), "analysis_states.json"), "r") as f:
+            with open(os.path.join(get_app_directory(), 'analysis_states.json'), 'r') as f:
                 self.analysis_states = json.load(f)
         except FileNotFoundError:
             # we already have an empty dictionary as the field value
-            print("No saved analysis states found.")
-    
-    def save_layer_last_updates(self):
+            print('No saved analysis states found.')
+
+    def save_layer_last_updates(self) -> None:
         """Saves the layer last updates to a file."""
-        with open(os.path.join(get_app_directory(), "layer_last_updates.json"), "w") as f:
+        with open(os.path.join(get_app_directory(), 'layer_last_updates.json'), 'w') as f:
             json.dump(self.layer_last_updates, f, default=lambda x: x.isoformat())
 
-    def load_layer_last_updates(self):
+    def load_layer_last_updates(self) -> None:
         """Loads the layer last updates from a file."""
         try:
-            with open(os.path.join(get_app_directory(), "layer_last_updates.json"), "r") as f:
+            with open(os.path.join(get_app_directory(), 'layer_last_updates.json'), 'r') as f:
                 self.layer_last_updates = json.load(f)
                 for cluster in self.layer_last_updates.values():
                     for state, timestamp in cluster.items():
                         cluster[state] = datetime.fromisoformat(timestamp) if timestamp else None
         except FileNotFoundError:
             # we already have an empty dictionary as the field value
-            print("No saved layer last updates found.")
-    
-    def save_synced_analyses(self):
+            print('No saved layer last updates found.')
+
+    def save_synced_analyses(self) -> None:
         """Saves the list of processed analyses to a file."""
-        with open(os.path.join(get_app_directory(), "synced_analyses.json"), "w") as f:
+        with open(os.path.join(get_app_directory(), 'synced_analyses.json'), 'w') as f:
             json.dump(list(self.synced_analyses), f)
-    
-    def load_synced_analyses(self):
+
+    def load_synced_analyses(self) -> None:
         """Loads the list of processed analyses from a file."""
         try:
-            with open(os.path.join(get_app_directory(), "synced_analyses.json"), "r") as f:
+            with open(os.path.join(get_app_directory(), 'synced_analyses.json'), 'r') as f:
                 self.synced_analyses = set(json.load(f))
         except FileNotFoundError:
             # we already have an empty set as the field value
-            print("No saved downloaded analyses found.")
+            print('No saved downloaded analyses found.')
+
+    def save_last_geosync_run(self) -> None:
+        """Saves the timestamp of the last successful geosync run."""
+        with open(os.path.join(get_app_directory(), 'last_geosync_run.json'), 'w') as f:
+            json.dump(self.last_geosync_run.isoformat() if self.last_geosync_run else None, f)
+
+    def load_last_geosync_run(self) -> None:
+        """Loads the timestamp of the last successful geosync run."""
+        try:
+            with open(os.path.join(get_app_directory(), 'last_geosync_run.json'), 'r') as f:
+                timestamp_str = json.load(f)
+                self.last_geosync_run = datetime.fromisoformat(timestamp_str) if timestamp_str else None
+        except FileNotFoundError:
+            # we already have None as the field value
+            print('No saved last geosync run timestamp found.')
+
+    def record_successful_geosync_run(self, start_time: datetime) -> None:
+        """Records the current time as the last successful geosync run."""
+        self.last_geosync_run = start_time
+        self.save_last_geosync_run()
 
-    def record_analyses_requested(self, start_analyses_result):
-        """Records the analyses that have been started, and where they were started."""
-        pattern = r"^startAnalysis_(?P<state>DE[1-9A-G])$"
-        for alias, analysis_metadata in start_analyses_result.__dict__.items(): 
+    def record_analyses_requested(self, start_analyses_result, analysis_inputs) -> None:
+        """Records the analyses that have been started, where they were started, and which layers were requested."""
+        pattern = r'^startAnalysis_(?P<state>DE[1-9A-G])$'
+        for alias, analysis_metadata in start_analyses_result.__dict__.items():
             match = re.match(pattern, alias)
             if not match:
                 continue
-            state = match.group("state")
+            state = match.group('state')
             # record where the analysis was started
             self.analysis_states[analysis_metadata.pk] = state
+            # record which layers were requested
+            requested_layers = set()
+            for request in analysis_inputs[state].specs.requests:
+                for layer in request.layers:
+                    requested_layers.add(layer.layer_name)
+            self.analysis_requested_layers[analysis_metadata.pk] = requested_layers
         self.save_analysis_states()
-    
-    def record_layers_unpacked(self, layers: Set[str], state: str, started_at: datetime):
+
+    def record_layers_unpacked(self, layers: Set[str], state: str, started_at: datetime) -> None:
         """Records the layers that have been unpacked, and when they were last updated."""
-        print(f"Recording layers {layers} as unpacked for state {state}")
+        print(f'Recording layers {layers} as unpacked for state {state}')
+
         for layer in layers:
             if layer not in self.layer_last_updates:
                 self.layer_last_updates[layer] = dict()
             self.layer_last_updates[layer][state] = started_at
         self.save_layer_last_updates()
-    
+
     def get_state_for_analysis(self, pk: str) -> str:
         """Returns the state where the analysis was started."""
         return self.analysis_states[pk]
-    
+
     def is_newer_than_saved(self, layer: str, state: str, timestamp: datetime) -> bool:
         """Checks if the layer needs to be unpacked."""
         if layer not in self.layer_last_updates:
@@ -112,9 +147,20 @@ class Journal:
             return True
         if not self.layer_last_updates[layer][state]:
             return True
-        return self.layer_last_updates[layer][state] < timestamp
-    
-    def record_analysis_synced(self, pk: str):
+
+        saved_timestamp = self.layer_last_updates[layer][state]
+
+        # Handle timezone comparison issues by making both timezone-aware
+        if saved_timestamp.tzinfo is None and timestamp.tzinfo is not None:
+            # Assume saved timestamp is UTC if it has no timezone info
+            saved_timestamp = saved_timestamp.replace(tzinfo=timezone.utc)
+        elif saved_timestamp.tzinfo is not None and timestamp.tzinfo is None:
+            # Make the API timestamp timezone-aware (assume UTC)
+            timestamp = timestamp.replace(tzinfo=timezone.utc)
+
+        return saved_timestamp < timestamp
+
+    def record_analysis_synced(self, pk: str) -> None:
         """Records that the analysis has been downloaded and unpacked."""
         self.synced_analyses.add(pk)
         self.save_synced_analyses()

nefino_geosync/layer_changelog.py

@@ -0,0 +1,240 @@
+"""Module for querying and logging layer changelog information."""
+
+import csv
+import os
+from .api_client import general_availability_operation, layer_changelog_operation
+from .config import Config
+from .graphql_errors import check_errors
+from .journal import Journal
+from .schema import LayerChangelogEntry
+from datetime import datetime, timezone
+from nefino_geosync.access_rule_filter import AccessRuleFilter
+from sgqlc.endpoint.http import HTTPEndpoint
+from typing import Any
+
+LayerChangelogResult = Any
+
+
+def query_layer_changelog(client: HTTPEndpoint, timestamp_start: str = None) -> LayerChangelogResult:
+    """Queries the layer changelog from the GraphQL API."""
+    changelog_op = layer_changelog_operation(timestamp_start)
+    changelog_data = client(changelog_op)
+    check_errors(changelog_data, 'Failed to fetch layer changelog')
+    return changelog_op + changelog_data
+
+
+def record_layer_changes_since_last_run(client: HTTPEndpoint) -> LayerChangelogResult:
+    """Logs all layer changes since the last successful geosync run and returns the changelog data."""
+    journal = Journal.singleton()
+
+    # Get the timestamp of the last successful run
+    timestamp_start = None
+    if journal.last_geosync_run:
+        # Format timestamp as required: 2025-09-19T10:30:20.383210+00:00
+        if journal.last_geosync_run.tzinfo is None:
+            # If timezone-naive, assume UTC
+
+            aware_timestamp = journal.last_geosync_run.replace(tzinfo=timezone.utc)
+        else:
+            aware_timestamp = journal.last_geosync_run
+
+        timestamp_start = aware_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
+        # Ensure the timezone format includes the colon (e.g., +00:00 not +0000)
+        if len(timestamp_start) >= 4 and timestamp_start[-4:].isdigit():
+            timestamp_start = timestamp_start[:-2] + ':' + timestamp_start[-2:]
+
+        print(f'Checking for layer changes since last geosync run: {timestamp_start}')
+    else:
+        print('No previous geosync run found, skipping changelog check')
+        return None
+
+    # Get available clusters to filter changelog results
+    try:
+        # First get general availability to determine accessible clusters
+        general_op = general_availability_operation()
+        general_data = client(general_op)
+        check_errors(general_data, 'Failed to fetch general availability for changelog filtering')
+        general_availability = general_op + general_data
+
+        # Use AccessRuleFilter to determine accessible clusters
+        rules = AccessRuleFilter(general_availability.access_rules)
+
+        # Get all places from access rules to check against
+        all_places = set()
+        for rule in general_availability.access_rules:
+            all_places.update(rule.places)
+
+        accessible_clusters = {
+            cluster.name
+            for cluster in general_availability.clusters
+            if cluster.has_access and any(rules.check(place, cluster.name) for place in all_places)
+        }
+        print(f'Accessible clusters: {accessible_clusters}')
+
+    except Exception as e:
+        print(f'Failed to fetch accessible clusters, showing all changelog entries: {e}')
+        accessible_clusters = set()  # Empty set means show all
+
+    # Query the changelog
+    try:
+        changelog_result = query_layer_changelog(client, timestamp_start)
+        log_changelog_entries(changelog_result, accessible_clusters)
+        return changelog_result
+    except Exception as e:
+        print(f'Failed to retrieve layer changelog: {e}')
+        return None
+
+
+def log_changelog_entries(changelog_result: LayerChangelogResult, accessible_clusters: set = None) -> None:
+    """Logs changelog entries, focusing on relevant changes and filtering by accessible clusters."""
+    if not hasattr(changelog_result, 'layer_changelog'):
+        print('No layer changelog data received')
+        return
+
+    changelog_entries = changelog_result.layer_changelog
+    if not changelog_entries:
+        print('✅ No layer changes detected for accessible clusters since last run')
+        return
+
+    # Filter entries by accessible clusters and collect those with relevant changes
+    relevant_entries = []
+    for entry in changelog_entries:
+        # Skip if not in accessible clusters
+        cluster_name = getattr(entry, 'cluster_name', None)
+        if accessible_clusters and (not cluster_name or cluster_name not in accessible_clusters):
+            continue
+
+        # Check for relevant changes
+        relevant_changes = _get_relevant_changes(entry)
+        if relevant_changes:
+            relevant_entries.append((entry, relevant_changes))
+
+    # If no relevant changes found, show success message
+    if not relevant_entries:
+        print('✅ No layer changes detected for accessible clusters since last run')
+        return
+
+    print(f'📋 Found {len(relevant_entries)} layer change(s) for accessible clusters since last run:')
+
+    for entry, relevant_changes in relevant_entries:
+        _log_entry_details(entry, relevant_changes)
+
+    # Save to CSV (use original filtered entries for CSV)
+    filtered_entries = [entry for entry, _ in relevant_entries]
+    save_changelog_to_csv(filtered_entries)
+
+
+def _get_relevant_changes(entry: LayerChangelogEntry) -> list:
+    """Extract relevant changes from a changelog entry."""
+    relevant_changes = []
+    if hasattr(entry, 'changed_fields') and entry.changed_fields:
+        for field in entry.changed_fields:
+            if field in ['attributes', 'layer_name', 'cluster_name']:
+                relevant_changes.append(field)
+    return relevant_changes
+
+
+def _log_entry_details(entry: LayerChangelogEntry, relevant_changes: list) -> None:
+    """Log details for a single changelog entry."""
+    layer_name = getattr(entry, 'layer_name', 'Unknown')
+    cluster_name = getattr(entry, 'cluster_name', 'Unknown')
+    action = getattr(entry, 'action', 'Unknown')
+    timestamp = getattr(entry, 'timestamp', 'Unknown')
+
+    print(f"  📦 Layer '{layer_name}' (cluster: {cluster_name})")
+    print(f'      Action: {action}')
+    print(f'      Changed fields: {", ".join(relevant_changes)}')
+    print(f'      Timestamp: {timestamp}')
+
+    # If attributes changed, log the attributes
+    if 'attributes' in relevant_changes and hasattr(entry, 'attributes') and entry.attributes:
+        print(f'      New attributes: {", ".join(entry.attributes)}')
+
+    print('')  # Empty line for readability
+
+
+def save_changelog_to_csv(filtered_entries: list) -> None:
+    """Saves changelog entries to a CSV file in the output directory."""
+    if not filtered_entries:
+        return
+
+    config = Config.singleton()
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    csv_filename = f'layer_changelog_{timestamp}.csv'
+    csv_path = os.path.join(config.output_path, csv_filename)
+
+    # Ensure output directory exists
+    os.makedirs(config.output_path, exist_ok=True)
+
+    # Define CSV headers
+    headers = ['timestamp', 'layer_name', 'cluster_name', 'action', 'changed_fields', 'attributes']
+
+    try:
+        with open(csv_path, 'w', newline='', encoding='utf-8') as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=headers)
+            writer.writeheader()
+
+            for entry in filtered_entries:
+                # Filter for relevant changes
+                relevant_changes = []
+                if hasattr(entry, 'changed_fields') and entry.changed_fields:
+                    for field in entry.changed_fields:
+                        if field in ['attributes', 'layer_name', 'cluster_name']:
+                            relevant_changes.append(field)
+
+                if not relevant_changes:
+                    continue  # Skip entries without relevant changes
+
+                # Extract entry data
+                layer_name = getattr(entry, 'layer_name', 'Unknown')
+                cluster_name = getattr(entry, 'cluster_name', 'Unknown')
+                action = getattr(entry, 'action', 'Unknown')
+                timestamp_str = getattr(entry, 'timestamp', 'Unknown')
+                attributes = ''
+                if hasattr(entry, 'attributes') and entry.attributes:
+                    attributes = ', '.join(entry.attributes)
+
+                writer.writerow(
+                    {
+                        'timestamp': timestamp_str,
+                        'layer_name': layer_name,
+                        'cluster_name': cluster_name,
+                        'action': action,
+                        'changed_fields': ', '.join(relevant_changes),
+                        'attributes': attributes,
+                    }
+                )
+
+        print(f'📊 Changelog saved to CSV: {csv_path}')
+    except Exception as e:
+        print(f'⚠️  Failed to save changelog to CSV: {e}')
+
+
+def record_successful_geosync_completion(start_time: datetime) -> None:
+    """Records that a geosync run completed successfully."""
+    journal = Journal.singleton()
+    journal.record_successful_geosync_run(start_time)
+    print('✅ Geosync completed successfully')
+
+
+def layer_has_relevant_changes_in_changelog(
+    changelog_result: LayerChangelogResult, layer_name: str, cluster_name: str
+) -> bool:
+    """Check if a layer has relevant changes in the provided changelog data."""
+    if not changelog_result or not hasattr(changelog_result, 'layer_changelog'):
+        return False
+
+    if not changelog_result.layer_changelog:
+        return False
+
+    # Check if this specific layer has relevant changes
+    for entry in changelog_result.layer_changelog:
+        entry_layer_name = getattr(entry, 'layer_name', None)
+        entry_cluster_name = getattr(entry, 'cluster_name', None)
+
+        if entry_layer_name == layer_name and entry_cluster_name == cluster_name:
+            relevant_changes = _get_relevant_changes(entry)
+            if relevant_changes:
+                return True
+
+    return False

nefino_geosync/run.py

@@ -1,28 +1,59 @@
 """This is the main entry point of the application."""
+
+import atexit
 import os
 from .api_client import get_client
-from .start_analyses import start_analyses
-from .download_completed_analyses import download_completed_analyses
 from .config import Config
+from .download_completed_analyses import download_completed_analyses
+from .layer_changelog import (
+    record_layer_changes_since_last_run,
+    record_successful_geosync_completion,
+)
 from .parse_args import parse_args
+from .session_logger import start_session_logging, stop_session_logging
+from .start_analyses import start_analyses
+from datetime import UTC, datetime
+
+
+def main() -> None:
+    # Start session-wide logging
+    print('Starting Nefino GeoSync...')
+    start_session_logging()
+    start_time = datetime.now(tz=UTC)
+    # Ensure logging stops when the program exits
+    atexit.register(stop_session_logging)
+
+    try:
+        args = parse_args()
+
+        if args.configure:
+            config = Config.singleton()
+            # if you are running with --configure on the first run (you don't need to)
+            # you will be prompted to configure the app by the config singleton init.
+            # In that case, don't prompt the user again.
+            if not config.already_prompted:
+                config.run_config_prompts()
+
+        client = get_client(api_host=os.getenv('NEFINO_API_HOST', default='https://api.nefino.li'))
+
+        # Check for layer changes since last run before starting new analyses
+        changelog_result = record_layer_changes_since_last_run(client)
 
-def main():
-    args = parse_args()
+        if not args.resume:
+            start_analyses(client, changelog_result)
+        else:
+            download_completed_analyses(client)
 
-    if args.configure:
-        config = Config.singleton()
-        # if you are running with --configure on the first run (you don't need to)
-        # you will be prompted to configure the app by the config singleton init.
-        # In that case, don't prompt the user again.
-        if not config.already_prompted:
-            config.run_config_prompts()
+        # Record successful completion
+        record_successful_geosync_completion(start_time)
 
-    client = get_client(api_host=os.getenv("NEFINO_API_HOST", default="https://api.nefino.li"))
+    except Exception as e:
+        print(f'Fatal error: {e}')
+        raise
+    finally:
+        # Ensure logging stops even if there's an error
+        stop_session_logging()
 
-    if not args.resume:
-        start_analyses(client)
-    else:
-        download_completed_analyses(client)
 
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main()