windborne 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

windborne/__init__.py

@@ -8,11 +8,11 @@ from .utils import (
 
 # Import Data API functions
 from .data_api import (
-    get_observations,
-    get_super_observations,
+    get_observations_page,
+    observations,
 
-    poll_super_observations,
-    poll_observations,
+    get_super_observations_page,
+    super_observations,
 
     get_flying_missions,
     get_mission_launch_site,
@@ -44,10 +44,11 @@ __all__ = [
     "convert_to_netcdf",
     "sync_to_s3",
 
-    "get_observations",
-    "get_super_observations",
-    "poll_super_observations",
-    "poll_observations",
+    "get_observations_page",
+    "observations",
+
+    "get_super_observations_page",
+    "super_observations",
 
     "get_flying_missions",
     "get_mission_launch_site",

windborne/cli.py

@@ -1,10 +1,10 @@
 import argparse
 
 from . import (
-    poll_super_observations,
-    poll_observations,
-    get_observations,
-    get_super_observations,
+    super_observations,
+    observations,
+    get_observations_page,
+    get_super_observations_page,
     get_flying_missions,
     get_mission_launch_site,
     get_predicted_path,
@@ -35,55 +35,57 @@ def main():
     ####################################################################################################################
     # DATA API FUNCTIONS
     ####################################################################################################################
-    # Poll Super Observations Command
-    poll_super_parser = subparsers.add_parser('poll-super-observations', help='Poll super observations within a time range')
-    poll_super_parser.add_argument('start_time', help='Starting time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    poll_super_parser.add_argument('end_time', help='End time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)', nargs='?', default=None)
-    poll_super_parser.add_argument('-i', '--interval', type=int, default=60, help='Polling interval in seconds')
-    poll_super_parser.add_argument('-b', '--bucket-hours', type=float, default=6.0, help='Hours per bucket')
-    poll_super_parser.add_argument('output', help='Save output to a single file (filename.csv, filename.json or filename.little_r) or to multiple files (csv or little_r)')
-
-    # Poll Observations Command
-    poll_parser = subparsers.add_parser('poll-observations', help='Poll observations within a time range')
-    poll_parser.add_argument('start_time', help='Starting time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    poll_parser.add_argument('end_time', help='End time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)', nargs='?', default=None)
-    poll_parser.add_argument('-m', '--mission-id', help='Filter observations by mission ID')
-    poll_parser.add_argument('-ml', '--min-latitude', type=float, help='Minimum latitude filter')
-    poll_parser.add_argument('-xl', '--max-latitude', type=float, help='Maximum latitude filter')
-    poll_parser.add_argument('-mg', '--min-longitude', type=float, help='Minimum longitude filter')
-    poll_parser.add_argument('-xg', '--max-longitude', type=float, help='Maximum longitude filter')
-    poll_parser.add_argument('-id', '--include-ids', action='store_true', help='Include observation IDs')
-    poll_parser.add_argument('-u', '--include-updated-at', action='store_true', help='Include update timestamps')
-    poll_parser.add_argument('-i', '--interval', type=int, default=60, help='Polling interval in seconds')
-    poll_parser.add_argument('-b', '--bucket-hours', type=float, default=6.0, help='Hours per bucket')
-    poll_parser.add_argument('output', help='Save output to a single file (filename.csv, filename.json or filename.little_r) or to multiple files (csv or little_r)')
-
-
-    # Get Observations Command
-    obs_parser = subparsers.add_parser('observations', help='Get observations with filters')
-    obs_parser.add_argument('since', help='Get observations since this time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    obs_parser.add_argument('-mt', '--min-time', help='Minimum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    obs_parser.add_argument('-xt', '--max-time', help='Maximum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    obs_parser.add_argument('-m', '--mission-id', help='Filter by mission ID')
+    # Super Observations Command
+    super_obs_parser = subparsers.add_parser('super-observations', help='Poll super observations within a time range')
+    super_obs_parser.add_argument('start_time', help='Starting time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    super_obs_parser.add_argument('end_time', help='End time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)', nargs='?', default=None)
+    super_obs_parser.add_argument('-i', '--interval', type=int, default=60, help='Polling interval in seconds')
+    super_obs_parser.add_argument('-b', '--bucket-hours', type=float, default=6.0, help='Hours per bucket')
+    super_obs_parser.add_argument('-d', '--output-dir', help='Directory path where the separate files should be saved. If not provided, files will be saved in current directory.')
+    super_obs_parser.add_argument('output', help='Save output to a single file (filename.csv, filename.json or filename.little_r) or to multiple files (csv or little_r)')
+
+    # Observations Command
+    obs_parser = subparsers.add_parser('observations', help='Poll observations within a time range')
+    obs_parser.add_argument('start_time', help='Starting time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    obs_parser.add_argument('end_time', help='End time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)', nargs='?', default=None)
+    obs_parser.add_argument('-m', '--mission-id', help='Filter observations by mission ID')
     obs_parser.add_argument('-ml', '--min-latitude', type=float, help='Minimum latitude filter')
     obs_parser.add_argument('-xl', '--max-latitude', type=float, help='Maximum latitude filter')
     obs_parser.add_argument('-mg', '--min-longitude', type=float, help='Minimum longitude filter')
     obs_parser.add_argument('-xg', '--max-longitude', type=float, help='Maximum longitude filter')
     obs_parser.add_argument('-id', '--include-ids', action='store_true', help='Include observation IDs')
-    obs_parser.add_argument('-mn', '--include-mission-name', action='store_true', help='Include mission names')
     obs_parser.add_argument('-u', '--include-updated-at', action='store_true', help='Include update timestamps')
-    obs_parser.add_argument('output', nargs='?', help='Output file')
+    obs_parser.add_argument('-i', '--interval', type=int, default=60, help='Polling interval in seconds')
+    obs_parser.add_argument('-b', '--bucket-hours', type=float, default=6.0, help='Hours per bucket')
+    obs_parser.add_argument('-d', '--output-dir', help='Directory path where the separate files should be saved. If not provided, files will be saved in current directory.')
+    obs_parser.add_argument('output', help='Save output to a single file (filename.csv, filename.json or filename.little_r) or to multiple files (csv or little_r)')
+
+
+    # Get Observations Page Command
+    obs_page_parser = subparsers.add_parser('observations-page', help='Get observations page with filters')
+    obs_page_parser.add_argument('since', help='Get observations since this time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    obs_page_parser.add_argument('-mt', '--min-time', help='Minimum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    obs_page_parser.add_argument('-xt', '--max-time', help='Maximum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    obs_page_parser.add_argument('-m', '--mission-id', help='Filter by mission ID')
+    obs_page_parser.add_argument('-ml', '--min-latitude', type=float, help='Minimum latitude filter')
+    obs_page_parser.add_argument('-xl', '--max-latitude', type=float, help='Maximum latitude filter')
+    obs_page_parser.add_argument('-mg', '--min-longitude', type=float, help='Minimum longitude filter')
+    obs_page_parser.add_argument('-xg', '--max-longitude', type=float, help='Maximum longitude filter')
+    obs_page_parser.add_argument('-id', '--include-ids', action='store_true', help='Include observation IDs')
+    obs_page_parser.add_argument('-mn', '--include-mission-name', action='store_true', help='Include mission names')
+    obs_page_parser.add_argument('-u', '--include-updated-at', action='store_true', help='Include update timestamps')
+    obs_page_parser.add_argument('output', nargs='?', help='Output file')
 
     # Get Super Observations Command
-    super_obs_parser = subparsers.add_parser('super-observations', help='Get super observations with filters')
-    super_obs_parser.add_argument('since', help='Get super observations since this time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    super_obs_parser.add_argument('-mt', '--min-time', help='Minimum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    super_obs_parser.add_argument('-xt', '--max-time', help='Maximum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
-    super_obs_parser.add_argument('-m', '--mission-id', help='Filter by mission ID')
-    super_obs_parser.add_argument('-id', '--include-ids', action='store_true', help='Include observation IDs')
-    super_obs_parser.add_argument('-mn', '--include-mission-name', action='store_true', help='Include mission names')
-    super_obs_parser.add_argument('-u', '--include-updated-at', action='store_true', help='Include update timestamps')
-    super_obs_parser.add_argument('output', nargs='?', help='Output file')
+    super_obs_page_parser = subparsers.add_parser('super-observations-page', help='Get super observations page with filters')
+    super_obs_page_parser.add_argument('since', help='Get super observations page since this time (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    super_obs_page_parser.add_argument('-mt', '--min-time', help='Minimum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    super_obs_page_parser.add_argument('-xt', '--max-time', help='Maximum time filter (YYYY-MM-DD_HH:MM, "YYYY-MM-DD HH:MM:SS" or YYYY-MM-DDTHH:MM:SS.fffZ)')
+    super_obs_page_parser.add_argument('-m', '--mission-id', help='Filter by mission ID')
+    super_obs_page_parser.add_argument('-id', '--include-ids', action='store_true', help='Include observation IDs')
+    super_obs_page_parser.add_argument('-mn', '--include-mission-name', action='store_true', help='Include mission names')
+    super_obs_page_parser.add_argument('-u', '--include-updated-at', action='store_true', help='Include update timestamps')
+    super_obs_page_parser.add_argument('output', nargs='?', help='Output file')
 
     # Get Flying Missions Command
     flying_parser = subparsers.add_parser('flying-missions', help='Get currently flying missions')
@@ -198,8 +200,8 @@ def main():
     ####################################################################################################################
     # DATA API FUNCTIONS CALLED
     ####################################################################################################################
-    if args.command == 'poll-super-observations':
-        # Error handling is performed within poll_super_observations
+    if args.command == 'super-observations':
+        # Error handling is performed within super_observations
         # and we display the appropriate error messages
         # No need to implement them here
 
@@ -207,22 +209,25 @@ def main():
         if '.' in args.output:
             save_to_file = args.output
             output_format = None
+            output_dir = None
         # In case user wants separate file for each data from missions (buckets)
         else:
             save_to_file = None
             output_format = args.output
+            output_dir = args.output_dir
 
-        poll_super_observations(
+        super_observations(
             start_time=args.start_time,
             end_time=args.end_time,
             interval=args.interval,
             save_to_file=save_to_file,
             bucket_hours=args.bucket_hours,
+            output_dir=output_dir,
             output_format=output_format
         )
 
-    elif args.command == 'poll-observations':
-        # Error handling is performed within poll_observations
+    elif args.command == 'observations':
+        # Error handling is performed within observations
         # and we display the appropriate error messages
         # No need to implement them here
 
@@ -230,12 +235,14 @@ def main():
         if '.' in args.output:
             save_to_file = args.output
             output_format = None
+            output_dir = None
         # In case user wants separate file for each data from missions (buckets)
         else:
             save_to_file = None
             output_format = args.output
+            output_dir = args.output_dir
 
-        poll_observations(
+        observations(
             start_time=args.start_time,
             end_time=args.end_time,
             include_ids=args.include_ids,
@@ -248,36 +255,63 @@ def main():
             interval=args.interval,
             save_to_file=save_to_file,
             bucket_hours=args.bucket_hours,
+            output_dir=output_dir,
             output_format=output_format
         )
 
-    elif args.command == 'observations':
-        get_observations(
-            since=args.since,
-            min_time=args.min_time,
-            max_time=args.max_time,
-            include_ids=args.include_ids,
-            include_mission_name=args.include_mission_name,
-            include_updated_at=args.include_updated_at,
-            mission_id=args.mission_id,
-            min_latitude=args.min_latitude,
-            max_latitude=args.max_latitude,
-            min_longitude=args.min_longitude,
-            max_longitude=args.max_longitude,
-            save_to_file=args.output
-        )
-
-    elif args.command == 'super-observations':
-        get_super_observations(
-            since=args.since,
-            min_time=args.min_time,
-            max_time=args.max_time,
-            include_ids=args.include_ids,
-            include_mission_name=args.include_mission_name,
-            include_updated_at=args.include_updated_at,
-            mission_id=args.mission_id,
-            save_to_file=args.output
-        )
+    elif args.command == 'observations-page':
+        if not args.output:
+            pprint(get_observations_page(
+                since=args.since,
+                min_time=args.min_time,
+                max_time=args.max_time,
+                include_ids=args.include_ids,
+                include_mission_name=args.include_mission_name,
+                include_updated_at=args.include_updated_at,
+                mission_id=args.mission_id,
+                min_latitude=args.min_latitude,
+                max_latitude=args.max_latitude,
+                min_longitude=args.min_longitude,
+                max_longitude=args.max_longitude
+            ))
+        else:
+            get_observations_page(
+                since=args.since,
+                min_time=args.min_time,
+                max_time=args.max_time,
+                include_ids=args.include_ids,
+                include_mission_name=args.include_mission_name,
+                include_updated_at=args.include_updated_at,
+                mission_id=args.mission_id,
+                min_latitude=args.min_latitude,
+                max_latitude=args.max_latitude,
+                min_longitude=args.min_longitude,
+                max_longitude=args.max_longitude,
+                save_to_file=args.output
+            )
+
+    elif args.command == 'super-observations-page':
+        if not args.output:
+            pprint(get_super_observations_page(
+                since=args.since,
+                min_time=args.min_time,
+                max_time=args.max_time,
+                include_ids=args.include_ids,
+                include_mission_name=args.include_mission_name,
+                include_updated_at=args.include_updated_at,
+                mission_id=args.mission_id
+            ))
+        else:
+            get_super_observations_page(
+                since=args.since,
+                min_time=args.min_time,
+                max_time=args.max_time,
+                include_ids=args.include_ids,
+                include_mission_name=args.include_mission_name,
+                include_updated_at=args.include_updated_at,
+                mission_id=args.mission_id,
+                save_to_file=args.output
+            )
 
     elif args.command == 'flying-missions':
         get_flying_missions(cli=True, save_to_file=args.output)

windborne/data_api.py

@@ -8,9 +8,15 @@ from datetime import datetime, timezone, timedelta
 import csv
 import json
 
-def get_observations(since=None, min_time=None, max_time=None, include_ids=None, include_mission_name=True, include_updated_at=None, mission_id=None, min_latitude=None, max_latitude=None, min_longitude=None, max_longitude=None, save_to_file=None):
+# ------------
+# CORE RESOURCES
+# ------------
+
+# Observations
+# ------------
+def get_observations_page(since=None, min_time=None, max_time=None, include_ids=None, include_mission_name=True, include_updated_at=None, mission_id=None, min_latitude=None, max_latitude=None, min_longitude=None, max_longitude=None, save_to_file=None):
     """
-    Retrieves observations based on specified filters including geographical bounds.
+    Retrieves observations page based on specified filters including geographical bounds.
 
     Args:
         since (str): Filter observations after this timestamp.
@@ -70,52 +76,7 @@ def get_observations(since=None, min_time=None, max_time=None, include_ids=None,
     
     return response
 
-def get_super_observations(since=None, min_time=None, max_time=None, include_ids=None, include_mission_name=None, include_updated_at=None, mission_id=None, save_to_file=None):
-    """
-    Retrieves super observations based on specified filters.
-
-    Args:
-        since (str): Filter observations after this timestamp.
-        min_time (str): Minimum timestamp for observations.
-        max_time (str): Maximum timestamp for observations.
-        include_ids (bool): Include observation IDs in response.
-        include_mission_name (bool): Include mission names in response.
-        include_updated_at (bool): Include update timestamps in response.
-        mission_id (str): Filter observations by mission ID.
-        save_to_file (str): Optional path to save the response data.
-                           If provided, saves the data in CSV format.
-
-    Returns:
-        dict: The API response containing filtered super observations.
-    """
-
-    url = f"{DATA_API_BASE_URL}/super_observations.json"
-    
-    params = {}
-    if since:
-        params["since"] = to_unix_timestamp(since)
-    if min_time:
-        params["min_time"] = to_unix_timestamp(min_time)
-    if max_time:
-        params["max_time"] = to_unix_timestamp(max_time)
-    if mission_id:
-        params["mission_id"] = mission_id
-    if include_ids:
-        params["include_ids"] = True
-    if include_mission_name:
-        params["include_mission_name"] = True
-    if include_updated_at:
-        params["include_updated_at"] = True
-    
-    params = {k: v for k, v in params.items() if v is not None}
-    
-    response = make_api_request(url, params=params)
-    if save_to_file:
-        save_csv_json(save_to_file, response, csv_data_key='observations')
-    
-    return response
-
-def poll_observations(start_time, end_time=None, include_ids=None, include_updated_at=None, mission_id=None, min_latitude=None, max_latitude=None, min_longitude=None, max_longitude=None, interval=60, save_to_file=None, bucket_hours=6.0, output_format=None, callback=None):
+def observations(start_time, end_time=None, include_ids=None, include_updated_at=None, mission_id=None, min_latitude=None, max_latitude=None, min_longitude=None, max_longitude=None, interval=60, save_to_file=None, bucket_hours=6.0, output_format=None, output_dir=None, callback=None):
     """
     Fetches observations between a start time and an optional end time and saves to files in specified format.
     Files are broken up into time buckets, with filenames containing the time at the mid-point of the bucket.
@@ -140,6 +101,7 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
                             Supported formats are '.csv', '.json', '.little_r' and '.nc'
         bucket_hours (int): Optional. Size of time buckets in hours. Defaults to 6 hours.
         output_format (str): Optional. Format to save data in separate files. Supported formats are 'json, 'csv', 'little_r' and 'netcdf'.
+        output_dir (str): Optional. Directory path where the separate files should be saved. If not provided, files will be saved in current directory.
         callback (callable): Optional callback function that receives (super observations, metadata) before saving.
                              This allows custom processing or saving in custom formats.
     """
@@ -165,7 +127,7 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
         return
 
     # Supported formats for saving into a single file:
-    # NOTE: for poll_observations we handle .csv saving within poll_observations and not using save_csv_json
+    # NOTE: for observations we handle .csv saving within observations and not using save_csv_json
     #   - .csv
     #   - .json
     #   - .little_r
@@ -201,12 +163,13 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
     # Initialize the polling loop
     current_timestamp = start_time
     has_next_page = True
+    fetced_so_far = 0
 
 
     while has_next_page:
         try:
             # Fetch observations
-            observations_page = get_observations(
+            observations_page = get_observations_page(
                 since=current_timestamp,
                 min_latitude=min_latitude,
                 max_latitude=max_latitude,
@@ -226,11 +189,15 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
                 continue
 
             observations = observations_page.get('observations', [])
-            print(f"Fetched {len(observations)} observation(s)")
+            fetced_so_far = fetced_so_far + len(observations)
+            print_current_timestamp = current_timestamp if current_timestamp < 1e11 else current_timestamp / 1e9
+            print(f"Fetched {fetced_so_far} observation(s)")
+            print(f"Current time: {datetime.fromtimestamp(print_current_timestamp).strftime('%Y-%m-%d %H:%M:%S')}")
+            print("-----------------------------------------------------")
 
             # Invoke the callback with fetched observations
             if callback:
-                print("/nCallback/n")
+                print("\nCallback\n")
                 callback(observations)
 
             for obs in observations:
@@ -281,7 +248,7 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
 
             if not has_next_page or not next_timestamp or next_timestamp <= current_timestamp:
                 print("-----------------------------------------------------\n")
-                print("No more pages available or reached end of time range.")
+                print("Fetching complete.")
                 print("\n-----------------------------------------------------")
                 break
 
@@ -293,12 +260,20 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
 
     # Save data to a single file
     if save_to_file:
+        # Create directory path if it doesn't exist
+        directory = os.path.dirname(save_to_file)
+        if directory and not os.path.isdir(directory):
+            os.makedirs(directory, exist_ok=True)
         filtered_observations = {obs_id: obs for obs_id, obs in all_observations.items()
                                  if float(obs['timestamp']) >= start_time}
         # Sort by timestamp
         sorted_observations = dict(sorted(filtered_observations.items(),
                                           key=lambda x: float(x[1]['timestamp'])))
 
+        print(f"Saving {len(sorted_observations)} {'observation' if len(sorted_observations) == 1 else 'observations'} to {save_to_file}")
+        print("This may take a while...")
+        print("-----------------------------------------------------\n")
+
         if save_to_file.endswith('.nc'):
             first_obs_timestamp = float(next(iter(sorted_observations.values()))['timestamp'])
             convert_to_netcdf(sorted_observations, first_obs_timestamp, output_filename=save_to_file)
@@ -321,6 +296,15 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
 
     # Save data to multiple file
     elif output_format:
+        # Create output directory if specified
+        if output_dir:
+            os.makedirs(output_dir, exist_ok=True)
+            print(f"Files will be saved to {output_dir}")
+        else:
+            print(f"Files will be saved to {os.getcwd()}")
+        print(f"Processing {fetced_so_far} {'observation' if fetced_so_far == 1 else 'observations'} and save them over multiple files.")
+        print("This may take a while...")
+        print("-----------------------------------------------------\n")
         # Track statistics per mission
         mission_stats = {}  # {mission_name: {'files': 0, 'observations': 0}}
         total_observations_written = 0
@@ -329,48 +313,38 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
         for (bucket_center, mission_name), observations in buckets.items():
             if observations:
                 # Format hour to be the actual bucket center
-                bucket_hour = int((bucket_center.hour + bucket_hours/2) % 24)
+                bucket_hour = int((bucket_center.hour + bucket_hours / 2) % 24)
 
-                if output_format == 'netcdf':
-                    convert_to_netcdf(observations, bucket_center.timestamp())
+                # Generate file name based on output format
+                file_name_format = {
+                    'csv': f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.csv",
+                    'json': f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.json",
+                    'little_r': f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d-00_%dh.little_r"
+                }
+                file_name = file_name_format[output_format] % (
+                    bucket_center.year, bucket_center.month, bucket_center.day,
+                    bucket_hour, bucket_hours)
 
-                if output_format == 'csv':
-                    output_file = (f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.csv" %
-                                   (bucket_center.year, bucket_center.month, bucket_center.day,
-                                    bucket_hour, bucket_hours))
+                output_file = os.path.join(output_dir or '.', file_name)
 
-                    os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True)
+                # Sort observations by timestamp within each bucket
+                sorted_obs = sorted(observations.values(), key=lambda x: int(x['timestamp']))
 
-                    # Sort observations by timestamp within each bucket
-                    sorted_obs = sorted(observations.values(), key=lambda x: int(x['timestamp']))
+                if output_format == 'netcdf':
+                    convert_to_netcdf(sorted_obs, bucket_center.timestamp())
 
+                elif output_format == 'csv':
                     with open(output_file, mode='w', newline='') as file:
                         writer = csv.DictWriter(file, fieldnames=headers)
                         writer.writeheader()
                         writer.writerows(sorted_obs)
 
                 elif output_format == 'json':
-                    output_file = (f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.json" %
-                                   (bucket_center.year, bucket_center.month, bucket_center.day,
-                                    bucket_hour, bucket_hours))
-
-                    os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True)
-
-                    # Sort observations by timestamp within each bucket
-                    sorted_obs = dict(sorted(observations.items(), key=lambda x: int(x[1]['timestamp'])))
-
+                    sorted_obs_dict = {k: v for k, v in sorted(observations.items(), key=lambda x: int(x[1]['timestamp']))}
                     with open(output_file, 'w', encoding='utf-8') as file:
-                        json.dump(sorted_obs, file, indent=4)
+                        json.dump(sorted_obs_dict, file, indent=4)
 
                 elif output_format == 'little_r':
-                    output_file = (f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d-00_%dh.little_r" %
-                                   (bucket_center.year, bucket_center.month, bucket_center.day,
-                                    bucket_hour, bucket_hours))
-
-                    os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True)
-
-                    sorted_obs = sorted(observations.items(), key=lambda x: int(x[1]['timestamp']))
-
                     little_r_records = format_little_r(sorted_obs)
                     with open(output_file, 'w') as file:
                         file.write('\n'.join(little_r_records))
@@ -382,7 +356,7 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
                 mission_stats[mission_name]['files'] += 1
                 mission_stats[mission_name]['observations'] += len(observations)
         # Print total observations written
-        print(f"Total {'observation' if total_observations_written == 1 else 'observations'} written: {total_observations_written}")
+        print(f"Saved {total_observations_written} {'observation.' if total_observations_written == 1 else 'observations.'}")
         print("-----------------------------------------------------")
 
         # Print summary for each mission
@@ -392,7 +366,55 @@ def poll_observations(start_time, end_time=None, include_ids=None, include_updat
     print("-----------------------------------------------------")
     print("All observations have been processed and saved.")
 
-def poll_super_observations(start_time, end_time=None, interval=60, save_to_file=None, bucket_hours=6.0, output_format=None, callback=None):
+
+# Super Observations
+# ------------
+def get_super_observations_page(since=None, min_time=None, max_time=None, include_ids=None, include_mission_name=None, include_updated_at=None, mission_id=None, save_to_file=None):
+    """
+    Retrieves super observations page based on specified filters.
+
+    Args:
+        since (str): Filter observations after this timestamp.
+        min_time (str): Minimum timestamp for observations.
+        max_time (str): Maximum timestamp for observations.
+        include_ids (bool): Include observation IDs in response.
+        include_mission_name (bool): Include mission names in response.
+        include_updated_at (bool): Include update timestamps in response.
+        mission_id (str): Filter observations by mission ID.
+        save_to_file (str): Optional path to save the response data.
+                           If provided, saves the data in CSV format.
+
+    Returns:
+        dict: The API response containing filtered super observations.
+    """
+
+    url = f"{DATA_API_BASE_URL}/super_observations.json"
+    
+    params = {}
+    if since:
+        params["since"] = to_unix_timestamp(since)
+    if min_time:
+        params["min_time"] = to_unix_timestamp(min_time)
+    if max_time:
+        params["max_time"] = to_unix_timestamp(max_time)
+    if mission_id:
+        params["mission_id"] = mission_id
+    if include_ids:
+        params["include_ids"] = True
+    if include_mission_name:
+        params["include_mission_name"] = True
+    if include_updated_at:
+        params["include_updated_at"] = True
+    
+    params = {k: v for k, v in params.items() if v is not None}
+    
+    response = make_api_request(url, params=params)
+    if save_to_file:
+        save_csv_json(save_to_file, response, csv_data_key='observations')
+    
+    return response
+
+def super_observations(start_time, end_time=None, interval=60, save_to_file=None, bucket_hours=6.0, output_format=None, output_dir=None, callback=None):
     """
     Fetches super observations between a start time and an optional end time and saves to files in specified format.
     Files are broken up into time buckets, with filenames containing the time at the mid-point of the bucket.
@@ -408,6 +430,7 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
                             Supported formats are '.csv', '.json', '.little_r' and '.nc'
         bucket_hours (int): Optional. Size of time buckets in hours. Defaults to 6 hours.
         output_format (str): Optional. Format to save data in separate files. Supported formats are 'json, 'csv', 'little_r' and 'netcdf'.
+        output_dir (str): Optional. Directory path where the separate files should be saved. If not provided, files will be saved in current directory.
         callback (callable): Optional callback function that receives (super observations, metadata) before saving.
                              This allows custom processing or saving in custom formats.
     """
@@ -469,12 +492,13 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
     # Initialize the polling loop
     current_timestamp = start_time
     has_next_page = True
+    fetced_so_far = 0
 
 
     while has_next_page:
         try:
             # Fetch observations
-            observations_page = get_super_observations(
+            observations_page = get_super_observations_page(
                 since=current_timestamp,
                 min_time=start_time,
                 max_time=end_time,
@@ -490,7 +514,11 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
                 continue
 
             observations = observations_page.get('observations', [])
-            print(f"Fetched {len(observations)} super observation(s)")
+            fetced_so_far = fetced_so_far + len(observations)
+            print_current_timestamp = current_timestamp if current_timestamp < 1e11 else current_timestamp / 1e9
+            print(f"Fetched {fetced_so_far} super observation(s)")
+            print(f"Current time: {datetime.fromtimestamp(print_current_timestamp).strftime('%Y-%m-%d %H:%M:%S')}")
+            print("-----------------------------------------------------")
 
             # Invoke the callback with fetched observations
             if callback:
@@ -547,7 +575,7 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
 
             if not has_next_page or not next_timestamp or next_timestamp <= current_timestamp:
                 print("-----------------------------------------------------\n")
-                print("No more pages available or reached end of time range.")
+                print("Fetching complete.")
                 print("\n-----------------------------------------------------")
                 break
 
@@ -559,12 +587,20 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
 
     # Save data to a single file
     if save_to_file:
+        # Create directory path if it doesn't exist
+        directory = os.path.dirname(save_to_file)
+        if directory and not os.path.isdir(directory):
+            os.makedirs(directory, exist_ok=True)
         filtered_observations = {obs_id: obs for obs_id, obs in all_observations.items()
                                  if float(obs['timestamp']) >= start_time}
         # Sort by timestamp
         sorted_observations = dict(sorted(filtered_observations.items(),
                                           key=lambda x: float(x[1]['timestamp'])))
 
+        print(f"Saving {len(sorted_observations)} super {'observation' if len(sorted_observations) == 1 else 'observations'} to {save_to_file}")
+        print("This may take a while...")
+        print("-----------------------------------------------------\n")
+
         if save_to_file.endswith('.nc'):
             first_obs_timestamp = float(next(iter(sorted_observations.values()))['timestamp'])
             convert_to_netcdf(sorted_observations, first_obs_timestamp, output_filename=save_to_file)
@@ -588,6 +624,16 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
 
     # Save data to multiple file
     elif output_format:
+        # Create output directory if specified
+        if output_dir:
+            os.makedirs(output_dir, exist_ok=True)
+            print(f"Files will be saved to {output_dir}")
+        else:
+            print(f"Files will be saved to {os.getcwd()}")
+
+        print(f"Processing {fetced_so_far} super {'observation' if fetced_so_far == 1 else 'observations'} and save them over multiple files.")
+        print("This may take a while...")
+        print("-----------------------------------------------------\n")
         # Track statistics per mission
         mission_stats = {}  # {mission_name: {'files': 0, 'observations': 0}}
         total_observations_written = 0
@@ -598,46 +644,36 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
                 # Format hour to be the actual bucket center
                 bucket_hour = int((bucket_center.hour + bucket_hours/2) % 24)
 
-                if output_format == 'netcdf':
-                    convert_to_netcdf(observations, bucket_center.timestamp())
+                # Generate file name based on output format
+                file_name_format = {
+                    'csv': f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.csv",
+                    'json': f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.json",
+                    'little_r': f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d-00_%dh.little_r"
+                }
+                file_name = file_name_format[output_format] % (
+                    bucket_center.year, bucket_center.month, bucket_center.day,
+                    bucket_hour, bucket_hours)
 
-                if output_format == 'csv':
-                    output_file = (f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.csv" %
-                                   (bucket_center.year, bucket_center.month, bucket_center.day,
-                                    bucket_hour, bucket_hours))
+                output_file = os.path.join(output_dir or '.', file_name)
 
-                    os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True)
+                # Sort observations by timestamp within each bucket
+                sorted_obs = sorted(observations.values(), key=lambda x: int(x['timestamp']))
 
-                    # Sort observations by timestamp within each bucket
-                    sorted_obs = sorted(observations.values(), key=lambda x: int(x['timestamp']))
+                if output_format == 'netcdf':
+                    convert_to_netcdf(sorted_obs, bucket_center.timestamp())
 
+                elif output_format == 'csv':
                     with open(output_file, mode='w', newline='') as file:
                         writer = csv.DictWriter(file, fieldnames=headers)
                         writer.writeheader()
                         writer.writerows(sorted_obs)
 
                 elif output_format == 'json':
-                    output_file = (f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d_%dh.json" %
-                                   (bucket_center.year, bucket_center.month, bucket_center.day,
-                                    bucket_hour, bucket_hours))
-
-                    os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True)
-
-                    # Sort observations by timestamp within each bucket
-                    sorted_obs = dict(sorted(observations.items(), key=lambda x: int(x[1]['timestamp'])))
-
+                    sorted_obs_dict = {k: v for k, v in sorted(observations.items(), key=lambda x: int(x[1]['timestamp']))}
                     with open(output_file, 'w', encoding='utf-8') as file:
-                        json.dump(sorted_obs, file, indent=4)
+                        json.dump(sorted_obs_dict, file, indent=4)
 
                 elif output_format == 'little_r':
-                    output_file = (f"WindBorne_{mission_name}_%04d-%02d-%02d_%02d-00_%dh.little_r" %
-                                   (bucket_center.year, bucket_center.month, bucket_center.day,
-                                    bucket_hour, bucket_hours))
-
-                    os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True)
-
-                    sorted_obs = sorted(observations.items(), key=lambda x: int(x[1]['timestamp']))
-
                     little_r_records = format_little_r(sorted_obs)
                     with open(output_file, 'w') as file:
                         file.write('\n'.join(little_r_records))
@@ -659,6 +695,10 @@ def poll_super_observations(start_time, end_time=None, interval=60, save_to_file
     print("-----------------------------------------------------")
     print("All super observations have been processed and saved.")
 
+
+# ------------
+# METADATA
+# ------------
 def get_flying_missions(cli=None, save_to_file=None):
     """
     Retrieves a list of currently flying missions.

windborne/utils.py

@@ -1,5 +1,6 @@
 from .config import CLIENT_ID, API_KEY
 
+import os
 import requests
 import jwt
 import time
@@ -58,6 +59,20 @@ def make_api_request(url, params=None, return_type=None):
         print("--------------------------------------")
         print("To get an API key, email data@windbornesystems.com.")
         exit(91)
+    # Check if credentials are swapped
+    elif len(CLIENT_ID) in [32, 35]:
+        print("Your Client ID and API Key are swapped.")
+        print("--------------------------------------")
+        print("Swap them or modify them accordingly to get access to WindBorne API.")
+        print("--------------------------------------")
+        print("You may refer to https://windbornesystems.com/docs/api/cli#introduction\n"
+              "for instructions on how to set your credentials as environment variables for CLI and Code usage\n\n"
+              "and to https://windbornesystems.com/docs/api/pip_data#introduction\n"
+              "for instruction on how to set your credentials for code usage.")
+        print("--------------------------------------")
+        print(f"Current Client ID: {CLIENT_ID}")
+        print(f"Current API Key: {API_KEY}")
+        exit(95)
 
     # Validate WB_CLIENT_ID format
     if not (is_valid_uuid_v4(CLIENT_ID) or is_valid_client_id_format(CLIENT_ID)):
@@ -263,6 +278,11 @@ def save_csv_json(save_to_file, response, csv_data_key=None):
         response (dict or list): The response data to save.
         csv_data_key (str, optional): Key to extract data for CSV. Defaults to None.
     """
+    # Create directory path if it doesn't exist
+    directory = os.path.dirname(save_to_file)
+    if directory and not os.path.isdir(directory):
+        os.makedirs(directory, exist_ok=True)
+
     if '.' not in save_to_file:
         print("You have to provide a file type for your filename.")
         print("Supported formats:")
@@ -357,8 +377,6 @@ def convert_to_netcdf(data, curtime, output_filename=None):
         if col in df.columns:
             df[col] = pd.to_numeric(df[col], errors='coerce')
 
-    df['id'] = pd.to_numeric(df['id'], downcast='integer')
-
     ds = xr.Dataset.from_dataframe(df)
 
     # Build the filename and save some variables for use later
@@ -392,6 +410,9 @@ def convert_to_netcdf(data, curtime, output_filename=None):
 
     # Now that calculations are done, remove variables not needed in the netcdf output
     variables_to_drop = ['humidity', 'speed_x', 'speed_y', 'timestamp']
+    if 'id' in ds and pd.isna(ds['id']).all():
+        variables_to_drop.append('id')
+
     existing_vars = [var for var in variables_to_drop if var in ds]
     ds = ds.drop_vars(existing_vars)
 

{windborne-1.0.4.dist-info → windborne-1.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: windborne
-Version: 1.0.4
+Version: 1.0.6
 Summary: A Python library for interacting with WindBorne Data and Forecasts API
 Author-email: WindBorne Systems <data@windbornesystems.com>
 Classifier: Programming Language :: Python :: 3

windborne-1.0.6.dist-info/RECORD

@@ -0,0 +1,11 @@
+windborne/__init__.py,sha256=tNnX9BrdgFNjy9NY6ucobCVAgV3KYwUydEAuwxdaiqQ,1784
+windborne/cli.py,sha256=YlIz9KgsJumaCxz64uwAxfUUTJzc2rPEwzwGajCT9Jw,32838
+windborne/config.py,sha256=FYIBRiIuii5igAFQlOsHUa6u2i1kKnO1yZE7QfQJvUg,1688
+windborne/data_api.py,sha256=uggo2Y5U36ptvpSYgCUHNFwbNlczsCx7OUrf_OwlvtE,37629
+windborne/forecasts_api.py,sha256=AYuhFRls_XvzuNB55NF0w3y-_ocYwPxmI6C1lIyFkgM,16865
+windborne/utils.py,sha256=Zp9oTWfbISmJ9nO893RrW6MkqOwCDaFgsszmYaRgJSg,39670
+windborne-1.0.6.dist-info/METADATA,sha256=Fx7tFudf2QWvgLmkzfmTr4BNy2id_rKoOj-J5W4ZqVQ,1264
+windborne-1.0.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+windborne-1.0.6.dist-info/entry_points.txt,sha256=j_YrqdCDrCd7p5MIwQ2BYwNXEi95VNANzLRJmcXEg1U,49
+windborne-1.0.6.dist-info/top_level.txt,sha256=PE9Lauriu5S5REf7JKhXprufZ_V5RiZ_TnfnrLGJrmE,10
+windborne-1.0.6.dist-info/RECORD,,

windborne-1.0.4.dist-info/RECORD

@@ -1,11 +0,0 @@
-windborne/__init__.py,sha256=Dv2RK6qcZa3YDrTo_W_N9YROKOQV1_HFa0OWnZSBKAc,1783
-windborne/cli.py,sha256=ZNqeIlWqlKE4iCo8kptUH6Wbish0T8O7-Hljt-gEi9Q,31031
-windborne/config.py,sha256=FYIBRiIuii5igAFQlOsHUa6u2i1kKnO1yZE7QfQJvUg,1688
-windborne/data_api.py,sha256=dnF54sLPYRU5dect3WxmonbHKUjihfE9dVdpaT-fylk,35560
-windborne/forecasts_api.py,sha256=AYuhFRls_XvzuNB55NF0w3y-_ocYwPxmI6C1lIyFkgM,16865
-windborne/utils.py,sha256=srFToLi3vtgrmqorwpJSIoG5KMyJgK_GggEinYfF02k,38572
-windborne-1.0.4.dist-info/METADATA,sha256=tKud7EN6Dwm4BADJoYHu3cwYB_wvSlAtJSl4gta6yzI,1264
-windborne-1.0.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-windborne-1.0.4.dist-info/entry_points.txt,sha256=j_YrqdCDrCd7p5MIwQ2BYwNXEi95VNANzLRJmcXEg1U,49
-windborne-1.0.4.dist-info/top_level.txt,sha256=PE9Lauriu5S5REf7JKhXprufZ_V5RiZ_TnfnrLGJrmE,10
-windborne-1.0.4.dist-info/RECORD,,