anta-database 0.3.3__tar.gz → 0.3.4__tar.gz

{anta_database-0.3.3 → anta_database-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anta_database
-Version: 0.3.3
+Version: 0.3.4
 Summary: SQLite database for the AntArchitecture Community Data
 Author: Antoine Hermant
 Author-email: antoine.hermant@etik.com
@@ -48,12 +48,12 @@ Visit the [Home Page](https://antoinehermant.github.io/anta_database/intro) for
 
 The AntADatabase contains all traced and dated Internal Reflective Horizons (isochrones) that have been published at that date. All the data is organized by flight transect for each dataset. The data is encoded in HDF5 (Hierarchical Data Format) which provides convenient data structure and performant read speeds. 
 Variables per file (when exist):
-- x(point)
-- y(point)
-- distance(point)
+- PSX(point)
+- PSY(point)
+- Distance(point)
 - IRH_DEPTH(point, age)
-- IRH_DENS(point, age)
-- ICE_THCK(point, age)
+- IRH_NUM(point, age)
+- ICE_THK(point)
 - SURF_ELEV(point, age)
 - BED_ELEV(point, age)
 
@@ -66,7 +66,7 @@ This Python module provides SQL indexing for the AntADatabase, as well as quick
 - age
 - acquisition_year (year of radar acquisition)
 - region
-- basin
+- IMBIE_basin
 - var (variable)
 - flight id
 

{anta_database-0.3.3 → anta_database-0.3.4}/README.md

@@ -16,12 +16,12 @@ Visit the [Home Page](https://antoinehermant.github.io/anta_database/intro) for
 
 The AntADatabase contains all traced and dated Internal Reflective Horizons (isochrones) that have been published at that date. All the data is organized by flight transect for each dataset. The data is encoded in HDF5 (Hierarchical Data Format) which provides convenient data structure and performant read speeds. 
 Variables per file (when exist):
-- x(point)
-- y(point)
-- distance(point)
+- PSX(point)
+- PSY(point)
+- Distance(point)
 - IRH_DEPTH(point, age)
-- IRH_DENS(point, age)
-- ICE_THCK(point, age)
+- IRH_NUM(point, age)
+- ICE_THK(point)
 - SURF_ELEV(point, age)
 - BED_ELEV(point, age)
 
@@ -34,7 +34,7 @@ This Python module provides SQL indexing for the AntADatabase, as well as quick
 - age
 - acquisition_year (year of radar acquisition)
 - region
-- basin
+- IMBIE_basin
 - var (variable)
 - flight id
 

anta_database-0.3.4/ROADMAP.md

@@ -0,0 +1,12 @@
+# Roadmap
+
+Here is a todo list of features that we want to include in the AntADatabase.
+
+## Important
+- [x] Implement HDF5 format (In process, for next release)
+- [x] Filter by radar system
+- [x] Create shapefiles, e.g. for QAntarctica (In process)
+
+## To discuss
+- [ ] A netCDF version 
+

{anta_database-0.3.3 → anta_database-0.3.4}/anta_database/compiling/compile.py

@@ -294,7 +294,7 @@ class CompileDatabase:
                             header=0,
                             sep=sep,
                             # usecols=original_new_columns.columns,
-                            na_values=['-9999', '-9999.0', 'NaN', 'nan', ''],
+                            # na_values=['-9999', '-9999.0', 'NaN', 'nan', ''],
                             dtype=str
                             )
 
@@ -304,6 +304,26 @@ class CompileDatabase:
         ds = ds[ds.columns.intersection(original_new_columns.columns)]
         ds.columns = original_new_columns[ds.columns].iloc[0].values  # renaming the columns
 
+        values_to_replace = ['-9999', '-9999.0', 'NaN', 'nan', '']
+
+        if 'PSX' in ds.columns and 'PSY' in ds.columns:
+            # Create a mask for rows where both PSX and PSY are in values_to_replace
+            mask = (
+                ds['PSX'].isin(values_to_replace) &
+                ds['PSY'].isin(values_to_replace)
+            )
+            if mask.any():
+                problematic_indices = ds.index[mask].tolist()
+                warnings.warn(
+                    f"Found {len(problematic_indices)} rows where both PSX and PSY are in {values_to_replace}. "
+                    f"Indices: {problematic_indices}. Dropping these rows."
+                )
+
+            ds = ds[~mask].copy()
+
+        cols_to_replace = [col for col in ds.columns if col not in ['PSX', 'PSY']]
+        ds[cols_to_replace] = ds[cols_to_replace].replace(values_to_replace, pd.NA)
+
         pattern_values = original_new_columns[1:]
         pattern_header = original_new_columns.iloc[0]
         pattern_values.columns = pattern_header
@@ -873,9 +893,9 @@ class CompileDatabase:
 
                     df = gpd.GeoDataFrame({
                         "geometry": [Point(x, y) for x, y in zip(ds.PSX.values, ds.PSY.values)],
-                        "irh_num": N.astype('int16'),
+                        "irh_num": N.astype('int8'),
                         "max_age": age_max_depth.astype('int32'),
-                        "max_depth": max_depth.astype('int16'),
+                        "max_depth": np.rint(max_depth).astype('int16'),
                         # "flight_id": [flight_id] * len(N),
                         # "doi_data": [doi_data] * len(N),
                     }, crs="EPSG:3031")

{anta_database-0.3.3 → anta_database-0.3.4}/anta_database/database/database.py

@@ -715,7 +715,7 @@ class Database:
         self,
         metadata: Union[None, Dict, 'MetadataResult'] = None,
         data_dir: Optional[str] = None,
-        downscale_factor: Optional[str] = None,
+        downsampling_factor: Optional[str] = None,
         disable_tqdm: bool = False,
     ) -> Generator[Tuple[pd.DataFrame, Dict]]:
         """
@@ -744,35 +744,43 @@ class Database:
         file_paths = self._get_file_paths_from_metadata(metadata=md)
         file_paths = np.unique(file_paths) # Be carefull as many pointers point to the same file
 
+        if disable_tqdm or self._disable_tqdm:
+            disable_tqdm = True
+
         for file_path in tqdm(file_paths, desc='Generating dataframes', total=len(file_paths), unit='file', disable=disable_tqdm):
             full_path = os.path.join(data_dir, file_path)
             file_md = self._get_file_metadata(file_path)
             ds = h5py.File(full_path, 'r')
-            df = pd.DataFrame({'PSX': ds['PSX'][::downscale_factor],
-                                'PSY': ds['PSY'][::downscale_factor]})
+            df = pd.DataFrame({'PSX': ds['PSX'][::downsampling_factor],
+                                'PSY': ds['PSY'][::downsampling_factor]})
 
             if 'Distance' in ds.keys():
-                df['Distance'] = ds['Distance'][::downscale_factor]
+                df['Distance'] = ds['Distance'][::downsampling_factor]
 
+            var_impl = []
+            age_impl = []
             for var in md['var']:
                 if var == 'IRH_DEPTH':
+                    var_impl.append(var)
                     for age in md['age']:
                         irh_values = ds['IRH_AGE'][:]
                         irh_index = np.where(irh_values == int(age))[0]
                         if len(irh_index) == 0:
                             continue
                         irh_index = irh_index[0]
+                        age_impl.append(age)
 
-                        df[int(age)] = ds[var][::downscale_factor, irh_index]
+                        df[int(age)] = ds[var][::downsampling_factor, irh_index]
 
                 else:
                     if var in ds.keys():
-                        df[var] = ds[var][::downscale_factor]
+                        var_impl.append(var)
+                        df[var] = ds[var][::downsampling_factor]
 
             metadata = {
                 'dataset': file_md['dataset'],
-                'var': md['var'],
-                'age': md['age'],
+                'var': var_impl,
+                'age': age_impl,
                 'flight_id': file_md['flight_id'],
                 'institute': file_md['institute'],
                 'project': file_md['project'],
@@ -789,105 +797,6 @@ class Database:
 
             yield df, metadata
 
-    # def return_dataset(
-    #     self,
-    #     metadata: Union[None, Dict, 'MetadataResult'] = None,
-    #     data_dir: Optional[str] = None,
-    #     downscale_factor: Optional[str] = None,
-    # ) -> Generator[Tuple[pd.DataFrame, Dict]]:
-    #     """
-    #     Generates xarray Datasets from HDF5 files, one at a time, with lazy loading.
-
-    #     Args:
-    #         metadata: Metadata for filtering files.
-    #         data_dir: Directory containing the data files.
-    #         vars_to_load: List of variables to load from each file.
-
-    #     Yields:
-    #         Tuple[xr.Dataset, Dict]: A lazy-loaded xarray Dataset and its metadata.
-    #     """
-    #     # Resolve metadata
-    #     md = metadata or self._md
-    #     if not md:
-    #         print('Please provide metadata of the files you want to generate the data from. Exiting...')
-    #         return
-
-    #     # Resolve data directory
-    #     data_dir = data_dir or self._db_dir
-    #     if not data_dir:
-    #         print('No data directory provided. Exiting...')
-    #         return
-
-    #     if len(md['var']) > 1:
-    #         print(f'WARNING: you requested multiple variables ({md['var']}), this will return individual dataframes for each variable in each transect.')
-
-    #     file_paths = self._get_file_paths_from_metadata(metadata=md)
-    #     file_paths = np.unique(file_paths) # Be carefull as many pointers point to the same file
-
-    #     for file_path in file_paths:
-    #         full_path = os.path.join(data_dir, file_path)
-    #         file_md = self._get_file_metadata(file_path)
-    #         for var in md['var']:
-    #             if var == 'IRH_DEPTH':
-    #                 for age in md['age']:
-    #                     with h5py.File(full_path, 'r') as ds:
-    #                         irh_values = ds['age'][:]
-    #                         irh_index = np.where(irh_values == int(age))[0]
-    #                         if len(irh_index) == 0:
-    #                             continue
-    #                         irh_index = irh_index[0]
-    #                         df = pd.DataFrame({'x': ds['x'][::downscale_factor],
-    #                                         'y': ds['y'][::downscale_factor],
-    #                                         'distance': ds['distance'][::downscale_factor],
-    #                                         var: ds[var][::downscale_factor, irh_index]})
-
-    #                         metadata = {
-    #                             'dataset': file_md['dataset'],
-    #                             'var': var,
-    #                             'age': age,
-    #                             'flight_id': file_md['flight_id'],
-    #                             'institute': file_md['institute'],
-    #                             'project': file_md['project'],
-    #                             'acquisition_year': file_md['acquisition_year'],
-    #                             'age_unc': file_md['age'],
-    #                             'reference': file_md['reference'],
-    #                             'DOI_dataset': file_md['DOI_dataset'],
-    #                             'DOI_publication': file_md['DOI_publication'],
-    #                             'flight_id': file_md['flight_id'],
-    #                             'region': file_md['region'],
-    #                             'IMBIE_basin': file_md['IMBIE_basin'],
-    #                             'radar_instrument': file_md['radar_instrument'],
-    #                         }
-
-    #                         yield df, metadata
-
-    #                 else:
-    #                     ds = h5py.File(full_path, 'r')
-    #                     df = pd.DataFrame({'PSX': ds['PSX'][::downscale_factor],
-    #                                     'PSY': ds['PSY'][::downscale_factor],
-    #                                     'Distance': ds['Distance'][::downscale_factor],
-    #                                     var: ds[var][::downscale_factor]})
-
-    #                     metadata = {
-    #                         'dataset': file_md['dataset'],
-    #                         'var': var,
-    #                         'age': None,
-    #                         'flight_id': file_md['flight_id'],
-    #                         'institute': file_md['institute'],
-    #                         'project': file_md['project'],
-    #                         'acquisition_year': file_md['acquisition_year'],
-    #                         'age_unc': file_md['age'],
-    #                         'reference': file_md['reference'],
-    #                         'DOI_dataset': file_md['DOI_dataset'],
-    #                         'DOI_publication': file_md['DOI_publication'],
-    #                         'flight_id': file_md['flight_id'],
-    #                         'region': file_md['region'],
-    #                         'IMBIE_basin': file_md['IMBIE_basin'],
-    #                         'radar_instrument': file_md['radar_instrument'],
-    #                     }
-
-    #                     yield df, metadata
-
     @property
     def plot(self):
         if self._plotting is None:

{anta_database-0.3.3 → anta_database-0.3.4}/anta_database/plotting/plotting.py

@@ -62,7 +62,7 @@ class Plotting:
     def dataset(
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
-            downscale_factor: Optional[int] = None,
+            downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -82,7 +82,7 @@ class Plotting:
         self._base_plot(
             color_by='dataset',
             metadata=metadata,
-            downscale_factor=downscale_factor,
+            downsampling_factor=downsampling_factor,
             marker_size=marker_size,
             title=title,
             xlim=xlim,
@@ -100,7 +100,7 @@ class Plotting:
     def institute(
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
-            downscale_factor: Optional[int] = None,
+            downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -120,7 +120,7 @@ class Plotting:
         self._base_plot(
             color_by='institute',
             metadata=metadata,
-            downscale_factor=downscale_factor,
+            downsampling_factor=downsampling_factor,
             marker_size=marker_size,
             title=title,
             xlim=xlim,
@@ -138,7 +138,7 @@ class Plotting:
     def flight_id(
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
-            downscale_factor: Optional[int] = None,
+            downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -158,7 +158,7 @@ class Plotting:
         self._base_plot(
             color_by='flight_id',
             metadata=metadata,
-            downscale_factor=downscale_factor,
+            downsampling_factor=downsampling_factor,
             scale_factor=scale_factor,
             marker_size=marker_size,
             title=title,
@@ -176,7 +176,7 @@ class Plotting:
     def var(
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
-            downscale_factor: Optional[int] = None,
+            downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -196,7 +196,7 @@ class Plotting:
         self._base_plot(
             color_by='var',
             metadata=metadata,
-            downscale_factor=downscale_factor,
+            downsampling_factor=downsampling_factor,
             title=title,
             xlim=xlim,
             ylim=ylim,
@@ -215,7 +215,7 @@ class Plotting:
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
             elevation: Optional[bool] = False,
-            downscale_factor: Optional[int] = None,
+            downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -234,7 +234,7 @@ class Plotting:
             color_by='transect_1D',
             elevation=elevation,
             metadata=metadata,
-            downscale_factor=downscale_factor,
+            downsampling_factor=downsampling_factor,
             title=title,
             xlim=xlim,
             ylim=ylim,
@@ -261,7 +261,7 @@ class Plotting:
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
             elevation: Optional[bool] = False,
-            downscale_factor: Optional[int] = None,
+            downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -314,8 +314,8 @@ class Plotting:
         if color_by == 'dataset':
             if not title:
                 title = f'AntADatabase by datasets'
-            if downscale_factor == None:
-                downscale_factor = 1
+            if downsampling_factor == None:
+                downsampling_factor = 1
             datasets = list(metadata['dataset'])
             bedmap_entries = {'BEDMAP1', 'BEDMAP2', 'BEDMAP3'}
             bedmap_colors = {
@@ -341,8 +341,8 @@ class Plotting:
                 for f in file_paths:
                     full_path = os.path.join(self._db._db_dir, f)
                     with h5py.File(full_path, 'r') as ds:
-                        all_x.append(ds['PSX'][::downscale_factor])
-                        all_y.append(ds['PSY'][::downscale_factor])
+                        all_x.append(ds['PSX'][::downsampling_factor])
+                        all_y.append(ds['PSY'][::downsampling_factor])
                 df = pd.DataFrame({'PSX': np.concatenate(all_x),
                                 'PSY': np.concatenate(all_y)})
                 plt.scatter(df['PSX']/1000, df['PSY']/1000, color=colors[dataset], s=marker_size, zorder=zorder, linewidths=0)
@@ -359,8 +359,8 @@ class Plotting:
         if color_by == 'institute':
             if not title:
                 title = f'AntADatabase by institutes'
-            if downscale_factor == None:
-                downscale_factor = 1
+            if downsampling_factor == None:
+                downsampling_factor = 1
             institutes = list(metadata['institute'])
 
             color_indices = np.linspace(0.1, 0.9, len(institutes))
@@ -378,8 +378,8 @@ class Plotting:
                 for f in file_paths:
                     full_path = os.path.join(self._db._db_dir, f)
                     with h5py.File(full_path, 'r') as ds:
-                        all_x.append(ds['PSX'][::downscale_factor])
-                        all_y.append(ds['PSY'][::downscale_factor])
+                        all_x.append(ds['PSX'][::downsampling_factor])
+                        all_y.append(ds['PSY'][::downsampling_factor])
                 df = pd.DataFrame({'PSX': np.concatenate(all_x),
                                 'PSY': np.concatenate(all_y)})
                 plt.scatter(df['PSX']/1000, df['PSY']/1000, color=colors[institute], s=marker_size, linewidths=0)
@@ -404,7 +404,7 @@ class Plotting:
 
 
             all_dfs = []
-            for ds, _ in tqdm(self._db.data_generator(metadata, downscale_factor=downscale_factor, disable_tqdm=True), desc="Plotting", total=total_traces, unit='trace', disable=self._disable_tqdm):
+            for ds, _ in tqdm(self._db.data_generator(metadata, downsampling_factor=downsampling_factor, disable_tqdm=True), desc="Plotting", total=total_traces, unit='trace', disable=self._disable_tqdm):
                 all_dfs.append(ds)
             df = pd.concat(all_dfs)
 
@@ -448,32 +448,32 @@ class Plotting:
                     if cmap == None:
                         cmap = cmaps.bukavu
                     extend = 'both'
-                    if not vmin: vmin = -1000
-                    if not vmax: vmax = 1000
+                    if vmin is None: vmin = -1000
+                    if vmax is None: vmax = 1000
                     if not title:
                         title = f'AntADatabase Bed Elevation'
                 elif var == 'ICE_THK':
                     if cmap == None:
                         cmap = cmaps.torch_r
                     extend = 'max'
-                    if not vmin: vmin = 0
-                    if not vmax: vmax = 4000
+                    if vmin is None: vmin = 0
+                    if vmax is None: vmax = 4000
                     if not title:
                         title = f'AntADatabase Ice Thickness'
                 elif var == 'SURF_ELEV':
                     if cmap == None:
-                        cmap = cmaps.torch_r
+                        cmap = cmaps.ice_r
                     extend = 'max'
-                    if not vmin: vmin = 2000
-                    if not vmax: vmax = 4000
+                    if vmin is None: vmin = 1000
+                    if vmax is None: vmax = 4000
                     if not title:
                         title = f'AntADatabase Surface Elevation'
                 elif var == 'BASAL_UNIT':
                     if cmap == None:
                         cmap = cmaps.torch_r
                     extend = 'both'
-                    if not vmin: vmin = 2000
-                    if not vmax: vmax = 400
+                    if vmin is None: vmin = 2000
+                    if vmax is None: vmax = 400
                     if not title:
                         title = f'AntADatabase Basal Unit'
                 scatter = plt.scatter(df['PSX']/1000, df['PSY']/1000, c=df[var], cmap=cmap, s=marker_size, vmin=vmin, vmax=vmax, linewidths=0, rasterized=True)
@@ -605,13 +605,14 @@ class Plotting:
                         all_y.append(ds['PSY'][:])
                 df = pd.DataFrame({'PSX': np.concatenate(all_x),
                                 'PSY': np.concatenate(all_y)})
-                plt.scatter(df['PSX'][::downscale_factor]/1000, df['PSY'][::downscale_factor]/1000, color=colors[flight_id], s=marker_size, linewidths=0)
+                plt.scatter(df['PSX'][::downsampling_factor]/1000, df['PSY'][::downsampling_factor]/1000, color=colors[flight_id], s=marker_size, linewidths=0)
 
                 plt.plot([], [], color=colors[flight_id], label=flight_id, linewidth=3)
             ncol = 2 if len(flight_ids) > 40 else 1
 
         # --- Format Figure ---
-        print('Formatting ...')
+        if not self._disable_tqdm:
+            print('Formatting ...')
 
         ax.set_xlim(*xlim)
         ax.set_ylim(*ylim)
@@ -710,7 +711,7 @@ class Plotting:
             site_coords = pd.read_pickle(self._site_coords_path)
             for i in site_coords.index:
                 site = site_coords.loc[i]
-                ax.scatter(site['x']/1000, site['y']/1000, color='red', s=50, marker='^', edgecolor='black', linewidth=1.5)
+                ax.scatter(site['x']/1000, site['y']/1000, color='red', s=50, marker='^', edgecolor='black', linewidth=1.5, zorder=50)
         # --- Save/Show ---
         with self._plot_context():
             if save:

{anta_database-0.3.3 → anta_database-0.3.4}/anta_database.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anta_database
-Version: 0.3.3
+Version: 0.3.4
 Summary: SQLite database for the AntArchitecture Community Data
 Author: Antoine Hermant
 Author-email: antoine.hermant@etik.com
@@ -48,12 +48,12 @@ Visit the [Home Page](https://antoinehermant.github.io/anta_database/intro) for
 
 The AntADatabase contains all traced and dated Internal Reflective Horizons (isochrones) that have been published at that date. All the data is organized by flight transect for each dataset. The data is encoded in HDF5 (Hierarchical Data Format) which provides convenient data structure and performant read speeds. 
 Variables per file (when exist):
-- x(point)
-- y(point)
-- distance(point)
+- PSX(point)
+- PSY(point)
+- Distance(point)
 - IRH_DEPTH(point, age)
-- IRH_DENS(point, age)
-- ICE_THCK(point, age)
+- IRH_NUM(point, age)
+- ICE_THK(point)
 - SURF_ELEV(point, age)
 - BED_ELEV(point, age)
 
@@ -66,7 +66,7 @@ This Python module provides SQL indexing for the AntADatabase, as well as quick
 - age
 - acquisition_year (year of radar acquisition)
 - region
-- basin
+- IMBIE_basin
 - var (variable)
 - flight id
 

{anta_database-0.3.3 → anta_database-0.3.4}/book/_config.yml

@@ -9,6 +9,7 @@ logo: logo.png
 # See https://jupyterbook.org/content/execute.html
 execute:
   execute_notebooks: force
+  timeout: 600  # Timeout in seconds (e.g., 600 = 10 minutes)
 
 # Define the name of the latex output file for PDF builds
 latex:

{anta_database-0.3.3 → anta_database-0.3.4}/book/contribute.md

@@ -6,3 +6,5 @@ Maybe first contact me so I know that people are interested and on which feature
 
 The project is hosted on [GitHub](https://github.com/antoinehermant/anta_database). First you should fork the repository, create your own features branches and submit pull request.
 
+Check the [Roadmap](https://github.com/antoinehermant/anta_database/blob/main/ROADMAP.md).
+

{anta_database-0.3.3 → anta_database-0.3.4}/book/intro.md

@@ -1,30 +1,36 @@
 # AntADatabase
 
-AntADatabase is a Python-powered SQLite database designed for browsing, visualizing and processing Internal Reflecting Horizons (isochrones) across Antarctica, curated by the AntArchitecture action group. It is specifically designed for ice dynamic modelers who need a fast, memory-efficient data structure to constrain their models.
+AntADatabase a Python-powered SQLite database designed for browsing, visualizing and processing Internal Reflecting Horizons (isochrones) across Antarctica, curated by the AntArchitecture action group. It is specifically designed for ice dynamic modelers who need a fast, memory-efficient data structure to constrain their models.
 
-### Overview
+## Overview
 ![Figure](figures/Overview_all.png) 
 **Figures** created using plotting functions from this module
 
 ### Datasets currently included:
-- Franke et al. 2025, https://doi.org/10.1594/PANGAEA.973266
-- Sanderson et al. 2024, https://doi.org/10.5285/cfafb639-991a-422f-9caa-7793c195d316
+- Ashmore et al. 2020, https://doi.org/10.1029/2019GL086663
+- BEDMAP1: Lythe et al. (2022), https://doi.org/10.5285/f64815ec-4077-4432-9f55-0ce230f46029
+- BEDMAP2: Fretwell et al. (2022), https://doi.org/10.5285/2fd95199-365e-4da1-ae26-3b6d48b3e6ac
+- BEDMAP3: Frémand et al. (2022), https://doi.org/10.5285/91523ff9-d621-46b3-87f7-ffb6efcd1847
+- Beem et al. 2021, https://doi.org/10.15784/601437
+- Bodart et al. 2021, https://doi.org/10.5285/F2DE31AF-9F83-44F8-9584-F0190A2CC3EB
+- Bodart and Sutter 2025a, https://doi.org/10.5281/zenodo.17348976
+- Bodart and Sutter 2025b, https://doi.org/10.5281/zenodo.17348094
+- Cavitte et al. 2020, https://doi.org/10.15784/601411
 - Chung et al. 2023 https://doi.pangaea.de/10.1594/PANGAEA.957176
+- Franke et al. 2025, https://doi.org/10.1594/PANGAEA.973266
+- Jacobel and Welch 2005, https://doi.org/10.7265/N5R20Z9T
+- Leysinger-Vieli et al. 2011, https://doi.org/10.5281/zenodo.15516203
 - Muldoon et al. 2023, https://doi.org/10.15784/601673
 - Mulvaney et al. 2023, https://doi.pangaea.de/10.1594/PANGAEA.963470
+- Sanderson et al. 2024, https://doi.org/10.5285/cfafb639-991a-422f-9caa-7793c195d316
+- Siegert and Payne 2024, https://onlinelibrary.wiley.com/doi/10.1002/esp.1238
 - Wang et al. 2023, https://doi.org/10.1594/PANGAEA.958462
-- Bodart et al. 2021, https://doi.org/10.5285/F2DE31AF-9F83-44F8-9584-F0190A2CC3EB
-- Beem et al. 2021, https://doi.org/10.15784/601437
-- Ashmore et al. 2020, https://doi.org/10.1029/2019GL086663
-- Cavitte et al. 2020, https://doi.org/10.15784/601411
 - Winter et al. 2018, https://doi.org/10.1594/PANGAEA.895528
-- Leysinger et al. 2011, https://doi.org/10.1029/2010JF001785
-- Jacobel et al 2005, https://doi.org/10.7265/N5R20Z9T
-- BEDMAP version 1, 2 and 3
+- Yan et al. 2025, https://doi.org/10.5281/zenodo.14962526
 
 ## Key Features
 - Standardized HDF5 format
-- Efficient SQLite indexing
+- SQLite indexing
 - Quick visualization on Antarctica map
 - Generate lazy data for later use
 

{anta_database-0.3.3 → anta_database-0.3.4}/book/quick_start.ipynb

@@ -78,8 +78,8 @@
     "- acquisition_year: year(s) in which the radar data were acquired\n",
     "- age: age(s) in yrs before present of the layer(s) of interest\n",
     "- region: region(s) of interest. E.g.: 'EAIS', 'WAIS'\n",
-    "- basin: IMBIE basin(s) of interest. E.g.: 'G-H', 'Ap-B'\n",
-    "- var: variables(s) of interest. Possible variables currently are: 'IRH_DEPTH', 'IRH_DENS', 'ICE_THCK', 'BED_ELEV', 'SURF_ELEV'\n",
+    "- IMBIE_basin: IMBIE basin(s) of interest. E.g.: 'G-H', 'Ap-B'.\n",
+    "- var: variables(s) of interest. Possible variables currently are: 'IRH_DEPTH', 'IRH_DENS', 'ICE_THK', 'BED_ELEV', 'SURF_ELEV'\n",
     "- flight_id: ID of a particular flight line. This is useful for explicit flight IDs such as 'DC_LDC_DIVIDE'\n",
     "\n",
     "One can also combine field queries as well as providing lists. Use '%' for searching by regular expressions. Here are a few examples:"
@@ -95,10 +95,10 @@
     "#Examples of queries:\n",
     "db.query(dataset='Cavitte_2020') # all data from Cavitte et al. 2020\n",
     "db.query(age='38100') # all datasets with the 38.1ka isochrone\n",
-    "db.query(var='ICE_THCK') # all datasets with IceThk variable\n",
-    "db.query(basin='G-H') # all flight lines that cross the G-H basin\n",
-    "db.query(trace_id='DC_LDC_DIVIDE') # all layers with the trace ID DC_LDC_DIVIDE\n",
-    "db.query(trace_id='%WSB%') # all traces with WSB in the trace ID\n",
+    "db.query(var='ICE_THK') # all datasets with IceThk variable\n",
+    "db.query(IMBIE_basin='G-H') # all flight lines that cross the G-H basin\n",
+    "db.query(flight_id='DC_LDC_DIVIDE') # all layers with the flight ID DC_LDC_DIVIDE\n",
+    "db.query(flight_id='%WSB%') # all flight lines with WSB in the flight ID\n",
     "db.query(dataset=['Franke_2025', 'Winter_2018'], age='38100') # example of multiple criteria"
    ]
   },
@@ -117,6 +117,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "%%capture\n",
     "db.filter_out(acquisition_year='<1990')  # filter out all data acquired before 1990\n",
     "db.query() # now all queries will exclude data acquired before 1990\n",
     "\n",
@@ -134,37 +135,60 @@
     "Use the results of the query in the plotting functions:"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "cf1e071c",
+   "metadata": {},
+   "source": [
+    "Current implemented plotting functions are:\n",
+    "- plot.dataset(): plots locations of the data, with different colors for the different datasets (example above)\n",
+    "- plot.institute(): plots locations of the data, with different colors for the different institutes\n",
+    "- plot.var(): color-coded scatter plot of the variable of insterest.\n",
+    "- plot.flight_id(): color-coded trace IDs. Useful for identifying specific traces of interest.\n",
+    "- plot.transect_1D(): plots depths of the IRH and Bed along a single flight line \n",
+    "\n",
+    "In Jupyter Notebook, use '%matplotlib qt' or '%matplotlib widget' depending on your IDE, to switch to the matplotlib widget that allows you to zoom in etc.\n",
+    "Use '%matplotlib inline' (default) to plot the figure in the notebook"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a07fe92e",
+   "metadata": {},
+   "source": [
+    "### Plot datasets"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "a8e3a247",
+   "id": "0c68119f",
    "metadata": {},
    "outputs": [],
    "source": [
-    "results = db.query(age=['38100','38000'])\n",
+    "results = db.query(IMBIE_basin='G-H')\n",
     "db.plot.dataset(results,\n",
-    "                downscale_factor=100, # downscale the datasets n times, which makes no visual difference but it is much lighter\n",
-    "                title='AntArchitecture 38(.1)ka layer',\n",
-    "                xlim=(-500, 2400), ylim=(-2200, 2200), # set the plot extent in km\n",
-    "                scale_factor=0.7, # adjust the size of the plot\n",
-    "                latex=True, # use latex compilers for plotting if you have them installed on your system\n",
-    "                # save='AntA_38ka.pdf', # Uncomment to save the figure, otherwise it we visualize with pyplot\n",
+    "                title='IRH data crossing the G-H IMBIE basin',\n",
+    "                xlim=(-2000, -500), # set the plot extent in km\n",
+    "                ylim=(-1000, 250),\n",
+    "                marker_size=1, # adjust the size of the markers\n",
     "                )"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "cf1e071c",
+   "id": "a0217254",
    "metadata": {},
    "source": [
-    "Current implemented plotting functions are:\n",
-    "- plot.dataset(): plot locations of the data, with different colors for the different datasets (example above)\n",
-    "- plot.institute(): plot locations of the data, with different colors for the different institutes\n",
-    "- plot.var(): color-coded scatter plot of the variable of insterest.\n",
-    "- plot.flight_id(): color-coded trace IDs. Useful for identifying specific traces of interest.\n",
-    "\n",
-    "In Jupyter Notebook, use '%matplotlib qt' or '%matplotlib widget' depending on your IDE, to switch to the matplotlib widget that allows you to zoom in etc.\n",
-    "Use '%matplotlib inline' (default) to plot the figure in the notebook"
+    "Note: all flight lines in the database are associated with one or several IMBIE basin(s). This depends if the flight line crosses one or multiple basins. The plot above shows all the flight lines which have traced IRH data and that crosses at some point the G-H basin."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e735309e",
+   "metadata": {},
+   "source": [
+    "### Plot variables"
    ]
   },
   {
@@ -178,13 +202,13 @@
    "source": [
     "# %matplotlib widget\n",
     "%matplotlib inline\n",
-    "results = db.query(age=['38100','38000'], var='IRH_DEPTH')\n",
+    "results = db.query(age=['37600', '38000', '38100', '38200', '38500'], var='IRH_DEPTH')\n",
     "db.plot.var(results, title='AntArchitecture 38(.1)ka isochrone depth',\n",
-    "                downscale_factor=100,\n",
+    "                downsampling_factor=10, # downscale the datasets n times, which makes little visual difference but lighter to plot\n",
     "                xlim=(-500, 2400),\n",
     "                ylim=(-2200, 2200),\n",
-    "                scale_factor=0.7,\n",
-    "                latex=True,\n",
+    "                scale_factor=0.7, # adjust the size of the plot \n",
+    "                marker_size=1.2,\n",
     "                # save='AntA_38ka_depth.png'\n",
     "                )"
    ]
@@ -196,14 +220,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "results = db.query(var='IRH_DENS')\n",
+    "results = db.query(var='IRH_NUM')\n",
     "db.plot.var(results, title='IRH Density over the Antarctic Ice Sheet',\n",
-    "                downscale_factor=100,\n",
+    "                downsampling_factor=100,\n",
     "                scale_factor=1,\n",
-    "                latex=True,\n",
+    "                marker_size=1.2,\n",
     "                )"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "9185b390",
+   "metadata": {},
+   "source": [
+    "### Plot flight IDs"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -211,22 +243,134 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "results = db.query(dataset='Winter%', var='IRH_DENS', trace_id=['EPICA%'])\n",
+    "results = db.query(dataset='Winter%', flight_id=['EPICA%'])\n",
     "db.plot.flight_id(results, title='Winter et al. 2018 - EPICA',\n",
     "                xlim=(-500, 1000),\n",
     "                ylim=(1000, 2200),\n",
-    "                latex=True,\n",
+    "                marker_size=1.2,\n",
     "                )"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "84b02a87",
+   "metadata": {},
+   "source": [
+    "### Plot layer depths along transect"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d0a358b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results = db.query(dataset='Cavitte_2020', flight_id='DC_LDC_DIVIDE')\n",
+    "db.plot.transect_1D(results)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd1d77ba",
+   "metadata": {},
+   "source": [
+    "Use the elevation=True option to plot the transect in absolute elevation above sea level:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "32c3457a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db.plot.transect_1D(results, elevation=True)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "3cee8359",
    "metadata": {},
    "source": [
     "Other possible arguments for the plot methods:\n",
-    "- cmap: provide your colormap of choice (as LinearSegmentedColormap). Tip: 'import colormaps as cmaps' for a large choice of colormaps (see [Colormaps docs](https://pratiman-91.github.io/colormaps/))\n",
-    "- marker_size: for setting larger or smaller markers on the scatter plots"
+    "- cmap: provide your colormap of choice (as LinearSegmentedColormap). Tip: 'import colormaps as cmaps' for a large choice of colormaps (see [Colormaps docs](https://pratiman-91.github.io/colormaps/)\n",
+    "- vmin and vmax: sets the minimum and maximum values for the colorbar."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "95dcf714",
+   "metadata": {},
+   "source": [
+    "## BEDMAP\n",
+    "\n",
+    "The AntADatabase now contains all the BEDMAP (1, 2 and 3) data. This is useful to see the whole extent of the existing radar data, or to reconnect the IRH datasets to BEDMAP in order to get the Bed Elevation or Ice Thickness when those are not included.\n",
+    "\n",
+    "By default, BEDMAP is not shown in the query, since it is not an IRH dataset. To include it, initialize the Database with the option:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6de501f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db = Database('/home/anthe/documents/data/isochrones/AntADatabase/', include_BEDMAP=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "28dcee6d",
+   "metadata": {},
+   "source": [
+    "## Get files from the database\n",
+    "\n",
+    "You may want to make your own plots or further process the data after querying the database. One option is to get the list of the files from your query and open them individually with either xarray or h5py. For this, use the get_files() function:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fc67b755",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import xarray as xr\n",
+    "import matplotlib.pyplot as plt\n",
+    "results = db.query(dataset='Cavitte_2020', flight_id='DC_LDC_DIVIDE')\n",
+    "file_list = db.get_files(results)\n",
+    "f = file_list[0]\n",
+    "ds = xr.open_dataset(f, engine='h5netcdf')\n",
+    "print(ds)\n",
+    "plt.plot(ds.Distance, ds.IRH_DEPTH)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "57c0f26c",
+   "metadata": {},
+   "source": [
+    "xarray provides a nice interface for interacting with the data. This works great when dealing with one file, and for example quickly plot all the layers (see above). But xarray adds some overload, which feels slow when reading many files. h5py on the other hand reads the underlying data right away, which is much more efficient:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a404f0ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import h5py\n",
+    "import matplotlib.pyplot as plt\n",
+    "results = db.query(dataset='Cavitte_2020')\n",
+    "file_list = db.get_files(results)\n",
+    "for f in file_list:\n",
+    "    with h5py.File(f, 'r') as ds:\n",
+    "        plt.scatter(ds['PSX'][:], ds['PSY'][:], c=ds['ICE_THK'][:], s=1)\n",
+    "plt.show()"
    ]
   },
   {
@@ -238,7 +382,7 @@
     "\n",
     "Note: This part could be developed further in the future if there is the need. But for now, I am designing a separate Python module for constraining my ice sheet model of use, which is tailored to this database and other parallel processing libraries.\n",
     "\n",
-    "The data_generator() function reads the query and 'yield' the dataframes for later use.\n",
+    "The data_generator() function reads the query and 'yield' the dataframes for later use. It uses h5py to read all the data efficiently, and creates pandas dataframes, including all the variables and ages from the query. Columns for IRH DEPTH are named after the age. Basically, it reads the data with h5py as shown above and restructure it as bit, which can be easier to manage layers than with h5py dimensions.\n",
     "Here is a quick example of how this can be used for computing the mean layer depth:"
    ]
   },
@@ -249,7 +393,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "results = db.query(age=['38100'])\n",
+    "results = db.query(age=['37600', '38000', '38100', '38200', '38500'], var='IRH_DEPTH')\n",
     "lazy_dfs = db.data_generator(results)\n",
     "\n",
     "import numpy as np\n",
@@ -257,7 +401,7 @@
     "min_depth = float('inf')\n",
     "max_depth = float('-inf')\n",
     "for df, md in lazy_dfs:\n",
-    "    depth_values = df['IRHDepth']\n",
+    "    depth_values = df[int(md['age'][0])].values\n",
     "    mean_depth_trs.append(np.mean(depth_values))\n",
     "    min_depth = min(min_depth, min(depth_values))\n",
     "    max_depth = max(max_depth, max(depth_values))\n",
@@ -276,35 +420,6 @@
     "In the for loop, 'df' is an individual dataframe from the database, corresponding to a single layer of a single trace from a single dataset. The 'md' stores the unique metadata from the current 'df'. One can then associate the metadata with the current df, for example:"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d0a358b2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "results = db.query(dataset='Cavitte_2020', trace_id='DC_LDC_DIVIDE')\n",
-    "lazy_dfs = db.data_generator(results)\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "# from matplotlib import rc\n",
-    "# rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})\n",
-    "# rc('text', usetex=True) # Do you use latex?\n",
-    "\n",
-    "for df, md in lazy_dfs:\n",
-    "    if md['age'] is not None:\n",
-    "        plt.plot(df.distance/1000, df['IRHDepth'], label=md['age'])\n",
-    "    if md['var'] == 'IceThk':\n",
-    "        plt.plot(df.distance/1000, df['IceThk'], linewidth=2, color='k', label='Bedrock')\n",
-    "\n",
-    "plt.legend(ncol=2)\n",
-    "plt.xlabel('Distance (km)')\n",
-    "plt.ylabel('Depth (m)')\n",
-    "plt.ylim(4000, 0)\n",
-    "plt.title('Isochronal structure along DC_LDC_DIVIDE transect')\n",
-    "plt.show()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "1051a179",
@@ -312,7 +427,7 @@
    "source": [
     "### Downscale the data\n",
     "\n",
-    "The downscale_factor argument in the data_generator reduces the size of the generated data by n times. This is useful when dealing with large data where the downscaling has a neglectable influence on the results.\n",
+    "The downsampling_factor argument in the data_generator reduces the size of the generated data by n times. This is useful when dealing with large data where the downscaling has a neglectable influence on the results.\n",
     "Another solution, probably more straightforward, is the downsample_distance option, which allows to provide a distance in meters by which the data should be average along the transects:"
    ]
   },
@@ -325,19 +440,19 @@
    "source": [
     "import matplotlib.pyplot as plt\n",
     "\n",
-    "results = db.query(dataset='Cavitte_2020', age='278000', trace_id='DC_LDC_DIVIDE')\n",
+    "results = db.query(dataset='Cavitte_2020', age='277900', flight_id='DC_LDC_DIVIDE')\n",
     "original = db.data_generator(results)\n",
-    "downscale_10 = db.data_generator(results, downscale_factor=100)\n",
-    "downsampled_by_distance = db.data_generator(results, downsample_distance=5000)\n",
+    "downscale_10 = db.data_generator(results, downsampling_factor=100)\n",
+    "# downsampled_by_distance = db.data_generator(results, downsample_distance=5000)\n",
     "\n",
-    "list = [original, downscale_10, downsampled_by_distance]\n",
-    "labels = ['original', 'downscale 100', 'downsample 5km']\n",
+    "list = [original, downscale_10] #, downsampled_by_distance]\n",
+    "labels = ['original', 'downscale 100'] #, 'downsample 5km']\n",
     "\n",
     "for i, lazy_dfs in enumerate(list):\n",
     "    for df, md in lazy_dfs:\n",
-    "        plt.plot(df.distance, df['IRHDepth'],label=labels[i], linestyle='--')\n",
+    "        plt.plot(df.Distance, df[277900],label=labels[i], linestyle='--')\n",
     "\n",
-    "plt.ylim(3000, 0)\n",
+    "plt.ylim(2600, 1800)\n",
     "plt.xlabel('Distance along DC_LDC_DIVIDE (km)')\n",
     "plt.ylabel('Depth (m)')\n",
     "plt.legend()\n",
@@ -363,7 +478,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "db.query(var='IRHDensity')\n",
+    "db.query(var='IRH_NUM')\n",
     "# db.plot.var() # This is equivalent to the overview plot above. Note that without downscaling it is very heavy to plot.\n",
     "\n",
     "import numpy as np\n",
@@ -371,7 +486,7 @@
     "min_density = float('inf')\n",
     "max_density = float('-inf')\n",
     "for df, md in db.data_generator():    # without explicitly passing results, it uses the last query.\n",
-    "    density_values = df['IRHDensity']\n",
+    "    density_values = df['IRH_NUM']\n",
     "    mean_density_trs.append(np.mean(density_values))\n",
     "    min_density = min(min_density, min(density_values))\n",
     "    max_density = max(max_density, max(density_values))\n",
@@ -383,13 +498,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "processing",
+   "display_name": "geo",
    "language": "python",
    "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
-   "version": "3.13.2"
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.9"
   }
  },
  "nbformat": 4,

anta_database-0.3.3/ROADMAP.md

@@ -1,12 +0,0 @@
-# Roadmap
-
-Here is a todo list of features that we want to include in the AntADatabase.
-
-## Important
-- [ ] Implement HDF5 format (In process, for next release)
-- [ ] Filter by radar system
-- [ ] Create shapefiles, e.g. for QAntarctica (In process)
-
-## To discuss
-- [ ] A netCDF version 
-