anta-database 0.3.2__tar.gz → 0.3.4__tar.gz

{anta_database-0.3.2 → anta_database-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anta_database
-Version: 0.3.2
+Version: 0.3.4
 Summary: SQLite database for the AntArchitecture Community Data
 Author: Antoine Hermant
 Author-email: antoine.hermant@etik.com
@@ -48,12 +48,12 @@ Visit the [Home Page](https://antoinehermant.github.io/anta_database/intro) for
 
 The AntADatabase contains all traced and dated Internal Reflective Horizons (isochrones) that have been published at that date. All the data is organized by flight transect for each dataset. The data is encoded in HDF5 (Hierarchical Data Format) which provides convenient data structure and performant read speeds. 
 Variables per file (when exist):
-- x(point)
-- y(point)
-- distance(point)
+- PSX(point)
+- PSY(point)
+- Distance(point)
 - IRH_DEPTH(point, age)
-- IRH_DENS(point, age)
-- ICE_THCK(point, age)
+- IRH_NUM(point, age)
+- ICE_THK(point)
 - SURF_ELEV(point, age)
 - BED_ELEV(point, age)
 
@@ -66,7 +66,7 @@ This Python module provides SQL indexing for the AntADatabase, as well as quick
 - age
 - acquisition_year (year of radar acquisition)
 - region
-- basin
+- IMBIE_basin
 - var (variable)
 - flight id
 

{anta_database-0.3.2 → anta_database-0.3.4}/README.md

@@ -16,12 +16,12 @@ Visit the [Home Page](https://antoinehermant.github.io/anta_database/intro) for
 
 The AntADatabase contains all traced and dated Internal Reflective Horizons (isochrones) that have been published at that date. All the data is organized by flight transect for each dataset. The data is encoded in HDF5 (Hierarchical Data Format) which provides convenient data structure and performant read speeds. 
 Variables per file (when exist):
-- x(point)
-- y(point)
-- distance(point)
+- PSX(point)
+- PSY(point)
+- Distance(point)
 - IRH_DEPTH(point, age)
-- IRH_DENS(point, age)
-- ICE_THCK(point, age)
+- IRH_NUM(point, age)
+- ICE_THK(point)
 - SURF_ELEV(point, age)
 - BED_ELEV(point, age)
 
@@ -34,7 +34,7 @@ This Python module provides SQL indexing for the AntADatabase, as well as quick
 - age
 - acquisition_year (year of radar acquisition)
 - region
-- basin
+- IMBIE_basin
 - var (variable)
 - flight id
 

anta_database-0.3.4/ROADMAP.md

@@ -0,0 +1,12 @@
+# Roadmap
+
+Here is a todo list of features that we want to include in the AntADatabase.
+
+## Important
+- [x] Implement HDF5 format (In process, for next release)
+- [x] Filter by radar system
+- [x] Create shapefiles, e.g. for QAntarctica (In process)
+
+## To discuss
+- [ ] A netCDF version 
+

{anta_database-0.3.2 → anta_database-0.3.4}/anta_database/compiling/compile.py

@@ -294,7 +294,7 @@ class CompileDatabase:
                             header=0,
                             sep=sep,
                             # usecols=original_new_columns.columns,
-                            na_values=['-9999', '-9999.0', 'NaN', 'nan', ''],
+                            # na_values=['-9999', '-9999.0', 'NaN', 'nan', ''],
                             dtype=str
                             )
 
@@ -304,6 +304,26 @@ class CompileDatabase:
         ds = ds[ds.columns.intersection(original_new_columns.columns)]
         ds.columns = original_new_columns[ds.columns].iloc[0].values  # renaming the columns
 
+        values_to_replace = ['-9999', '-9999.0', 'NaN', 'nan', '']
+
+        if 'PSX' in ds.columns and 'PSY' in ds.columns:
+            # Create a mask for rows where both PSX and PSY are in values_to_replace
+            mask = (
+                ds['PSX'].isin(values_to_replace) &
+                ds['PSY'].isin(values_to_replace)
+            )
+            if mask.any():
+                problematic_indices = ds.index[mask].tolist()
+                warnings.warn(
+                    f"Found {len(problematic_indices)} rows where both PSX and PSY are in {values_to_replace}. "
+                    f"Indices: {problematic_indices}. Dropping these rows."
+                )
+
+            ds = ds[~mask].copy()
+
+        cols_to_replace = [col for col in ds.columns if col not in ['PSX', 'PSY']]
+        ds[cols_to_replace] = ds[cols_to_replace].replace(values_to_replace, pd.NA)
+
         pattern_values = original_new_columns[1:]
         pattern_header = original_new_columns.iloc[0]
         pattern_values.columns = pattern_header
@@ -863,6 +883,8 @@ class CompileDatabase:
                             max_depth[i] = depth_slice[max_idx]
                             age_max_depth[i] = ds.IRH_AGE.values[max_idx]
 
+                    age_max_depth = np.abs(age_max_depth)
+                    max_depth = np.abs(max_depth)
                     flight_id = ds.attrs["flight ID"]
                     acq_year = ds.attrs["acquisition year"]
                     project = ds.attrs["project"]
@@ -871,11 +893,11 @@ class CompileDatabase:
 
                     df = gpd.GeoDataFrame({
                         "geometry": [Point(x, y) for x, y in zip(ds.PSX.values, ds.PSY.values)],
-                        "irh_num": N.astype('int16'),
-                        "max_age": age_max_depth.astype('int16'),
-                        "max_depth": max_depth.astype('int16'),
-                        "flight_id": [flight_id] * len(N),
-                        "doi_data": [doi_data] * len(N),
+                        "irh_num": N.astype('int8'),
+                        "max_age": age_max_depth.astype('int32'),
+                        "max_depth": np.rint(max_depth).astype('int16'),
+                        # "flight_id": [flight_id] * len(N),
+                        # "doi_data": [doi_data] * len(N),
                     }, crs="EPSG:3031")
 
                     all_dfs.append(df)

{anta_database-0.3.2 → anta_database-0.3.4}/anta_database/compiling/reconnect.py

@@ -43,7 +43,7 @@ class Reconnect:
 
 
     def process(self, df_connect: pd.DataFrame, tree, file_path: str):
-        data_dir = self._db.db_dir
+        data_dir = self._db._db_dir
         full_path = os.path.join(data_dir, file_path)
         with h5py.File(full_path, 'a') as f:
             with xr.open_dataset(f, engine='h5netcdf') as ds:

{anta_database-0.3.2 → anta_database-0.3.4}/anta_database/database/database.py

@@ -92,11 +92,13 @@ class Database:
             LEFT JOIN
                 variables v ON dv.variable_id = v.id
             LEFT JOIN
-                dataset_ages da ON d.id = da.dataset_id
+                dataset_ages da ON d.id = da.dataset_id AND v.name = 'IRH_DEPTH'
             LEFT JOIN
                 ages a ON da.age_id = a.id
             LEFT JOIN
-                regions r ON da.region_id = r.id
+                dataset_regions dr ON d.id = dr.dataset_id
+            LEFT JOIN
+                regions r ON dr.region_id = r.id
         '''
         conditions = []
         params = []
@@ -415,21 +417,21 @@ class Database:
         Helper method to build the SQL query and parameters for filtering.
         Returns the query string and parameters list.
         """
-        select_clause = 's.name AS dataset, \
+        select_clause = "s.name AS dataset, \
                         s.citation, \
                         s.DOI_dataset, \
                         s.DOI_publication, \
                         institutes_list.institutes, \
                         projects_list.projects, \
                         d.acq_year, \
-                        a.age, \
-                        a.age_unc, \
+                        CASE WHEN v.name = 'IRH_DEPTH' THEN a.age ELSE NULL END AS age, \
+                        CASE WHEN v.name = 'IRH_DEPTH' THEN a.age_unc ELSE NULL END AS age_unc, \
                         v.name AS var, \
                         d.flight_id, \
                         r.region, \
                         r.IMBIE_basin, \
                         radar_instruments_list.radar_instruments \
-        '
+        "
         query = f'''
             SELECT {select_clause}
             FROM
@@ -474,11 +476,13 @@ class Database:
             LEFT JOIN
                 variables v ON dv.variable_id = v.id
             LEFT JOIN
-                dataset_ages da ON d.id = da.dataset_id
+                dataset_ages da ON d.id = da.dataset_id AND v.name = 'IRH_DEPTH'
             LEFT JOIN
                 ages a ON da.age_id = a.id
             LEFT JOIN
-                regions r ON da.region_id = r.id
+                dataset_regions dr ON d.id = dr.dataset_id
+            LEFT JOIN
+                regions r ON dr.region_id = r.id
         '''
         conditions = []
         params = []
@@ -536,21 +540,21 @@ class Database:
               retain_query: Optional[bool] = True,
               ) -> 'MetadataResult':
 
-        select_clause = 's.name AS dataset, \
+        select_clause = "s.name AS dataset, \
                         s.citation, \
                         s.DOI_dataset, \
                         s.DOI_publication, \
                         institutes_list.institutes, \
                         projects_list.projects, \
                         d.acq_year, \
-                        a.age, \
-                        a.age_unc, \
+                        CASE WHEN v.name = 'IRH_DEPTH' THEN a.age ELSE NULL END AS age, \
+                        CASE WHEN v.name = 'IRH_DEPTH' THEN a.age_unc ELSE NULL END AS age_unc, \
                         v.name AS var, \
                         d.flight_id, \
                         r.region, \
                         r.IMBIE_basin, \
                         radar_instruments_list.radar_instruments \
-        '
+        "
         query, params = self._build_query_and_params(age, var, dataset, institute, project, acquisition_year, flight_id, region, IMBIE_basin, radar_instrument, select_clause)
 
         conn = sqlite3.connect(self._file_db_path)
@@ -686,18 +690,18 @@ class Database:
 
     def get_files(
         self,
-        metadata: Union[None, Dict, 'MetadataResult'] = None,
-        data_dir: Union[None, str] = None,
+        metadata: Optional[Union[None, Dict, 'MetadataResult']] = None,
+        data_dir: Optional[Union[None, str]] = None,
     ):
 
         md = metadata or self._md
         if not md:
-            print('Please provide metadata of the files you want to generate the data from. Exiting...')
+            print('Please provide metadata of the files you want to generate the data from.')
             return
 
         data_dir = data_dir or self._db_dir
         if not data_dir:
-            print('No data directory provided. Exiting...')
+            print('No data directory provided.')
             return
 
         file_paths = self._get_file_paths_from_metadata(metadata=md)
@@ -711,7 +715,7 @@ class Database:
         self,
         metadata: Union[None, Dict, 'MetadataResult'] = None,
         data_dir: Optional[str] = None,
-        downscale_factor: Optional[str] = None,
+        downsampling_factor: Optional[str] = None,
         disable_tqdm: bool = False,
     ) -> Generator[Tuple[pd.DataFrame, Dict]]:
         """
@@ -740,35 +744,43 @@ class Database:
         file_paths = self._get_file_paths_from_metadata(metadata=md)
         file_paths = np.unique(file_paths) # Be carefull as many pointers point to the same file
 
+        if disable_tqdm or self._disable_tqdm:
+            disable_tqdm = True
+
         for file_path in tqdm(file_paths, desc='Generating dataframes', total=len(file_paths), unit='file', disable=disable_tqdm):
             full_path = os.path.join(data_dir, file_path)
             file_md = self._get_file_metadata(file_path)
             ds = h5py.File(full_path, 'r')
-            df = pd.DataFrame({'PSX': ds['PSX'][::downscale_factor],
-                                'PSY': ds['PSY'][::downscale_factor]})
+            df = pd.DataFrame({'PSX': ds['PSX'][::downsampling_factor],
+                                'PSY': ds['PSY'][::downsampling_factor]})
 
             if 'Distance' in ds.keys():
-                df['Distance'] = ds['Distance'][::downscale_factor]
+                df['Distance'] = ds['Distance'][::downsampling_factor]
 
+            var_impl = []
+            age_impl = []
             for var in md['var']:
                 if var == 'IRH_DEPTH':
+                    var_impl.append(var)
                     for age in md['age']:
                         irh_values = ds['IRH_AGE'][:]
                         irh_index = np.where(irh_values == int(age))[0]
                         if len(irh_index) == 0:
                             continue
                         irh_index = irh_index[0]
+                        age_impl.append(age)
 
-                        df[int(age)] = ds[var][::downscale_factor, irh_index]
+                        df[int(age)] = ds[var][::downsampling_factor, irh_index]
 
                 else:
                     if var in ds.keys():
-                        df[var] = ds[var][::downscale_factor]
+                        var_impl.append(var)
+                        df[var] = ds[var][::downsampling_factor]
 
             metadata = {
                 'dataset': file_md['dataset'],
-                'var': md['var'],
-                'age': md['age'],
+                'var': var_impl,
+                'age': age_impl,
                 'flight_id': file_md['flight_id'],
                 'institute': file_md['institute'],
                 'project': file_md['project'],
@@ -785,105 +797,6 @@ class Database:
 
             yield df, metadata
 
-    # def return_dataset(
-    #     self,
-    #     metadata: Union[None, Dict, 'MetadataResult'] = None,
-    #     data_dir: Optional[str] = None,
-    #     downscale_factor: Optional[str] = None,
-    # ) -> Generator[Tuple[pd.DataFrame, Dict]]:
-    #     """
-    #     Generates xarray Datasets from HDF5 files, one at a time, with lazy loading.
-
-    #     Args:
-    #         metadata: Metadata for filtering files.
-    #         data_dir: Directory containing the data files.
-    #         vars_to_load: List of variables to load from each file.
-
-    #     Yields:
-    #         Tuple[xr.Dataset, Dict]: A lazy-loaded xarray Dataset and its metadata.
-    #     """
-    #     # Resolve metadata
-    #     md = metadata or self._md
-    #     if not md:
-    #         print('Please provide metadata of the files you want to generate the data from. Exiting...')
-    #         return
-
-    #     # Resolve data directory
-    #     data_dir = data_dir or self._db_dir
-    #     if not data_dir:
-    #         print('No data directory provided. Exiting...')
-    #         return
-
-    #     if len(md['var']) > 1:
-    #         print(f'WARNING: you requested multiple variables ({md['var']}), this will return individual dataframes for each variable in each transect.')
-
-    #     file_paths = self._get_file_paths_from_metadata(metadata=md)
-    #     file_paths = np.unique(file_paths) # Be carefull as many pointers point to the same file
-
-    #     for file_path in file_paths:
-    #         full_path = os.path.join(data_dir, file_path)
-    #         file_md = self._get_file_metadata(file_path)
-    #         for var in md['var']:
-    #             if var == 'IRH_DEPTH':
-    #                 for age in md['age']:
-    #                     with h5py.File(full_path, 'r') as ds:
-    #                         irh_values = ds['age'][:]
-    #                         irh_index = np.where(irh_values == int(age))[0]
-    #                         if len(irh_index) == 0:
-    #                             continue
-    #                         irh_index = irh_index[0]
-    #                         df = pd.DataFrame({'x': ds['x'][::downscale_factor],
-    #                                         'y': ds['y'][::downscale_factor],
-    #                                         'distance': ds['distance'][::downscale_factor],
-    #                                         var: ds[var][::downscale_factor, irh_index]})
-
-    #                         metadata = {
-    #                             'dataset': file_md['dataset'],
-    #                             'var': var,
-    #                             'age': age,
-    #                             'flight_id': file_md['flight_id'],
-    #                             'institute': file_md['institute'],
-    #                             'project': file_md['project'],
-    #                             'acquisition_year': file_md['acquisition_year'],
-    #                             'age_unc': file_md['age'],
-    #                             'reference': file_md['reference'],
-    #                             'DOI_dataset': file_md['DOI_dataset'],
-    #                             'DOI_publication': file_md['DOI_publication'],
-    #                             'flight_id': file_md['flight_id'],
-    #                             'region': file_md['region'],
-    #                             'IMBIE_basin': file_md['IMBIE_basin'],
-    #                             'radar_instrument': file_md['radar_instrument'],
-    #                         }
-
-    #                         yield df, metadata
-
-    #                 else:
-    #                     ds = h5py.File(full_path, 'r')
-    #                     df = pd.DataFrame({'PSX': ds['PSX'][::downscale_factor],
-    #                                     'PSY': ds['PSY'][::downscale_factor],
-    #                                     'Distance': ds['Distance'][::downscale_factor],
-    #                                     var: ds[var][::downscale_factor]})
-
-    #                     metadata = {
-    #                         'dataset': file_md['dataset'],
-    #                         'var': var,
-    #                         'age': None,
-    #                         'flight_id': file_md['flight_id'],
-    #                         'institute': file_md['institute'],
-    #                         'project': file_md['project'],
-    #                         'acquisition_year': file_md['acquisition_year'],
-    #                         'age_unc': file_md['age'],
-    #                         'reference': file_md['reference'],
-    #                         'DOI_dataset': file_md['DOI_dataset'],
-    #                         'DOI_publication': file_md['DOI_publication'],
-    #                         'flight_id': file_md['flight_id'],
-    #                         'region': file_md['region'],
-    #                         'IMBIE_basin': file_md['IMBIE_basin'],
-    #                         'radar_instrument': file_md['radar_instrument'],
-    #                     }
-
-    #                     yield df, metadata
-
     @property
     def plot(self):
         if self._plotting is None:

{anta_database-0.3.2 → anta_database-0.3.4}/anta_database/indexing/index_database.py

@@ -126,11 +126,9 @@ class IndexDatabase:
             CREATE TABLE IF NOT EXISTS dataset_ages (
                 dataset_id INTEGER,
                 age_id INTEGER,
-                region_id INTEGER,
                 FOREIGN KEY (dataset_id) REFERENCES datasets (id),
                 FOREIGN KEY (age_id) REFERENCES ages (id),
-                FOREIGN KEY (region_id) REFERENCES regions (id),
-                PRIMARY KEY (dataset_id, age_id, region_id)
+                PRIMARY KEY (dataset_id, age_id)
                 )
             ''')
 
@@ -164,6 +162,16 @@ class IndexDatabase:
                 )
             ''')
 
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS dataset_regions (
+                dataset_id INTEGER,
+                region_id INTEGER,
+                FOREIGN KEY (dataset_id) REFERENCES datasets (id),
+                FOREIGN KEY (region_id) REFERENCES regions (id),
+                PRIMARY KEY (dataset_id, region_id)
+                )
+            ''')
+
         for f in tqdm(h5_files, desc="Indexing files"):
             _, file_name = os.path.split(f)
 
@@ -281,7 +289,7 @@ class IndexDatabase:
             # Insert variables
             var_list = ['ICE_THK', 'SURF_ELEV', 'BED_ELEV', 'BASAL_UNIT', 'IRH_NUM', 'IRH_DEPTH']
             for var in var_list:
-                if var in ds_vars or var == 'IRH_DEPTH':
+                if var in ds_vars:
                     cursor.execute('INSERT OR IGNORE INTO variables (name) VALUES (?)', (var,))
                     var_id = cursor.execute('SELECT id FROM variables WHERE name = ?', (var,)).fetchone()[0]
                     cursor.execute('''
@@ -294,13 +302,18 @@ class IndexDatabase:
                 for age, age_unc in age_uncs.iterrows():
                     cursor.execute('INSERT OR IGNORE INTO ages (age, age_unc) VALUES (?, ?)', (str(age), str(age_unc['age_unc'])))
                     age_id = cursor.execute('SELECT id FROM ages WHERE age = ? AND age_unc = ?', (str(age), str(age_unc['age_unc']))).fetchone()[0]
-                    for basin, region in basin_mapping.items():
-                        cursor.execute('INSERT OR IGNORE INTO regions (region, IMBIE_basin) VALUES (?, ?)', (region, basin))
-                        region_id = cursor.execute('SELECT id FROM regions WHERE region = ? AND IMBIE_basin = ?', (region, basin)).fetchone()[0]
-                        cursor.execute('''
-                            INSERT OR IGNORE INTO dataset_ages (dataset_id, age_id, region_id)
-                            VALUES (?, ?, ?)
-                        ''', (dataset_row_id, age_id, region_id))
+                    cursor.execute('''
+                        INSERT OR IGNORE INTO dataset_ages (dataset_id, age_id)
+                        VALUES (?, ?)
+                    ''', (dataset_row_id, age_id))
+
+            for basin, region in basin_mapping.items():
+                cursor.execute('INSERT OR IGNORE INTO regions (region, IMBIE_basin) VALUES (?, ?)', (region, basin))
+                region_id = cursor.execute('SELECT id FROM regions WHERE region = ? AND IMBIE_basin = ?', (region, basin)).fetchone()[0]
+                cursor.execute('''
+                    INSERT OR IGNORE INTO dataset_regions (dataset_id, region_id)
+                    VALUES (?, ?)
+                ''', (dataset_row_id, region_id))
 
         conn.commit()
         conn.close()