anta-database 0.3.4__tar.gz → 0.3.6__tar.gz

{anta_database-0.3.4 → anta_database-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anta_database
-Version: 0.3.4
+Version: 0.3.6
 Summary: SQLite database for the AntArchitecture Community Data
 Author: Antoine Hermant
 Author-email: antoine.hermant@etik.com

{anta_database-0.3.4 → anta_database-0.3.6}/anta_database/compiling/compile.py

@@ -24,6 +24,7 @@ class CompileDatabase:
                  firn_correction: Optional[float] = None,
                  extract_data: bool = True,
                  hdf5: bool = True,
+                 netcdf: bool = False,
                  compute_distance: bool = True,
                  compute_IMBIE_basins: bool = True,
                  compute_IRH_density: bool = True,
@@ -42,6 +43,7 @@ class CompileDatabase:
         self._compute_IMBIE_basins = compute_IMBIE_basins
         self._set_attributes = set_attributes
         self._hdf5 = hdf5
+        self._netcdf = netcdf
         self._shapefiles = shapefiles
         self._geopackages = geopackages
         self._break_transects = break_transects
@@ -94,7 +96,7 @@ class CompileDatabase:
                 for file_dict in tqdm(all_files_list, desc="Processing", unit="file"):
                     self.extract(file_dict=file_dict)
 
-        if self._hdf5 is True:
+        if self._hdf5 is True or self._netcdf is True:
             all_dirs = []
             for dir_ in self._dir_list:
                 dirs = [d for d in glob.glob(f"{dir_}/pkl/*") if os.path.isdir(d)]
@@ -104,7 +106,7 @@ class CompileDatabase:
             num_workers = min(num_tasks, cpus)
 
             print('\n',
-                    'Will start creating the hdf5 for', len(all_dirs), 'traces\n'
+                    'Will start creating the hdf5/netcdf for', len(all_dirs), 'traces\n'
                     '\n   ', num_workers, 'worker(s) allocated out of', cpu_count(), 'available cpus\n')
 
             if num_workers > 1:
@@ -680,14 +682,25 @@ class CompileDatabase:
                 ds[var] = ds[var].chunk({'point': chunk_size_point})
                 encoding[var] = {'zlib': True, 'complevel': 1, 'chunksizes': (chunk_size_point)}
 
-        h5_dir = os.path.join(dataset_dir, 'h5')
-        os.makedirs(h5_dir, exist_ok=True)
-        if not isinstance(institute, (list, tuple, set)) and institute != 'nan':
-            h5_file = f'{h5_dir}/{institute}_{flight_id}.h5'
-        else:
-            h5_file = f'{h5_dir}/{flight_id}.h5'
+        if self._hdf5:
+            h5_dir = os.path.join(dataset_dir, 'h5')
+            os.makedirs(h5_dir, exist_ok=True)
+            if not isinstance(institute, (list, tuple, set)) and institute != 'nan':
+                h5_file = f'{h5_dir}/{institute}_{flight_id}.h5'
+            else:
+                h5_file = f'{h5_dir}/{flight_id}.h5'
+
+            ds.to_netcdf(h5_file, engine='h5netcdf', encoding=encoding, mode='w')
+
+        if self._netcdf:
+            nc_dir = os.path.join(dataset_dir, 'nc')
+            os.makedirs(nc_dir, exist_ok=True)
+            if not isinstance(institute, (list, tuple, set)) and institute != 'nan':
+                nc_file = f'{nc_dir}/{institute}_{flight_id}.nc'
+            else:
+                nc_file = f'{nc_dir}/{flight_id}.nc'
 
-        ds.to_netcdf(h5_file, engine='h5netcdf', encoding=encoding, mode='w')
+            ds.to_netcdf(nc_file, engine='netcdf4', encoding=encoding, mode='w')
 
     def do_break_transects(self, h5_file: str) -> None:
         h5_dir, name = os.path.split(h5_file)

{anta_database-0.3.4 → anta_database-0.3.6}/anta_database/database/database.py

@@ -8,9 +8,10 @@ from typing import Union, List, Dict, Tuple, Optional, Generator
 from tqdm import tqdm
 
 from anta_database.plotting.plotting import Plotting
+from anta_database.indexing.index_database import IndexDatabase
 
 class Database:
-    def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db', include_BEDMAP: bool = False, max_displayed_flight_ids: Optional[int] = 50) -> None:
+    def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db', index_database: bool = False, include_BEDMAP: bool = False, max_displayed_flight_ids: Optional[int] = 50) -> None:
         self._db_dir = database_dir
         self._file_db = file_db
         self._file_db_path = os.path.join(self._db_dir, file_db)
@@ -18,6 +19,7 @@ class Database:
         self._plotting = None
         self._max_displayed_flight_ids = max_displayed_flight_ids
         self._include_BM = include_BEDMAP
+        self._include_BM = index_database
         self._disable_tqdm = os.getenv("JUPYTER_BOOK_BUILD", False)
         self._excluded = {
             'dataset': [],
@@ -32,6 +34,10 @@ class Database:
             'radar_instrument': [],
         }
 
+        if index_database:
+            indexing = IndexDatabase(self._db_dir)
+            indexing.index_database()
+
     def _build_query_and_params(self,
                                 age: Optional[Union[str, List[str]]] = None,
                                 var: Optional[Union[str, List[str]]] = None,
@@ -177,38 +183,40 @@ class Database:
                         op, val = self._parse_range_query(field)
                         if op == '>':
                             conditions.append(f"("
-                                f"({column} > ?) OR "
-                                f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) > ?)"
-                                f")")
+                                            f"CAST({column} AS INTEGER) > ? OR "
+                                            f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-') + 1) AS INTEGER) > ?"
+                                            f")")
                             params.extend([val, val])
                         elif op == '<':
                             conditions.append(f"("
-                                f"({column} < ?) OR "
-                                f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) < ?)"
-                                f")")
+                                            f"CAST({column} AS INTEGER) < ? OR "
+                                            f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-') - 1) AS INTEGER) < ?"
+                                            f")")
                             params.extend([val, val])
                         elif op == '>=':
                             conditions.append(f"("
-                                f"({column} >= ?) OR "
-                                f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) >= ?)"
-                                f")")
+                                            f"CAST({column} AS INTEGER) >= ? OR "
+                                            f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-') + 1) AS INTEGER) >= ?"
+                                            f")")
                             params.extend([val, val])
                         elif op == '<=':
                             conditions.append(f"("
-                                f"({column} <= ?) OR "
-                                f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) <= ?)"
-                                f")")
+                                            f"CAST({column} AS INTEGER) <= ? OR "
+                                            f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-') - 1) AS INTEGER) <= ?"
+                                            f")")
                             params.extend([val, val])
                         elif op == '=':
                             conditions.append(f"("
-                                f"({column} = ?) OR "
+                                f"CAST({column} AS INTEGER) = ? OR "
                                 f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) <= ? AND CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) >= ?)"
                                 f")")
                             params.extend([val, val, val])
                     elif self._is_range_value(field):
                             start, end = self._parse_range_value(field)
+                            start = int(start)
+                            end = int(end)
                             conditions.append(f"("
-                                f"({column} BETWEEN ? AND ?) OR "
+                                f"CAST({column} AS INTEGER) >= ? AND CAST({column} AS INTEGER) <= ? OR "
                                 f"({column} LIKE '%-%' AND "
                                 f"CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) <= ? AND "
                                 f"CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) >= ?)"
@@ -240,31 +248,31 @@ class Database:
                             inverted_op = self._invert_range_operator(op)
                             if inverted_op == '>':
                                 conditions.append(f"("
-                                    f"({column} > ?) OR "
-                                    f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) > ?)"
-                                    f")")
+                                                f"CAST({column} AS INTEGER) > ? OR "
+                                                f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-') + 1) AS INTEGER) > ?"
+                                                f")")
                                 params.extend([val, val])
                             elif inverted_op == '<':
                                 conditions.append(f"("
-                                    f"({column} < ?) OR "
-                                    f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) < ?)"
-                                    f")")
+                                                f"CAST({column} AS INTEGER) < ? OR "
+                                                f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-') - 1) AS INTEGER) < ?"
+                                                f")")
                                 params.extend([val, val])
                             elif inverted_op == '>=':
                                 conditions.append(f"("
-                                    f"({column} >= ?) OR "
-                                    f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) >= ?)"
-                                    f")")
+                                                f"CAST({column} AS INTEGER) >= ? OR "
+                                                f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, INSTR({column}, '-') + 1) AS INTEGER) >= ?"
+                                                f")")
                                 params.extend([val, val])
                             elif inverted_op == '<=':
                                 conditions.append(f"("
-                                    f"({column} <= ?) OR "
-                                    f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) <= ?)"
-                                    f")")
+                                                f"CAST({column} AS INTEGER) <= ? OR "
+                                                f"{column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-') - 1) AS INTEGER) <= ?"
+                                                f")")
                                 params.extend([val, val])
                             elif inverted_op == '=':
                                 conditions.append(f"("
-                                    f"({column} = ?) OR "
+                                    f"CAST({column} AS INTEGER) = ? OR "
                                     f"({column} LIKE '%-%' AND CAST(SUBSTR({column}, 1, INSTR({column}, '-')-1) AS INTEGER) <= ? AND CAST(SUBSTR({column}, INSTR({column}, '-')+1) AS INTEGER) >= ?)"
                                     f")")
                                 params.extend([val, val, val])
@@ -642,21 +650,48 @@ class Database:
                 seen.add(age)
                 unique_pairs.append((age, unc))
 
+        unique_institutes = set()
+        for institutes in institutes_list:
+            for institute in institutes.split(', '):
+                unique_institutes.add(institute.strip())
+
+        unique_projects = set()
+        for projects in projects_list:
+            for project in projects.split(', '):
+                unique_projects.add(project.strip())
+
+        unique_radar_instruments = set()
+        for radar_instruments in radar_instruments_list:
+            for radar_instrument in radar_instruments.split(', '):
+                unique_radar_instruments.add(radar_instrument.strip())
+
+        unique_basins = set()
+        for basins in basin_list:
+            for basin in basins.split(', '):
+                unique_basins.add(basin.strip())
+
+        df_dataset_sorted = pd.DataFrame({
+            'dataset': metadata['dataset'],
+            'reference': metadata['reference'],
+            'DOI_dataset': metadata['DOI_dataset'],
+            'DOI_publication': metadata['DOI_publication']
+        }).sort_values('dataset')
+
         sorted_ages, sorted_age_unc = zip(*unique_pairs) if unique_pairs else ([], [])
         metadata['age'] = [str(age) for age in sorted_ages]
         metadata['age_unc'] = [str(age_unc) for age_unc in sorted_age_unc]
         metadata['var'] = sorted(set(vars_list))
-        metadata['institute'] = sorted(set(institutes_list))
-        metadata['project'] = sorted(set(projects_list))
+        metadata['institute'] = sorted(set(unique_institutes))
+        metadata['project'] = sorted(set(unique_projects))
         metadata['acquisition_year'] = sorted(set(acq_years_list))
-        metadata['dataset'] = list(dict.fromkeys(metadata['dataset']))
-        metadata['reference'] = list(dict.fromkeys(metadata['reference']))
-        metadata['DOI_dataset'] = list(dict.fromkeys(metadata['DOI_dataset']))
-        metadata['DOI_publication'] = list(dict.fromkeys(metadata['DOI_publication']))
+        metadata['dataset'] = list(dict.fromkeys(df_dataset_sorted['dataset']))
+        metadata['reference'] = list(dict.fromkeys(df_dataset_sorted['reference']))
+        metadata['DOI_dataset'] = list(dict.fromkeys(df_dataset_sorted['DOI_dataset']))
+        metadata['DOI_publication'] = list(dict.fromkeys(df_dataset_sorted['DOI_publication']))
         metadata['flight_id'] = list(set(metadata['flight_id']))
-        metadata['radar_instrument'] = sorted(set(radar_instruments_list))
+        metadata['radar_instrument'] = sorted(set(unique_radar_instruments))
         metadata['region'] = sorted(set(region_list))
-        metadata['IMBIE_basin'] = sorted(set(basin_list))
+        metadata['IMBIE_basin'] = sorted(set(unique_basins))
 
         if retain_query:
             self._md = metadata
@@ -717,6 +752,7 @@ class Database:
         data_dir: Optional[str] = None,
         downsampling_factor: Optional[str] = None,
         disable_tqdm: bool = False,
+        fraction_depth: Optional[bool] = False,
     ) -> Generator[Tuple[pd.DataFrame, Dict]]:
         """
         Generates xarray Datasets from HDF5 files, one at a time, with lazy loading.
@@ -770,13 +806,21 @@ class Database:
                         irh_index = irh_index[0]
                         age_impl.append(age)
 
-                        df[int(age)] = ds[var][::downsampling_factor, irh_index]
+                        df[age] = ds[var][::downsampling_factor, irh_index]
+                        if fraction_depth:
+                            if 'ICE_THK' in ds.keys():
+                                df[age] *= 100/ds['ICE_THK'][::downsampling_factor]
+                            else:
+                                df[age] = np.nan
+                                # print(f'WARNING: flight line {file_md['flight_id']} in {file_md['dataset']} does not have ICE_THK, cannot compute fraction depth, will return NaN')
 
                 else:
                     if var in ds.keys():
                         var_impl.append(var)
                         df[var] = ds[var][::downsampling_factor]
 
+            if 0 < len(age_impl) <= 1:
+                age_impl = list(age_impl)[0]
             metadata = {
                 'dataset': file_md['dataset'],
                 'var': var_impl,

{anta_database-0.3.4 → anta_database-0.3.6}/anta_database/indexing/index_database.py

@@ -3,23 +3,42 @@ from tqdm import tqdm
 import glob
 import pandas as pd
 import numpy as np
-import xarray as xr
 import sqlite3
 import h5py
 import json
 import ast
 
 class IndexDatabase:
-    def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db', index: str = 'references.json'):
+    def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db') -> None:
         self._db_dir = database_dir
         self._file_db = os.path.join(self._db_dir, file_db)
-        self._file_index = index
-        with open(os.path.join(self._db_dir, self._file_index), 'r', encoding='utf-8') as f:
-            self._index = json.load(f)
+        self._disable_tqdm = os.getenv("JUPYTER_BOOK_BUILD", False)
 
     def index_database(self):
         h5_files = []
-        for ref in self._index:
+
+        dirs = glob.glob(f'{self._db_dir}/*/')
+        self._dataset_list = []
+
+        for d in dirs:
+            with open(f'{d}/raw_md.json', 'r', encoding='utf-8') as f:
+                unique_data = []
+                seen = set()
+                index = json.load(f)
+                for item in index:
+                    combo = (item['dataset'], item['DOI publication'], item['DOI dataset'])
+                    if combo not in seen:
+                        seen.add(combo)
+                        unique_data.append({
+                            "dataset": item['dataset'],
+                            "citation": item['citation'],
+                            "DOI publication": item['DOI publication'],
+                            "DOI dataset": item['DOI dataset']
+                        })
+                for unique in unique_data:
+                    self._dataset_list.append(unique)
+
+        for ref in self._dataset_list:
             found_files = glob.glob(f'{self._db_dir}/{ref['dataset']}/h5/*.h5', recursive=False)
             if found_files:
                 h5_files.extend(found_files)
@@ -44,11 +63,11 @@ class IndexDatabase:
             )
         ''')
 
-        for row in self._index:
+        for row in self._dataset_list:
             try:
                 cursor.execute(
                     'INSERT INTO sources (name, citation, DOI_dataset, DOI_publication) VALUES (?, ?, ?, ?)',
-                    (row['dataset'], row['citation'], row['DOI_dataset'], row['DOI_publication'])
+                    (row['dataset'], row['citation'], row['DOI dataset'], row['DOI publication'])
                 )
             except sqlite3.IntegrityError:
                 # dataset already exists, skip
@@ -172,7 +191,7 @@ class IndexDatabase:
                 )
             ''')
 
-        for f in tqdm(h5_files, desc="Indexing files"):
+        for f in tqdm(h5_files, desc="Indexing files", disable=self._disable_tqdm):
             _, file_name = os.path.split(f)
 
             with h5py.File(f, 'r') as f:

{anta_database-0.3.4 → anta_database-0.3.6}/anta_database/plotting/plotting.py

@@ -177,6 +177,7 @@ class Plotting:
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
             downsampling_factor: Optional[int] = None,
+            fraction_depth: Optional[bool] = False,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
             ylim: Optional[tuple] = (None, None),
@@ -197,6 +198,7 @@ class Plotting:
             color_by='var',
             metadata=metadata,
             downsampling_factor=downsampling_factor,
+            fraction_depth=fraction_depth,
             title=title,
             xlim=xlim,
             ylim=ylim,
@@ -261,6 +263,7 @@ class Plotting:
             self,
             metadata: Union[None, Dict, 'MetadataResult'] = None,
             elevation: Optional[bool] = False,
+            fraction_depth: Optional[bool] = False,
             downsampling_factor: Optional[int] = None,
             title: Optional[str] = None,
             xlim: Optional[tuple] = (None, None),
@@ -402,9 +405,8 @@ class Plotting:
             else:
                 var = var[0]
 
-
             all_dfs = []
-            for ds, _ in tqdm(self._db.data_generator(metadata, downsampling_factor=downsampling_factor, disable_tqdm=True), desc="Plotting", total=total_traces, unit='trace', disable=self._disable_tqdm):
+            for ds, _ in tqdm(self._db.data_generator(metadata, downsampling_factor=downsampling_factor, disable_tqdm=True, fraction_depth=fraction_depth), desc="Plotting", total=total_traces, unit='trace', disable=self._disable_tqdm):
                 all_dfs.append(ds)
             df = pd.concat(all_dfs)
 
@@ -478,37 +480,31 @@ class Plotting:
                         title = f'AntADatabase Basal Unit'
                 scatter = plt.scatter(df['PSX']/1000, df['PSY']/1000, c=df[var], cmap=cmap, s=marker_size, vmin=vmin, vmax=vmax, linewidths=0, rasterized=True)
 
-            elif var in ['IRH_DEPTH', 'IRH_FRAC_DEPTH']:
-                if var == 'IRH_DEPTH':
-                    label = f'IRH Depth [m]'
-                    if cmap == None:
-                        cmap = cmaps.torch_r
-                    extend = 'both'
-                    age = list(metadata['age'])
-                    if not title:
+            elif var in ['IRH_DEPTH']:
+                if cmap == None:
+                    cmap = cmaps.torch_r
+                extend = 'both'
+                age = list(metadata['age'])
+                if not title:
+                    if fraction_depth:
+                        title = f'AntADatabase IRH Fraction Depth'
+                    else:
                         title = f'AntADatabase IRH Depth'
-                    if len(age) > 1:
-                        print('WARNING: Multiple layers provided: ', age,
-                            '\nSelect a unique age for better results')
-                    elif len(age) == 0:
-                        print('No layer provided, please provide a valid age.')
-                        return
-                elif var == 'IRH_FRAC_DEPTH':
-                    if cmap == None:
-                        cmap = cmaps.torch_r
-                    extend = 'both'
-                    age = list(metadata['age'])
-                    if len(age) > 1:
-                        print('WARNING: Multiple layers provided: ', age,
-                            '\nSelect a unique age for better results')
-                    elif len(age) == 0:
-                        print('No layer provided, please provide a valid age.')
-                        return
-                    age = age[0]
+                if len(age) > 1:
+                    print('WARNING: Multiple layers provided: ', age,
+                        '\nSelect a unique age for better results')
+                elif len(age) == 0:
+                    print('No layer provided, please provide a valid age.')
+                    return
+                if fraction_depth:
                     label = r'IRH Fractional Depth [\%]'
+                    if vmin is None: vmin = 0
+                    if vmax is None: vmax = 100
+                else:
+                    label = f'IRH Depth [m]'
 
                 for age in metadata['age']:
-                    scatter = plt.scatter(df['PSX']/1000, df['PSY']/1000, c=df[int(age)], cmap=cmap, s=marker_size, vmin=vmin, vmax=vmax, linewidths=0, rasterized=True)
+                    scatter = plt.scatter(df['PSX']/1000, df['PSY']/1000, c=df[age], cmap=cmap, s=marker_size, vmin=vmin, vmax=vmax, linewidths=0, rasterized=True)
 
         if color_by == 'transect_1D':
             flight_id = list(metadata['flight_id'])

{anta_database-0.3.4 → anta_database-0.3.6}/anta_database.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anta_database
-Version: 0.3.4
+Version: 0.3.6
 Summary: SQLite database for the AntArchitecture Community Data
 Author: Antoine Hermant
 Author-email: antoine.hermant@etik.com

{anta_database-0.3.4 → anta_database-0.3.6}/anta_database.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ book/contact.md
 book/contribute.md
 book/intro.md
 book/logo.png
+book/pism_example.ipynb
 book/quick_start.ipynb
 book/references.bib
 book/requirements.txt

{anta_database-0.3.4 → anta_database-0.3.6}/book/_toc.yml

@@ -5,6 +5,7 @@ format: jb-book
 root: intro
 chapters:
 - file: quick_start
+- file: pism_example
 - file: advanced
 - file: contact
 - file: contribute

anta_database-0.3.6/book/advanced.md

@@ -0,0 +1,154 @@
+---
+jupytext:
+  formats: md:myst
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.11.5
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+---
+
+# Advanced: Managing the database
+
+## Reindexing
+
+You may want for example to update the age of the layers in a particular dataset. For this, you just need to modify the ages in the file called `raw_md.json` located under a dataset directory. Then, reindex with the `IndexDatabase` class:
+
+```
+from anta_database import IndexDatabase
+
+db_path = '/home/anthe/documents/data/isochrones/AntADatabase/' 
+indexing = IndexDatabase(db_path)
+indexing.index_database() 
+```
+
+Or simply use the `index_database=True` argument when initializing the `Database`, which runs the IndexDatabase class itself: 
+```
+from anta_database import Database
+
+db_path = '/home/anthe/documents/data/isochrones/AntADatabase/' 
+db = Database(db_path, index_database=True)
+```
+
+## (Re)compile the database
+
+You can (re)compile the database, if for example you modify some data in the raw directories or if you add a dataset. For this, make sure to follow the structure: 
+
+```
+AntADatabase/
+├── AntADatabase.db
+├── FirstAuthor_YYYY
+    ├── raw_md.json # file containing information about the dataset, take example on an existing dataset and see the section below
+    ├── original_new_column_names.csv # first row: names of columns to keep from raw files, second row: how the columns should be renamed (see section below)
+    ├── raw/ #Directory with the original files to process
+    └── h5/ # Directory were the processed files will be written (it will be created in the process)
+```
+
+Note that the header information in the raw files should be commented out with '#'. The column names of the dataframe should be kept.
+Then use the `CompileDatabase class to compile the database:
+
+```
+from anta_database import CompileDatabase
+
+dir_path_list = [ # list of the dataset subdirectories to compile
+    './Winter_2018',
+    './Sanderson_2024',
+    './Franke_2025',
+    './Cavitte_2020',
+    './Beem_2021',
+    './Bodart_2021/',
+    './Muldoon_2023/',
+    './Ashmore_2020/',
+]
+
+compiler = CompileDatabase(dir_path_list)
+compiler.compile()
+```
+
+Then reindex (see above). Furthermore, if the depth is not given in meters but TWT, you should set the wave\_speed (units should match values in the file) for conversion and firn\_correction (meters):
+
+```
+dir_path = './Wang_2023'
+compiler = CompileDatabase(dir_path, wave_speed=0.1685, firn_correction=15.5)
+compiler.compile()
+```
+
+## The JSON file
+
+The `raw_md.json` defines all the metadata of a raw file which we be associated to the corresponding data in the database. This will allow to query the database with all the implemented filters. One should provide all the information that exists if possible:
+One important field is the `data type`, which specifies if the raw files are organised per layers or per flight lines. There are then two possibilities:
+
+`data type`: "layer"
+| field         | Description                                                                                     | Example Values                     |
+|-------------------|-------------------------------------------------------------------------------------------------|-------------------------------------|
+| `data type`       | `layer`                                                                                                       | `'layer'` |
+| `raw file`        | Name of the raw file in the raw folder with the correct extension (csv, txt, tab).               | `'OIA_EDC_IRH1.csv'` |
+| `dataset`         | Name of the dataset corresponding dataset (usually same as the folder name)                       | `'Cavitte_2020'` |
+| `institute`       | Institute(s) that produced the data.                                                           | `'BAS'`, `['AWI', 'NASA']`            |
+| `project`         | Project(s) under which the data were collected.                                                | `'OIB'`                             |
+| `acquisition year`| Year(s) in which the radar data were acquired. Can be a range.                                        | `'2000-2010'`, `'2005'` |
+| `age`             | Age in years before present of the layer.               | `'10000'`   |
+| `age uncertainty` | Uncertainty on the age of the layer                                                         | `'800'` |
+| `citation` | How the dataset should be cited                                                                       | `'Cavitte et al. 2020'` |
+| `DOI dataset` | DOI that directly links to the original dataset (e.g zenodo, USAP-DC...)                          | `'https://doi.org/10.15784/601411'` |
+| `DOI publication` | DOI of the associated publication that published the data                                     | `'https://doi.org/10.5194/essd-13-4759-2021'` |
+Example of such a dataset is Cavitte_2020
+
+`data type`: "flight line"
+Replace the `raw file` entry by `IRH name` as they are named in the raw files. Then the compilation will use the file name for flight ID.
+| field         | Description                                                                                     | Example Values                     |
+|-------------------|-------------------------------------------------------------------------------------------------|-------------------------------------|
+| `data type`       | `flight line`    | `'flight line'` |
+| `IRH name`        | Name of the IRH layer as is it named in the raw file (e.g. column name)                       | `'IRH1'` |
+| `age`             | Age in years before present of the layer.                                                | `'10000'`  |
+Example of such a dataset is Wang_2023
+
+## The CSV column file
+
+The original_new_column_names.csv pairs the columns in the raw file with associated variables in the database. The minimal information in the file should be:
+- Polar Stereographic coordinates PSX and PSY or lon, lat (will be converted to PSX, PSY)
+- The IRH depth
+
+More information can be provided such as flight ID, Ice thickness, acquisition year etc. But the columns should be renamed following the database convention:
+| variable         | variable name convention in the Database                                                                                     |
+|-------------------|-------------------------------------------------------------------------------------------------|
+| IRH depth       | `IRH_DEPTH`                                                                     |
+| Ice thickness   | `ICE_THK`     |
+| Surface elevation       |     `SURF_ELEV`                   |
+| Bed elevation       |    `BED_ELEV`                        |
+| Distance along transect         |     `DIST`                     |
+| `acquisition year`|         `acq_year`                 |
+| `Flight ID`             | `Flight_ID`   |
+
+Note if the acquisition year column is not directly a year YYYY but a string that contains the year, add a third row to provide the pattern to extract the acquisition year from the string.
+
+Example of original_new_column_names.csv file:
+```
+timestamp,x,y,thk_m,bedelv_mABSL,IRHdepth_m,line
+acq_year,PSX,PSY,ICE_THK,BED_ELEV,IRH_DEPTH,Flight_ID
+YYYYxxxxxxxxxxxxxxxxxxxxxx
+```
+
+For a dataset organized by flight lines, so many IRH depth in the raw files, the original_new_column_names.csv should look like:
+```
+Longitude,Latitude,TWT [ns] (of bed),TWT [ns] (of H1),TWT [ns] (of H2),TWT [ns] (of H3),TWT [ns] (of H4),TWT [ns] (of H5),TWT [ns] (of H6),TWT [ns] (of H7)
+lon,lat,ICE_THK,IRH1,IRH2,IRH3,IRH4,IRH5,IRH6,IRH7
+```
+Where the new column names for the layers correspond to the names you provided in the JSON file.
+
+
+## Notes
+### Multiprocessing
+The compilation uses multiprocessing tools for parallel processing. It first finds all the raw files to process and then distribute the processes on multiple processors. By default, it uses all available cpus on the machine minus 1 (to not completely freeze the machine). However, if there are fewer tasks than cpus (fewer files to process), it will use only as many cpus as there are tasks. 
+To manually fix the number of cpus used during the compilation:
+
+```
+compiler.compile(cpus=2) # Or any integer of choice
+```
+
+### Compilation process
+Needs to be updated.