anta-database 0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anta_database/__init__.py +3 -0
- anta_database/anta_database.py +3 -0
- anta_database/compiling/__init__.py +1 -0
- anta_database/compiling/compile.py +192 -0
- anta_database/database/__init__.py +1 -0
- anta_database/database/database.py +288 -0
- anta_database/database/plotting/GL.pkl +0 -0
- anta_database/database/plotting/__init__.py +1 -0
- anta_database/indexing/__init__.py +1 -0
- anta_database/indexing/index_database.py +101 -0
- anta_database-0.1.dist-info/METADATA +20 -0
- anta_database-0.1.dist-info/RECORD +15 -0
- anta_database-0.1.dist-info/WHEEL +5 -0
- anta_database-0.1.dist-info/licenses/LICENSE +19 -0
- anta_database-0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .compile import *
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import numpy as np
|
|
5
|
+
import glob
|
|
6
|
+
from pyproj import Transformer
|
|
7
|
+
from typing import Union
|
|
8
|
+
from multiprocessing import Pool, cpu_count
|
|
9
|
+
|
|
10
|
+
class CompileDatabase:
|
|
11
|
+
def __init__(self, dir_list: Union[str, list[str]], file_type: str = 'layer', wave_speed: Union[None, float] = None, firn_correction: Union[None, float] = None) -> None:
|
|
12
|
+
self.dir_list = dir_list
|
|
13
|
+
self.wave_speed = wave_speed
|
|
14
|
+
self.firn_correction = firn_correction
|
|
15
|
+
self.file_type = file_type
|
|
16
|
+
|
|
17
|
+
def get_dict_ages(self, tab_file) -> dict:
|
|
18
|
+
ages = pd.read_csv(tab_file, header=None, sep='\t', names=['file', 'age'])
|
|
19
|
+
return dict(zip(ages['file'], ages['age']))
|
|
20
|
+
|
|
21
|
+
def compile(self, cpus: int = cpu_count()-1) -> None:
|
|
22
|
+
if not isinstance(self.dir_list, list):
|
|
23
|
+
self.dir_list = [self.dir_list]
|
|
24
|
+
|
|
25
|
+
num_tasks = len(self.dir_list)
|
|
26
|
+
num_workers = min(num_tasks, cpus)
|
|
27
|
+
|
|
28
|
+
print('\n',
|
|
29
|
+
'Will start compiling', len(self.dir_list), 'directories\n'
|
|
30
|
+
'\n ', num_workers, 'worker(s) allocated out of', cpu_count(), 'available cpus\n')
|
|
31
|
+
|
|
32
|
+
if num_workers > 1:
|
|
33
|
+
with Pool(num_workers) as pool:
|
|
34
|
+
pool.map(self._compile, self.dir_list)
|
|
35
|
+
else:
|
|
36
|
+
for dir_path in self.dir_list:
|
|
37
|
+
self._compile(dir_path=dir_path)
|
|
38
|
+
|
|
39
|
+
def _compile(self, dir_path: str) -> None:
|
|
40
|
+
raw_files = glob.glob(f'{dir_path}/raw/*.*')
|
|
41
|
+
|
|
42
|
+
_, ext = os.path.splitext(raw_files[0])
|
|
43
|
+
|
|
44
|
+
ages = self.get_dict_ages(f'{dir_path}/IRH_ages.tab')
|
|
45
|
+
original_new_columns = pd.read_csv(f'{dir_path}/original_new_column_names.csv')
|
|
46
|
+
|
|
47
|
+
if ext == '.tab':
|
|
48
|
+
sep='\t'
|
|
49
|
+
elif ext == '.csv':
|
|
50
|
+
sep=','
|
|
51
|
+
else:
|
|
52
|
+
print('File type not supported, exiting ...')
|
|
53
|
+
sys.exit()
|
|
54
|
+
|
|
55
|
+
for i, file in enumerate(raw_files):
|
|
56
|
+
print('Processing', file, f'({i}/{len(raw_files)})')
|
|
57
|
+
ds = pd.read_csv(file, comment="#", header=0, sep=sep)
|
|
58
|
+
_, file_name = os.path.split(file)
|
|
59
|
+
file_name_, ext = os.path.splitext(file_name)
|
|
60
|
+
|
|
61
|
+
ds = ds[ds.columns.intersection(original_new_columns.columns)]
|
|
62
|
+
ds.columns = original_new_columns[ds.columns].iloc[0].values # renaming the columns
|
|
63
|
+
|
|
64
|
+
if 'IceThk' in ds.columns and 'SurfElev' in ds.columns and not 'BedElev' in ds.columns:
|
|
65
|
+
ds['BedElev'] = ds['SurfElev'] - ds['IceThk']
|
|
66
|
+
if 'IceThk' in ds.columns and 'BedElev' in ds.columns and not 'SurfElev' in ds.columns:
|
|
67
|
+
ds['SurfElev'] = ds['BedElev'] + ds['IceThk']
|
|
68
|
+
if 'SurfElev' in ds.columns and 'BedElev' in ds.columns and not 'IceThk' in ds.columns:
|
|
69
|
+
ds['IceThk'] = ds['SurfElev'] - ds['BedElev']
|
|
70
|
+
|
|
71
|
+
if self.wave_speed:
|
|
72
|
+
for var in ['IceThk', 'BedElev']:
|
|
73
|
+
if var in ds.columns:
|
|
74
|
+
ds[var] *= self.wave_speed
|
|
75
|
+
if self.firn_correction:
|
|
76
|
+
for var in ['IceThk', 'BedElev']:
|
|
77
|
+
if var in ds.columns:
|
|
78
|
+
ds[var] += self.firn_correction
|
|
79
|
+
|
|
80
|
+
if 'x' not in ds.columns and 'y' not in ds.columns:
|
|
81
|
+
if 'lon' in ds.columns and 'lat' in ds.columns:
|
|
82
|
+
transformer = Transformer.from_proj(
|
|
83
|
+
"EPSG:4326", # source: WGS84 (lon/lat)
|
|
84
|
+
"+proj=stere +lon_0=0 +lat_0=-90 +lat_ts=-71 +datum=WGS84 +units=m +no_defs", # target: polar
|
|
85
|
+
always_xy=True
|
|
86
|
+
)
|
|
87
|
+
ds['x'], ds['y'] = transformer.transform(ds['lon'].values, ds['lat'].values)
|
|
88
|
+
elif 'lon' not in ds.columns and 'lat' not in ds.columns:
|
|
89
|
+
if 'x' in ds.columns and 'y' in ds.columns:
|
|
90
|
+
inverse_transformer = Transformer.from_proj(
|
|
91
|
+
"+proj=stere +lon_0=0 +lat_0=-90 +lat_ts=-71 +datum=WGS84 +units=m +no_defs", # source: polar
|
|
92
|
+
"EPSG:4326", # target: WGS84 (lon/lat)
|
|
93
|
+
always_xy=True
|
|
94
|
+
)
|
|
95
|
+
ds['lon'], ds['lat'] = inverse_transformer.transform(ds['x'].values, ds['y'].values)
|
|
96
|
+
elif 'lon' in ds.columns and 'lat' in ds.columns and 'x' in ds.columns and 'y' in ds.columns:
|
|
97
|
+
pass
|
|
98
|
+
else:
|
|
99
|
+
print('No coordinates found in the dataset, exiting ....')
|
|
100
|
+
sys.exit()
|
|
101
|
+
|
|
102
|
+
if self.file_type == 'layer':
|
|
103
|
+
age = str(ages[file_name_])
|
|
104
|
+
ds = ds.rename(columns={'IRHdepth': age})
|
|
105
|
+
if self.wave_speed:
|
|
106
|
+
ds[age] *= self.wave_speed
|
|
107
|
+
if self.firn_correction:
|
|
108
|
+
ds[age] += self.firn_correction
|
|
109
|
+
|
|
110
|
+
ds['Trace_ID'] = ds['Trace_ID'].astype(str)
|
|
111
|
+
ds['Trace_ID'] = ds['Trace_ID'].str.replace(r'/\s+', '_') # Replace slashes with underscores, otherwise the paths can get messy
|
|
112
|
+
ds['Trace_ID'] = ds['Trace_ID'].str.replace('/', '_')
|
|
113
|
+
|
|
114
|
+
ds.set_index('Trace_ID', inplace=True)
|
|
115
|
+
|
|
116
|
+
for trace_id in np.unique(ds.index):
|
|
117
|
+
ds_trace = ds.loc[trace_id].copy()
|
|
118
|
+
if 'distance' not in ds_trace.columns:
|
|
119
|
+
x = ds_trace[['x', 'y']]
|
|
120
|
+
distances = np.sqrt(np.sum(np.diff(x, axis=0)**2, axis=1))
|
|
121
|
+
cumulative_distance = np.concatenate([[0], np.cumsum(distances)])
|
|
122
|
+
ds_trace['distance'] = cumulative_distance
|
|
123
|
+
|
|
124
|
+
ds_trace_file = f'{dir_path}/pkl/{trace_id}/{file_name_}.pkl'
|
|
125
|
+
os.makedirs(f'{dir_path}/pkl/{trace_id}' , exist_ok=True)
|
|
126
|
+
ds_trace.to_pickle(ds_trace_file)
|
|
127
|
+
print(ds_trace_file)
|
|
128
|
+
|
|
129
|
+
for var in ['IceThk', 'BedElev', 'SurfElev']:
|
|
130
|
+
if var in ds.columns:
|
|
131
|
+
ds_var = ds_trace[['x', 'y', 'distance', var]]
|
|
132
|
+
ds_var_file = f'{dir_path}/pkl/{trace_id}/{var}.pkl'
|
|
133
|
+
if os.path.exists(ds_var_file):
|
|
134
|
+
var_data = pd.read_pickle(ds_var_file)
|
|
135
|
+
merged_data = pd.concat([var_data, ds_var]).drop_duplicates(subset=['x', 'y'])
|
|
136
|
+
|
|
137
|
+
merged_data.to_pickle(ds_var_file)
|
|
138
|
+
print(ds_var_file)
|
|
139
|
+
else:
|
|
140
|
+
ds_var.to_pickle(ds_var_file)
|
|
141
|
+
print(ds_var_file)
|
|
142
|
+
|
|
143
|
+
elif self.file_type == 'trace':
|
|
144
|
+
if 'distance' not in ds.columns:
|
|
145
|
+
x = ds[['x', 'y']]
|
|
146
|
+
distances = np.sqrt(np.sum(np.diff(x, axis=0)**2, axis=1))
|
|
147
|
+
cumulative_distance = np.concatenate([[0], np.cumsum(distances)])
|
|
148
|
+
ds['distance'] = cumulative_distance
|
|
149
|
+
|
|
150
|
+
trace_id = file_name_
|
|
151
|
+
os.makedirs(f'{dir_path}/pkl/{trace_id}' , exist_ok=True)
|
|
152
|
+
|
|
153
|
+
for var in ['IceThk', 'BedElev', 'SurfElev']:
|
|
154
|
+
if var in ds.columns:
|
|
155
|
+
ds_var = ds[['x', 'y', 'distance', var]]
|
|
156
|
+
ds_var_file = f'{dir_path}/pkl/{trace_id}/{var}.pkl'
|
|
157
|
+
if os.path.exists(ds_var_file):
|
|
158
|
+
var_data = pd.read_pickle(ds_var_file)
|
|
159
|
+
merged_data = pd.concat([var_data, ds_var]).drop_duplicates(subset=['x', 'y'])
|
|
160
|
+
|
|
161
|
+
merged_data.to_pickle(ds_var_file)
|
|
162
|
+
print(ds_var_file)
|
|
163
|
+
else:
|
|
164
|
+
ds_var.to_pickle(ds_var_file)
|
|
165
|
+
print(ds_var_file)
|
|
166
|
+
|
|
167
|
+
ages = {key: ages[key] for key in ds.columns if key in ages}
|
|
168
|
+
|
|
169
|
+
for IRH in ages:
|
|
170
|
+
age = str(ages.get(IRH))
|
|
171
|
+
ds_IRH = ds[IRH]
|
|
172
|
+
ds_IRH = pd.DataFrame({
|
|
173
|
+
'lon': ds['lon'],
|
|
174
|
+
'lat': ds['lat'],
|
|
175
|
+
'x': ds['x'],
|
|
176
|
+
'y': ds['y'],
|
|
177
|
+
'distance': ds['distance'],
|
|
178
|
+
age: ds_IRH,
|
|
179
|
+
})
|
|
180
|
+
if self.wave_speed:
|
|
181
|
+
ds_IRH[age] *= self.wave_speed
|
|
182
|
+
if self.firn_correction:
|
|
183
|
+
ds_IRH += self.firn_correction
|
|
184
|
+
|
|
185
|
+
for var in ['IceThk', 'BedElev', 'SurfElev']:
|
|
186
|
+
if var in ds.columns:
|
|
187
|
+
ds_IRH[var] = ds[var]
|
|
188
|
+
|
|
189
|
+
ds_trace_file = f'{dir_path}/pkl/{trace_id}/{IRH}.pkl'
|
|
190
|
+
|
|
191
|
+
ds_IRH.to_pickle(ds_trace_file)
|
|
192
|
+
print(ds_trace_file)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .database import *
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import os, sys
|
|
2
|
+
import sqlite3
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import numpy as np
|
|
5
|
+
import matplotlib.pyplot as plt
|
|
6
|
+
from matplotlib.colors import LinearSegmentedColormap
|
|
7
|
+
import seaborn as sns
|
|
8
|
+
from typing import Union, List, Dict, Tuple
|
|
9
|
+
|
|
10
|
+
class Database:
|
|
11
|
+
def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db'):
|
|
12
|
+
module_dir = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
+
self.gl = pd.read_pickle(os.path.join(module_dir, 'plotting', 'GL.pkl'))
|
|
14
|
+
self.db_dir = database_dir
|
|
15
|
+
self.file_db = file_db
|
|
16
|
+
self.file_db_path = os.path.join(self.db_dir, file_db)
|
|
17
|
+
self.md = None
|
|
18
|
+
|
|
19
|
+
def _build_query_and_params(self, age: Union[None, str, List[str]]=None, var: Union[None, str, List[str]]=None, author: Union[None, str, List[str]]=None, line: Union[None, str, List[str]]=None, select_clause='') -> Tuple[str, List[Union[str, int]]]:
|
|
20
|
+
"""
|
|
21
|
+
Helper method to build the SQL query and parameters for filtering.
|
|
22
|
+
Returns the query string and parameters list.
|
|
23
|
+
"""
|
|
24
|
+
query = f'''
|
|
25
|
+
SELECT {select_clause}
|
|
26
|
+
FROM datasets d
|
|
27
|
+
JOIN authors a ON d.author = a.id
|
|
28
|
+
'''
|
|
29
|
+
conditions = []
|
|
30
|
+
params = []
|
|
31
|
+
|
|
32
|
+
for field, column in [
|
|
33
|
+
(age, 'd.age'),
|
|
34
|
+
(var, 'd.var'),
|
|
35
|
+
(author, 'a.name'),
|
|
36
|
+
(line, 'd.trace_id')
|
|
37
|
+
]:
|
|
38
|
+
if field is not None:
|
|
39
|
+
if isinstance(field, list):
|
|
40
|
+
placeholders = ','.join(['?'] * len(field))
|
|
41
|
+
conditions.append(f'{column} IN ({placeholders})')
|
|
42
|
+
params.extend(field)
|
|
43
|
+
else:
|
|
44
|
+
conditions.append(f'{column} = ?')
|
|
45
|
+
params.append(field)
|
|
46
|
+
|
|
47
|
+
if conditions:
|
|
48
|
+
query += ' WHERE ' + ' AND '.join(conditions)
|
|
49
|
+
|
|
50
|
+
query += 'ORDER BY CAST(d.age AS INTEGER) ASC'
|
|
51
|
+
return query, params
|
|
52
|
+
|
|
53
|
+
def _get_file_metadata(self, file_path) -> Dict:
|
|
54
|
+
"""
|
|
55
|
+
Helper method to build the SQL query and parameters for filtering.
|
|
56
|
+
Returns the query string and parameters list.
|
|
57
|
+
"""
|
|
58
|
+
select_clause = 'a.name, d.age, d.var, d.trace_id, d.file_path'
|
|
59
|
+
query = f'''
|
|
60
|
+
SELECT {select_clause}
|
|
61
|
+
FROM datasets d
|
|
62
|
+
JOIN authors a ON d.author = a.id
|
|
63
|
+
'''
|
|
64
|
+
conditions = []
|
|
65
|
+
params = []
|
|
66
|
+
|
|
67
|
+
if file_path is not None:
|
|
68
|
+
if isinstance(file_path, list):
|
|
69
|
+
placeholders = ','.join(['?'] * len(file_path))
|
|
70
|
+
conditions.append(f'd.file_path IN ({placeholders})')
|
|
71
|
+
params.extend(file_path)
|
|
72
|
+
else:
|
|
73
|
+
conditions.append(f'd.file_path = ?')
|
|
74
|
+
params.append(file_path)
|
|
75
|
+
|
|
76
|
+
if conditions:
|
|
77
|
+
query += ' WHERE ' + ' AND '.join(conditions)
|
|
78
|
+
|
|
79
|
+
conn = sqlite3.connect(self.file_db_path)
|
|
80
|
+
cursor = conn.cursor()
|
|
81
|
+
cursor.execute(query, params)
|
|
82
|
+
results = cursor.fetchall()
|
|
83
|
+
conn.close()
|
|
84
|
+
|
|
85
|
+
metadata = {
|
|
86
|
+
'author': results[0][0],
|
|
87
|
+
'age': results[0][1],
|
|
88
|
+
'var': results[0][2],
|
|
89
|
+
'trace_id': results[0][3],
|
|
90
|
+
'file_path': results[0][4],
|
|
91
|
+
'database_path': self.db_dir,
|
|
92
|
+
'file_db': self.file_db,
|
|
93
|
+
}
|
|
94
|
+
return metadata
|
|
95
|
+
|
|
96
|
+
def query(self, age: Union[None, str, List[str]]=None, var: Union[None, str, List[str]]=None,author: Union[None, str, List[str]]=None, trace_id: Union[None, str, List[str]]=None) -> Dict:
|
|
97
|
+
select_clause = 'a.name, a.citation, a.doi, d.age, d.var, d.trace_id'
|
|
98
|
+
query, params = self._build_query_and_params(age, var, author, trace_id, select_clause)
|
|
99
|
+
|
|
100
|
+
conn = sqlite3.connect(self.file_db_path)
|
|
101
|
+
cursor = conn.cursor()
|
|
102
|
+
cursor.execute(query, params)
|
|
103
|
+
results = cursor.fetchall()
|
|
104
|
+
conn.close()
|
|
105
|
+
|
|
106
|
+
metadata = {
|
|
107
|
+
'author': [],
|
|
108
|
+
'age': [],
|
|
109
|
+
'var': [],
|
|
110
|
+
'reference': [],
|
|
111
|
+
'doi': [],
|
|
112
|
+
'trace_id': [],
|
|
113
|
+
'_query_params': {'age': age, 'var': var, 'author': author, 'trace_id': trace_id},
|
|
114
|
+
'database_path': self.db_dir,
|
|
115
|
+
'file_db': self.file_db,
|
|
116
|
+
}
|
|
117
|
+
ages_list = []
|
|
118
|
+
vars_list = []
|
|
119
|
+
for author_name, citations, doi, ages, vars, trace_id in results:
|
|
120
|
+
metadata['author'].append(author_name)
|
|
121
|
+
metadata['reference'].append(citations)
|
|
122
|
+
metadata['doi'].append(doi)
|
|
123
|
+
metadata['trace_id'].append(trace_id)
|
|
124
|
+
# Check if the age is numeric
|
|
125
|
+
if ages is not None and ages.isdigit():
|
|
126
|
+
ages_list.append(int(ages))
|
|
127
|
+
if vars is not None:
|
|
128
|
+
vars_list.append(vars)
|
|
129
|
+
metadata['age'] = sorted({str(age) for age in set(ages_list)}, key=int)
|
|
130
|
+
metadata['var'] = sorted(set(vars_list))
|
|
131
|
+
metadata['author'] = list(dict.fromkeys(metadata['author']))
|
|
132
|
+
metadata['reference'] = list(dict.fromkeys(metadata['reference']))
|
|
133
|
+
metadata['doi'] = list(dict.fromkeys(metadata['doi']))
|
|
134
|
+
metadata['trace_id'] = list(set(metadata['trace_id']))
|
|
135
|
+
|
|
136
|
+
self.md = metadata
|
|
137
|
+
return metadata
|
|
138
|
+
|
|
139
|
+
def _get_file_paths_from_metadata(self, metadata) -> List:
|
|
140
|
+
query_params = metadata.get('_query_params', {})
|
|
141
|
+
age = query_params.get('age')
|
|
142
|
+
var = query_params.get('var')
|
|
143
|
+
author = query_params.get('author')
|
|
144
|
+
line = query_params.get('trace_id')
|
|
145
|
+
|
|
146
|
+
select_clause = 'd.file_path'
|
|
147
|
+
query, params = self._build_query_and_params(age, var, author, line, select_clause)
|
|
148
|
+
|
|
149
|
+
conn = sqlite3.connect(self.file_db_path)
|
|
150
|
+
cursor = conn.cursor()
|
|
151
|
+
cursor.execute(query, params)
|
|
152
|
+
file_paths = [row[0] for row in cursor.fetchall()]
|
|
153
|
+
conn.close()
|
|
154
|
+
|
|
155
|
+
return file_paths
|
|
156
|
+
|
|
157
|
+
def data_generator(self, metadata: Union[None, Dict] = None, data_dir: Union[None, str] = None, downscale_factor: Union[None, int] = None, downsample_distance: Union[None, float, int] = None):
|
|
158
|
+
"""
|
|
159
|
+
Generates DataFrames and their associated author names from the database based on the provided metadata.
|
|
160
|
+
|
|
161
|
+
This method queries the database using the filter parameters stored in the metadata,
|
|
162
|
+
retrieves the file paths and author names, and yields each DataFrame along with its author.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
metadata: the results from the query()
|
|
166
|
+
"""
|
|
167
|
+
if metadata:
|
|
168
|
+
md = metadata
|
|
169
|
+
elif self.md:
|
|
170
|
+
md = self.md
|
|
171
|
+
else:
|
|
172
|
+
print('Please provide metadata of the files you want to generate the data from. Exiting ...')
|
|
173
|
+
sys.exit()
|
|
174
|
+
|
|
175
|
+
query_params = md.get('_query_params', {})
|
|
176
|
+
age = query_params.get('age')
|
|
177
|
+
var = query_params.get('var')
|
|
178
|
+
author = query_params.get('author')
|
|
179
|
+
trace_id = query_params.get('trace_id')
|
|
180
|
+
|
|
181
|
+
select_clause = 'DISTINCT d.file_path, d.age'
|
|
182
|
+
query, params = self._build_query_and_params(age, var, author, trace_id, select_clause)
|
|
183
|
+
|
|
184
|
+
conn = sqlite3.connect(self.file_db_path)
|
|
185
|
+
cursor = conn.cursor()
|
|
186
|
+
cursor.execute(query, params)
|
|
187
|
+
results = cursor.fetchall()
|
|
188
|
+
conn.close()
|
|
189
|
+
|
|
190
|
+
if data_dir:
|
|
191
|
+
data_dir = data_dir
|
|
192
|
+
elif self.db_dir:
|
|
193
|
+
data_dir = self.db_dir
|
|
194
|
+
else:
|
|
195
|
+
print('No data dir provided, do not know where to look for data, exiting ...')
|
|
196
|
+
sys.exit()
|
|
197
|
+
|
|
198
|
+
for file_path, age in results:
|
|
199
|
+
df = pd.read_pickle(os.path.join(data_dir, file_path))
|
|
200
|
+
if downscale_factor:
|
|
201
|
+
df = df[::downscale_factor]
|
|
202
|
+
if downsample_distance:
|
|
203
|
+
df['bin'] = np.floor(df['distance'] / downsample_distance) * downsample_distance
|
|
204
|
+
df = df.groupby('bin').mean().reset_index()
|
|
205
|
+
df.drop(columns=['bin'], inplace=True)
|
|
206
|
+
metadata = self._get_file_metadata(file_path)
|
|
207
|
+
yield df, metadata
|
|
208
|
+
|
|
209
|
+
def custom_cmap(self):
|
|
210
|
+
|
|
211
|
+
flare = plt.colormaps['magma_r'] # Sequential (yellow/orange/red)
|
|
212
|
+
crest = plt.colormaps['mako'] # Diverging (blue/white/red)
|
|
213
|
+
|
|
214
|
+
flare_colors = flare(np.linspace(0, 0.8, 256)) # Shape: (256, 4) RGBA
|
|
215
|
+
crest_colors = crest(np.linspace(0.2, 1, 256)) # Shape: (256, 4) RGBA
|
|
216
|
+
combined_colors = np.vstack((flare_colors, crest_colors))
|
|
217
|
+
custom_cmap = LinearSegmentedColormap.from_list(
|
|
218
|
+
'flare_crest', # Name your colormap
|
|
219
|
+
combined_colors,
|
|
220
|
+
N=512 # Double the resolution (256 + 256)
|
|
221
|
+
)
|
|
222
|
+
return custom_cmap
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def plotXY(self,
|
|
226
|
+
metadata: Union[None, Dict] = None,
|
|
227
|
+
downscale_factor: Union[None, int] = None,
|
|
228
|
+
title: str = '',
|
|
229
|
+
xlim: tuple = (None, None),
|
|
230
|
+
ylim: tuple = (None, None),
|
|
231
|
+
scale_factor: float = 1.0,
|
|
232
|
+
latex:bool = False,
|
|
233
|
+
save: Union[str, None] = None) -> None:
|
|
234
|
+
|
|
235
|
+
if metadata:
|
|
236
|
+
metadata = metadata
|
|
237
|
+
elif self.md:
|
|
238
|
+
metadata = self.md
|
|
239
|
+
else:
|
|
240
|
+
print('Please provide metadata of the files you want to generate the data from. Exiting ...')
|
|
241
|
+
sys.exit()
|
|
242
|
+
|
|
243
|
+
cmaps = self.custom_cmap()
|
|
244
|
+
authors = list(metadata['author'])
|
|
245
|
+
color_indices = np.linspace(0.1, 0.9, len(authors))
|
|
246
|
+
colors = {author: cmaps(i) for i, author in zip(color_indices, authors)}
|
|
247
|
+
|
|
248
|
+
if latex:
|
|
249
|
+
from matplotlib import rc
|
|
250
|
+
rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
|
|
251
|
+
rc('text', usetex=True)
|
|
252
|
+
|
|
253
|
+
plt.figure()
|
|
254
|
+
plt.plot(self.gl.x/1000, self.gl.y/1000, linewidth=1, color='k')
|
|
255
|
+
for df, md in self.data_generator(metadata, downscale_factor=downscale_factor):
|
|
256
|
+
author = md['author']
|
|
257
|
+
plt.plot(df.x/1000, df.y/1000, linewidth=0.8, color=colors[author])
|
|
258
|
+
|
|
259
|
+
ax = plt.gca()
|
|
260
|
+
x0, x1 = ax.get_xlim() if xlim == (None, None) else xlim
|
|
261
|
+
y0, y1 = ax.get_ylim() if ylim == (None, None) else ylim
|
|
262
|
+
|
|
263
|
+
for author in authors:
|
|
264
|
+
citation = self.query(author=author)['reference']
|
|
265
|
+
plt.plot([], [], color=colors[author], label=citation)
|
|
266
|
+
plt.legend(loc='lower left')
|
|
267
|
+
|
|
268
|
+
x_extent = x1 - x0
|
|
269
|
+
y_extent = y1 - y0
|
|
270
|
+
|
|
271
|
+
aspect_ratio = y_extent / x_extent
|
|
272
|
+
|
|
273
|
+
# Set the figure size based on the aspect ratio and a scale factor
|
|
274
|
+
plt.gcf().set_size_inches(8 * scale_factor, 8 * aspect_ratio * scale_factor)
|
|
275
|
+
ax.set_xlabel('x [km]')
|
|
276
|
+
ax.set_ylabel('y [km]')
|
|
277
|
+
ax.set_xlim(*xlim)
|
|
278
|
+
ax.set_ylim(*ylim)
|
|
279
|
+
# self.scale_axes(ax, factor=1000)
|
|
280
|
+
plt.title(title, fontsize=30*scale_factor)
|
|
281
|
+
plt.tight_layout()
|
|
282
|
+
if save:
|
|
283
|
+
plt.savefig(save, dpi=200)
|
|
284
|
+
plt.close()
|
|
285
|
+
print('Figure saved as', save)
|
|
286
|
+
else:
|
|
287
|
+
plt.show()
|
|
288
|
+
plt.close()
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .index_database import *
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import glob
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import sqlite3
|
|
5
|
+
|
|
6
|
+
class IndexDatabase:
|
|
7
|
+
def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db', index: str = 'database_index.csv'):
|
|
8
|
+
self.db_dir = database_dir
|
|
9
|
+
self.file_db = os.path.join(self.db_dir, file_db)
|
|
10
|
+
self.file_index = index
|
|
11
|
+
self.index = pd.read_csv(f'{self.db_dir}/{self.file_index}', header=0)
|
|
12
|
+
|
|
13
|
+
def get_dict_ages(self, tab_file) -> dict:
|
|
14
|
+
ages = pd.read_csv(tab_file, header=None, sep='\t', names=['file', 'age'])
|
|
15
|
+
return dict(zip(ages['file'], ages['age']))
|
|
16
|
+
|
|
17
|
+
def get_dict_vars(self, tab_file) -> dict:
|
|
18
|
+
ages = pd.read_csv(tab_file, header=None, sep='\t', names=['file', 'var'])
|
|
19
|
+
return dict(zip(ages['file'], ages['var']))
|
|
20
|
+
|
|
21
|
+
def index_database(self):
|
|
22
|
+
Authors_ages = {}
|
|
23
|
+
for _, row in self.index.iterrows():
|
|
24
|
+
ages = self.get_dict_ages(f"{self.db_dir}/{row.directory}/IRH_ages.tab")
|
|
25
|
+
Authors_ages.update({f"{row.directory}": ages})
|
|
26
|
+
|
|
27
|
+
Extra_vars = {}
|
|
28
|
+
for _, row in self.index.iterrows():
|
|
29
|
+
vars = self.get_dict_ages(f"{self.db_dir}/{row.directory}/vars.tab")
|
|
30
|
+
Extra_vars.update({f"{row.directory}": vars})
|
|
31
|
+
|
|
32
|
+
if os.path.exists(self.file_db):
|
|
33
|
+
os.remove(self.file_db)
|
|
34
|
+
|
|
35
|
+
conn = sqlite3.connect(self.file_db)
|
|
36
|
+
cursor = conn.cursor()
|
|
37
|
+
# Create a table for original reference to datasets
|
|
38
|
+
cursor.execute('''
|
|
39
|
+
CREATE TABLE IF NOT EXISTS authors (
|
|
40
|
+
id INTEGER PRIMARY KEY,
|
|
41
|
+
name TEXT UNIQUE,
|
|
42
|
+
citation TEXT,
|
|
43
|
+
doi TEXT
|
|
44
|
+
)
|
|
45
|
+
''')
|
|
46
|
+
|
|
47
|
+
for _, row in self.index.iterrows():
|
|
48
|
+
try:
|
|
49
|
+
cursor.execute(
|
|
50
|
+
'INSERT INTO authors (name, citation, doi) VALUES (?, ?, ?)',
|
|
51
|
+
(row.directory, row.citation, row.doi)
|
|
52
|
+
)
|
|
53
|
+
except sqlite3.IntegrityError:
|
|
54
|
+
# Author already exists, skip
|
|
55
|
+
continue
|
|
56
|
+
|
|
57
|
+
# Create a table to store the metadata for each dataset
|
|
58
|
+
cursor.execute('''
|
|
59
|
+
CREATE TABLE IF NOT EXISTS datasets (
|
|
60
|
+
id INTEGER PRIMARY KEY,
|
|
61
|
+
file_path TEXT,
|
|
62
|
+
author TEXT,
|
|
63
|
+
age TEXT,
|
|
64
|
+
var TEXT,
|
|
65
|
+
trace_id TEXT,
|
|
66
|
+
FOREIGN KEY (author) REFERENCES authors (id)
|
|
67
|
+
)
|
|
68
|
+
''')
|
|
69
|
+
|
|
70
|
+
# Insert the metadata for each dataset into the table
|
|
71
|
+
for file in glob.glob(f'{self.db_dir}/**/**/*.pkl', recursive=True):
|
|
72
|
+
dir_name, file_name = os.path.split(file)
|
|
73
|
+
pkl_dir, trace_id = os.path.split(dir_name)
|
|
74
|
+
author_dir, _ = os.path.split(pkl_dir)
|
|
75
|
+
|
|
76
|
+
author = os.path.basename(author_dir)
|
|
77
|
+
file_name_, ext = os.path.splitext(file_name)
|
|
78
|
+
relative_file_path = f'{author}/pkl/{trace_id}/{file_name}'
|
|
79
|
+
print(relative_file_path)
|
|
80
|
+
|
|
81
|
+
# Get the author's ID from the authors table
|
|
82
|
+
cursor.execute('SELECT id FROM authors WHERE name = ?', (author,))
|
|
83
|
+
author_id = cursor.fetchone()[0]
|
|
84
|
+
|
|
85
|
+
if file_name_ in Authors_ages[author]:
|
|
86
|
+
age = Authors_ages[author][file_name_]
|
|
87
|
+
else:
|
|
88
|
+
age = None
|
|
89
|
+
|
|
90
|
+
if file_name_ in Extra_vars[author]:
|
|
91
|
+
var = Extra_vars[author][file_name_]
|
|
92
|
+
else:
|
|
93
|
+
var = None
|
|
94
|
+
|
|
95
|
+
cursor.execute('''
|
|
96
|
+
INSERT INTO datasets (file_path, author, age, var, trace_id)
|
|
97
|
+
VALUES (?, ?, ?, ?, ?)
|
|
98
|
+
''', (relative_file_path, author_id, age, var, trace_id))
|
|
99
|
+
|
|
100
|
+
conn.commit()
|
|
101
|
+
conn.close()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: anta_database
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Summary: SQLite database for the AntArchitecture Community Data
|
|
5
|
+
Home-page: https://github.com/antoinehermant/anta_database
|
|
6
|
+
Author: Antoine Hermant
|
|
7
|
+
Author-email: antoine.hermant@etik.com
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pandas
|
|
11
|
+
Requires-Dist: pyproj
|
|
12
|
+
Requires-Dist: matplotlib
|
|
13
|
+
Requires-Dist: seaborn
|
|
14
|
+
Dynamic: author
|
|
15
|
+
Dynamic: author-email
|
|
16
|
+
Dynamic: home-page
|
|
17
|
+
Dynamic: license
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
Dynamic: requires-dist
|
|
20
|
+
Dynamic: summary
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
anta_database/__init__.py,sha256=za_EMw8x6VUrEDpgx6Hbfc9ct7MP_MFBeIrU8ObqI78,73
|
|
2
|
+
anta_database/anta_database.py,sha256=mvUwa-U1YtfRp4M9P86mQWc8dz7TOc9iL1_FUyUSemM,70
|
|
3
|
+
anta_database/compiling/__init__.py,sha256=0LSKE1W-rYk4yt8GVjCvT7bDUwlg9adnV027iMYTMVw,23
|
|
4
|
+
anta_database/compiling/compile.py,sha256=LCNxvZhOcX-nt_eYWPpbzc8V_eVaVFO59F1dYcRkPOY,8856
|
|
5
|
+
anta_database/database/__init__.py,sha256=ML02DLn1T9HlTMe_4jjzz6WcocOvU5a4XtJRDMxZmSI,24
|
|
6
|
+
anta_database/database/database.py,sha256=N7WQm38BfniwWWBN4F9L_Ba3E1NwQ-6FfF0RNQTCoUA,10959
|
|
7
|
+
anta_database/database/plotting/GL.pkl,sha256=aBfkVyZX8Tbufy7jgNzWVtjmWRSA6xr5O3qjy3RKq_E,959263
|
|
8
|
+
anta_database/database/plotting/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
9
|
+
anta_database/indexing/__init__.py,sha256=P-lFx-wi8VkwLjZcQ3YpSw93Ilx9pOc20huuXI9s51Q,30
|
|
10
|
+
anta_database/indexing/index_database.py,sha256=bq8qeuq9DVOVVdiuXDbI-yg-LCO3_fN-Hzy3l2a6Cyc,3711
|
|
11
|
+
anta_database-0.1.dist-info/licenses/LICENSE,sha256=5vkb0ozrwOYpRAelL2l6d2GLL7-sUgy5Zhgf_oDp5jA,1059
|
|
12
|
+
anta_database-0.1.dist-info/METADATA,sha256=pi7-mi6VdMyM8Z1NbWhLgT7chtIURbhIWHzMzu2WmBs,505
|
|
13
|
+
anta_database-0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
14
|
+
anta_database-0.1.dist-info/top_level.txt,sha256=hq6yvmnnXEKKrxD7Nq_e4pM0KabQBXHhK5MGnm_mWsE,14
|
|
15
|
+
anta_database-0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Copyright (c) 2025 Antoine Hermant
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
anta_database
|