anta-database 0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ from .database import *
2
+ from .indexing import *
3
+ from .compiling import *
@@ -0,0 +1,3 @@
1
+ from database import *
2
+ from indexing import *
3
+ from compiling import *
@@ -0,0 +1 @@
1
+ from .compile import *
@@ -0,0 +1,192 @@
1
+ import os
2
+ import sys
3
+ import pandas as pd
4
+ import numpy as np
5
+ import glob
6
+ from pyproj import Transformer
7
+ from typing import Union
8
+ from multiprocessing import Pool, cpu_count
9
+
10
+ class CompileDatabase:
11
+ def __init__(self, dir_list: Union[str, list[str]], file_type: str = 'layer', wave_speed: Union[None, float] = None, firn_correction: Union[None, float] = None) -> None:
12
+ self.dir_list = dir_list
13
+ self.wave_speed = wave_speed
14
+ self.firn_correction = firn_correction
15
+ self.file_type = file_type
16
+
17
+ def get_dict_ages(self, tab_file) -> dict:
18
+ ages = pd.read_csv(tab_file, header=None, sep='\t', names=['file', 'age'])
19
+ return dict(zip(ages['file'], ages['age']))
20
+
21
+ def compile(self, cpus: int = cpu_count()-1) -> None:
22
+ if not isinstance(self.dir_list, list):
23
+ self.dir_list = [self.dir_list]
24
+
25
+ num_tasks = len(self.dir_list)
26
+ num_workers = min(num_tasks, cpus)
27
+
28
+ print('\n',
29
+ 'Will start compiling', len(self.dir_list), 'directories\n'
30
+ '\n ', num_workers, 'worker(s) allocated out of', cpu_count(), 'available cpus\n')
31
+
32
+ if num_workers > 1:
33
+ with Pool(num_workers) as pool:
34
+ pool.map(self._compile, self.dir_list)
35
+ else:
36
+ for dir_path in self.dir_list:
37
+ self._compile(dir_path=dir_path)
38
+
39
+ def _compile(self, dir_path: str) -> None:
40
+ raw_files = glob.glob(f'{dir_path}/raw/*.*')
41
+
42
+ _, ext = os.path.splitext(raw_files[0])
43
+
44
+ ages = self.get_dict_ages(f'{dir_path}/IRH_ages.tab')
45
+ original_new_columns = pd.read_csv(f'{dir_path}/original_new_column_names.csv')
46
+
47
+ if ext == '.tab':
48
+ sep='\t'
49
+ elif ext == '.csv':
50
+ sep=','
51
+ else:
52
+ print('File type not supported, exiting ...')
53
+ sys.exit()
54
+
55
+ for i, file in enumerate(raw_files):
56
+ print('Processing', file, f'({i}/{len(raw_files)})')
57
+ ds = pd.read_csv(file, comment="#", header=0, sep=sep)
58
+ _, file_name = os.path.split(file)
59
+ file_name_, ext = os.path.splitext(file_name)
60
+
61
+ ds = ds[ds.columns.intersection(original_new_columns.columns)]
62
+ ds.columns = original_new_columns[ds.columns].iloc[0].values # renaming the columns
63
+
64
+ if 'IceThk' in ds.columns and 'SurfElev' in ds.columns and not 'BedElev' in ds.columns:
65
+ ds['BedElev'] = ds['SurfElev'] - ds['IceThk']
66
+ if 'IceThk' in ds.columns and 'BedElev' in ds.columns and not 'SurfElev' in ds.columns:
67
+ ds['SurfElev'] = ds['BedElev'] + ds['IceThk']
68
+ if 'SurfElev' in ds.columns and 'BedElev' in ds.columns and not 'IceThk' in ds.columns:
69
+ ds['IceThk'] = ds['SurfElev'] - ds['BedElev']
70
+
71
+ if self.wave_speed:
72
+ for var in ['IceThk', 'BedElev']:
73
+ if var in ds.columns:
74
+ ds[var] *= self.wave_speed
75
+ if self.firn_correction:
76
+ for var in ['IceThk', 'BedElev']:
77
+ if var in ds.columns:
78
+ ds[var] += self.firn_correction
79
+
80
+ if 'x' not in ds.columns and 'y' not in ds.columns:
81
+ if 'lon' in ds.columns and 'lat' in ds.columns:
82
+ transformer = Transformer.from_proj(
83
+ "EPSG:4326", # source: WGS84 (lon/lat)
84
+ "+proj=stere +lon_0=0 +lat_0=-90 +lat_ts=-71 +datum=WGS84 +units=m +no_defs", # target: polar
85
+ always_xy=True
86
+ )
87
+ ds['x'], ds['y'] = transformer.transform(ds['lon'].values, ds['lat'].values)
88
+ elif 'lon' not in ds.columns and 'lat' not in ds.columns:
89
+ if 'x' in ds.columns and 'y' in ds.columns:
90
+ inverse_transformer = Transformer.from_proj(
91
+ "+proj=stere +lon_0=0 +lat_0=-90 +lat_ts=-71 +datum=WGS84 +units=m +no_defs", # source: polar
92
+ "EPSG:4326", # target: WGS84 (lon/lat)
93
+ always_xy=True
94
+ )
95
+ ds['lon'], ds['lat'] = inverse_transformer.transform(ds['x'].values, ds['y'].values)
96
+ elif 'lon' in ds.columns and 'lat' in ds.columns and 'x' in ds.columns and 'y' in ds.columns:
97
+ pass
98
+ else:
99
+ print('No coordinates found in the dataset, exiting ....')
100
+ sys.exit()
101
+
102
+ if self.file_type == 'layer':
103
+ age = str(ages[file_name_])
104
+ ds = ds.rename(columns={'IRHdepth': age})
105
+ if self.wave_speed:
106
+ ds[age] *= self.wave_speed
107
+ if self.firn_correction:
108
+ ds[age] += self.firn_correction
109
+
110
+ ds['Trace_ID'] = ds['Trace_ID'].astype(str)
111
+ ds['Trace_ID'] = ds['Trace_ID'].str.replace(r'/\s+', '_') # Replace slashes with underscores, otherwise the paths can get messy
112
+ ds['Trace_ID'] = ds['Trace_ID'].str.replace('/', '_')
113
+
114
+ ds.set_index('Trace_ID', inplace=True)
115
+
116
+ for trace_id in np.unique(ds.index):
117
+ ds_trace = ds.loc[trace_id].copy()
118
+ if 'distance' not in ds_trace.columns:
119
+ x = ds_trace[['x', 'y']]
120
+ distances = np.sqrt(np.sum(np.diff(x, axis=0)**2, axis=1))
121
+ cumulative_distance = np.concatenate([[0], np.cumsum(distances)])
122
+ ds_trace['distance'] = cumulative_distance
123
+
124
+ ds_trace_file = f'{dir_path}/pkl/{trace_id}/{file_name_}.pkl'
125
+ os.makedirs(f'{dir_path}/pkl/{trace_id}' , exist_ok=True)
126
+ ds_trace.to_pickle(ds_trace_file)
127
+ print(ds_trace_file)
128
+
129
+ for var in ['IceThk', 'BedElev', 'SurfElev']:
130
+ if var in ds.columns:
131
+ ds_var = ds_trace[['x', 'y', 'distance', var]]
132
+ ds_var_file = f'{dir_path}/pkl/{trace_id}/{var}.pkl'
133
+ if os.path.exists(ds_var_file):
134
+ var_data = pd.read_pickle(ds_var_file)
135
+ merged_data = pd.concat([var_data, ds_var]).drop_duplicates(subset=['x', 'y'])
136
+
137
+ merged_data.to_pickle(ds_var_file)
138
+ print(ds_var_file)
139
+ else:
140
+ ds_var.to_pickle(ds_var_file)
141
+ print(ds_var_file)
142
+
143
+ elif self.file_type == 'trace':
144
+ if 'distance' not in ds.columns:
145
+ x = ds[['x', 'y']]
146
+ distances = np.sqrt(np.sum(np.diff(x, axis=0)**2, axis=1))
147
+ cumulative_distance = np.concatenate([[0], np.cumsum(distances)])
148
+ ds['distance'] = cumulative_distance
149
+
150
+ trace_id = file_name_
151
+ os.makedirs(f'{dir_path}/pkl/{trace_id}' , exist_ok=True)
152
+
153
+ for var in ['IceThk', 'BedElev', 'SurfElev']:
154
+ if var in ds.columns:
155
+ ds_var = ds[['x', 'y', 'distance', var]]
156
+ ds_var_file = f'{dir_path}/pkl/{trace_id}/{var}.pkl'
157
+ if os.path.exists(ds_var_file):
158
+ var_data = pd.read_pickle(ds_var_file)
159
+ merged_data = pd.concat([var_data, ds_var]).drop_duplicates(subset=['x', 'y'])
160
+
161
+ merged_data.to_pickle(ds_var_file)
162
+ print(ds_var_file)
163
+ else:
164
+ ds_var.to_pickle(ds_var_file)
165
+ print(ds_var_file)
166
+
167
+ ages = {key: ages[key] for key in ds.columns if key in ages}
168
+
169
+ for IRH in ages:
170
+ age = str(ages.get(IRH))
171
+ ds_IRH = ds[IRH]
172
+ ds_IRH = pd.DataFrame({
173
+ 'lon': ds['lon'],
174
+ 'lat': ds['lat'],
175
+ 'x': ds['x'],
176
+ 'y': ds['y'],
177
+ 'distance': ds['distance'],
178
+ age: ds_IRH,
179
+ })
180
+ if self.wave_speed:
181
+ ds_IRH[age] *= self.wave_speed
182
+ if self.firn_correction:
183
+ ds_IRH += self.firn_correction
184
+
185
+ for var in ['IceThk', 'BedElev', 'SurfElev']:
186
+ if var in ds.columns:
187
+ ds_IRH[var] = ds[var]
188
+
189
+ ds_trace_file = f'{dir_path}/pkl/{trace_id}/{IRH}.pkl'
190
+
191
+ ds_IRH.to_pickle(ds_trace_file)
192
+ print(ds_trace_file)
@@ -0,0 +1 @@
1
+ from .database import *
@@ -0,0 +1,288 @@
1
+ import os, sys
2
+ import sqlite3
3
+ import pandas as pd
4
+ import numpy as np
5
+ import matplotlib.pyplot as plt
6
+ from matplotlib.colors import LinearSegmentedColormap
7
+ import seaborn as sns
8
+ from typing import Union, List, Dict, Tuple
9
+
10
+ class Database:
11
+ def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db'):
12
+ module_dir = os.path.dirname(os.path.abspath(__file__))
13
+ self.gl = pd.read_pickle(os.path.join(module_dir, 'plotting', 'GL.pkl'))
14
+ self.db_dir = database_dir
15
+ self.file_db = file_db
16
+ self.file_db_path = os.path.join(self.db_dir, file_db)
17
+ self.md = None
18
+
19
+ def _build_query_and_params(self, age: Union[None, str, List[str]]=None, var: Union[None, str, List[str]]=None, author: Union[None, str, List[str]]=None, line: Union[None, str, List[str]]=None, select_clause='') -> Tuple[str, List[Union[str, int]]]:
20
+ """
21
+ Helper method to build the SQL query and parameters for filtering.
22
+ Returns the query string and parameters list.
23
+ """
24
+ query = f'''
25
+ SELECT {select_clause}
26
+ FROM datasets d
27
+ JOIN authors a ON d.author = a.id
28
+ '''
29
+ conditions = []
30
+ params = []
31
+
32
+ for field, column in [
33
+ (age, 'd.age'),
34
+ (var, 'd.var'),
35
+ (author, 'a.name'),
36
+ (line, 'd.trace_id')
37
+ ]:
38
+ if field is not None:
39
+ if isinstance(field, list):
40
+ placeholders = ','.join(['?'] * len(field))
41
+ conditions.append(f'{column} IN ({placeholders})')
42
+ params.extend(field)
43
+ else:
44
+ conditions.append(f'{column} = ?')
45
+ params.append(field)
46
+
47
+ if conditions:
48
+ query += ' WHERE ' + ' AND '.join(conditions)
49
+
50
+ query += 'ORDER BY CAST(d.age AS INTEGER) ASC'
51
+ return query, params
52
+
53
+ def _get_file_metadata(self, file_path) -> Dict:
54
+ """
55
+ Helper method to build the SQL query and parameters for filtering.
56
+ Returns the query string and parameters list.
57
+ """
58
+ select_clause = 'a.name, d.age, d.var, d.trace_id, d.file_path'
59
+ query = f'''
60
+ SELECT {select_clause}
61
+ FROM datasets d
62
+ JOIN authors a ON d.author = a.id
63
+ '''
64
+ conditions = []
65
+ params = []
66
+
67
+ if file_path is not None:
68
+ if isinstance(file_path, list):
69
+ placeholders = ','.join(['?'] * len(file_path))
70
+ conditions.append(f'd.file_path IN ({placeholders})')
71
+ params.extend(file_path)
72
+ else:
73
+ conditions.append(f'd.file_path = ?')
74
+ params.append(file_path)
75
+
76
+ if conditions:
77
+ query += ' WHERE ' + ' AND '.join(conditions)
78
+
79
+ conn = sqlite3.connect(self.file_db_path)
80
+ cursor = conn.cursor()
81
+ cursor.execute(query, params)
82
+ results = cursor.fetchall()
83
+ conn.close()
84
+
85
+ metadata = {
86
+ 'author': results[0][0],
87
+ 'age': results[0][1],
88
+ 'var': results[0][2],
89
+ 'trace_id': results[0][3],
90
+ 'file_path': results[0][4],
91
+ 'database_path': self.db_dir,
92
+ 'file_db': self.file_db,
93
+ }
94
+ return metadata
95
+
96
+ def query(self, age: Union[None, str, List[str]]=None, var: Union[None, str, List[str]]=None,author: Union[None, str, List[str]]=None, trace_id: Union[None, str, List[str]]=None) -> Dict:
97
+ select_clause = 'a.name, a.citation, a.doi, d.age, d.var, d.trace_id'
98
+ query, params = self._build_query_and_params(age, var, author, trace_id, select_clause)
99
+
100
+ conn = sqlite3.connect(self.file_db_path)
101
+ cursor = conn.cursor()
102
+ cursor.execute(query, params)
103
+ results = cursor.fetchall()
104
+ conn.close()
105
+
106
+ metadata = {
107
+ 'author': [],
108
+ 'age': [],
109
+ 'var': [],
110
+ 'reference': [],
111
+ 'doi': [],
112
+ 'trace_id': [],
113
+ '_query_params': {'age': age, 'var': var, 'author': author, 'trace_id': trace_id},
114
+ 'database_path': self.db_dir,
115
+ 'file_db': self.file_db,
116
+ }
117
+ ages_list = []
118
+ vars_list = []
119
+ for author_name, citations, doi, ages, vars, trace_id in results:
120
+ metadata['author'].append(author_name)
121
+ metadata['reference'].append(citations)
122
+ metadata['doi'].append(doi)
123
+ metadata['trace_id'].append(trace_id)
124
+ # Check if the age is numeric
125
+ if ages is not None and ages.isdigit():
126
+ ages_list.append(int(ages))
127
+ if vars is not None:
128
+ vars_list.append(vars)
129
+ metadata['age'] = sorted({str(age) for age in set(ages_list)}, key=int)
130
+ metadata['var'] = sorted(set(vars_list))
131
+ metadata['author'] = list(dict.fromkeys(metadata['author']))
132
+ metadata['reference'] = list(dict.fromkeys(metadata['reference']))
133
+ metadata['doi'] = list(dict.fromkeys(metadata['doi']))
134
+ metadata['trace_id'] = list(set(metadata['trace_id']))
135
+
136
+ self.md = metadata
137
+ return metadata
138
+
139
+ def _get_file_paths_from_metadata(self, metadata) -> List:
140
+ query_params = metadata.get('_query_params', {})
141
+ age = query_params.get('age')
142
+ var = query_params.get('var')
143
+ author = query_params.get('author')
144
+ line = query_params.get('trace_id')
145
+
146
+ select_clause = 'd.file_path'
147
+ query, params = self._build_query_and_params(age, var, author, line, select_clause)
148
+
149
+ conn = sqlite3.connect(self.file_db_path)
150
+ cursor = conn.cursor()
151
+ cursor.execute(query, params)
152
+ file_paths = [row[0] for row in cursor.fetchall()]
153
+ conn.close()
154
+
155
+ return file_paths
156
+
157
+ def data_generator(self, metadata: Union[None, Dict] = None, data_dir: Union[None, str] = None, downscale_factor: Union[None, int] = None, downsample_distance: Union[None, float, int] = None):
158
+ """
159
+ Generates DataFrames and their associated author names from the database based on the provided metadata.
160
+
161
+ This method queries the database using the filter parameters stored in the metadata,
162
+ retrieves the file paths and author names, and yields each DataFrame along with its author.
163
+
164
+ Args:
165
+ metadata: the results from the query()
166
+ """
167
+ if metadata:
168
+ md = metadata
169
+ elif self.md:
170
+ md = self.md
171
+ else:
172
+ print('Please provide metadata of the files you want to generate the data from. Exiting ...')
173
+ sys.exit()
174
+
175
+ query_params = md.get('_query_params', {})
176
+ age = query_params.get('age')
177
+ var = query_params.get('var')
178
+ author = query_params.get('author')
179
+ trace_id = query_params.get('trace_id')
180
+
181
+ select_clause = 'DISTINCT d.file_path, d.age'
182
+ query, params = self._build_query_and_params(age, var, author, trace_id, select_clause)
183
+
184
+ conn = sqlite3.connect(self.file_db_path)
185
+ cursor = conn.cursor()
186
+ cursor.execute(query, params)
187
+ results = cursor.fetchall()
188
+ conn.close()
189
+
190
+ if data_dir:
191
+ data_dir = data_dir
192
+ elif self.db_dir:
193
+ data_dir = self.db_dir
194
+ else:
195
+ print('No data dir provided, do not know where to look for data, exiting ...')
196
+ sys.exit()
197
+
198
+ for file_path, age in results:
199
+ df = pd.read_pickle(os.path.join(data_dir, file_path))
200
+ if downscale_factor:
201
+ df = df[::downscale_factor]
202
+ if downsample_distance:
203
+ df['bin'] = np.floor(df['distance'] / downsample_distance) * downsample_distance
204
+ df = df.groupby('bin').mean().reset_index()
205
+ df.drop(columns=['bin'], inplace=True)
206
+ metadata = self._get_file_metadata(file_path)
207
+ yield df, metadata
208
+
209
+ def custom_cmap(self):
210
+
211
+ flare = plt.colormaps['magma_r'] # Sequential (yellow/orange/red)
212
+ crest = plt.colormaps['mako'] # Diverging (blue/white/red)
213
+
214
+ flare_colors = flare(np.linspace(0, 0.8, 256)) # Shape: (256, 4) RGBA
215
+ crest_colors = crest(np.linspace(0.2, 1, 256)) # Shape: (256, 4) RGBA
216
+ combined_colors = np.vstack((flare_colors, crest_colors))
217
+ custom_cmap = LinearSegmentedColormap.from_list(
218
+ 'flare_crest', # Name your colormap
219
+ combined_colors,
220
+ N=512 # Double the resolution (256 + 256)
221
+ )
222
+ return custom_cmap
223
+
224
+
225
+ def plotXY(self,
226
+ metadata: Union[None, Dict] = None,
227
+ downscale_factor: Union[None, int] = None,
228
+ title: str = '',
229
+ xlim: tuple = (None, None),
230
+ ylim: tuple = (None, None),
231
+ scale_factor: float = 1.0,
232
+ latex:bool = False,
233
+ save: Union[str, None] = None) -> None:
234
+
235
+ if metadata:
236
+ metadata = metadata
237
+ elif self.md:
238
+ metadata = self.md
239
+ else:
240
+ print('Please provide metadata of the files you want to generate the data from. Exiting ...')
241
+ sys.exit()
242
+
243
+ cmaps = self.custom_cmap()
244
+ authors = list(metadata['author'])
245
+ color_indices = np.linspace(0.1, 0.9, len(authors))
246
+ colors = {author: cmaps(i) for i, author in zip(color_indices, authors)}
247
+
248
+ if latex:
249
+ from matplotlib import rc
250
+ rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
251
+ rc('text', usetex=True)
252
+
253
+ plt.figure()
254
+ plt.plot(self.gl.x/1000, self.gl.y/1000, linewidth=1, color='k')
255
+ for df, md in self.data_generator(metadata, downscale_factor=downscale_factor):
256
+ author = md['author']
257
+ plt.plot(df.x/1000, df.y/1000, linewidth=0.8, color=colors[author])
258
+
259
+ ax = plt.gca()
260
+ x0, x1 = ax.get_xlim() if xlim == (None, None) else xlim
261
+ y0, y1 = ax.get_ylim() if ylim == (None, None) else ylim
262
+
263
+ for author in authors:
264
+ citation = self.query(author=author)['reference']
265
+ plt.plot([], [], color=colors[author], label=citation)
266
+ plt.legend(loc='lower left')
267
+
268
+ x_extent = x1 - x0
269
+ y_extent = y1 - y0
270
+
271
+ aspect_ratio = y_extent / x_extent
272
+
273
+ # Set the figure size based on the aspect ratio and a scale factor
274
+ plt.gcf().set_size_inches(8 * scale_factor, 8 * aspect_ratio * scale_factor)
275
+ ax.set_xlabel('x [km]')
276
+ ax.set_ylabel('y [km]')
277
+ ax.set_xlim(*xlim)
278
+ ax.set_ylim(*ylim)
279
+ # self.scale_axes(ax, factor=1000)
280
+ plt.title(title, fontsize=30*scale_factor)
281
+ plt.tight_layout()
282
+ if save:
283
+ plt.savefig(save, dpi=200)
284
+ plt.close()
285
+ print('Figure saved as', save)
286
+ else:
287
+ plt.show()
288
+ plt.close()
Binary file
@@ -0,0 +1 @@
1
+ from .index_database import *
@@ -0,0 +1,101 @@
1
+ import os
2
+ import glob
3
+ import pandas as pd
4
+ import sqlite3
5
+
6
+ class IndexDatabase:
7
+ def __init__(self, database_dir: str, file_db: str = 'AntADatabase.db', index: str = 'database_index.csv'):
8
+ self.db_dir = database_dir
9
+ self.file_db = os.path.join(self.db_dir, file_db)
10
+ self.file_index = index
11
+ self.index = pd.read_csv(f'{self.db_dir}/{self.file_index}', header=0)
12
+
13
+ def get_dict_ages(self, tab_file) -> dict:
14
+ ages = pd.read_csv(tab_file, header=None, sep='\t', names=['file', 'age'])
15
+ return dict(zip(ages['file'], ages['age']))
16
+
17
+ def get_dict_vars(self, tab_file) -> dict:
18
+ ages = pd.read_csv(tab_file, header=None, sep='\t', names=['file', 'var'])
19
+ return dict(zip(ages['file'], ages['var']))
20
+
21
+ def index_database(self):
22
+ Authors_ages = {}
23
+ for _, row in self.index.iterrows():
24
+ ages = self.get_dict_ages(f"{self.db_dir}/{row.directory}/IRH_ages.tab")
25
+ Authors_ages.update({f"{row.directory}": ages})
26
+
27
+ Extra_vars = {}
28
+ for _, row in self.index.iterrows():
29
+ vars = self.get_dict_ages(f"{self.db_dir}/{row.directory}/vars.tab")
30
+ Extra_vars.update({f"{row.directory}": vars})
31
+
32
+ if os.path.exists(self.file_db):
33
+ os.remove(self.file_db)
34
+
35
+ conn = sqlite3.connect(self.file_db)
36
+ cursor = conn.cursor()
37
+ # Create a table for original reference to datasets
38
+ cursor.execute('''
39
+ CREATE TABLE IF NOT EXISTS authors (
40
+ id INTEGER PRIMARY KEY,
41
+ name TEXT UNIQUE,
42
+ citation TEXT,
43
+ doi TEXT
44
+ )
45
+ ''')
46
+
47
+ for _, row in self.index.iterrows():
48
+ try:
49
+ cursor.execute(
50
+ 'INSERT INTO authors (name, citation, doi) VALUES (?, ?, ?)',
51
+ (row.directory, row.citation, row.doi)
52
+ )
53
+ except sqlite3.IntegrityError:
54
+ # Author already exists, skip
55
+ continue
56
+
57
+ # Create a table to store the metadata for each dataset
58
+ cursor.execute('''
59
+ CREATE TABLE IF NOT EXISTS datasets (
60
+ id INTEGER PRIMARY KEY,
61
+ file_path TEXT,
62
+ author TEXT,
63
+ age TEXT,
64
+ var TEXT,
65
+ trace_id TEXT,
66
+ FOREIGN KEY (author) REFERENCES authors (id)
67
+ )
68
+ ''')
69
+
70
+ # Insert the metadata for each dataset into the table
71
+ for file in glob.glob(f'{self.db_dir}/**/**/*.pkl', recursive=True):
72
+ dir_name, file_name = os.path.split(file)
73
+ pkl_dir, trace_id = os.path.split(dir_name)
74
+ author_dir, _ = os.path.split(pkl_dir)
75
+
76
+ author = os.path.basename(author_dir)
77
+ file_name_, ext = os.path.splitext(file_name)
78
+ relative_file_path = f'{author}/pkl/{trace_id}/{file_name}'
79
+ print(relative_file_path)
80
+
81
+ # Get the author's ID from the authors table
82
+ cursor.execute('SELECT id FROM authors WHERE name = ?', (author,))
83
+ author_id = cursor.fetchone()[0]
84
+
85
+ if file_name_ in Authors_ages[author]:
86
+ age = Authors_ages[author][file_name_]
87
+ else:
88
+ age = None
89
+
90
+ if file_name_ in Extra_vars[author]:
91
+ var = Extra_vars[author][file_name_]
92
+ else:
93
+ var = None
94
+
95
+ cursor.execute('''
96
+ INSERT INTO datasets (file_path, author, age, var, trace_id)
97
+ VALUES (?, ?, ?, ?, ?)
98
+ ''', (relative_file_path, author_id, age, var, trace_id))
99
+
100
+ conn.commit()
101
+ conn.close()
@@ -0,0 +1,20 @@
1
+ Metadata-Version: 2.4
2
+ Name: anta_database
3
+ Version: 0.1
4
+ Summary: SQLite database for the AntArchitecture Community Data
5
+ Home-page: https://github.com/antoinehermant/anta_database
6
+ Author: Antoine Hermant
7
+ Author-email: antoine.hermant@etik.com
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Requires-Dist: pandas
11
+ Requires-Dist: pyproj
12
+ Requires-Dist: matplotlib
13
+ Requires-Dist: seaborn
14
+ Dynamic: author
15
+ Dynamic: author-email
16
+ Dynamic: home-page
17
+ Dynamic: license
18
+ Dynamic: license-file
19
+ Dynamic: requires-dist
20
+ Dynamic: summary
@@ -0,0 +1,15 @@
1
+ anta_database/__init__.py,sha256=za_EMw8x6VUrEDpgx6Hbfc9ct7MP_MFBeIrU8ObqI78,73
2
+ anta_database/anta_database.py,sha256=mvUwa-U1YtfRp4M9P86mQWc8dz7TOc9iL1_FUyUSemM,70
3
+ anta_database/compiling/__init__.py,sha256=0LSKE1W-rYk4yt8GVjCvT7bDUwlg9adnV027iMYTMVw,23
4
+ anta_database/compiling/compile.py,sha256=LCNxvZhOcX-nt_eYWPpbzc8V_eVaVFO59F1dYcRkPOY,8856
5
+ anta_database/database/__init__.py,sha256=ML02DLn1T9HlTMe_4jjzz6WcocOvU5a4XtJRDMxZmSI,24
6
+ anta_database/database/database.py,sha256=N7WQm38BfniwWWBN4F9L_Ba3E1NwQ-6FfF0RNQTCoUA,10959
7
+ anta_database/database/plotting/GL.pkl,sha256=aBfkVyZX8Tbufy7jgNzWVtjmWRSA6xr5O3qjy3RKq_E,959263
8
+ anta_database/database/plotting/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
9
+ anta_database/indexing/__init__.py,sha256=P-lFx-wi8VkwLjZcQ3YpSw93Ilx9pOc20huuXI9s51Q,30
10
+ anta_database/indexing/index_database.py,sha256=bq8qeuq9DVOVVdiuXDbI-yg-LCO3_fN-Hzy3l2a6Cyc,3711
11
+ anta_database-0.1.dist-info/licenses/LICENSE,sha256=5vkb0ozrwOYpRAelL2l6d2GLL7-sUgy5Zhgf_oDp5jA,1059
12
+ anta_database-0.1.dist-info/METADATA,sha256=pi7-mi6VdMyM8Z1NbWhLgT7chtIURbhIWHzMzu2WmBs,505
13
+ anta_database-0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
14
+ anta_database-0.1.dist-info/top_level.txt,sha256=hq6yvmnnXEKKrxD7Nq_e4pM0KabQBXHhK5MGnm_mWsE,14
15
+ anta_database-0.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (80.9.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,19 @@
1
+ Copyright (c) 2025 Antoine Hermant
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in all
11
+ copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ anta_database