atriumdb 2.7.0__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. atriumdb/__init__.py +37 -0
  2. atriumdb/adb_functions.py +1688 -0
  3. atriumdb/adb_remote.py +32 -0
  4. atriumdb/atrium_sdk.py +6184 -0
  5. atriumdb/bin/libTSC.dll +0 -0
  6. atriumdb/binary_signal_converter.py +35 -0
  7. atriumdb/block.py +963 -0
  8. atriumdb/block_wrapper.py +297 -0
  9. atriumdb/cli/__init__.py +17 -0
  10. atriumdb/cli/atriumdb_cli.py +462 -0
  11. atriumdb/cli/hello.py +23 -0
  12. atriumdb/cli/sdk.py +59 -0
  13. atriumdb/file_api.py +191 -0
  14. atriumdb/helpers/__init__.py +30 -0
  15. atriumdb/helpers/block_calculations.py +105 -0
  16. atriumdb/helpers/block_constants.py +26 -0
  17. atriumdb/helpers/config.toml +8 -0
  18. atriumdb/helpers/mariadb_env_cred.py +22 -0
  19. atriumdb/helpers/settings.py +22 -0
  20. atriumdb/helpers/sql_helpers.py +23 -0
  21. atriumdb/helpers/sql_tables.py +99 -0
  22. atriumdb/helpers/sql_wait_wrapper.py +38 -0
  23. atriumdb/helpers/type_helpers.py +85 -0
  24. atriumdb/interval_overlap.py +119 -0
  25. atriumdb/intervals/__init__.py +18 -0
  26. atriumdb/intervals/compact.py +24 -0
  27. atriumdb/intervals/difference.py +28 -0
  28. atriumdb/intervals/intersection.py +51 -0
  29. atriumdb/intervals/intervals.py +250 -0
  30. atriumdb/intervals/union.py +65 -0
  31. atriumdb/mem_cache.py +34 -0
  32. atriumdb/pytorch_integrations.py +116 -0
  33. atriumdb/sql_handler/__init__.py +17 -0
  34. atriumdb/sql_handler/maria/__init__.py +17 -0
  35. atriumdb/sql_handler/maria/maria_functions.py +73 -0
  36. atriumdb/sql_handler/maria/maria_handler.py +826 -0
  37. atriumdb/sql_handler/maria/maria_tables.py +355 -0
  38. atriumdb/sql_handler/sql_constants.py +19 -0
  39. atriumdb/sql_handler/sql_handler.py +1245 -0
  40. atriumdb/sql_handler/sql_helper.py +24 -0
  41. atriumdb/sql_handler/sqlite/__init__.py +17 -0
  42. atriumdb/sql_handler/sqlite/sqlite_functions.py +86 -0
  43. atriumdb/sql_handler/sqlite/sqlite_handler.py +774 -0
  44. atriumdb/sql_handler/sqlite/sqlite_tables.py +338 -0
  45. atriumdb/transfer/__init__.py +17 -0
  46. atriumdb/transfer/adb/__init__.py +17 -0
  47. atriumdb/transfer/adb/csv.py +47 -0
  48. atriumdb/transfer/adb/dataset.py +635 -0
  49. atriumdb/transfer/adb/datestring_conversion.py +46 -0
  50. atriumdb/transfer/adb/definition.py +133 -0
  51. atriumdb/transfer/adb/devices.py +45 -0
  52. atriumdb/transfer/adb/labels.py +29 -0
  53. atriumdb/transfer/adb/measures.py +74 -0
  54. atriumdb/transfer/adb/numpy.py +21 -0
  55. atriumdb/transfer/adb/parquet.py +48 -0
  56. atriumdb/transfer/adb/patients.py +398 -0
  57. atriumdb/transfer/adb/tsc.py +44 -0
  58. atriumdb/transfer/adb/wfdb.py +34 -0
  59. atriumdb/transfer/cohort/__init__.py +17 -0
  60. atriumdb/transfer/cohort/cohort_yaml.py +46 -0
  61. atriumdb/transfer/formats/__init__.py +17 -0
  62. atriumdb/transfer/formats/dataset.py +116 -0
  63. atriumdb/transfer/formats/export_data.py +100 -0
  64. atriumdb/transfer/formats/formats.py +19 -0
  65. atriumdb/transfer/formats/import_data.py +97 -0
  66. atriumdb/transfer/formats/json_file_metadata.py +54 -0
  67. atriumdb/windowing/__init__.py +16 -0
  68. atriumdb/windowing/cross_validation.py +282 -0
  69. atriumdb/windowing/dataset_iterator.py +424 -0
  70. atriumdb/windowing/definition.py +864 -0
  71. atriumdb/windowing/definition_builder.py +163 -0
  72. atriumdb/windowing/definition_combine.py +256 -0
  73. atriumdb/windowing/definition_splitter.py +711 -0
  74. atriumdb/windowing/filtered_iterator.py +101 -0
  75. atriumdb/windowing/map_definition_sources.py +119 -0
  76. atriumdb/windowing/mapped_iterator.py +309 -0
  77. atriumdb/windowing/verify_definition.py +292 -0
  78. atriumdb/windowing/window.py +75 -0
  79. atriumdb/windowing/window_config.py +130 -0
  80. atriumdb/windowing/windowing_functions.py +266 -0
  81. atriumdb/write_buffer.py +119 -0
  82. atriumdb-2.7.0.dist-info/METADATA +143 -0
  83. atriumdb-2.7.0.dist-info/RECORD +86 -0
  84. atriumdb-2.7.0.dist-info/WHEEL +5 -0
  85. atriumdb-2.7.0.dist-info/entry_points.txt +4 -0
  86. atriumdb-2.7.0.dist-info/licenses/LICENSE +226 -0
@@ -0,0 +1,1688 @@
1
+ # AtriumDB is a timeseries database software designed to best handle the unique features and
2
+ # challenges that arise from clinical waveform data.
3
+ # Copyright (C) 2023 The Hospital for Sick Children
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <https://www.gnu.org/licenses/>.
17
+ import warnings
18
+ import os
19
+
20
+ import numpy as np
21
+ from tqdm import tqdm
22
+ import time
23
+ from urllib.parse import urlsplit, urlunsplit
24
+ from numpy.lib.stride_tricks import sliding_window_view
25
+ from atriumdb.helpers.block_calculations import freq_nhz_to_period_ns
26
+ import logging
27
+ import bisect
28
+
29
+ from atriumdb.helpers.block_calculations import calc_time_by_freq
30
+ from atriumdb.helpers.block_constants import TIME_TYPES
31
+ from atriumdb.intervals.union import intervals_union_list
32
+ from atriumdb.block_wrapper import get_period_ns_from_header
33
+
34
+ ALLOWED_TIME_TYPES = [1, 2]
35
+ time_unit_options = {
36
+ "s": 1_000_000_000,
37
+ "ms": 1_000_000,
38
+ "us": 1_000,
39
+ "ns": 1
40
+ }
41
+ freq_unit_options = {
42
+ "MHz": 1_000_000_000_000_000,
43
+ "kHz": 1_000_000_000_000,
44
+ "Hz": 1_000_000_000,
45
+ "mHz": 1_000_000,
46
+ "uHz": 1_000,
47
+ "nHz": 1
48
+ }
49
+ allowed_interval_index_modes = ["fast", "merge", "disable"]
50
+
51
+
52
+ def get_block_and_interval_data(measure_id, device_id, metadata, start_bytes, intervals, interval_gap_tolerance=0):
53
+ block_data = []
54
+ for header_i, header in enumerate(metadata):
55
+ block_data.append({
56
+ "measure_id": measure_id,
57
+ "device_id": device_id,
58
+ "start_byte": int(start_bytes[header_i]),
59
+ "num_bytes": header.meta_num_bytes + header.t_num_bytes + header.v_num_bytes,
60
+ "start_time_n": header.start_n,
61
+ "end_time_n": header.end_n,
62
+ "num_values": header.num_vals,
63
+ })
64
+
65
+ interval_data = []
66
+ intervals = sorted(intervals)
67
+ for interval in intervals:
68
+ if interval_data and int(interval[0]) - interval_data[-1]["end_time_n"] <= interval_gap_tolerance:
69
+ interval_data[-1]["end_time_n"] = int(interval[1])
70
+ else:
71
+ interval_data.append({
72
+ "measure_id": measure_id,
73
+ "device_id": device_id,
74
+ "start_time_n": int(interval[0]),
75
+ "end_time_n": int(interval[1]),
76
+ })
77
+ return block_data, interval_data
78
+
79
+
80
+ def condense_byte_read_list(block_list):
81
+ result = []
82
+
83
+ for row in block_list:
84
+ if len(result) == 0 or result[-1][2] != row[3] or result[-1][3] + result[-1][4] != row[4]:
85
+ # append measure_id, device_id, file_id, start_byte and num_bytes
86
+ result.append([row[1], row[2], row[3], row[4], row[5]])
87
+ else:
88
+ # if the blocks are continuous merge the reads together by adding the size of the next block to the
89
+ # num_bytes field
90
+ result[-1][4] += row[5]
91
+
92
+ return result
93
+
94
+
95
+ def find_intervals(freq_nhz=None, raw_time_type=None, time_data=None, data_start_time=None, num_values=None, *, period_ns=None):
96
+ _validate_freq_period_params(freq_nhz, period_ns)
97
+ if freq_nhz is not None:
98
+ current_period_ns = int((10 ** 18) / freq_nhz)
99
+ else:
100
+ current_period_ns = period_ns
101
+
102
+ if raw_time_type == TIME_TYPES['TIME_ARRAY_INT64_NS']:
103
+ intervals = [[time_data[0], 0]]
104
+ time_deltas = time_data[1:] - time_data[:-1]
105
+ for time_arr_i in range(time_data.size - 1):
106
+ if time_deltas[time_arr_i] > current_period_ns:
107
+ intervals[-1][-1] = time_data[time_arr_i] + current_period_ns
108
+ intervals.append([time_data[time_arr_i + 1], 0])
109
+
110
+ intervals[-1][-1] = time_data[-1] + current_period_ns
111
+
112
+ elif raw_time_type == TIME_TYPES['START_TIME_NUM_SAMPLES']:
113
+ intervals = [[time_data[0], time_data[0] + ((time_data[1] - 1) * current_period_ns)]]
114
+
115
+ for interval_data_i in range(1, time_data.size // 2):
116
+ start_time = time_data[2 * interval_data_i]
117
+ end_time = time_data[2 * interval_data_i] + ((time_data[(2 * interval_data_i) + 1] - 1) * current_period_ns)
118
+
119
+ if start_time <= intervals[-1][-1] + current_period_ns:
120
+ intervals[-1][-1] = end_time
121
+ else:
122
+ intervals.append([start_time, end_time])
123
+
124
+ elif raw_time_type == TIME_TYPES['GAP_ARRAY_INT64_INDEX_DURATION_NS']:
125
+ if freq_nhz is not None:
126
+ intervals = [[data_start_time, data_start_time + calc_time_by_freq(freq_nhz, num_values)]]
127
+ else:
128
+ intervals = [[data_start_time, data_start_time + calc_time_by_period(period_ns, num_values)]]
129
+ last_id = 0
130
+
131
+ for sample_id, duration in time_data.reshape((-1, 2)):
132
+ if freq_nhz is not None:
133
+ intervals[-1][-1] = intervals[-1][0] + calc_time_by_freq(freq_nhz, sample_id - last_id)
134
+ else:
135
+ intervals[-1][-1] = intervals[-1][0] + calc_time_by_period(period_ns, sample_id - last_id)
136
+ last_id = sample_id
137
+
138
+ if freq_nhz is not None:
139
+ next_duration = calc_time_by_freq(freq_nhz, num_values - last_id)
140
+ else:
141
+ next_duration = calc_time_by_period(period_ns, num_values - last_id)
142
+
143
+ intervals.append([intervals[-1][-1] + duration,
144
+ intervals[-1][-1] + duration + next_duration])
145
+
146
+ else:
147
+ raise ValueError("raw_time_type not one of {}.".format(
148
+ [TIME_TYPES['TIME_ARRAY_INT64_NS'], TIME_TYPES['START_TIME_NUM_SAMPLES']]))
149
+
150
+ return intervals
151
+
152
+
153
+ # if you want to just use this to sort data will have to add default vals for start/end time and skip bisect
154
+ def sort_data(times, values, headers, start_time, end_time, allow_duplicates=True):
155
+ start_bench = time.perf_counter()
156
+ if len(headers) == 0:
157
+ return times, values
158
+
159
+ block_info = np.zeros((len(headers), 4), dtype=np.int64)
160
+ block_info[:] = [[h.start_n, h.end_n, h.num_vals, 0] for h in headers]
161
+ np.cumsum(block_info.T[2], out=block_info.T[3])
162
+ block_info.T[2] = block_info.T[3] - block_info.T[2]
163
+
164
+ end_bench = time.perf_counter()
165
+ logging.debug(f"rearrange block data info {(end_bench - start_bench) * 1000} ms")
166
+
167
+ # check if the start times are sorted
168
+ start_times_sorted = np.all(np.greater(block_info.T[0][1:], block_info.T[0][:-1]))
169
+ # check if the blocks overlap each other
170
+ start_end_times_dont_intersect = np.all(np.greater_equal(block_info.T[0][1:], block_info.T[1][:-1]))
171
+ if start_end_times_dont_intersect and start_times_sorted:
172
+ logging.debug("Blocks already Sorted and don't intersect.")
173
+
174
+ left, right = bisect.bisect_left(times, start_time), bisect.bisect_left(times, end_time)
175
+ times, values = times[left:right], values[left:right]
176
+
177
+ # if duplicates are allowed then just return times and values
178
+ if allow_duplicates:
179
+ return times, values
180
+ # if duplicates are not allowed then remove them using np.unique
181
+ sorted_times, sorted_time_indices = np.unique(times, return_index=True)
182
+ return sorted_times, values[sorted_time_indices]
183
+
184
+ start_bench = time.perf_counter()
185
+ # if the start times were not sorted, sort them if they were then don't bother running the sort
186
+ if not start_times_sorted:
187
+ # use quicksort as it will be faster with smaller arrays
188
+ sorted_block_i = np.argsort(block_info.T[0], kind='quicksort')
189
+ block_info = block_info[sorted_block_i]
190
+ # recheck if the blocks intersect
191
+ start_end_times_dont_intersect = np.all(np.greater_equal(block_info.T[0][1:], block_info.T[1][:-1]))
192
+
193
+ end_bench = time.perf_counter()
194
+ logging.debug(f"sort data by block {(end_bench - start_bench) * 1000} ms")
195
+
196
+ if allow_duplicates:
197
+ if start_end_times_dont_intersect:
198
+ # Blocks don't intersect each other.
199
+ start_bench = time.perf_counter()
200
+
201
+ # Original Index Creation
202
+ sorted_time_indices = np.concatenate([np.arange(i_start, i_end) for _, _, i_start, i_end in block_info])
203
+
204
+ times, values = times[sorted_time_indices], values[sorted_time_indices]
205
+ end_bench = time.perf_counter()
206
+ logging.debug(f"Blocks don't intersect, sort by blocks {(end_bench - start_bench) * 1000} ms")
207
+
208
+ else:
209
+ # Blocks do intersect each other, so sort every value.
210
+ start_bench = time.perf_counter()
211
+
212
+ # If allowing duplicates use argsort to get the indices so duplicates aren't removed
213
+ # Use mergesort as it will be faster than quicksort on large arrays and since its mostly sorted already
214
+ sorted_time_indices = np.argsort(times, kind='mergesort')
215
+
216
+ end_bench = time.perf_counter()
217
+ logging.debug(f"Blocks intersect, sort every value {(end_bench - start_bench) * 1000} ms")
218
+
219
+ times, values = times[sorted_time_indices], values[sorted_time_indices]
220
+ else:
221
+ # if duplicates are not allowed then remove them using np.unique
222
+ times, sorted_time_indices = np.unique(times, return_index=True)
223
+ values = values[sorted_time_indices]
224
+
225
+ # if the start or end time is in the middle of a block, more times/values will be decoded than are needed so
226
+ # bisect the array on the left and right sides to truncate the data
227
+ left, right = bisect.bisect_left(times, start_time), bisect.bisect_left(times, end_time)
228
+ times, values = times[left:right], values[left:right]
229
+
230
+ return times, values
231
+
232
+
233
+ def yield_data(r_times, r_values, window_size, step_size, get_last_window, total_query_index):
234
+ if window_size is not None:
235
+ time_sliding_window_view = sliding_window_view(r_times, window_size)
236
+ value_sliding_window_view = sliding_window_view(r_values, window_size)
237
+ index_arr = np.arange(0, value_sliding_window_view.shape[0], step_size) + total_query_index
238
+
239
+ # print("r_values.size, window_size")
240
+ # print(r_values.size, window_size)
241
+ # print()
242
+ # print("index_arr.shape")
243
+ # print(index_arr.shape)
244
+ # print("value_sliding_window_view[::step_size, :].shape")
245
+ # print(value_sliding_window_view[::step_size, :].shape)
246
+ assert index_arr.size == value_sliding_window_view[::step_size, :].shape[0]
247
+
248
+ yield (index_arr,
249
+ time_sliding_window_view[::step_size, :],
250
+ value_sliding_window_view[::step_size, :])
251
+
252
+ if get_last_window and ((value_sliding_window_view.shape[0] - 1) % step_size != 0):
253
+ yield (np.array([(value_sliding_window_view.shape[0] - 1) + total_query_index], dtype=np.int64),
254
+ time_sliding_window_view[-1::, :],
255
+ value_sliding_window_view[-1::, :])
256
+ else:
257
+ yield total_query_index, r_times, r_values
258
+
259
+
260
+ def convert_to_nanoseconds(data, time_units):
261
+ """
262
+ Converts time data (array or scalar) to nanoseconds (int64).
263
+ """
264
+ if time_units not in time_unit_options:
265
+ raise ValueError(f"Invalid time units. Expected one of: {list(time_unit_options.keys())}")
266
+
267
+ multiplier = time_unit_options[time_units]
268
+
269
+ # Handle scalar float/int (e.g., period)
270
+ if np.isscalar(data):
271
+ return int(round(data * multiplier))
272
+
273
+ # Copy to avoid altering user data
274
+ data_ns = data.copy() * multiplier
275
+ return np.around(data_ns).astype(np.int64)
276
+
277
+
278
+ def convert_from_nanoseconds(data_ns, target_units):
279
+ """
280
+ Converts nanosecond data (array or scalar) back to target units.
281
+ Returns float for non-ns units, int for ns units.
282
+ """
283
+ if target_units not in time_unit_options:
284
+ raise ValueError(f"Invalid target units. Expected one of: {list(time_unit_options.keys())}")
285
+
286
+ divisor = time_unit_options[target_units]
287
+
288
+ # If target is NS, return integer (no division needed)
289
+ if target_units == "ns":
290
+ if np.isscalar(data_ns):
291
+ return int(data_ns)
292
+ return data_ns.astype(np.int64)
293
+
294
+ # For other units, return float
295
+ return data_ns / divisor
296
+
297
+
298
+ def convert_value_to_nanoseconds(time_value, time_units):
299
+ if time_units not in time_unit_options:
300
+ raise ValueError(f"Invalid time units. Expected one of {time_unit_options}.")
301
+
302
+ return round(time_value * time_unit_options[time_units])
303
+
304
+
305
+ def convert_to_nanohz(freq, freq_units):
306
+ if freq_units not in freq_unit_options.keys():
307
+ raise ValueError("Invalid frequency units. Expected one of: %s" % freq_unit_options)
308
+
309
+ freq *= freq_unit_options[freq_units]
310
+
311
+ return round(freq)
312
+
313
+
314
+ def default_freq_units(freq, freq_units):
315
+ if freq is not None and freq_units is None:
316
+ warnings.warn("freq_units was not given, so freq is read as nHz. The default changes to Hz in "
317
+ "AtriumDB 2.8; pass freq_units explicitly.", FutureWarning, stacklevel=3)
318
+ return "nHz"
319
+ return freq_units
320
+
321
+
322
+ def convert_from_nanohz(freq_nhz, freq_units):
323
+ if freq_units not in freq_unit_options.keys():
324
+ raise ValueError("Invalid frequency units. Expected one of: %s" % freq_unit_options)
325
+
326
+ freq = freq_nhz / freq_unit_options[freq_units]
327
+
328
+ if freq == np.floor(freq):
329
+ freq = int(freq)
330
+
331
+ return freq
332
+
333
+ def detect_period(times: np.ndarray, threshold_ratio: float = 0.3):
334
+ """
335
+ Analyzes a time array to detect the dominant sampling period.
336
+
337
+ If fewer than 2 timestamps, defaults to 1_000_000_000 ns (1 second)
338
+ with a warning. If no dominant period (below threshold), uses the mode
339
+ delta as a best-effort estimate with a warning. Never returns -1.
340
+ """
341
+ if len(times) < 2:
342
+ warnings.warn(
343
+ "Cannot detect period from fewer than 2 timestamps. "
344
+ "Defaulting period to 1,000,000,000 ns (1 second). "
345
+ "Consider explicitly providing 'period' or 'freq' for accurate results.",
346
+ UserWarning
347
+ )
348
+ return 1_000_000_000
349
+
350
+ # Calculate deltas
351
+ deltas = np.diff(times)
352
+
353
+ # Find the mode of the deltas
354
+ unique_deltas, counts = np.unique(deltas, return_counts=True)
355
+ max_count_index = np.argmax(counts)
356
+ mode_delta = unique_deltas[max_count_index]
357
+ mode_count = counts[max_count_index]
358
+
359
+ # Handle Duplicate Values (Mode is 0)
360
+ if mode_delta == 0:
361
+ warnings.warn(
362
+ f"Dominant delta is 0 ({mode_count} occurrences). "
363
+ "Duplicates detected. Redetecting period using unique timestamps.",
364
+ UserWarning
365
+ )
366
+ # Recursive call with unique, sorted timestamps
367
+ return detect_period(np.unique(times), threshold_ratio)
368
+
369
+ # Check against threshold - if below, warn but still return best estimate
370
+ total_intervals = len(deltas)
371
+ if (mode_count / total_intervals) < threshold_ratio:
372
+ warnings.warn(
373
+ f"Automatic period detection: no single time delta accounts for "
374
+ f">{threshold_ratio*100:.0f}% of intervals. "
375
+ f"Using best-effort estimate of {mode_delta} "
376
+ f"(mode of deltas, {mode_count}/{total_intervals} intervals). "
377
+ f"For more accurate results, explicitly provide 'period' or 'freq'.",
378
+ UserWarning
379
+ )
380
+
381
+ # Return based on input type
382
+ if np.issubdtype(times.dtype, np.integer):
383
+ return int(mode_delta)
384
+
385
+ return float(mode_delta)
386
+
387
+
388
+ def parse_metadata_uri(metadata_uri):
389
+ parsed_uri = urlsplit(metadata_uri)
390
+
391
+ if not parsed_uri.scheme or not parsed_uri.netloc:
392
+ raise ValueError(f"Invalid metadata_uri format: expected 'sqltype://username:password@host:port[/dbname]', got {metadata_uri}")
393
+
394
+ sqltype = parsed_uri.scheme
395
+
396
+ if not parsed_uri.username or not parsed_uri.password:
397
+ raise ValueError(f"Invalid metadata_uri format: expected 'sqltype://username:password@host:port[/dbname]', got {metadata_uri}")
398
+
399
+ username = parsed_uri.username
400
+ password = parsed_uri.password
401
+
402
+ if not parsed_uri.hostname or not parsed_uri.port:
403
+ raise ValueError(f"Invalid metadata_uri format: expected 'sqltype://username:password@host:port[/dbname]', got {metadata_uri}")
404
+
405
+ host = parsed_uri.hostname
406
+ port = parsed_uri.port
407
+
408
+ dbname = parsed_uri.path.lstrip('/')
409
+
410
+ return {
411
+ 'sqltype': sqltype,
412
+ 'user': username,
413
+ 'password': password,
414
+ 'host': host,
415
+ 'port': int(port),
416
+ 'database': dbname if dbname else None
417
+ }
418
+
419
+
420
+ def generate_metadata_uri(metadata):
421
+ sqltype = metadata['sqltype']
422
+ username = metadata['user']
423
+ password = metadata['password']
424
+ host = metadata['host']
425
+ port = metadata['port']
426
+ dbname = metadata.get('database')
427
+
428
+ netloc = f"{username}:{password}@{host}:{port}"
429
+ path = f"/{dbname}" if dbname else ""
430
+
431
+ return urlunsplit((sqltype, netloc, path, "", ""))
432
+
433
+
434
+ def convert_gap_data_to_timestamps(headers, r_times, r_values, start_time_n=None, end_time_n=None, sort=True, allow_duplicates=True):
435
+ """
436
+ Convert gap data to timestamps.
437
+
438
+ :param headers: A list of headers containing information about the data.
439
+ :param r_times: An array of times in nanoseconds with gaps.
440
+ :param r_values: An array of values corresponding to the times in r_times.
441
+ :param start_time_n: The start time in nanoseconds. Only needs to be specified if sorting result.
442
+ :param end_time_n: The end time in nanoseconds. Only needs to be specified if sorting result.
443
+ :param bool sort: Whether to sort the returned data.
444
+ :param bool allow_duplicates: Whether to allow duplicate times in the sorted returned data if they exist. Does
445
+ nothing if sort is false.
446
+ :return: A tuple containing an array of timestamps and an array of values.
447
+ """
448
+ # # Start performance benchmark
449
+ # start_bench = time.perf_counter()
450
+
451
+ # Check if all times are integers
452
+ is_int_times = all([(10 ** 18) % h.freq_nhz == 0 for h in headers])
453
+
454
+ # Set the data type for the full_timestamps array based on is_int_times
455
+ time_dtype = np.int64 if is_int_times else np.float64
456
+
457
+ # Create an empty array for the full timestamps
458
+ full_timestamps = np.zeros(r_values.size, dtype=time_dtype)
459
+
460
+ # Initialize the current index and gap variables
461
+ cur_index, cur_gap = 0, 0
462
+
463
+ # Fill full_timestamps array with times based on headers and gaps
464
+ if is_int_times:
465
+ # Loop through the headers and calculate the timestamps for integer times
466
+ for block_i, h in enumerate(headers):
467
+ period_ns = freq_nhz_to_period_ns(h.freq_nhz)
468
+ full_timestamps[cur_index:cur_index + h.num_vals] = \
469
+ np.arange(h.start_n, h.start_n + (h.num_vals * period_ns), period_ns)
470
+
471
+ # Apply the gaps to the timestamps
472
+ for _ in range(h.num_gaps):
473
+ full_timestamps[cur_index + r_times[cur_gap]:cur_index + h.num_vals] += r_times[cur_gap + 1]
474
+ cur_gap += 2
475
+
476
+ # Update the current index
477
+ cur_index += h.num_vals
478
+ else:
479
+ # Loop through the headers and calculate the timestamps for non-integer times
480
+ for block_i, h in enumerate(headers):
481
+ period_ns = float(10 ** 18) / float(h.freq_nhz)
482
+ full_timestamps[cur_index:cur_index + h.num_vals] = \
483
+ np.linspace(h.start_n, h.start_n + (h.num_vals * period_ns), num=h.num_vals, endpoint=False)
484
+
485
+ # Apply the gaps to the timestamps
486
+ for _ in range(h.num_gaps):
487
+ full_timestamps[cur_index + r_times[cur_gap]:cur_index + h.num_vals] += r_times[cur_gap + 1]
488
+ cur_gap += 2
489
+
490
+ # Update the current index
491
+ cur_index += h.num_vals
492
+
493
+ # # End performance benchmark and log the time taken
494
+ # end_bench = time.perf_counter()
495
+ # _LOGGER.debug(f"Expand Gap Data {(end_bench - start_bench) * 1000} ms")
496
+
497
+ # Sort the data based on the timestamps if sort is true
498
+ if sort and start_time_n is not None and end_time_n is not None:
499
+ r_times, r_values = sort_data(r_times, r_values, headers, start_time_n, end_time_n, allow_duplicates)
500
+
501
+ return full_timestamps, r_values
502
+
503
+
504
+ def get_best_measure_id(sdk, measure_tag, freq, units, freq_units):
505
+ measure_dict = {'tag': measure_tag}
506
+ if freq is not None:
507
+ freq_units = default_freq_units(freq, freq_units)
508
+ if freq and freq_units and freq_units != "nHz":
509
+ freq = convert_to_nanohz(freq, freq_units)
510
+ measure_dict['freq_nhz'] = freq
511
+ if units is not None:
512
+ measure_dict['units'] = units
513
+ measure_id_list = get_measure_id_from_generic_measure(sdk, measure_dict, measure_tag_match_rule="best")
514
+ if len(measure_id_list) == 0:
515
+ raise ValueError(f"No matching measure found for: {measure_dict}")
516
+ new_measure_id = measure_id_list[0]
517
+ return new_measure_id
518
+
519
+
520
+ def get_measure_id_from_generic_measure(sdk, measure, measure_tag_match_rule="best", freq_units="Hz"):
521
+ measure_ids = []
522
+
523
+ if isinstance(measure, int):
524
+ measure_ids.append(measure)
525
+ elif isinstance(measure, str):
526
+ matching_ids = sdk.get_measure_id_list_from_tag(measure, approx=True)
527
+ if len(matching_ids) == 0:
528
+ raise ValueError(f"No matching measures found for: {measure}")
529
+ if measure_tag_match_rule == "best":
530
+ measure_ids.append(matching_ids[0])
531
+ elif measure_tag_match_rule == "all":
532
+ measure_ids.extend(matching_ids)
533
+ elif isinstance(measure, dict):
534
+ assert 'tag' in measure, "tag not in measure dictionary"
535
+ freq = measure.get('freq_nhz') or measure.get('freq_hz')
536
+ units = measure.get('units')
537
+ dict_freq_units = "nHz" if 'freq_nhz' in measure else "Hz" if 'freq_hz' in measure else None
538
+
539
+ if freq and units:
540
+ # Use sdk.get_measure_id for a unique match when both freq and units are specified
541
+ measure_id = sdk.get_measure_id(measure['tag'], freq=freq, units=units, freq_units=dict_freq_units)
542
+ if measure_id is not None:
543
+ measure_ids.append(measure_id)
544
+ else:
545
+ raise ValueError(f"No unique measure found for {measure['tag']} with specified frequency and units.")
546
+ else:
547
+ # Use get_measure_id_list_from_tag when either freq or units is not specified
548
+ matching_ids = sdk.get_measure_id_list_from_tag(measure['tag'], approx=True)
549
+ if len(matching_ids) == 0:
550
+ raise ValueError(f"No matching measures found for: {measure['tag']}")
551
+ if measure_tag_match_rule == "best":
552
+ measure_ids.append(matching_ids[0])
553
+ elif measure_tag_match_rule == "all":
554
+ measure_ids.extend(matching_ids)
555
+ elif isinstance(measure, tuple):
556
+ tag, freq, units = measure
557
+ measure_id = sdk.get_measure_id(tag, freq=freq, units=units, freq_units=freq_units)
558
+ if measure_id is not None:
559
+ measure_ids.append(measure_id)
560
+ else:
561
+ raise ValueError(f"No unique measure found for tuple: {measure} assuming freq_units = {freq_units}.")
562
+ else:
563
+ raise ValueError(f"measure type {type(measure)} not supported.")
564
+
565
+ if not measure_ids:
566
+ raise ValueError(f"Measure {measure} not found or multiple matches found without a clear selection rule.")
567
+
568
+ return measure_ids
569
+
570
+
571
+ def collect_all_descendant_ids(label_set_ids, sql_handler):
572
+ """
573
+ Collects all unique descendant label set IDs for a given list of label set IDs,
574
+ along with their closest requested ancestor ID.
575
+
576
+ Args:
577
+ - label_set_ids (List[int]): List of label set IDs to start with.
578
+ - sql_handler (SQLHandler): An instance of SQLHandler for database queries.
579
+
580
+ Returns:
581
+ - Tuple[Set[int], Dict[int, int]]: A set of all unique descendant label set IDs and
582
+ a dictionary mapping each unique descendant ID to its closest requested ancestor ID.
583
+ """
584
+ all_descendants = set(label_set_ids)
585
+ closest_ancestor_dict = {id: id for id in label_set_ids} # Maps each ID to itself initially
586
+ to_process = set(label_set_ids)
587
+
588
+ while to_process:
589
+ current_id = to_process.pop()
590
+ child_records = sql_handler.select_label_name_children(label_set_id=current_id)
591
+ for child_id, *_ in child_records:
592
+ if child_id not in all_descendants:
593
+ all_descendants.add(child_id)
594
+ closest_ancestor_dict[child_id] = closest_ancestor_dict.get(current_id, current_id)
595
+ to_process.add(child_id)
596
+
597
+ return all_descendants, closest_ancestor_dict
598
+
599
+
600
+ def merge_gap_data(values_1, gap_array_1, start_time_1, values_2, gap_array_2, start_time_2, freq_nhz=None, *,
601
+ period_ns=None):
602
+ """
603
+ Merge two sets of gap-array data into one, in time order. Where they overlap, values from the second
604
+ set replace those from the first.
605
+
606
+ :param numpy.ndarray values_1: Values of the first set.
607
+ :param numpy.ndarray gap_array_1: Flat gap array of the first set.
608
+ :param int start_time_1: Start time of the first set in nanoseconds.
609
+ :param numpy.ndarray values_2: Values of the second set (same dtype as ``values_1``).
610
+ :param numpy.ndarray gap_array_2: Flat gap array of the second set.
611
+ :param int start_time_2: Start time of the second set in nanoseconds.
612
+ :param int freq_nhz: Sample frequency in nanohertz. Give exactly one of ``freq_nhz`` and ``period_ns``.
613
+ :param int period_ns: Keyword-only. Sample period in nanoseconds.
614
+ :return: Tuple ``(values, gap_array, start_time)`` of the merged data.
615
+ :rtype: tuple
616
+ :raises ValueError: If an input array is not a numpy array, the value dtypes differ, or not exactly one
617
+ of ``freq_nhz`` and ``period_ns`` is given.
618
+ """
619
+ _validate_freq_period_params(freq_nhz, period_ns)
620
+
621
+ if not all(isinstance(arr, np.ndarray) for arr in [values_1, gap_array_1, values_2, gap_array_2]):
622
+ raise ValueError(f"All input value and gap arrays must be numpy arrays.")
623
+
624
+ if not values_1.dtype == values_2.dtype:
625
+ raise ValueError(f"Values 1 and 2 have different dtypes {values_1.dtype}, {values_2.dtype}. Cannot merge.")
626
+
627
+ end_time_1 = _calc_end_time_from_gap_data(values_1.size, gap_array_1, start_time_1, freq_nhz=freq_nhz,
628
+ period_ns=period_ns)
629
+ end_time_2 = _calc_end_time_from_gap_data(values_2.size, gap_array_2, start_time_2, freq_nhz=freq_nhz,
630
+ period_ns=period_ns)
631
+
632
+ overlap = (start_time_1 < end_time_2) and (end_time_1 > start_time_2)
633
+
634
+ # If there's no overlap, you can simply concatenate the data.
635
+ if is_gap_data_sorted(gap_array_1, freq_nhz=freq_nhz, period_ns=period_ns) and is_gap_data_sorted(gap_array_2,
636
+ freq_nhz=freq_nhz,
637
+ period_ns=period_ns) and not overlap:
638
+ return _concatenate_gap_data(
639
+ values_1, gap_array_1, start_time_1, values_2, gap_array_2, start_time_2, freq_nhz=freq_nhz,
640
+ period_ns=period_ns)
641
+
642
+ # if starts, values and gaps are equal, then just return the 1's
643
+ if np.array_equal(values_1, values_2) and \
644
+ np.array_equal(gap_array_1, gap_array_2) and \
645
+ start_time_1 == start_time_2:
646
+ return values_1, gap_array_1, start_time_1
647
+
648
+ # Convert both gap_data into messages
649
+ message_starts_1, message_sizes_1 = reconstruct_messages(
650
+ start_time_1, gap_array_1, num_values=int(values_1.size), freq_nhz=freq_nhz, period_ns=period_ns)
651
+
652
+ message_starts_2, message_sizes_2 = reconstruct_messages(
653
+ start_time_2, gap_array_2, num_values=int(values_2.size), freq_nhz=freq_nhz, period_ns=period_ns)
654
+
655
+ # Sort both message lists + values, and copy values to not mess with the originals
656
+ values_1, values_2 = values_1.copy(), values_2.copy()
657
+ sort_message_time_values(message_starts_1, message_sizes_1, values_1)
658
+ sort_message_time_values(message_starts_2, message_sizes_2, values_2)
659
+
660
+ # Merge lists and Overwrite 2 over 1 if overlapping
661
+ merged_starts, merged_sizes, merged_values = merge_sorted_messages(
662
+ message_starts_1, message_sizes_1, values_1,
663
+ message_starts_2, message_sizes_2, values_2, freq_nhz=freq_nhz, period_ns=period_ns)
664
+
665
+ # Convert back into gap data
666
+ merged_gap_data = create_gap_arr_from_variable_messages(merged_starts, merged_sizes, freq_nhz=freq_nhz,
667
+ period_ns=period_ns)
668
+
669
+ return merged_values, merged_gap_data, int(merged_starts[0])
670
+
671
+
672
+ def _concatenate_gap_data(values_1, gap_array_1, start_time_1, values_2, gap_array_2, start_time_2, freq_nhz=None, *,
673
+ period_ns=None):
674
+ _validate_freq_period_params(freq_nhz, period_ns)
675
+
676
+ # if start_time_2 < start_time_1, swap 1's with 2's so that 1 is always temporally first.
677
+ if start_time_2 < start_time_1:
678
+ values_1, values_2 = values_2, values_1
679
+ gap_array_1, gap_array_2 = gap_array_2, gap_array_1
680
+ start_time_1, start_time_2 = start_time_2, start_time_1
681
+
682
+ # Calculate the gap between blocks
683
+ end_time_1 = _calc_end_time_from_gap_data(values_1.size, gap_array_1, start_time_1, freq_nhz=freq_nhz,
684
+ period_ns=period_ns)
685
+ new_gap_index, new_gap_duration = int(values_1.size), start_time_2 - end_time_1
686
+
687
+ # Calculate positions depending on if we need to add a new gap between blocks.
688
+ if new_gap_duration == 0:
689
+ merged_gap_size = gap_array_1.size + gap_array_2.size
690
+ gap_array_2_index = gap_array_1.size
691
+ else:
692
+ merged_gap_size = gap_array_1.size + gap_array_2.size + 2
693
+ gap_array_2_index = gap_array_1.size + 2
694
+
695
+ merged_gap_array = np.empty(merged_gap_size, dtype=np.int64)
696
+ merged_gap_array[:gap_array_1.size] = gap_array_1
697
+
698
+ if new_gap_duration != 0:
699
+ merged_gap_array[gap_array_1.size:gap_array_1.size + 2] = [new_gap_index, new_gap_duration]
700
+
701
+ merged_gap_array[gap_array_2_index:] = gap_array_2
702
+ # move the gap_array_2 indices forward.
703
+ merged_gap_array[gap_array_2_index::2] += values_1.size
704
+
705
+ # concatenate the values.
706
+ merged_values = np.empty(values_1.size + values_2.size, dtype=values_1.dtype)
707
+ merged_values[:values_1.size] = values_1
708
+ merged_values[values_1.size:] = values_2
709
+ return merged_values, merged_gap_array, start_time_1
710
+
711
+
712
+ def _calc_end_time_from_gap_data(values_size, gap_array, start_time, freq_nhz=None, *, period_ns=None):
713
+ _validate_freq_period_params(freq_nhz, period_ns)
714
+
715
+ if freq_nhz is not None:
716
+ if (int(values_size) * (10 ** 18)) % freq_nhz != 0:
717
+ warnings.warn(f"Blocking starting on epoch {start_time} doesn't end on an integer number of nanoseconds, "
718
+ f"merge will be approximate.")
719
+ sample_duration = (int(values_size) * (10 ** 18)) // freq_nhz
720
+ else:
721
+ sample_duration = calc_time_by_period(period_ns, int(values_size))
722
+
723
+ gap_total = int(np.sum(gap_array[1::2]))
724
+ return start_time + sample_duration + gap_total
725
+
726
+
727
+ def create_timestamps_from_gap_data(values_size, gap_array, start_time, freq_nhz=None, *, period_ns=None):
728
+ _validate_freq_period_params(freq_nhz, period_ns)
729
+
730
+ if freq_nhz is not None:
731
+ if (10 ** 18) % freq_nhz != 0:
732
+ raise ValueError(f"Cannot create perfect timestamps from frequency_nhz = {freq_nhz}")
733
+ current_period_ns = freq_nhz_to_period_ns(freq_nhz)
734
+ else:
735
+ current_period_ns = period_ns
736
+
737
+ timestamps = np.arange(values_size, dtype=np.int64)
738
+ timestamps *= current_period_ns
739
+ timestamps += start_time
740
+ for i in range(gap_array.size // 2):
741
+ gap_index, gap_duration = gap_array[2 * i], gap_array[(2 * i) + 1]
742
+ timestamps[gap_index:] += gap_duration
743
+
744
+ return timestamps
745
+
746
+
747
+ def is_gap_data_sorted(gap_data, freq_nhz=None, *, period_ns=None):
748
+ _validate_freq_period_params(freq_nhz, period_ns)
749
+
750
+ if freq_nhz is not None:
751
+ current_period_ns = freq_nhz_to_period_ns(freq_nhz)
752
+ else:
753
+ current_period_ns = period_ns
754
+
755
+ gap_durations = gap_data[1::2]
756
+ return np.all(gap_durations >= -current_period_ns)
757
+
758
+
759
+ def create_gap_arr_from_variable_messages(message_start_epoch_array, message_size_array, freq_nhz=None, *,
760
+ period_ns=None):
761
+ _validate_freq_period_params(freq_nhz, period_ns)
762
+
763
+ result_list = []
764
+ current_sample = 0
765
+
766
+ for i in range(1, len(message_start_epoch_array)):
767
+ # Compute the time difference between consecutive messages
768
+ delta_t = message_start_epoch_array[i] - message_start_epoch_array[i - 1]
769
+
770
+ # Calculate the message period for the current message based on its size
771
+ current_message_size = int(message_size_array[i - 1])
772
+
773
+ if freq_nhz is not None:
774
+ current_message_period_ns = ((10 ** 18) * current_message_size) // int(freq_nhz)
775
+ else:
776
+ current_message_period_ns = _message_size_period_to_durations_ns(current_message_size, period_ns)
777
+
778
+ # Check if the time difference doesn't match the expected message period
779
+ if delta_t != current_message_period_ns:
780
+ # Compute the extra duration (time gap) and the starting index of the gap
781
+ time_gap = delta_t - current_message_period_ns
782
+ gap_start_index = current_sample + current_message_size
783
+
784
+ # Add the gap information to the result list
785
+ result_list.extend([gap_start_index, time_gap])
786
+
787
+ # Update the current sample index for the next iteration
788
+ current_sample += current_message_size
789
+
790
+ # Convert the result list to a NumPy array of integers
791
+ return np.array(result_list, dtype=np.int64)
792
+
793
+
794
+ def reconstruct_messages(start_time_nano_epoch, gap_data_array, freq_nhz=None, num_values=None, *, period_ns=None):
795
+ _validate_freq_period_params(freq_nhz, period_ns)
796
+
797
+ message_sizes = np.empty((gap_data_array.size // 2) + 2, dtype=np.int64)
798
+ message_sizes[0] = 0
799
+ message_sizes[-1] = num_values
800
+ message_sizes[1:-1] = gap_data_array[::2]
801
+ message_sizes = np.diff(message_sizes)
802
+
803
+ message_starts = np.empty((gap_data_array.size // 2) + 1, dtype=np.int64)
804
+
805
+ message_starts[0] = start_time_nano_epoch
806
+
807
+ if freq_nhz is not None:
808
+ if any(((10 ** 18) * int(m_size)) % freq_nhz != 0 for m_size in message_sizes[:-1]):
809
+ warnings.warn(
810
+ "Not all messages durations can be expressed as a perfect nanosecond integer, some rounding has occured")
811
+ message_starts[1:] = [_message_size_to_duration_ns(int(m_size), freq_nhz) for m_size in message_sizes[:-1]]
812
+ else:
813
+ message_starts[1:] = [_message_size_period_to_durations_ns(int(m_size), period_ns) for m_size in
814
+ message_sizes[:-1]]
815
+
816
+ message_starts[1:] += gap_data_array[1::2]
817
+ message_starts = np.cumsum(message_starts)
818
+
819
+ return message_starts, message_sizes
820
+
821
+
822
+ def reconstruct_messages_multi(headers, gap_data, freq_nhz=None, *, period_ns=None):
823
+ _validate_freq_period_params(freq_nhz, period_ns)
824
+
825
+ message_starts_list = []
826
+ message_sizes_list = []
827
+
828
+ gap_idx = 0
829
+ for header in headers:
830
+ num_vals = header.num_vals
831
+ num_gaps = header.num_gaps
832
+ start_n = header.start_n # starting time in nanoseconds since epoch
833
+
834
+ gap_data_block = gap_data[gap_idx: gap_idx + 2 * num_gaps]
835
+
836
+ # Process the block to get message_starts and message_sizes
837
+ message_starts, message_sizes = reconstruct_messages(
838
+ start_n, gap_data_block, freq_nhz=freq_nhz, num_values=num_vals, period_ns=period_ns)
839
+
840
+ message_starts_list.append(message_starts)
841
+ message_sizes_list.append(message_sizes)
842
+
843
+ # Update indices for the next block
844
+ gap_idx += 2 * num_gaps
845
+
846
+ # Concatenate all message starts and sizes from all blocks
847
+ all_message_starts = np.concatenate(message_starts_list)
848
+ all_message_sizes = np.concatenate(message_sizes_list)
849
+
850
+ return all_message_starts, all_message_sizes
851
+
852
+
853
+ def sort_message_time_values(message_starts: np.ndarray, message_sizes: np.ndarray, value_array: np.ndarray):
854
+ # The start and end index of each message within the value_array
855
+ message_end_indices, message_start_indices = _get_message_indices(message_sizes)
856
+
857
+ sorted_message_i_arr = np.argsort(message_starts)
858
+ if np.all(sorted_message_i_arr[:-1] <= sorted_message_i_arr[1:]):
859
+ return
860
+
861
+ message_starts[:] = message_starts[sorted_message_i_arr]
862
+ message_sizes[:] = message_sizes[sorted_message_i_arr]
863
+
864
+ values_copy = value_array.copy()
865
+
866
+ start_idx = 0
867
+ for message_i in range(message_start_indices.size):
868
+ sorted_left = message_start_indices[sorted_message_i_arr][message_i]
869
+ sorted_right = message_end_indices[sorted_message_i_arr][message_i]
870
+ message_size = message_sizes[message_i]
871
+
872
+ value_array[start_idx:start_idx+message_size] = values_copy[sorted_left:sorted_right]
873
+ start_idx += message_size
874
+
875
+
876
+ def _get_message_indices(message_sizes):
877
+ message_end_indices = np.cumsum(message_sizes)
878
+ message_start_indices = message_end_indices - message_sizes
879
+ return message_end_indices, message_start_indices
880
+
881
+
882
+ def truncate_messages(value_data, message_starts, message_sizes, freq_nhz=None, trunc_start_nano=None, trunc_end_nano=None, *,
883
+ period_ns=None):
884
+ _validate_freq_period_params(freq_nhz, period_ns)
885
+
886
+ truncated_value_data = []
887
+ truncated_message_starts = []
888
+ truncated_message_sizes = []
889
+ cumulative_sample_index = 0 # To track the index in value_data
890
+
891
+ if freq_nhz is not None:
892
+ current_period_ns = 10 ** 18 // freq_nhz # Period in nanoseconds per sample
893
+ else:
894
+ current_period_ns = period_ns
895
+
896
+ for i in range(len(message_starts)):
897
+ msg_start_ns = message_starts[i]
898
+ msg_size = message_sizes[i]
899
+
900
+ if freq_nhz is not None:
901
+ msg_duration_ns = _message_size_freq_to_duration_ns(msg_size, freq_nhz)
902
+ else:
903
+ msg_duration_ns = _message_size_period_to_durations_ns(msg_size, period_ns)
904
+
905
+ msg_end_ns = msg_start_ns + msg_duration_ns
906
+
907
+ # Calculate overlapping region with truncation window
908
+ overlap_start_ns = max(msg_start_ns, trunc_start_nano)
909
+ overlap_end_ns = min(msg_end_ns, trunc_end_nano)
910
+
911
+ if overlap_end_ns <= overlap_start_ns:
912
+ # No overlap with the truncation window
913
+ cumulative_sample_index += msg_size
914
+ continue
915
+
916
+ # Calculate the sample indices within the message
917
+ samples_before_overlap_start = ((overlap_start_ns - msg_start_ns) + current_period_ns - 1) // current_period_ns
918
+ samples_until_overlap_end = ((overlap_end_ns - msg_start_ns) + current_period_ns - 1) // current_period_ns
919
+
920
+ num_samples_to_keep = samples_until_overlap_end - samples_before_overlap_start
921
+
922
+ if num_samples_to_keep <= 0:
923
+ cumulative_sample_index += msg_size
924
+ continue
925
+
926
+ # Calculate the indices in value_data
927
+ start_idx = cumulative_sample_index + samples_before_overlap_start
928
+ end_idx = start_idx + num_samples_to_keep
929
+
930
+ # Truncate the value_data
931
+ truncated_value_data.append(value_data[start_idx:end_idx])
932
+
933
+ # Adjust message start time
934
+ adjusted_msg_start_ns = msg_start_ns + samples_before_overlap_start * current_period_ns
935
+
936
+ truncated_message_starts.append(adjusted_msg_start_ns)
937
+ truncated_message_sizes.append(num_samples_to_keep)
938
+
939
+ cumulative_sample_index += msg_size
940
+
941
+ # Concatenate all truncated data
942
+ truncated_value_data = np.concatenate(truncated_value_data) if truncated_value_data else np.array([])
943
+ truncated_message_starts = np.array(truncated_message_starts)
944
+ truncated_message_sizes = np.array(truncated_message_sizes)
945
+
946
+ return truncated_value_data, truncated_message_starts, truncated_message_sizes
947
+
948
+
949
+ def merge_sorted_messages(message_starts_1, message_sizes_1, values_1,
950
+ message_starts_2, message_sizes_2, values_2, freq_nhz=None, *, period_ns=None):
951
+ _validate_freq_period_params(freq_nhz, period_ns)
952
+
953
+ # Find the smallest start time from both inputs
954
+ min_start = min(np.min(message_starts_1), np.min(message_starts_2))
955
+
956
+ # Create relative start times
957
+ rel_starts_1 = message_starts_1 - min_start
958
+ rel_starts_2 = message_starts_2 - min_start
959
+
960
+ # Calculate period in nanoseconds for one sample
961
+ if freq_nhz is not None:
962
+ current_period_ns = float((10 ** 18) / int(freq_nhz))
963
+ else:
964
+ current_period_ns = float(period_ns)
965
+
966
+ timestamps_1 = []
967
+ timestamps_2 = []
968
+
969
+ # Create timestamps for dataset 1
970
+ for start, size in zip(rel_starts_1, message_sizes_1):
971
+ timestamps_1.append(start + (np.arange(size) * current_period_ns))
972
+
973
+ # Create timestamps for dataset 2
974
+ for start, size in zip(rel_starts_2, message_sizes_2):
975
+ timestamps_2.append(start + (np.arange(size) * current_period_ns))
976
+
977
+ # Convert lists to numpy arrays
978
+ timestamps_1 = np.concatenate(timestamps_1, dtype=np.float64)
979
+ timestamps_2 = np.concatenate(timestamps_2, dtype=np.float64)
980
+
981
+ # Combine and sort the timestamp and value arrays
982
+ combined_timestamps = np.concatenate((timestamps_1, timestamps_2))
983
+ combined_values = np.concatenate((values_1, values_2))
984
+
985
+ # Create array indicators to ensure stable sorting where array 2 overwrites array 1
986
+ array_indicators = np.concatenate((np.zeros(len(timestamps_1), dtype=int),
987
+ np.ones(len(timestamps_2), dtype=int)))
988
+
989
+ # Sort by timestamp first, then by array indicator (so array 2 comes after array 1 for ties)
990
+ sort_keys = np.lexsort((array_indicators, combined_timestamps))
991
+ sorted_timestamps = combined_timestamps[sort_keys]
992
+ sorted_values = combined_values[sort_keys]
993
+
994
+ # Convert the sorted timestamps into "sample times"
995
+ sample_times = sorted_timestamps / current_period_ns
996
+
997
+ # Find differences in sample times
998
+ diff_sample_times = np.diff(sample_times)
999
+
1000
+ # Calculate the max floating point imprecision
1001
+ max_timestamp = np.max(sorted_timestamps)
1002
+ max_diff_sample_times = np.max(diff_sample_times)
1003
+
1004
+ max_imprecision = np.finfo(np.float64).eps * (max_timestamp / current_period_ns + max_diff_sample_times)
1005
+
1006
+ # Identify where timestamps are close to 1 (consecutive message elements)
1007
+ close_to_one = np.isclose(diff_sample_times, 1, atol=max_imprecision, rtol=0)
1008
+
1009
+ # Identify where timestamps are close to 0 (duplicates)
1010
+ close_to_zero = np.isclose(diff_sample_times, 0, atol=max_imprecision, rtol=0)
1011
+
1012
+ # Reconstruct the merged messages, handling duplicates by skipping them
1013
+ merged_starts = []
1014
+ merged_sizes = []
1015
+ merged_values = []
1016
+
1017
+ current_message_start = sorted_timestamps[0]
1018
+ current_message_values = [sorted_values[0]]
1019
+
1020
+ for i in range(1, sample_times.size):
1021
+ # Skip over duplicates by looking ahead until we find a non-duplicate
1022
+ if close_to_zero[i - 1]:
1023
+ # If the next value is a duplicate, keep overwriting current value until the last one
1024
+ current_message_values[-1] = sorted_values[i]
1025
+ else:
1026
+ if close_to_one[i - 1]:
1027
+ # This value continues the current message
1028
+ current_message_values.append(sorted_values[i])
1029
+ else:
1030
+ # End of the current message, start a new one
1031
+ merged_starts.append(current_message_start)
1032
+ merged_sizes.append(len(current_message_values))
1033
+ merged_values.append(np.array(current_message_values))
1034
+
1035
+ current_message_start = sorted_timestamps[i]
1036
+ current_message_values = [sorted_values[i]]
1037
+
1038
+ # Append the last message
1039
+ if len(current_message_values) > 0:
1040
+ merged_starts.append(current_message_start)
1041
+ merged_sizes.append(len(current_message_values))
1042
+ merged_values.append(np.array(current_message_values))
1043
+
1044
+ # Convert to final outputs
1045
+ merged_starts = np.array(merged_starts, dtype=np.int64) + min_start
1046
+ merged_sizes = np.array(merged_sizes, dtype=np.int64)
1047
+ merged_values = np.concatenate(merged_values) if merged_values else np.array([], dtype=np.float64)
1048
+
1049
+ return merged_starts, merged_sizes, merged_values
1050
+
1051
+
1052
+ def _message_size_freq_to_duration_ns(m_size, freq_nhz):
1053
+ return ((10 ** 18) * int(m_size)) // freq_nhz
1054
+
1055
+ def _message_size_period_to_durations_ns(m_size, period_ns):
1056
+ return m_size * period_ns
1057
+
1058
+ def _message_size_to_duration_ns(m_size, freq_nhz):
1059
+ return ((10 ** 18) * int(m_size)) // freq_nhz
1060
+
1061
+ def calc_time_by_period(period_ns, num_samples):
1062
+ return int(num_samples) * period_ns
1063
+
1064
+
1065
+ def merge_timestamp_data(values_1, times_1, values_2, times_2):
1066
+ # concatenate the time and value arrays
1067
+ concatenated_times = np.concatenate((times_1, times_2))
1068
+ concatenated_values = np.concatenate((values_1, values_2))
1069
+
1070
+ # remove duplicate times and get indices to sort values in time order
1071
+ time_data, index_unique = np.unique(concatenated_times, return_index=True)
1072
+
1073
+ # return the sorted arrays
1074
+ return time_data, concatenated_values[index_unique]
1075
+
1076
+
1077
+ def generate_single_messages_from_big_message(message_start_1, message_end_1, start_index_1, end_index_1, message_values_1,
1078
+ message_start_2, message_end_2, start_index_2, end_index_2, message_values_2,
1079
+ freq_nhz):
1080
+ if (10 ** 18) % freq_nhz != 0:
1081
+ warnings.warn("Out of phase data is being merged with a freq that doesn't create perfect nanosecond integer "
1082
+ "timestamps, some rounding will occur in the stored data.")
1083
+
1084
+ # Calculate the period based on frequency
1085
+ period_ns = (10 ** 18) // freq_nhz
1086
+
1087
+ # Initialize lists for both data sets
1088
+ starts_1, ends_1, values_list_1 = [], [], []
1089
+ starts_2, ends_2, values_list_2 = [], [], []
1090
+
1091
+ # Process the first set of parameters
1092
+ cur_time = message_start_1
1093
+ for index in range(start_index_1, end_index_1):
1094
+ starts_1.append(cur_time)
1095
+ values_list_1.append(message_values_1[index:index+1])
1096
+ cur_time += period_ns
1097
+ ends_1.append(cur_time)
1098
+
1099
+ # Process the second set of parameters
1100
+ cur_time = message_start_2
1101
+ for index in range(start_index_2, end_index_2):
1102
+ starts_2.append(cur_time)
1103
+ values_list_2.append(message_values_2[index:index+1])
1104
+ cur_time += period_ns
1105
+ ends_2.append(cur_time)
1106
+
1107
+ # Now merge the two sets in order of ascending starts
1108
+ starts, ends, values_list = [], [], []
1109
+ index_1, index_2 = 0, 0
1110
+
1111
+ while index_1 < len(starts_1) and index_2 < len(starts_2):
1112
+ if starts_1[index_1] <= starts_2[index_2]:
1113
+ starts.append(starts_1[index_1])
1114
+ ends.append(ends_1[index_1])
1115
+ values_list.append(values_list_1[index_1])
1116
+ index_1 += 1
1117
+ else:
1118
+ starts.append(starts_2[index_2])
1119
+ ends.append(ends_2[index_2])
1120
+ values_list.append(values_list_2[index_2])
1121
+ index_2 += 1
1122
+
1123
+ # Append remaining elements from the first list
1124
+ while index_1 < len(starts_1):
1125
+ starts.append(starts_1[index_1])
1126
+ ends.append(ends_1[index_1])
1127
+ values_list.append(values_list_1[index_1])
1128
+ index_1 += 1
1129
+
1130
+ # Append remaining elements from the second list
1131
+ while index_2 < len(starts_2):
1132
+ starts.append(starts_2[index_2])
1133
+ ends.append(ends_2[index_2])
1134
+ values_list.append(values_list_2[index_2])
1135
+ index_2 += 1
1136
+
1137
+ return starts, ends, values_list
1138
+
1139
+
1140
+ def get_headers(sdk, measure_id: int = None, start_time: int = None, end_time: int = None, device_id: int = None,
1141
+ patient_id=None, time_units: str = None, mrn: int = None):
1142
+ """
1143
+ The method for querying block headers from the dataset, indexed by signal type (measure_id or measure_tag with freq and units),
1144
+ time (start_time_n and end_time_n), and data source (device_id, device_tag, patient_id, or mrn).
1145
+
1146
+ If measure_id is None, measure_tag along with freq and units must not be None, and vice versa.
1147
+ Similarly, if device_id is None, device_tag must not be None, and if patient_id is None, mrn must not be None.
1148
+
1149
+ :param AtriumSDK sdk: The AtriumSDK object.
1150
+ :param int measure_id: The measure identifier. If None, measure_tag must be provided.
1151
+ :param int start_time: The start epoch in nanoseconds of the data you would like to query.
1152
+ :param int end_time: The end epoch in nanoseconds. The end time is not inclusive.
1153
+ :param int device_id: The device identifier. If None, device_tag must be provided.
1154
+ :param int patient_id: The patient identifier. If None, mrn must be provided.
1155
+ """
1156
+
1157
+ # check that a correct unit type was entered
1158
+ time_units = "ns" if time_units is None else time_units
1159
+
1160
+ if time_units not in time_unit_options.keys():
1161
+ raise ValueError("Invalid time units. Expected one of: %s" % time_unit_options)
1162
+
1163
+ # convert start and end time to nanoseconds
1164
+ start_time = int(start_time * time_unit_options[time_units])
1165
+ end_time = int(end_time * time_unit_options[time_units])
1166
+
1167
+ if patient_id is None and mrn is not None:
1168
+ patient_id = sdk.get_patient_id(mrn)
1169
+
1170
+ # If we don't already have the blocks
1171
+ # Select all blocks from the block_index (sql table) that match params.
1172
+ block_list = sdk.sql_handler.select_blocks(int(measure_id), int(start_time), int(end_time), device_id,
1173
+ patient_id)
1174
+
1175
+ # Concatenate continuous byte intervals to cut down on total number of reads.
1176
+ read_list = condense_byte_read_list(block_list)
1177
+
1178
+ # if no matching block ids
1179
+ if len(read_list) == 0:
1180
+ return [], np.array([]), np.array([])
1181
+
1182
+ # Map file_ids to filenames and return a dictionary.
1183
+ file_id_list = [row[2] for row in read_list]
1184
+ filename_dict = sdk.get_filename_dict(file_id_list)
1185
+
1186
+ # Condense the block list for optimized reading
1187
+ read_list = condense_byte_read_list(block_list)
1188
+
1189
+ # Read the data from the files using the read list
1190
+ encoded_bytes = sdk.file_api.read_file_list(read_list, filename_dict)
1191
+
1192
+ num_bytes_list = [row[5] for row in block_list]
1193
+ byte_start_array = np.cumsum(num_bytes_list, dtype=np.uint64)
1194
+ byte_start_array = np.concatenate([np.array([0], dtype=np.uint64), byte_start_array[:-1]], axis=None)
1195
+ headers = sdk.block.decode_headers(encoded_bytes, byte_start_array)
1196
+
1197
+ return headers
1198
+
1199
+
1200
+ def delete_unreferenced_tsc_files(sdk):
1201
+ """
1202
+ The method is for removing unreferenced tsc files. When you write a lot of data with merge=True
1203
+ eventually a lot of unused tsc files will build up since the function is copy on write. This
1204
+ Will use up a lot of space unnecessarily and this function will help you free up that space.
1205
+
1206
+ :param AtriumSDK sdk: The AtriumSDK object.
1207
+ """
1208
+ # find tsc files in the file_index that have no references to them in the block_index
1209
+ files = sdk.sql_handler.find_unreferenced_tsc_files()
1210
+
1211
+ # if there are no tsc files to remove just return
1212
+ if len(files) == 0:
1213
+ return
1214
+
1215
+ # extract file names from files and make it a set so we can do a set intersection later
1216
+ file_names = {file[1] for file in files}
1217
+ # extract the ids and put them in a tuple so we can remove them from the sql table later
1218
+ file_ids = [(file[0],) for file in files]
1219
+
1220
+ # walk the tsc directory looking for files to delete (walk is a generator for memory efficiency)
1221
+ for root, dir_names, files in sdk.file_api.walk(sdk.file_api.top_level_dir):
1222
+ # remove any dirs that are not digits (device_id or measure_id) so walk doesn't traverse those directories
1223
+ dir_names[:] = [d for d in dir_names if d.isdigit()]
1224
+
1225
+ # check if there is a match between any of the tsc file names to be deleted and files in the current directory
1226
+ matches = set(files) & file_names
1227
+
1228
+ # if you find a match remove the file from disk
1229
+ for m in matches:
1230
+ print(f"Deleting tsc file {m} from disk")
1231
+ sdk.file_api.remove(os.path.join(root, m))
1232
+
1233
+ # free up memory
1234
+ del file_names
1235
+
1236
+ # remove them from the file_index
1237
+ sdk.sql_handler.delete_files_by_ids(file_ids)
1238
+ print("Completed removal of unreferenced tsc files")
1239
+
1240
+
1241
+ def get_all_blocks(sdk, measure_id, device_id):
1242
+ query = """
1243
+ SELECT id, measure_id, device_id, file_id, start_byte, num_bytes, start_time_n, end_time_n, num_values
1244
+ FROM block_index
1245
+ WHERE measure_id = ? AND device_id = ?
1246
+ ORDER BY start_time_n ASC;
1247
+ """
1248
+
1249
+ with sdk.sql_handler.connection() as (conn, cursor):
1250
+ cursor.execute(query, (measure_id, device_id))
1251
+ block_query_result = cursor.fetchall()
1252
+
1253
+ return block_query_result
1254
+
1255
+
1256
+ def reencode_dataset(sdk, values_per_block=131072, blocks_per_file=2048, interval_gap_tolerance_nano=0):
1257
+ # WARNING:
1258
+ # This function does not preserve duplicate values. (uses np.unique)
1259
+ original_block_size = sdk.block.block_size
1260
+ sdk.block.block_size = values_per_block
1261
+
1262
+ measures = sdk.get_all_measures()
1263
+ devices = sdk.get_all_devices()
1264
+
1265
+ measure_data = list(measures.items())
1266
+ device_data = list(devices.items())
1267
+
1268
+ for device_id, device_info in tqdm(device_data, desc="Devices"):
1269
+ for measure_id, measure_info in tqdm(measure_data, desc="Measures"):
1270
+ sorted_block_list_by_time = get_all_blocks(sdk, measure_id, device_id)
1271
+ if len(sorted_block_list_by_time) == 0:
1272
+ continue
1273
+
1274
+ file_id_list = list(set([row[3] for row in sorted_block_list_by_time]))
1275
+ filename_dict = sdk.get_filename_dict(file_id_list)
1276
+
1277
+ total_measure_device_interval_array = []
1278
+
1279
+ for block_group in group_sorted_block_list(
1280
+ sorted_block_list_by_time, num_values_per_group=values_per_block * blocks_per_file):
1281
+
1282
+ r_headers, r_times, r_values = sdk.get_data_from_blocks(
1283
+ block_list=block_group,
1284
+ filename_dict=filename_dict,
1285
+ start_time_n=0,
1286
+ end_time_n=2 ** 62,
1287
+ analog=False,
1288
+ time_type="encoded",
1289
+ sort=False,
1290
+ allow_duplicates=True
1291
+ )
1292
+
1293
+ if r_values.size == 0:
1294
+ continue
1295
+
1296
+ # Prepare segments for encode_blocks_from_multiple_segments
1297
+ segments = []
1298
+ block_group_interval_data = []
1299
+
1300
+ # Group data by scale factor, freq, time type
1301
+ for group_headers, group_times, group_values in group_headers_by_scale_factor_freq_time_type(
1302
+ r_headers, r_times, r_values):
1303
+ group_time_type = int(group_headers[0].t_raw_type)
1304
+ group_encoded_time_type = int(group_headers[0].t_encoded_type)
1305
+
1306
+ # Check TSC version to determine if freq_nhz is actually period_ns
1307
+ version_code = group_headers[0].tsc_version_num * 10 + group_headers[0].tsc_version_ext
1308
+ is_version_24_plus = version_code >= 24
1309
+
1310
+ if is_version_24_plus:
1311
+ group_period_ns = int(group_headers[0].freq_nhz) # Actually period_ns for v2.4+
1312
+ group_freq_nhz = None
1313
+ else:
1314
+ group_freq_nhz = int(group_headers[0].freq_nhz)
1315
+ group_period_ns = None
1316
+
1317
+ # Sort and merge the group data
1318
+ if group_time_type == 1:
1319
+ period_ns = group_period_ns if is_version_24_plus else (10 ** 18) // group_freq_nhz
1320
+
1321
+ group_times, sorted_time_indices = np.unique(group_times, return_index=True)
1322
+ group_values = group_values[sorted_time_indices]
1323
+ group_interval_data = get_interval_list_from_ordered_timestamps(group_times, period_ns)
1324
+ group_start_time = int(group_times[0])
1325
+ elif group_time_type == 2:
1326
+ # Merge the gap data into messages (segments)
1327
+ message_starts, message_sizes = reconstruct_messages_multi(
1328
+ group_headers, group_times, freq_nhz=group_freq_nhz, period_ns=group_period_ns)
1329
+
1330
+ # Sort the message data
1331
+ sort_message_time_values(message_starts, message_sizes, group_values)
1332
+ # Revert back to gap array
1333
+ gap_data = create_gap_arr_from_variable_messages(
1334
+ message_starts, message_sizes, freq_nhz=group_freq_nhz, period_ns=group_period_ns)
1335
+
1336
+ group_times = gap_data
1337
+
1338
+ group_interval_data = get_interval_list_from_message_starts_and_sizes(
1339
+ message_starts, message_sizes, freq_nhz=group_freq_nhz, period_ns=group_period_ns)
1340
+
1341
+ group_start_time = int(group_interval_data[0][0])
1342
+ else:
1343
+ raise ValueError("time_type must be 1 or 2")
1344
+
1345
+ block_group_interval_data.append(group_interval_data)
1346
+
1347
+ if np.issubdtype(group_values.dtype, np.integer):
1348
+ raw_value_type = 1
1349
+ encoded_value_type = 3
1350
+ else:
1351
+ raw_value_type = 2
1352
+ encoded_value_type = 2
1353
+
1354
+ # Create segment dictionary
1355
+ segment = {
1356
+ 'times': group_times,
1357
+ 'values': group_values,
1358
+ 'start_ns': group_start_time,
1359
+ 'raw_time_type': group_time_type,
1360
+ 'encoded_time_type': group_encoded_time_type,
1361
+ 'raw_value_type': raw_value_type,
1362
+ 'encoded_value_type': encoded_value_type,
1363
+ 'scale_m': group_headers[0].scale_m,
1364
+ 'scale_b': group_headers[0].scale_b,
1365
+ 'freq_nhz': group_freq_nhz,
1366
+ 'period_ns': group_period_ns
1367
+ }
1368
+
1369
+ segments.append(segment)
1370
+
1371
+ # Now, call encode_blocks_from_multiple_segments
1372
+ encoded_bytes, encoded_headers, byte_start_array = (
1373
+ sdk.block.encode_blocks_from_multiple_segments(segments))
1374
+
1375
+ # Handle the outputs
1376
+ total_encoded_bytes = encoded_bytes
1377
+ total_encoded_headers = encoded_headers
1378
+ total_byte_start_array = byte_start_array
1379
+
1380
+ # Process the intervals
1381
+ block_group_interval_data = intervals_union_list(block_group_interval_data)
1382
+
1383
+ # Write the encoded bytes to disk
1384
+ filename = sdk.file_api.write_bytes(measure_id, device_id, total_encoded_bytes)
1385
+
1386
+ block_data = []
1387
+ for header_i, header in enumerate(total_encoded_headers):
1388
+ block_data.append({
1389
+ "measure_id": measure_id,
1390
+ "device_id": device_id,
1391
+ "start_byte": int(total_byte_start_array[header_i]),
1392
+ "num_bytes": header.meta_num_bytes + header.t_num_bytes + header.v_num_bytes,
1393
+ "start_time_n": header.start_n,
1394
+ "end_time_n": header.end_n,
1395
+ "num_values": header.num_vals,
1396
+ })
1397
+
1398
+ # Write new and delete old SQL blocks
1399
+ sdk.sql_handler.insert_and_delete_tsc_file_data(
1400
+ filename, block_data, [block[0] for block in block_group])
1401
+
1402
+ total_measure_device_interval_array.append(block_group_interval_data)
1403
+
1404
+ # Combine collected intervals and replace the old intervals with the collected.
1405
+ total_measure_device_interval_array = intervals_union_list(
1406
+ total_measure_device_interval_array, gap_tolerance_nano=interval_gap_tolerance_nano)
1407
+
1408
+ sdk.sql_handler.replace_intervals(measure_id, device_id, total_measure_device_interval_array)
1409
+
1410
+ # Remove TSC files from SQL and disk
1411
+ sdk.sql_handler.delete_files_by_ids(file_id_list)
1412
+
1413
+ for filename in filename_dict.values():
1414
+ file_path = sdk.file_api.to_abs_path(filename, measure_id, device_id)
1415
+ sdk.file_api.remove(file_path)
1416
+
1417
+ sdk.block.block_size = original_block_size
1418
+
1419
+
1420
+ def rebuild_intervals_from_existing_blocks(sdk, interval_gap_tolerance_nano=0, max_values_in_ram=1_000_000):
1421
+ """
1422
+ Rebuilds intervals for all measures and devices based on existing block data.
1423
+ """
1424
+ measures = sdk.get_all_measures()
1425
+ devices = sdk.get_all_devices()
1426
+
1427
+ measure_data = list(measures.items())
1428
+ device_data = list(devices.items())
1429
+
1430
+ for device_id, device_info in tqdm(device_data, desc="Devices"):
1431
+ for measure_id, measure_info in tqdm(measure_data, desc="Measures"):
1432
+ sorted_block_list_by_time = get_all_blocks(sdk, measure_id, device_id)
1433
+ if len(sorted_block_list_by_time) == 0:
1434
+ continue
1435
+
1436
+ file_id_list = list(set([row[3] for row in sorted_block_list_by_time]))
1437
+ filename_dict = sdk.get_filename_dict(file_id_list)
1438
+
1439
+ total_measure_device_interval_array = []
1440
+
1441
+ # Process all blocks to extract interval information
1442
+ for block_group in group_sorted_block_list(
1443
+ sorted_block_list_by_time, num_values_per_group=max_values_in_ram):
1444
+
1445
+ r_headers, r_times, r_values = sdk.get_data_from_blocks(
1446
+ block_list=block_group,
1447
+ filename_dict=filename_dict,
1448
+ start_time_n=0,
1449
+ end_time_n=2 ** 62,
1450
+ analog=False,
1451
+ time_type="encoded",
1452
+ sort=False,
1453
+ allow_duplicates=True
1454
+ )
1455
+
1456
+ if r_values.size == 0:
1457
+ continue
1458
+
1459
+ block_interval_data = []
1460
+
1461
+ # Group data by scale factor, freq, time type
1462
+ for group_headers, group_times, group_values in group_headers_by_scale_factor_freq_time_type(
1463
+ r_headers, r_times, r_values):
1464
+ group_time_type = int(group_headers[0].t_raw_type)
1465
+ period_ns = int(get_period_ns_from_header(group_headers[0]))
1466
+
1467
+ # Calculate intervals based on time type
1468
+ if group_time_type == 1:
1469
+ group_times, sorted_time_indices = np.unique(group_times, return_index=True)
1470
+ group_interval_data = get_interval_list_from_ordered_timestamps(group_times, period_ns)
1471
+ elif group_time_type == 2:
1472
+ # Merge the gap data into messages (segments)
1473
+ message_starts, message_sizes = reconstruct_messages_multi(
1474
+ group_headers, group_times, period_ns=period_ns)
1475
+ group_interval_data = get_interval_list_from_message_starts_and_sizes(
1476
+ message_starts, message_sizes, period_ns=period_ns)
1477
+ else:
1478
+ raise ValueError("time_type must be 1 or 2")
1479
+
1480
+ block_interval_data.append(group_interval_data)
1481
+
1482
+ # Union intervals within this block
1483
+ block_interval_data = intervals_union_list(block_interval_data)
1484
+ total_measure_device_interval_array.append(block_interval_data)
1485
+
1486
+ # Combine all collected intervals for this measure/device
1487
+ total_measure_device_interval_array = intervals_union_list(
1488
+ total_measure_device_interval_array, gap_tolerance_nano=interval_gap_tolerance_nano)
1489
+
1490
+ # Replace the intervals in the database
1491
+ sdk.sql_handler.replace_intervals(measure_id, device_id, total_measure_device_interval_array)
1492
+
1493
+ def inplace_block_time_fix(
1494
+ sdk,
1495
+ batch_size: int = 500
1496
+ ) -> int:
1497
+ """
1498
+ Scan every block in the SDK’s block_index table, decode it (time_type=1),
1499
+ and if the decoded start/end times differ from what the SQL row reports,
1500
+ update them in batches via SQLHandler.update_block_times.
1501
+
1502
+ :param AtriumSDK sdk: the SDK instance to fix
1503
+ :param int batch_size: how many mismatches to accumulate before flushing
1504
+ :return: total number of blocks updated
1505
+ """
1506
+ block_rows = []
1507
+ for device_id in sdk.get_all_devices():
1508
+ cur_block_rows = sdk.sql_handler.select_blocks_for_device(device_id)
1509
+ block_rows.extend(cur_block_rows)
1510
+
1511
+ total_rows = len(block_rows)
1512
+ with tqdm(total=total_rows, desc="Fixing Block Times", unit="block") as pbar:
1513
+ for block_group in group_sorted_block_list(
1514
+ block_rows,
1515
+ num_values_per_group=sdk.block.block_size * batch_size,
1516
+ src_sdk_mode=sdk.mode
1517
+ ):
1518
+ file_id_list = list({row[3] for row in block_group})
1519
+ filename_dict = sdk.get_filename_dict(file_id_list)
1520
+ read_list = condense_byte_read_list(block_group)
1521
+ encoded_bytes = sdk.file_api.read_file_list(read_list, filename_dict)
1522
+ block_num_bytes = [block[5] for block in block_group]
1523
+ block_num_values = [block[8] for block in block_group]
1524
+
1525
+ encode_group_times, encode_group_values, _ = sdk.block.decode_blocks(
1526
+ encoded_bytes, block_num_bytes, analog=False, time_type=1)
1527
+
1528
+ left = 0
1529
+ right = None
1530
+ to_fix_ids = []
1531
+ to_fix_ranges = []
1532
+ for block in block_group:
1533
+ right = left + block[8]
1534
+ block_times = encode_group_times[left:right]
1535
+ true_start = int(np.min(block_times))
1536
+ true_end = int(np.max(block_times))
1537
+ orig_start = int(block[6])
1538
+ orig_end = int(block[7])
1539
+ blk_id = int(block[0])
1540
+
1541
+ if true_start != orig_start or true_end != orig_end:
1542
+ to_fix_ids.append(blk_id)
1543
+ to_fix_ranges.append((true_start, true_end))
1544
+ left = right
1545
+
1546
+ if to_fix_ids:
1547
+ sdk.sql_handler.update_block_times(to_fix_ids, to_fix_ranges)
1548
+
1549
+ pbar.update(len(block_group))
1550
+
1551
+
1552
+ def group_sorted_block_list(sorted_block_list, num_values_per_group=8388608, src_sdk_mode=None):
1553
+ next_group = []
1554
+ total_group_size = 0
1555
+
1556
+ for block in sorted_block_list:
1557
+ if total_group_size >= num_values_per_group:
1558
+ yield next_group
1559
+ next_group = []
1560
+ total_group_size = 0
1561
+
1562
+ num_values = block["num_values"] if src_sdk_mode == "api" else block[8]
1563
+ next_group.append(block)
1564
+ total_group_size += num_values
1565
+
1566
+ if total_group_size > 0:
1567
+ yield next_group
1568
+
1569
+
1570
+ def group_headers_by_scale_factor_freq_time_type(headers, times_array, values_array):
1571
+ if len(headers) == 0:
1572
+ return
1573
+
1574
+ # Initialize the first header group variables
1575
+ start_idx_vals = 0
1576
+ start_idx_times = 0
1577
+ current_scale_m = headers[0].scale_m
1578
+ current_scale_b = headers[0].scale_b
1579
+ current_t_raw_type = headers[0].t_raw_type
1580
+ current_freq_nhz = headers[0].freq_nhz
1581
+ current_t_encoded_type = headers[0].t_encoded_type
1582
+ current_group = [headers[0]]
1583
+
1584
+ for h in headers[1:]:
1585
+ # Check if header properties match the current group's properties
1586
+ if (h.scale_m == current_scale_m and
1587
+ h.scale_b == current_scale_b and
1588
+ h.t_raw_type == current_t_raw_type and
1589
+ h.freq_nhz == current_freq_nhz and
1590
+ h.t_encoded_type == current_t_encoded_type):
1591
+ # Same grouping factors, add to current group
1592
+ current_group.append(h)
1593
+ else:
1594
+ # Different grouping factors, yield the current group
1595
+ end_idx_vals = start_idx_vals + sum(header.num_vals for header in current_group)
1596
+ if current_t_raw_type == 1:
1597
+ end_idx_times = start_idx_times + sum(header.num_vals for header in current_group)
1598
+ elif current_t_raw_type == 2:
1599
+ end_idx_times = start_idx_times + sum(header.num_gaps * 2 for header in current_group)
1600
+ else:
1601
+ raise ValueError(f"Unknown t_raw_type: {current_t_raw_type}")
1602
+
1603
+ yield (current_group,
1604
+ times_array[start_idx_times:end_idx_times],
1605
+ values_array[start_idx_vals:end_idx_vals])
1606
+
1607
+ # Start a new group
1608
+ start_idx_vals = end_idx_vals
1609
+ start_idx_times = end_idx_times
1610
+ current_scale_m = h.scale_m
1611
+ current_scale_b = h.scale_b
1612
+ current_t_raw_type = h.t_raw_type
1613
+ current_freq_nhz = h.freq_nhz
1614
+ current_t_encoded_type = h.t_encoded_type
1615
+ current_group = [h]
1616
+
1617
+ # After the loop, yield any remaining group
1618
+ if current_group:
1619
+ end_idx_vals = start_idx_vals + sum(header.num_vals for header in current_group)
1620
+ if current_t_raw_type == 1:
1621
+ end_idx_times = start_idx_times + sum(header.num_vals for header in current_group)
1622
+ elif current_t_raw_type == 2:
1623
+ end_idx_times = start_idx_times + sum(header.num_gaps * 2 for header in current_group)
1624
+ else:
1625
+ raise ValueError(f"Unknown t_raw_type: {current_t_raw_type}")
1626
+
1627
+ yield (current_group,
1628
+ times_array[start_idx_times:end_idx_times],
1629
+ values_array[start_idx_vals:end_idx_vals])
1630
+
1631
+
1632
+ def get_interval_list_from_ordered_timestamps(timestamps, period_ns):
1633
+ if timestamps.size == 0:
1634
+ return []
1635
+
1636
+ diffs = np.diff(timestamps)
1637
+ break_indices = np.where(diffs > period_ns)[0]
1638
+
1639
+ # Start indices of each region ([0] + break_indices + 1)
1640
+ start_indices = np.insert(break_indices + 1, 0, 0)
1641
+ # End indices of each region (break_indices + [len(timestamps) - 1])
1642
+ end_indices = np.append(break_indices, timestamps.size - 1)
1643
+
1644
+ start_times = timestamps[start_indices]
1645
+ end_times = timestamps[end_indices] + period_ns
1646
+
1647
+ regions = np.column_stack((start_times, end_times))
1648
+
1649
+ return regions
1650
+
1651
+
1652
+ def get_interval_list_from_message_starts_and_sizes(message_starts, message_sizes, freq_nhz=None, *, period_ns=None):
1653
+ _validate_freq_period_params(freq_nhz, period_ns)
1654
+
1655
+ if message_starts.size == 0:
1656
+ return np.array([], dtype=np.int64)
1657
+
1658
+ merged_intervals = []
1659
+ current_start = message_starts[0]
1660
+
1661
+ if freq_nhz is not None:
1662
+ current_end = current_start + _message_size_freq_to_duration_ns(int(message_sizes[0]), freq_nhz)
1663
+ else:
1664
+ current_end = current_start + _message_size_period_to_durations_ns(int(message_sizes[0]), period_ns)
1665
+
1666
+ for start, message_size in zip(message_starts[1:], message_sizes[1:]):
1667
+ if freq_nhz is not None:
1668
+ end = start + _message_size_freq_to_duration_ns(int(message_size), freq_nhz)
1669
+ else:
1670
+ end = start + _message_size_period_to_durations_ns(int(message_size), period_ns)
1671
+
1672
+ if start <= current_end:
1673
+ # Overlapping intervals, update the end if needed
1674
+ current_end = max(current_end, end)
1675
+ else:
1676
+ # No overlap, append the current interval and reset
1677
+ merged_intervals.append([current_start, current_end])
1678
+ current_start, current_end = start, end
1679
+ merged_intervals.append([current_start, current_end])
1680
+
1681
+ return np.array(merged_intervals, dtype=np.int64)
1682
+
1683
+
1684
+ def _validate_freq_period_params(freq_nhz, period_ns):
1685
+ if freq_nhz is not None and period_ns is not None:
1686
+ raise ValueError("freq_nhz and period_ns are mutually exclusive")
1687
+ if freq_nhz is None and period_ns is None:
1688
+ raise ValueError("Either freq_nhz or period_ns must be provided")