atriumdb 2.7.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atriumdb/__init__.py +37 -0
- atriumdb/adb_functions.py +1688 -0
- atriumdb/adb_remote.py +32 -0
- atriumdb/atrium_sdk.py +6184 -0
- atriumdb/bin/libTSC.dll +0 -0
- atriumdb/binary_signal_converter.py +35 -0
- atriumdb/block.py +963 -0
- atriumdb/block_wrapper.py +297 -0
- atriumdb/cli/__init__.py +17 -0
- atriumdb/cli/atriumdb_cli.py +462 -0
- atriumdb/cli/hello.py +23 -0
- atriumdb/cli/sdk.py +59 -0
- atriumdb/file_api.py +191 -0
- atriumdb/helpers/__init__.py +30 -0
- atriumdb/helpers/block_calculations.py +105 -0
- atriumdb/helpers/block_constants.py +26 -0
- atriumdb/helpers/config.toml +8 -0
- atriumdb/helpers/mariadb_env_cred.py +22 -0
- atriumdb/helpers/settings.py +22 -0
- atriumdb/helpers/sql_helpers.py +23 -0
- atriumdb/helpers/sql_tables.py +99 -0
- atriumdb/helpers/sql_wait_wrapper.py +38 -0
- atriumdb/helpers/type_helpers.py +85 -0
- atriumdb/interval_overlap.py +119 -0
- atriumdb/intervals/__init__.py +18 -0
- atriumdb/intervals/compact.py +24 -0
- atriumdb/intervals/difference.py +28 -0
- atriumdb/intervals/intersection.py +51 -0
- atriumdb/intervals/intervals.py +250 -0
- atriumdb/intervals/union.py +65 -0
- atriumdb/mem_cache.py +34 -0
- atriumdb/pytorch_integrations.py +116 -0
- atriumdb/sql_handler/__init__.py +17 -0
- atriumdb/sql_handler/maria/__init__.py +17 -0
- atriumdb/sql_handler/maria/maria_functions.py +73 -0
- atriumdb/sql_handler/maria/maria_handler.py +826 -0
- atriumdb/sql_handler/maria/maria_tables.py +355 -0
- atriumdb/sql_handler/sql_constants.py +19 -0
- atriumdb/sql_handler/sql_handler.py +1245 -0
- atriumdb/sql_handler/sql_helper.py +24 -0
- atriumdb/sql_handler/sqlite/__init__.py +17 -0
- atriumdb/sql_handler/sqlite/sqlite_functions.py +86 -0
- atriumdb/sql_handler/sqlite/sqlite_handler.py +774 -0
- atriumdb/sql_handler/sqlite/sqlite_tables.py +338 -0
- atriumdb/transfer/__init__.py +17 -0
- atriumdb/transfer/adb/__init__.py +17 -0
- atriumdb/transfer/adb/csv.py +47 -0
- atriumdb/transfer/adb/dataset.py +635 -0
- atriumdb/transfer/adb/datestring_conversion.py +46 -0
- atriumdb/transfer/adb/definition.py +133 -0
- atriumdb/transfer/adb/devices.py +45 -0
- atriumdb/transfer/adb/labels.py +29 -0
- atriumdb/transfer/adb/measures.py +74 -0
- atriumdb/transfer/adb/numpy.py +21 -0
- atriumdb/transfer/adb/parquet.py +48 -0
- atriumdb/transfer/adb/patients.py +398 -0
- atriumdb/transfer/adb/tsc.py +44 -0
- atriumdb/transfer/adb/wfdb.py +34 -0
- atriumdb/transfer/cohort/__init__.py +17 -0
- atriumdb/transfer/cohort/cohort_yaml.py +46 -0
- atriumdb/transfer/formats/__init__.py +17 -0
- atriumdb/transfer/formats/dataset.py +116 -0
- atriumdb/transfer/formats/export_data.py +100 -0
- atriumdb/transfer/formats/formats.py +19 -0
- atriumdb/transfer/formats/import_data.py +97 -0
- atriumdb/transfer/formats/json_file_metadata.py +54 -0
- atriumdb/windowing/__init__.py +16 -0
- atriumdb/windowing/cross_validation.py +282 -0
- atriumdb/windowing/dataset_iterator.py +424 -0
- atriumdb/windowing/definition.py +864 -0
- atriumdb/windowing/definition_builder.py +163 -0
- atriumdb/windowing/definition_combine.py +256 -0
- atriumdb/windowing/definition_splitter.py +711 -0
- atriumdb/windowing/filtered_iterator.py +101 -0
- atriumdb/windowing/map_definition_sources.py +119 -0
- atriumdb/windowing/mapped_iterator.py +309 -0
- atriumdb/windowing/verify_definition.py +292 -0
- atriumdb/windowing/window.py +75 -0
- atriumdb/windowing/window_config.py +130 -0
- atriumdb/windowing/windowing_functions.py +266 -0
- atriumdb/write_buffer.py +119 -0
- atriumdb-2.7.0.dist-info/METADATA +143 -0
- atriumdb-2.7.0.dist-info/RECORD +86 -0
- atriumdb-2.7.0.dist-info/WHEEL +5 -0
- atriumdb-2.7.0.dist-info/entry_points.txt +4 -0
- atriumdb-2.7.0.dist-info/licenses/LICENSE +226 -0
|
@@ -0,0 +1,1688 @@
|
|
|
1
|
+
# AtriumDB is a timeseries database software designed to best handle the unique features and
|
|
2
|
+
# challenges that arise from clinical waveform data.
|
|
3
|
+
# Copyright (C) 2023 The Hospital for Sick Children
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
import warnings
|
|
18
|
+
import os
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
from tqdm import tqdm
|
|
22
|
+
import time
|
|
23
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
24
|
+
from numpy.lib.stride_tricks import sliding_window_view
|
|
25
|
+
from atriumdb.helpers.block_calculations import freq_nhz_to_period_ns
|
|
26
|
+
import logging
|
|
27
|
+
import bisect
|
|
28
|
+
|
|
29
|
+
from atriumdb.helpers.block_calculations import calc_time_by_freq
|
|
30
|
+
from atriumdb.helpers.block_constants import TIME_TYPES
|
|
31
|
+
from atriumdb.intervals.union import intervals_union_list
|
|
32
|
+
from atriumdb.block_wrapper import get_period_ns_from_header
|
|
33
|
+
|
|
34
|
+
ALLOWED_TIME_TYPES = [1, 2]
|
|
35
|
+
time_unit_options = {
|
|
36
|
+
"s": 1_000_000_000,
|
|
37
|
+
"ms": 1_000_000,
|
|
38
|
+
"us": 1_000,
|
|
39
|
+
"ns": 1
|
|
40
|
+
}
|
|
41
|
+
freq_unit_options = {
|
|
42
|
+
"MHz": 1_000_000_000_000_000,
|
|
43
|
+
"kHz": 1_000_000_000_000,
|
|
44
|
+
"Hz": 1_000_000_000,
|
|
45
|
+
"mHz": 1_000_000,
|
|
46
|
+
"uHz": 1_000,
|
|
47
|
+
"nHz": 1
|
|
48
|
+
}
|
|
49
|
+
allowed_interval_index_modes = ["fast", "merge", "disable"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def get_block_and_interval_data(measure_id, device_id, metadata, start_bytes, intervals, interval_gap_tolerance=0):
|
|
53
|
+
block_data = []
|
|
54
|
+
for header_i, header in enumerate(metadata):
|
|
55
|
+
block_data.append({
|
|
56
|
+
"measure_id": measure_id,
|
|
57
|
+
"device_id": device_id,
|
|
58
|
+
"start_byte": int(start_bytes[header_i]),
|
|
59
|
+
"num_bytes": header.meta_num_bytes + header.t_num_bytes + header.v_num_bytes,
|
|
60
|
+
"start_time_n": header.start_n,
|
|
61
|
+
"end_time_n": header.end_n,
|
|
62
|
+
"num_values": header.num_vals,
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
interval_data = []
|
|
66
|
+
intervals = sorted(intervals)
|
|
67
|
+
for interval in intervals:
|
|
68
|
+
if interval_data and int(interval[0]) - interval_data[-1]["end_time_n"] <= interval_gap_tolerance:
|
|
69
|
+
interval_data[-1]["end_time_n"] = int(interval[1])
|
|
70
|
+
else:
|
|
71
|
+
interval_data.append({
|
|
72
|
+
"measure_id": measure_id,
|
|
73
|
+
"device_id": device_id,
|
|
74
|
+
"start_time_n": int(interval[0]),
|
|
75
|
+
"end_time_n": int(interval[1]),
|
|
76
|
+
})
|
|
77
|
+
return block_data, interval_data
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def condense_byte_read_list(block_list):
|
|
81
|
+
result = []
|
|
82
|
+
|
|
83
|
+
for row in block_list:
|
|
84
|
+
if len(result) == 0 or result[-1][2] != row[3] or result[-1][3] + result[-1][4] != row[4]:
|
|
85
|
+
# append measure_id, device_id, file_id, start_byte and num_bytes
|
|
86
|
+
result.append([row[1], row[2], row[3], row[4], row[5]])
|
|
87
|
+
else:
|
|
88
|
+
# if the blocks are continuous merge the reads together by adding the size of the next block to the
|
|
89
|
+
# num_bytes field
|
|
90
|
+
result[-1][4] += row[5]
|
|
91
|
+
|
|
92
|
+
return result
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def find_intervals(freq_nhz=None, raw_time_type=None, time_data=None, data_start_time=None, num_values=None, *, period_ns=None):
|
|
96
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
97
|
+
if freq_nhz is not None:
|
|
98
|
+
current_period_ns = int((10 ** 18) / freq_nhz)
|
|
99
|
+
else:
|
|
100
|
+
current_period_ns = period_ns
|
|
101
|
+
|
|
102
|
+
if raw_time_type == TIME_TYPES['TIME_ARRAY_INT64_NS']:
|
|
103
|
+
intervals = [[time_data[0], 0]]
|
|
104
|
+
time_deltas = time_data[1:] - time_data[:-1]
|
|
105
|
+
for time_arr_i in range(time_data.size - 1):
|
|
106
|
+
if time_deltas[time_arr_i] > current_period_ns:
|
|
107
|
+
intervals[-1][-1] = time_data[time_arr_i] + current_period_ns
|
|
108
|
+
intervals.append([time_data[time_arr_i + 1], 0])
|
|
109
|
+
|
|
110
|
+
intervals[-1][-1] = time_data[-1] + current_period_ns
|
|
111
|
+
|
|
112
|
+
elif raw_time_type == TIME_TYPES['START_TIME_NUM_SAMPLES']:
|
|
113
|
+
intervals = [[time_data[0], time_data[0] + ((time_data[1] - 1) * current_period_ns)]]
|
|
114
|
+
|
|
115
|
+
for interval_data_i in range(1, time_data.size // 2):
|
|
116
|
+
start_time = time_data[2 * interval_data_i]
|
|
117
|
+
end_time = time_data[2 * interval_data_i] + ((time_data[(2 * interval_data_i) + 1] - 1) * current_period_ns)
|
|
118
|
+
|
|
119
|
+
if start_time <= intervals[-1][-1] + current_period_ns:
|
|
120
|
+
intervals[-1][-1] = end_time
|
|
121
|
+
else:
|
|
122
|
+
intervals.append([start_time, end_time])
|
|
123
|
+
|
|
124
|
+
elif raw_time_type == TIME_TYPES['GAP_ARRAY_INT64_INDEX_DURATION_NS']:
|
|
125
|
+
if freq_nhz is not None:
|
|
126
|
+
intervals = [[data_start_time, data_start_time + calc_time_by_freq(freq_nhz, num_values)]]
|
|
127
|
+
else:
|
|
128
|
+
intervals = [[data_start_time, data_start_time + calc_time_by_period(period_ns, num_values)]]
|
|
129
|
+
last_id = 0
|
|
130
|
+
|
|
131
|
+
for sample_id, duration in time_data.reshape((-1, 2)):
|
|
132
|
+
if freq_nhz is not None:
|
|
133
|
+
intervals[-1][-1] = intervals[-1][0] + calc_time_by_freq(freq_nhz, sample_id - last_id)
|
|
134
|
+
else:
|
|
135
|
+
intervals[-1][-1] = intervals[-1][0] + calc_time_by_period(period_ns, sample_id - last_id)
|
|
136
|
+
last_id = sample_id
|
|
137
|
+
|
|
138
|
+
if freq_nhz is not None:
|
|
139
|
+
next_duration = calc_time_by_freq(freq_nhz, num_values - last_id)
|
|
140
|
+
else:
|
|
141
|
+
next_duration = calc_time_by_period(period_ns, num_values - last_id)
|
|
142
|
+
|
|
143
|
+
intervals.append([intervals[-1][-1] + duration,
|
|
144
|
+
intervals[-1][-1] + duration + next_duration])
|
|
145
|
+
|
|
146
|
+
else:
|
|
147
|
+
raise ValueError("raw_time_type not one of {}.".format(
|
|
148
|
+
[TIME_TYPES['TIME_ARRAY_INT64_NS'], TIME_TYPES['START_TIME_NUM_SAMPLES']]))
|
|
149
|
+
|
|
150
|
+
return intervals
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# if you want to just use this to sort data will have to add default vals for start/end time and skip bisect
|
|
154
|
+
def sort_data(times, values, headers, start_time, end_time, allow_duplicates=True):
|
|
155
|
+
start_bench = time.perf_counter()
|
|
156
|
+
if len(headers) == 0:
|
|
157
|
+
return times, values
|
|
158
|
+
|
|
159
|
+
block_info = np.zeros((len(headers), 4), dtype=np.int64)
|
|
160
|
+
block_info[:] = [[h.start_n, h.end_n, h.num_vals, 0] for h in headers]
|
|
161
|
+
np.cumsum(block_info.T[2], out=block_info.T[3])
|
|
162
|
+
block_info.T[2] = block_info.T[3] - block_info.T[2]
|
|
163
|
+
|
|
164
|
+
end_bench = time.perf_counter()
|
|
165
|
+
logging.debug(f"rearrange block data info {(end_bench - start_bench) * 1000} ms")
|
|
166
|
+
|
|
167
|
+
# check if the start times are sorted
|
|
168
|
+
start_times_sorted = np.all(np.greater(block_info.T[0][1:], block_info.T[0][:-1]))
|
|
169
|
+
# check if the blocks overlap each other
|
|
170
|
+
start_end_times_dont_intersect = np.all(np.greater_equal(block_info.T[0][1:], block_info.T[1][:-1]))
|
|
171
|
+
if start_end_times_dont_intersect and start_times_sorted:
|
|
172
|
+
logging.debug("Blocks already Sorted and don't intersect.")
|
|
173
|
+
|
|
174
|
+
left, right = bisect.bisect_left(times, start_time), bisect.bisect_left(times, end_time)
|
|
175
|
+
times, values = times[left:right], values[left:right]
|
|
176
|
+
|
|
177
|
+
# if duplicates are allowed then just return times and values
|
|
178
|
+
if allow_duplicates:
|
|
179
|
+
return times, values
|
|
180
|
+
# if duplicates are not allowed then remove them using np.unique
|
|
181
|
+
sorted_times, sorted_time_indices = np.unique(times, return_index=True)
|
|
182
|
+
return sorted_times, values[sorted_time_indices]
|
|
183
|
+
|
|
184
|
+
start_bench = time.perf_counter()
|
|
185
|
+
# if the start times were not sorted, sort them if they were then don't bother running the sort
|
|
186
|
+
if not start_times_sorted:
|
|
187
|
+
# use quicksort as it will be faster with smaller arrays
|
|
188
|
+
sorted_block_i = np.argsort(block_info.T[0], kind='quicksort')
|
|
189
|
+
block_info = block_info[sorted_block_i]
|
|
190
|
+
# recheck if the blocks intersect
|
|
191
|
+
start_end_times_dont_intersect = np.all(np.greater_equal(block_info.T[0][1:], block_info.T[1][:-1]))
|
|
192
|
+
|
|
193
|
+
end_bench = time.perf_counter()
|
|
194
|
+
logging.debug(f"sort data by block {(end_bench - start_bench) * 1000} ms")
|
|
195
|
+
|
|
196
|
+
if allow_duplicates:
|
|
197
|
+
if start_end_times_dont_intersect:
|
|
198
|
+
# Blocks don't intersect each other.
|
|
199
|
+
start_bench = time.perf_counter()
|
|
200
|
+
|
|
201
|
+
# Original Index Creation
|
|
202
|
+
sorted_time_indices = np.concatenate([np.arange(i_start, i_end) for _, _, i_start, i_end in block_info])
|
|
203
|
+
|
|
204
|
+
times, values = times[sorted_time_indices], values[sorted_time_indices]
|
|
205
|
+
end_bench = time.perf_counter()
|
|
206
|
+
logging.debug(f"Blocks don't intersect, sort by blocks {(end_bench - start_bench) * 1000} ms")
|
|
207
|
+
|
|
208
|
+
else:
|
|
209
|
+
# Blocks do intersect each other, so sort every value.
|
|
210
|
+
start_bench = time.perf_counter()
|
|
211
|
+
|
|
212
|
+
# If allowing duplicates use argsort to get the indices so duplicates aren't removed
|
|
213
|
+
# Use mergesort as it will be faster than quicksort on large arrays and since its mostly sorted already
|
|
214
|
+
sorted_time_indices = np.argsort(times, kind='mergesort')
|
|
215
|
+
|
|
216
|
+
end_bench = time.perf_counter()
|
|
217
|
+
logging.debug(f"Blocks intersect, sort every value {(end_bench - start_bench) * 1000} ms")
|
|
218
|
+
|
|
219
|
+
times, values = times[sorted_time_indices], values[sorted_time_indices]
|
|
220
|
+
else:
|
|
221
|
+
# if duplicates are not allowed then remove them using np.unique
|
|
222
|
+
times, sorted_time_indices = np.unique(times, return_index=True)
|
|
223
|
+
values = values[sorted_time_indices]
|
|
224
|
+
|
|
225
|
+
# if the start or end time is in the middle of a block, more times/values will be decoded than are needed so
|
|
226
|
+
# bisect the array on the left and right sides to truncate the data
|
|
227
|
+
left, right = bisect.bisect_left(times, start_time), bisect.bisect_left(times, end_time)
|
|
228
|
+
times, values = times[left:right], values[left:right]
|
|
229
|
+
|
|
230
|
+
return times, values
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def yield_data(r_times, r_values, window_size, step_size, get_last_window, total_query_index):
|
|
234
|
+
if window_size is not None:
|
|
235
|
+
time_sliding_window_view = sliding_window_view(r_times, window_size)
|
|
236
|
+
value_sliding_window_view = sliding_window_view(r_values, window_size)
|
|
237
|
+
index_arr = np.arange(0, value_sliding_window_view.shape[0], step_size) + total_query_index
|
|
238
|
+
|
|
239
|
+
# print("r_values.size, window_size")
|
|
240
|
+
# print(r_values.size, window_size)
|
|
241
|
+
# print()
|
|
242
|
+
# print("index_arr.shape")
|
|
243
|
+
# print(index_arr.shape)
|
|
244
|
+
# print("value_sliding_window_view[::step_size, :].shape")
|
|
245
|
+
# print(value_sliding_window_view[::step_size, :].shape)
|
|
246
|
+
assert index_arr.size == value_sliding_window_view[::step_size, :].shape[0]
|
|
247
|
+
|
|
248
|
+
yield (index_arr,
|
|
249
|
+
time_sliding_window_view[::step_size, :],
|
|
250
|
+
value_sliding_window_view[::step_size, :])
|
|
251
|
+
|
|
252
|
+
if get_last_window and ((value_sliding_window_view.shape[0] - 1) % step_size != 0):
|
|
253
|
+
yield (np.array([(value_sliding_window_view.shape[0] - 1) + total_query_index], dtype=np.int64),
|
|
254
|
+
time_sliding_window_view[-1::, :],
|
|
255
|
+
value_sliding_window_view[-1::, :])
|
|
256
|
+
else:
|
|
257
|
+
yield total_query_index, r_times, r_values
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def convert_to_nanoseconds(data, time_units):
|
|
261
|
+
"""
|
|
262
|
+
Converts time data (array or scalar) to nanoseconds (int64).
|
|
263
|
+
"""
|
|
264
|
+
if time_units not in time_unit_options:
|
|
265
|
+
raise ValueError(f"Invalid time units. Expected one of: {list(time_unit_options.keys())}")
|
|
266
|
+
|
|
267
|
+
multiplier = time_unit_options[time_units]
|
|
268
|
+
|
|
269
|
+
# Handle scalar float/int (e.g., period)
|
|
270
|
+
if np.isscalar(data):
|
|
271
|
+
return int(round(data * multiplier))
|
|
272
|
+
|
|
273
|
+
# Copy to avoid altering user data
|
|
274
|
+
data_ns = data.copy() * multiplier
|
|
275
|
+
return np.around(data_ns).astype(np.int64)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def convert_from_nanoseconds(data_ns, target_units):
|
|
279
|
+
"""
|
|
280
|
+
Converts nanosecond data (array or scalar) back to target units.
|
|
281
|
+
Returns float for non-ns units, int for ns units.
|
|
282
|
+
"""
|
|
283
|
+
if target_units not in time_unit_options:
|
|
284
|
+
raise ValueError(f"Invalid target units. Expected one of: {list(time_unit_options.keys())}")
|
|
285
|
+
|
|
286
|
+
divisor = time_unit_options[target_units]
|
|
287
|
+
|
|
288
|
+
# If target is NS, return integer (no division needed)
|
|
289
|
+
if target_units == "ns":
|
|
290
|
+
if np.isscalar(data_ns):
|
|
291
|
+
return int(data_ns)
|
|
292
|
+
return data_ns.astype(np.int64)
|
|
293
|
+
|
|
294
|
+
# For other units, return float
|
|
295
|
+
return data_ns / divisor
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def convert_value_to_nanoseconds(time_value, time_units):
|
|
299
|
+
if time_units not in time_unit_options:
|
|
300
|
+
raise ValueError(f"Invalid time units. Expected one of {time_unit_options}.")
|
|
301
|
+
|
|
302
|
+
return round(time_value * time_unit_options[time_units])
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def convert_to_nanohz(freq, freq_units):
|
|
306
|
+
if freq_units not in freq_unit_options.keys():
|
|
307
|
+
raise ValueError("Invalid frequency units. Expected one of: %s" % freq_unit_options)
|
|
308
|
+
|
|
309
|
+
freq *= freq_unit_options[freq_units]
|
|
310
|
+
|
|
311
|
+
return round(freq)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def default_freq_units(freq, freq_units):
|
|
315
|
+
if freq is not None and freq_units is None:
|
|
316
|
+
warnings.warn("freq_units was not given, so freq is read as nHz. The default changes to Hz in "
|
|
317
|
+
"AtriumDB 2.8; pass freq_units explicitly.", FutureWarning, stacklevel=3)
|
|
318
|
+
return "nHz"
|
|
319
|
+
return freq_units
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def convert_from_nanohz(freq_nhz, freq_units):
|
|
323
|
+
if freq_units not in freq_unit_options.keys():
|
|
324
|
+
raise ValueError("Invalid frequency units. Expected one of: %s" % freq_unit_options)
|
|
325
|
+
|
|
326
|
+
freq = freq_nhz / freq_unit_options[freq_units]
|
|
327
|
+
|
|
328
|
+
if freq == np.floor(freq):
|
|
329
|
+
freq = int(freq)
|
|
330
|
+
|
|
331
|
+
return freq
|
|
332
|
+
|
|
333
|
+
def detect_period(times: np.ndarray, threshold_ratio: float = 0.3):
|
|
334
|
+
"""
|
|
335
|
+
Analyzes a time array to detect the dominant sampling period.
|
|
336
|
+
|
|
337
|
+
If fewer than 2 timestamps, defaults to 1_000_000_000 ns (1 second)
|
|
338
|
+
with a warning. If no dominant period (below threshold), uses the mode
|
|
339
|
+
delta as a best-effort estimate with a warning. Never returns -1.
|
|
340
|
+
"""
|
|
341
|
+
if len(times) < 2:
|
|
342
|
+
warnings.warn(
|
|
343
|
+
"Cannot detect period from fewer than 2 timestamps. "
|
|
344
|
+
"Defaulting period to 1,000,000,000 ns (1 second). "
|
|
345
|
+
"Consider explicitly providing 'period' or 'freq' for accurate results.",
|
|
346
|
+
UserWarning
|
|
347
|
+
)
|
|
348
|
+
return 1_000_000_000
|
|
349
|
+
|
|
350
|
+
# Calculate deltas
|
|
351
|
+
deltas = np.diff(times)
|
|
352
|
+
|
|
353
|
+
# Find the mode of the deltas
|
|
354
|
+
unique_deltas, counts = np.unique(deltas, return_counts=True)
|
|
355
|
+
max_count_index = np.argmax(counts)
|
|
356
|
+
mode_delta = unique_deltas[max_count_index]
|
|
357
|
+
mode_count = counts[max_count_index]
|
|
358
|
+
|
|
359
|
+
# Handle Duplicate Values (Mode is 0)
|
|
360
|
+
if mode_delta == 0:
|
|
361
|
+
warnings.warn(
|
|
362
|
+
f"Dominant delta is 0 ({mode_count} occurrences). "
|
|
363
|
+
"Duplicates detected. Redetecting period using unique timestamps.",
|
|
364
|
+
UserWarning
|
|
365
|
+
)
|
|
366
|
+
# Recursive call with unique, sorted timestamps
|
|
367
|
+
return detect_period(np.unique(times), threshold_ratio)
|
|
368
|
+
|
|
369
|
+
# Check against threshold - if below, warn but still return best estimate
|
|
370
|
+
total_intervals = len(deltas)
|
|
371
|
+
if (mode_count / total_intervals) < threshold_ratio:
|
|
372
|
+
warnings.warn(
|
|
373
|
+
f"Automatic period detection: no single time delta accounts for "
|
|
374
|
+
f">{threshold_ratio*100:.0f}% of intervals. "
|
|
375
|
+
f"Using best-effort estimate of {mode_delta} "
|
|
376
|
+
f"(mode of deltas, {mode_count}/{total_intervals} intervals). "
|
|
377
|
+
f"For more accurate results, explicitly provide 'period' or 'freq'.",
|
|
378
|
+
UserWarning
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
# Return based on input type
|
|
382
|
+
if np.issubdtype(times.dtype, np.integer):
|
|
383
|
+
return int(mode_delta)
|
|
384
|
+
|
|
385
|
+
return float(mode_delta)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def parse_metadata_uri(metadata_uri):
|
|
389
|
+
parsed_uri = urlsplit(metadata_uri)
|
|
390
|
+
|
|
391
|
+
if not parsed_uri.scheme or not parsed_uri.netloc:
|
|
392
|
+
raise ValueError(f"Invalid metadata_uri format: expected 'sqltype://username:password@host:port[/dbname]', got {metadata_uri}")
|
|
393
|
+
|
|
394
|
+
sqltype = parsed_uri.scheme
|
|
395
|
+
|
|
396
|
+
if not parsed_uri.username or not parsed_uri.password:
|
|
397
|
+
raise ValueError(f"Invalid metadata_uri format: expected 'sqltype://username:password@host:port[/dbname]', got {metadata_uri}")
|
|
398
|
+
|
|
399
|
+
username = parsed_uri.username
|
|
400
|
+
password = parsed_uri.password
|
|
401
|
+
|
|
402
|
+
if not parsed_uri.hostname or not parsed_uri.port:
|
|
403
|
+
raise ValueError(f"Invalid metadata_uri format: expected 'sqltype://username:password@host:port[/dbname]', got {metadata_uri}")
|
|
404
|
+
|
|
405
|
+
host = parsed_uri.hostname
|
|
406
|
+
port = parsed_uri.port
|
|
407
|
+
|
|
408
|
+
dbname = parsed_uri.path.lstrip('/')
|
|
409
|
+
|
|
410
|
+
return {
|
|
411
|
+
'sqltype': sqltype,
|
|
412
|
+
'user': username,
|
|
413
|
+
'password': password,
|
|
414
|
+
'host': host,
|
|
415
|
+
'port': int(port),
|
|
416
|
+
'database': dbname if dbname else None
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def generate_metadata_uri(metadata):
|
|
421
|
+
sqltype = metadata['sqltype']
|
|
422
|
+
username = metadata['user']
|
|
423
|
+
password = metadata['password']
|
|
424
|
+
host = metadata['host']
|
|
425
|
+
port = metadata['port']
|
|
426
|
+
dbname = metadata.get('database')
|
|
427
|
+
|
|
428
|
+
netloc = f"{username}:{password}@{host}:{port}"
|
|
429
|
+
path = f"/{dbname}" if dbname else ""
|
|
430
|
+
|
|
431
|
+
return urlunsplit((sqltype, netloc, path, "", ""))
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def convert_gap_data_to_timestamps(headers, r_times, r_values, start_time_n=None, end_time_n=None, sort=True, allow_duplicates=True):
|
|
435
|
+
"""
|
|
436
|
+
Convert gap data to timestamps.
|
|
437
|
+
|
|
438
|
+
:param headers: A list of headers containing information about the data.
|
|
439
|
+
:param r_times: An array of times in nanoseconds with gaps.
|
|
440
|
+
:param r_values: An array of values corresponding to the times in r_times.
|
|
441
|
+
:param start_time_n: The start time in nanoseconds. Only needs to be specified if sorting result.
|
|
442
|
+
:param end_time_n: The end time in nanoseconds. Only needs to be specified if sorting result.
|
|
443
|
+
:param bool sort: Whether to sort the returned data.
|
|
444
|
+
:param bool allow_duplicates: Whether to allow duplicate times in the sorted returned data if they exist. Does
|
|
445
|
+
nothing if sort is false.
|
|
446
|
+
:return: A tuple containing an array of timestamps and an array of values.
|
|
447
|
+
"""
|
|
448
|
+
# # Start performance benchmark
|
|
449
|
+
# start_bench = time.perf_counter()
|
|
450
|
+
|
|
451
|
+
# Check if all times are integers
|
|
452
|
+
is_int_times = all([(10 ** 18) % h.freq_nhz == 0 for h in headers])
|
|
453
|
+
|
|
454
|
+
# Set the data type for the full_timestamps array based on is_int_times
|
|
455
|
+
time_dtype = np.int64 if is_int_times else np.float64
|
|
456
|
+
|
|
457
|
+
# Create an empty array for the full timestamps
|
|
458
|
+
full_timestamps = np.zeros(r_values.size, dtype=time_dtype)
|
|
459
|
+
|
|
460
|
+
# Initialize the current index and gap variables
|
|
461
|
+
cur_index, cur_gap = 0, 0
|
|
462
|
+
|
|
463
|
+
# Fill full_timestamps array with times based on headers and gaps
|
|
464
|
+
if is_int_times:
|
|
465
|
+
# Loop through the headers and calculate the timestamps for integer times
|
|
466
|
+
for block_i, h in enumerate(headers):
|
|
467
|
+
period_ns = freq_nhz_to_period_ns(h.freq_nhz)
|
|
468
|
+
full_timestamps[cur_index:cur_index + h.num_vals] = \
|
|
469
|
+
np.arange(h.start_n, h.start_n + (h.num_vals * period_ns), period_ns)
|
|
470
|
+
|
|
471
|
+
# Apply the gaps to the timestamps
|
|
472
|
+
for _ in range(h.num_gaps):
|
|
473
|
+
full_timestamps[cur_index + r_times[cur_gap]:cur_index + h.num_vals] += r_times[cur_gap + 1]
|
|
474
|
+
cur_gap += 2
|
|
475
|
+
|
|
476
|
+
# Update the current index
|
|
477
|
+
cur_index += h.num_vals
|
|
478
|
+
else:
|
|
479
|
+
# Loop through the headers and calculate the timestamps for non-integer times
|
|
480
|
+
for block_i, h in enumerate(headers):
|
|
481
|
+
period_ns = float(10 ** 18) / float(h.freq_nhz)
|
|
482
|
+
full_timestamps[cur_index:cur_index + h.num_vals] = \
|
|
483
|
+
np.linspace(h.start_n, h.start_n + (h.num_vals * period_ns), num=h.num_vals, endpoint=False)
|
|
484
|
+
|
|
485
|
+
# Apply the gaps to the timestamps
|
|
486
|
+
for _ in range(h.num_gaps):
|
|
487
|
+
full_timestamps[cur_index + r_times[cur_gap]:cur_index + h.num_vals] += r_times[cur_gap + 1]
|
|
488
|
+
cur_gap += 2
|
|
489
|
+
|
|
490
|
+
# Update the current index
|
|
491
|
+
cur_index += h.num_vals
|
|
492
|
+
|
|
493
|
+
# # End performance benchmark and log the time taken
|
|
494
|
+
# end_bench = time.perf_counter()
|
|
495
|
+
# _LOGGER.debug(f"Expand Gap Data {(end_bench - start_bench) * 1000} ms")
|
|
496
|
+
|
|
497
|
+
# Sort the data based on the timestamps if sort is true
|
|
498
|
+
if sort and start_time_n is not None and end_time_n is not None:
|
|
499
|
+
r_times, r_values = sort_data(r_times, r_values, headers, start_time_n, end_time_n, allow_duplicates)
|
|
500
|
+
|
|
501
|
+
return full_timestamps, r_values
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def get_best_measure_id(sdk, measure_tag, freq, units, freq_units):
|
|
505
|
+
measure_dict = {'tag': measure_tag}
|
|
506
|
+
if freq is not None:
|
|
507
|
+
freq_units = default_freq_units(freq, freq_units)
|
|
508
|
+
if freq and freq_units and freq_units != "nHz":
|
|
509
|
+
freq = convert_to_nanohz(freq, freq_units)
|
|
510
|
+
measure_dict['freq_nhz'] = freq
|
|
511
|
+
if units is not None:
|
|
512
|
+
measure_dict['units'] = units
|
|
513
|
+
measure_id_list = get_measure_id_from_generic_measure(sdk, measure_dict, measure_tag_match_rule="best")
|
|
514
|
+
if len(measure_id_list) == 0:
|
|
515
|
+
raise ValueError(f"No matching measure found for: {measure_dict}")
|
|
516
|
+
new_measure_id = measure_id_list[0]
|
|
517
|
+
return new_measure_id
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def get_measure_id_from_generic_measure(sdk, measure, measure_tag_match_rule="best", freq_units="Hz"):
|
|
521
|
+
measure_ids = []
|
|
522
|
+
|
|
523
|
+
if isinstance(measure, int):
|
|
524
|
+
measure_ids.append(measure)
|
|
525
|
+
elif isinstance(measure, str):
|
|
526
|
+
matching_ids = sdk.get_measure_id_list_from_tag(measure, approx=True)
|
|
527
|
+
if len(matching_ids) == 0:
|
|
528
|
+
raise ValueError(f"No matching measures found for: {measure}")
|
|
529
|
+
if measure_tag_match_rule == "best":
|
|
530
|
+
measure_ids.append(matching_ids[0])
|
|
531
|
+
elif measure_tag_match_rule == "all":
|
|
532
|
+
measure_ids.extend(matching_ids)
|
|
533
|
+
elif isinstance(measure, dict):
|
|
534
|
+
assert 'tag' in measure, "tag not in measure dictionary"
|
|
535
|
+
freq = measure.get('freq_nhz') or measure.get('freq_hz')
|
|
536
|
+
units = measure.get('units')
|
|
537
|
+
dict_freq_units = "nHz" if 'freq_nhz' in measure else "Hz" if 'freq_hz' in measure else None
|
|
538
|
+
|
|
539
|
+
if freq and units:
|
|
540
|
+
# Use sdk.get_measure_id for a unique match when both freq and units are specified
|
|
541
|
+
measure_id = sdk.get_measure_id(measure['tag'], freq=freq, units=units, freq_units=dict_freq_units)
|
|
542
|
+
if measure_id is not None:
|
|
543
|
+
measure_ids.append(measure_id)
|
|
544
|
+
else:
|
|
545
|
+
raise ValueError(f"No unique measure found for {measure['tag']} with specified frequency and units.")
|
|
546
|
+
else:
|
|
547
|
+
# Use get_measure_id_list_from_tag when either freq or units is not specified
|
|
548
|
+
matching_ids = sdk.get_measure_id_list_from_tag(measure['tag'], approx=True)
|
|
549
|
+
if len(matching_ids) == 0:
|
|
550
|
+
raise ValueError(f"No matching measures found for: {measure['tag']}")
|
|
551
|
+
if measure_tag_match_rule == "best":
|
|
552
|
+
measure_ids.append(matching_ids[0])
|
|
553
|
+
elif measure_tag_match_rule == "all":
|
|
554
|
+
measure_ids.extend(matching_ids)
|
|
555
|
+
elif isinstance(measure, tuple):
|
|
556
|
+
tag, freq, units = measure
|
|
557
|
+
measure_id = sdk.get_measure_id(tag, freq=freq, units=units, freq_units=freq_units)
|
|
558
|
+
if measure_id is not None:
|
|
559
|
+
measure_ids.append(measure_id)
|
|
560
|
+
else:
|
|
561
|
+
raise ValueError(f"No unique measure found for tuple: {measure} assuming freq_units = {freq_units}.")
|
|
562
|
+
else:
|
|
563
|
+
raise ValueError(f"measure type {type(measure)} not supported.")
|
|
564
|
+
|
|
565
|
+
if not measure_ids:
|
|
566
|
+
raise ValueError(f"Measure {measure} not found or multiple matches found without a clear selection rule.")
|
|
567
|
+
|
|
568
|
+
return measure_ids
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def collect_all_descendant_ids(label_set_ids, sql_handler):
|
|
572
|
+
"""
|
|
573
|
+
Collects all unique descendant label set IDs for a given list of label set IDs,
|
|
574
|
+
along with their closest requested ancestor ID.
|
|
575
|
+
|
|
576
|
+
Args:
|
|
577
|
+
- label_set_ids (List[int]): List of label set IDs to start with.
|
|
578
|
+
- sql_handler (SQLHandler): An instance of SQLHandler for database queries.
|
|
579
|
+
|
|
580
|
+
Returns:
|
|
581
|
+
- Tuple[Set[int], Dict[int, int]]: A set of all unique descendant label set IDs and
|
|
582
|
+
a dictionary mapping each unique descendant ID to its closest requested ancestor ID.
|
|
583
|
+
"""
|
|
584
|
+
all_descendants = set(label_set_ids)
|
|
585
|
+
closest_ancestor_dict = {id: id for id in label_set_ids} # Maps each ID to itself initially
|
|
586
|
+
to_process = set(label_set_ids)
|
|
587
|
+
|
|
588
|
+
while to_process:
|
|
589
|
+
current_id = to_process.pop()
|
|
590
|
+
child_records = sql_handler.select_label_name_children(label_set_id=current_id)
|
|
591
|
+
for child_id, *_ in child_records:
|
|
592
|
+
if child_id not in all_descendants:
|
|
593
|
+
all_descendants.add(child_id)
|
|
594
|
+
closest_ancestor_dict[child_id] = closest_ancestor_dict.get(current_id, current_id)
|
|
595
|
+
to_process.add(child_id)
|
|
596
|
+
|
|
597
|
+
return all_descendants, closest_ancestor_dict
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def merge_gap_data(values_1, gap_array_1, start_time_1, values_2, gap_array_2, start_time_2, freq_nhz=None, *,
|
|
601
|
+
period_ns=None):
|
|
602
|
+
"""
|
|
603
|
+
Merge two sets of gap-array data into one, in time order. Where they overlap, values from the second
|
|
604
|
+
set replace those from the first.
|
|
605
|
+
|
|
606
|
+
:param numpy.ndarray values_1: Values of the first set.
|
|
607
|
+
:param numpy.ndarray gap_array_1: Flat gap array of the first set.
|
|
608
|
+
:param int start_time_1: Start time of the first set in nanoseconds.
|
|
609
|
+
:param numpy.ndarray values_2: Values of the second set (same dtype as ``values_1``).
|
|
610
|
+
:param numpy.ndarray gap_array_2: Flat gap array of the second set.
|
|
611
|
+
:param int start_time_2: Start time of the second set in nanoseconds.
|
|
612
|
+
:param int freq_nhz: Sample frequency in nanohertz. Give exactly one of ``freq_nhz`` and ``period_ns``.
|
|
613
|
+
:param int period_ns: Keyword-only. Sample period in nanoseconds.
|
|
614
|
+
:return: Tuple ``(values, gap_array, start_time)`` of the merged data.
|
|
615
|
+
:rtype: tuple
|
|
616
|
+
:raises ValueError: If an input array is not a numpy array, the value dtypes differ, or not exactly one
|
|
617
|
+
of ``freq_nhz`` and ``period_ns`` is given.
|
|
618
|
+
"""
|
|
619
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
620
|
+
|
|
621
|
+
if not all(isinstance(arr, np.ndarray) for arr in [values_1, gap_array_1, values_2, gap_array_2]):
|
|
622
|
+
raise ValueError(f"All input value and gap arrays must be numpy arrays.")
|
|
623
|
+
|
|
624
|
+
if not values_1.dtype == values_2.dtype:
|
|
625
|
+
raise ValueError(f"Values 1 and 2 have different dtypes {values_1.dtype}, {values_2.dtype}. Cannot merge.")
|
|
626
|
+
|
|
627
|
+
end_time_1 = _calc_end_time_from_gap_data(values_1.size, gap_array_1, start_time_1, freq_nhz=freq_nhz,
|
|
628
|
+
period_ns=period_ns)
|
|
629
|
+
end_time_2 = _calc_end_time_from_gap_data(values_2.size, gap_array_2, start_time_2, freq_nhz=freq_nhz,
|
|
630
|
+
period_ns=period_ns)
|
|
631
|
+
|
|
632
|
+
overlap = (start_time_1 < end_time_2) and (end_time_1 > start_time_2)
|
|
633
|
+
|
|
634
|
+
# If there's no overlap, you can simply concatenate the data.
|
|
635
|
+
if is_gap_data_sorted(gap_array_1, freq_nhz=freq_nhz, period_ns=period_ns) and is_gap_data_sorted(gap_array_2,
|
|
636
|
+
freq_nhz=freq_nhz,
|
|
637
|
+
period_ns=period_ns) and not overlap:
|
|
638
|
+
return _concatenate_gap_data(
|
|
639
|
+
values_1, gap_array_1, start_time_1, values_2, gap_array_2, start_time_2, freq_nhz=freq_nhz,
|
|
640
|
+
period_ns=period_ns)
|
|
641
|
+
|
|
642
|
+
# if starts, values and gaps are equal, then just return the 1's
|
|
643
|
+
if np.array_equal(values_1, values_2) and \
|
|
644
|
+
np.array_equal(gap_array_1, gap_array_2) and \
|
|
645
|
+
start_time_1 == start_time_2:
|
|
646
|
+
return values_1, gap_array_1, start_time_1
|
|
647
|
+
|
|
648
|
+
# Convert both gap_data into messages
|
|
649
|
+
message_starts_1, message_sizes_1 = reconstruct_messages(
|
|
650
|
+
start_time_1, gap_array_1, num_values=int(values_1.size), freq_nhz=freq_nhz, period_ns=period_ns)
|
|
651
|
+
|
|
652
|
+
message_starts_2, message_sizes_2 = reconstruct_messages(
|
|
653
|
+
start_time_2, gap_array_2, num_values=int(values_2.size), freq_nhz=freq_nhz, period_ns=period_ns)
|
|
654
|
+
|
|
655
|
+
# Sort both message lists + values, and copy values to not mess with the originals
|
|
656
|
+
values_1, values_2 = values_1.copy(), values_2.copy()
|
|
657
|
+
sort_message_time_values(message_starts_1, message_sizes_1, values_1)
|
|
658
|
+
sort_message_time_values(message_starts_2, message_sizes_2, values_2)
|
|
659
|
+
|
|
660
|
+
# Merge lists and Overwrite 2 over 1 if overlapping
|
|
661
|
+
merged_starts, merged_sizes, merged_values = merge_sorted_messages(
|
|
662
|
+
message_starts_1, message_sizes_1, values_1,
|
|
663
|
+
message_starts_2, message_sizes_2, values_2, freq_nhz=freq_nhz, period_ns=period_ns)
|
|
664
|
+
|
|
665
|
+
# Convert back into gap data
|
|
666
|
+
merged_gap_data = create_gap_arr_from_variable_messages(merged_starts, merged_sizes, freq_nhz=freq_nhz,
|
|
667
|
+
period_ns=period_ns)
|
|
668
|
+
|
|
669
|
+
return merged_values, merged_gap_data, int(merged_starts[0])
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def _concatenate_gap_data(values_1, gap_array_1, start_time_1, values_2, gap_array_2, start_time_2, freq_nhz=None, *,
|
|
673
|
+
period_ns=None):
|
|
674
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
675
|
+
|
|
676
|
+
# if start_time_2 < start_time_1, swap 1's with 2's so that 1 is always temporally first.
|
|
677
|
+
if start_time_2 < start_time_1:
|
|
678
|
+
values_1, values_2 = values_2, values_1
|
|
679
|
+
gap_array_1, gap_array_2 = gap_array_2, gap_array_1
|
|
680
|
+
start_time_1, start_time_2 = start_time_2, start_time_1
|
|
681
|
+
|
|
682
|
+
# Calculate the gap between blocks
|
|
683
|
+
end_time_1 = _calc_end_time_from_gap_data(values_1.size, gap_array_1, start_time_1, freq_nhz=freq_nhz,
|
|
684
|
+
period_ns=period_ns)
|
|
685
|
+
new_gap_index, new_gap_duration = int(values_1.size), start_time_2 - end_time_1
|
|
686
|
+
|
|
687
|
+
# Calculate positions depending on if we need to add a new gap between blocks.
|
|
688
|
+
if new_gap_duration == 0:
|
|
689
|
+
merged_gap_size = gap_array_1.size + gap_array_2.size
|
|
690
|
+
gap_array_2_index = gap_array_1.size
|
|
691
|
+
else:
|
|
692
|
+
merged_gap_size = gap_array_1.size + gap_array_2.size + 2
|
|
693
|
+
gap_array_2_index = gap_array_1.size + 2
|
|
694
|
+
|
|
695
|
+
merged_gap_array = np.empty(merged_gap_size, dtype=np.int64)
|
|
696
|
+
merged_gap_array[:gap_array_1.size] = gap_array_1
|
|
697
|
+
|
|
698
|
+
if new_gap_duration != 0:
|
|
699
|
+
merged_gap_array[gap_array_1.size:gap_array_1.size + 2] = [new_gap_index, new_gap_duration]
|
|
700
|
+
|
|
701
|
+
merged_gap_array[gap_array_2_index:] = gap_array_2
|
|
702
|
+
# move the gap_array_2 indices forward.
|
|
703
|
+
merged_gap_array[gap_array_2_index::2] += values_1.size
|
|
704
|
+
|
|
705
|
+
# concatenate the values.
|
|
706
|
+
merged_values = np.empty(values_1.size + values_2.size, dtype=values_1.dtype)
|
|
707
|
+
merged_values[:values_1.size] = values_1
|
|
708
|
+
merged_values[values_1.size:] = values_2
|
|
709
|
+
return merged_values, merged_gap_array, start_time_1
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _calc_end_time_from_gap_data(values_size, gap_array, start_time, freq_nhz=None, *, period_ns=None):
|
|
713
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
714
|
+
|
|
715
|
+
if freq_nhz is not None:
|
|
716
|
+
if (int(values_size) * (10 ** 18)) % freq_nhz != 0:
|
|
717
|
+
warnings.warn(f"Blocking starting on epoch {start_time} doesn't end on an integer number of nanoseconds, "
|
|
718
|
+
f"merge will be approximate.")
|
|
719
|
+
sample_duration = (int(values_size) * (10 ** 18)) // freq_nhz
|
|
720
|
+
else:
|
|
721
|
+
sample_duration = calc_time_by_period(period_ns, int(values_size))
|
|
722
|
+
|
|
723
|
+
gap_total = int(np.sum(gap_array[1::2]))
|
|
724
|
+
return start_time + sample_duration + gap_total
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def create_timestamps_from_gap_data(values_size, gap_array, start_time, freq_nhz=None, *, period_ns=None):
|
|
728
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
729
|
+
|
|
730
|
+
if freq_nhz is not None:
|
|
731
|
+
if (10 ** 18) % freq_nhz != 0:
|
|
732
|
+
raise ValueError(f"Cannot create perfect timestamps from frequency_nhz = {freq_nhz}")
|
|
733
|
+
current_period_ns = freq_nhz_to_period_ns(freq_nhz)
|
|
734
|
+
else:
|
|
735
|
+
current_period_ns = period_ns
|
|
736
|
+
|
|
737
|
+
timestamps = np.arange(values_size, dtype=np.int64)
|
|
738
|
+
timestamps *= current_period_ns
|
|
739
|
+
timestamps += start_time
|
|
740
|
+
for i in range(gap_array.size // 2):
|
|
741
|
+
gap_index, gap_duration = gap_array[2 * i], gap_array[(2 * i) + 1]
|
|
742
|
+
timestamps[gap_index:] += gap_duration
|
|
743
|
+
|
|
744
|
+
return timestamps
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def is_gap_data_sorted(gap_data, freq_nhz=None, *, period_ns=None):
|
|
748
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
749
|
+
|
|
750
|
+
if freq_nhz is not None:
|
|
751
|
+
current_period_ns = freq_nhz_to_period_ns(freq_nhz)
|
|
752
|
+
else:
|
|
753
|
+
current_period_ns = period_ns
|
|
754
|
+
|
|
755
|
+
gap_durations = gap_data[1::2]
|
|
756
|
+
return np.all(gap_durations >= -current_period_ns)
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def create_gap_arr_from_variable_messages(message_start_epoch_array, message_size_array, freq_nhz=None, *,
|
|
760
|
+
period_ns=None):
|
|
761
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
762
|
+
|
|
763
|
+
result_list = []
|
|
764
|
+
current_sample = 0
|
|
765
|
+
|
|
766
|
+
for i in range(1, len(message_start_epoch_array)):
|
|
767
|
+
# Compute the time difference between consecutive messages
|
|
768
|
+
delta_t = message_start_epoch_array[i] - message_start_epoch_array[i - 1]
|
|
769
|
+
|
|
770
|
+
# Calculate the message period for the current message based on its size
|
|
771
|
+
current_message_size = int(message_size_array[i - 1])
|
|
772
|
+
|
|
773
|
+
if freq_nhz is not None:
|
|
774
|
+
current_message_period_ns = ((10 ** 18) * current_message_size) // int(freq_nhz)
|
|
775
|
+
else:
|
|
776
|
+
current_message_period_ns = _message_size_period_to_durations_ns(current_message_size, period_ns)
|
|
777
|
+
|
|
778
|
+
# Check if the time difference doesn't match the expected message period
|
|
779
|
+
if delta_t != current_message_period_ns:
|
|
780
|
+
# Compute the extra duration (time gap) and the starting index of the gap
|
|
781
|
+
time_gap = delta_t - current_message_period_ns
|
|
782
|
+
gap_start_index = current_sample + current_message_size
|
|
783
|
+
|
|
784
|
+
# Add the gap information to the result list
|
|
785
|
+
result_list.extend([gap_start_index, time_gap])
|
|
786
|
+
|
|
787
|
+
# Update the current sample index for the next iteration
|
|
788
|
+
current_sample += current_message_size
|
|
789
|
+
|
|
790
|
+
# Convert the result list to a NumPy array of integers
|
|
791
|
+
return np.array(result_list, dtype=np.int64)
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def reconstruct_messages(start_time_nano_epoch, gap_data_array, freq_nhz=None, num_values=None, *, period_ns=None):
|
|
795
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
796
|
+
|
|
797
|
+
message_sizes = np.empty((gap_data_array.size // 2) + 2, dtype=np.int64)
|
|
798
|
+
message_sizes[0] = 0
|
|
799
|
+
message_sizes[-1] = num_values
|
|
800
|
+
message_sizes[1:-1] = gap_data_array[::2]
|
|
801
|
+
message_sizes = np.diff(message_sizes)
|
|
802
|
+
|
|
803
|
+
message_starts = np.empty((gap_data_array.size // 2) + 1, dtype=np.int64)
|
|
804
|
+
|
|
805
|
+
message_starts[0] = start_time_nano_epoch
|
|
806
|
+
|
|
807
|
+
if freq_nhz is not None:
|
|
808
|
+
if any(((10 ** 18) * int(m_size)) % freq_nhz != 0 for m_size in message_sizes[:-1]):
|
|
809
|
+
warnings.warn(
|
|
810
|
+
"Not all messages durations can be expressed as a perfect nanosecond integer, some rounding has occured")
|
|
811
|
+
message_starts[1:] = [_message_size_to_duration_ns(int(m_size), freq_nhz) for m_size in message_sizes[:-1]]
|
|
812
|
+
else:
|
|
813
|
+
message_starts[1:] = [_message_size_period_to_durations_ns(int(m_size), period_ns) for m_size in
|
|
814
|
+
message_sizes[:-1]]
|
|
815
|
+
|
|
816
|
+
message_starts[1:] += gap_data_array[1::2]
|
|
817
|
+
message_starts = np.cumsum(message_starts)
|
|
818
|
+
|
|
819
|
+
return message_starts, message_sizes
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def reconstruct_messages_multi(headers, gap_data, freq_nhz=None, *, period_ns=None):
|
|
823
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
824
|
+
|
|
825
|
+
message_starts_list = []
|
|
826
|
+
message_sizes_list = []
|
|
827
|
+
|
|
828
|
+
gap_idx = 0
|
|
829
|
+
for header in headers:
|
|
830
|
+
num_vals = header.num_vals
|
|
831
|
+
num_gaps = header.num_gaps
|
|
832
|
+
start_n = header.start_n # starting time in nanoseconds since epoch
|
|
833
|
+
|
|
834
|
+
gap_data_block = gap_data[gap_idx: gap_idx + 2 * num_gaps]
|
|
835
|
+
|
|
836
|
+
# Process the block to get message_starts and message_sizes
|
|
837
|
+
message_starts, message_sizes = reconstruct_messages(
|
|
838
|
+
start_n, gap_data_block, freq_nhz=freq_nhz, num_values=num_vals, period_ns=period_ns)
|
|
839
|
+
|
|
840
|
+
message_starts_list.append(message_starts)
|
|
841
|
+
message_sizes_list.append(message_sizes)
|
|
842
|
+
|
|
843
|
+
# Update indices for the next block
|
|
844
|
+
gap_idx += 2 * num_gaps
|
|
845
|
+
|
|
846
|
+
# Concatenate all message starts and sizes from all blocks
|
|
847
|
+
all_message_starts = np.concatenate(message_starts_list)
|
|
848
|
+
all_message_sizes = np.concatenate(message_sizes_list)
|
|
849
|
+
|
|
850
|
+
return all_message_starts, all_message_sizes
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def sort_message_time_values(message_starts: np.ndarray, message_sizes: np.ndarray, value_array: np.ndarray):
|
|
854
|
+
# The start and end index of each message within the value_array
|
|
855
|
+
message_end_indices, message_start_indices = _get_message_indices(message_sizes)
|
|
856
|
+
|
|
857
|
+
sorted_message_i_arr = np.argsort(message_starts)
|
|
858
|
+
if np.all(sorted_message_i_arr[:-1] <= sorted_message_i_arr[1:]):
|
|
859
|
+
return
|
|
860
|
+
|
|
861
|
+
message_starts[:] = message_starts[sorted_message_i_arr]
|
|
862
|
+
message_sizes[:] = message_sizes[sorted_message_i_arr]
|
|
863
|
+
|
|
864
|
+
values_copy = value_array.copy()
|
|
865
|
+
|
|
866
|
+
start_idx = 0
|
|
867
|
+
for message_i in range(message_start_indices.size):
|
|
868
|
+
sorted_left = message_start_indices[sorted_message_i_arr][message_i]
|
|
869
|
+
sorted_right = message_end_indices[sorted_message_i_arr][message_i]
|
|
870
|
+
message_size = message_sizes[message_i]
|
|
871
|
+
|
|
872
|
+
value_array[start_idx:start_idx+message_size] = values_copy[sorted_left:sorted_right]
|
|
873
|
+
start_idx += message_size
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def _get_message_indices(message_sizes):
|
|
877
|
+
message_end_indices = np.cumsum(message_sizes)
|
|
878
|
+
message_start_indices = message_end_indices - message_sizes
|
|
879
|
+
return message_end_indices, message_start_indices
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
def truncate_messages(value_data, message_starts, message_sizes, freq_nhz=None, trunc_start_nano=None, trunc_end_nano=None, *,
|
|
883
|
+
period_ns=None):
|
|
884
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
885
|
+
|
|
886
|
+
truncated_value_data = []
|
|
887
|
+
truncated_message_starts = []
|
|
888
|
+
truncated_message_sizes = []
|
|
889
|
+
cumulative_sample_index = 0 # To track the index in value_data
|
|
890
|
+
|
|
891
|
+
if freq_nhz is not None:
|
|
892
|
+
current_period_ns = 10 ** 18 // freq_nhz # Period in nanoseconds per sample
|
|
893
|
+
else:
|
|
894
|
+
current_period_ns = period_ns
|
|
895
|
+
|
|
896
|
+
for i in range(len(message_starts)):
|
|
897
|
+
msg_start_ns = message_starts[i]
|
|
898
|
+
msg_size = message_sizes[i]
|
|
899
|
+
|
|
900
|
+
if freq_nhz is not None:
|
|
901
|
+
msg_duration_ns = _message_size_freq_to_duration_ns(msg_size, freq_nhz)
|
|
902
|
+
else:
|
|
903
|
+
msg_duration_ns = _message_size_period_to_durations_ns(msg_size, period_ns)
|
|
904
|
+
|
|
905
|
+
msg_end_ns = msg_start_ns + msg_duration_ns
|
|
906
|
+
|
|
907
|
+
# Calculate overlapping region with truncation window
|
|
908
|
+
overlap_start_ns = max(msg_start_ns, trunc_start_nano)
|
|
909
|
+
overlap_end_ns = min(msg_end_ns, trunc_end_nano)
|
|
910
|
+
|
|
911
|
+
if overlap_end_ns <= overlap_start_ns:
|
|
912
|
+
# No overlap with the truncation window
|
|
913
|
+
cumulative_sample_index += msg_size
|
|
914
|
+
continue
|
|
915
|
+
|
|
916
|
+
# Calculate the sample indices within the message
|
|
917
|
+
samples_before_overlap_start = ((overlap_start_ns - msg_start_ns) + current_period_ns - 1) // current_period_ns
|
|
918
|
+
samples_until_overlap_end = ((overlap_end_ns - msg_start_ns) + current_period_ns - 1) // current_period_ns
|
|
919
|
+
|
|
920
|
+
num_samples_to_keep = samples_until_overlap_end - samples_before_overlap_start
|
|
921
|
+
|
|
922
|
+
if num_samples_to_keep <= 0:
|
|
923
|
+
cumulative_sample_index += msg_size
|
|
924
|
+
continue
|
|
925
|
+
|
|
926
|
+
# Calculate the indices in value_data
|
|
927
|
+
start_idx = cumulative_sample_index + samples_before_overlap_start
|
|
928
|
+
end_idx = start_idx + num_samples_to_keep
|
|
929
|
+
|
|
930
|
+
# Truncate the value_data
|
|
931
|
+
truncated_value_data.append(value_data[start_idx:end_idx])
|
|
932
|
+
|
|
933
|
+
# Adjust message start time
|
|
934
|
+
adjusted_msg_start_ns = msg_start_ns + samples_before_overlap_start * current_period_ns
|
|
935
|
+
|
|
936
|
+
truncated_message_starts.append(adjusted_msg_start_ns)
|
|
937
|
+
truncated_message_sizes.append(num_samples_to_keep)
|
|
938
|
+
|
|
939
|
+
cumulative_sample_index += msg_size
|
|
940
|
+
|
|
941
|
+
# Concatenate all truncated data
|
|
942
|
+
truncated_value_data = np.concatenate(truncated_value_data) if truncated_value_data else np.array([])
|
|
943
|
+
truncated_message_starts = np.array(truncated_message_starts)
|
|
944
|
+
truncated_message_sizes = np.array(truncated_message_sizes)
|
|
945
|
+
|
|
946
|
+
return truncated_value_data, truncated_message_starts, truncated_message_sizes
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def merge_sorted_messages(message_starts_1, message_sizes_1, values_1,
|
|
950
|
+
message_starts_2, message_sizes_2, values_2, freq_nhz=None, *, period_ns=None):
|
|
951
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
952
|
+
|
|
953
|
+
# Find the smallest start time from both inputs
|
|
954
|
+
min_start = min(np.min(message_starts_1), np.min(message_starts_2))
|
|
955
|
+
|
|
956
|
+
# Create relative start times
|
|
957
|
+
rel_starts_1 = message_starts_1 - min_start
|
|
958
|
+
rel_starts_2 = message_starts_2 - min_start
|
|
959
|
+
|
|
960
|
+
# Calculate period in nanoseconds for one sample
|
|
961
|
+
if freq_nhz is not None:
|
|
962
|
+
current_period_ns = float((10 ** 18) / int(freq_nhz))
|
|
963
|
+
else:
|
|
964
|
+
current_period_ns = float(period_ns)
|
|
965
|
+
|
|
966
|
+
timestamps_1 = []
|
|
967
|
+
timestamps_2 = []
|
|
968
|
+
|
|
969
|
+
# Create timestamps for dataset 1
|
|
970
|
+
for start, size in zip(rel_starts_1, message_sizes_1):
|
|
971
|
+
timestamps_1.append(start + (np.arange(size) * current_period_ns))
|
|
972
|
+
|
|
973
|
+
# Create timestamps for dataset 2
|
|
974
|
+
for start, size in zip(rel_starts_2, message_sizes_2):
|
|
975
|
+
timestamps_2.append(start + (np.arange(size) * current_period_ns))
|
|
976
|
+
|
|
977
|
+
# Convert lists to numpy arrays
|
|
978
|
+
timestamps_1 = np.concatenate(timestamps_1, dtype=np.float64)
|
|
979
|
+
timestamps_2 = np.concatenate(timestamps_2, dtype=np.float64)
|
|
980
|
+
|
|
981
|
+
# Combine and sort the timestamp and value arrays
|
|
982
|
+
combined_timestamps = np.concatenate((timestamps_1, timestamps_2))
|
|
983
|
+
combined_values = np.concatenate((values_1, values_2))
|
|
984
|
+
|
|
985
|
+
# Create array indicators to ensure stable sorting where array 2 overwrites array 1
|
|
986
|
+
array_indicators = np.concatenate((np.zeros(len(timestamps_1), dtype=int),
|
|
987
|
+
np.ones(len(timestamps_2), dtype=int)))
|
|
988
|
+
|
|
989
|
+
# Sort by timestamp first, then by array indicator (so array 2 comes after array 1 for ties)
|
|
990
|
+
sort_keys = np.lexsort((array_indicators, combined_timestamps))
|
|
991
|
+
sorted_timestamps = combined_timestamps[sort_keys]
|
|
992
|
+
sorted_values = combined_values[sort_keys]
|
|
993
|
+
|
|
994
|
+
# Convert the sorted timestamps into "sample times"
|
|
995
|
+
sample_times = sorted_timestamps / current_period_ns
|
|
996
|
+
|
|
997
|
+
# Find differences in sample times
|
|
998
|
+
diff_sample_times = np.diff(sample_times)
|
|
999
|
+
|
|
1000
|
+
# Calculate the max floating point imprecision
|
|
1001
|
+
max_timestamp = np.max(sorted_timestamps)
|
|
1002
|
+
max_diff_sample_times = np.max(diff_sample_times)
|
|
1003
|
+
|
|
1004
|
+
max_imprecision = np.finfo(np.float64).eps * (max_timestamp / current_period_ns + max_diff_sample_times)
|
|
1005
|
+
|
|
1006
|
+
# Identify where timestamps are close to 1 (consecutive message elements)
|
|
1007
|
+
close_to_one = np.isclose(diff_sample_times, 1, atol=max_imprecision, rtol=0)
|
|
1008
|
+
|
|
1009
|
+
# Identify where timestamps are close to 0 (duplicates)
|
|
1010
|
+
close_to_zero = np.isclose(diff_sample_times, 0, atol=max_imprecision, rtol=0)
|
|
1011
|
+
|
|
1012
|
+
# Reconstruct the merged messages, handling duplicates by skipping them
|
|
1013
|
+
merged_starts = []
|
|
1014
|
+
merged_sizes = []
|
|
1015
|
+
merged_values = []
|
|
1016
|
+
|
|
1017
|
+
current_message_start = sorted_timestamps[0]
|
|
1018
|
+
current_message_values = [sorted_values[0]]
|
|
1019
|
+
|
|
1020
|
+
for i in range(1, sample_times.size):
|
|
1021
|
+
# Skip over duplicates by looking ahead until we find a non-duplicate
|
|
1022
|
+
if close_to_zero[i - 1]:
|
|
1023
|
+
# If the next value is a duplicate, keep overwriting current value until the last one
|
|
1024
|
+
current_message_values[-1] = sorted_values[i]
|
|
1025
|
+
else:
|
|
1026
|
+
if close_to_one[i - 1]:
|
|
1027
|
+
# This value continues the current message
|
|
1028
|
+
current_message_values.append(sorted_values[i])
|
|
1029
|
+
else:
|
|
1030
|
+
# End of the current message, start a new one
|
|
1031
|
+
merged_starts.append(current_message_start)
|
|
1032
|
+
merged_sizes.append(len(current_message_values))
|
|
1033
|
+
merged_values.append(np.array(current_message_values))
|
|
1034
|
+
|
|
1035
|
+
current_message_start = sorted_timestamps[i]
|
|
1036
|
+
current_message_values = [sorted_values[i]]
|
|
1037
|
+
|
|
1038
|
+
# Append the last message
|
|
1039
|
+
if len(current_message_values) > 0:
|
|
1040
|
+
merged_starts.append(current_message_start)
|
|
1041
|
+
merged_sizes.append(len(current_message_values))
|
|
1042
|
+
merged_values.append(np.array(current_message_values))
|
|
1043
|
+
|
|
1044
|
+
# Convert to final outputs
|
|
1045
|
+
merged_starts = np.array(merged_starts, dtype=np.int64) + min_start
|
|
1046
|
+
merged_sizes = np.array(merged_sizes, dtype=np.int64)
|
|
1047
|
+
merged_values = np.concatenate(merged_values) if merged_values else np.array([], dtype=np.float64)
|
|
1048
|
+
|
|
1049
|
+
return merged_starts, merged_sizes, merged_values
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def _message_size_freq_to_duration_ns(m_size, freq_nhz):
|
|
1053
|
+
return ((10 ** 18) * int(m_size)) // freq_nhz
|
|
1054
|
+
|
|
1055
|
+
def _message_size_period_to_durations_ns(m_size, period_ns):
|
|
1056
|
+
return m_size * period_ns
|
|
1057
|
+
|
|
1058
|
+
def _message_size_to_duration_ns(m_size, freq_nhz):
|
|
1059
|
+
return ((10 ** 18) * int(m_size)) // freq_nhz
|
|
1060
|
+
|
|
1061
|
+
def calc_time_by_period(period_ns, num_samples):
|
|
1062
|
+
return int(num_samples) * period_ns
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def merge_timestamp_data(values_1, times_1, values_2, times_2):
|
|
1066
|
+
# concatenate the time and value arrays
|
|
1067
|
+
concatenated_times = np.concatenate((times_1, times_2))
|
|
1068
|
+
concatenated_values = np.concatenate((values_1, values_2))
|
|
1069
|
+
|
|
1070
|
+
# remove duplicate times and get indices to sort values in time order
|
|
1071
|
+
time_data, index_unique = np.unique(concatenated_times, return_index=True)
|
|
1072
|
+
|
|
1073
|
+
# return the sorted arrays
|
|
1074
|
+
return time_data, concatenated_values[index_unique]
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
def generate_single_messages_from_big_message(message_start_1, message_end_1, start_index_1, end_index_1, message_values_1,
|
|
1078
|
+
message_start_2, message_end_2, start_index_2, end_index_2, message_values_2,
|
|
1079
|
+
freq_nhz):
|
|
1080
|
+
if (10 ** 18) % freq_nhz != 0:
|
|
1081
|
+
warnings.warn("Out of phase data is being merged with a freq that doesn't create perfect nanosecond integer "
|
|
1082
|
+
"timestamps, some rounding will occur in the stored data.")
|
|
1083
|
+
|
|
1084
|
+
# Calculate the period based on frequency
|
|
1085
|
+
period_ns = (10 ** 18) // freq_nhz
|
|
1086
|
+
|
|
1087
|
+
# Initialize lists for both data sets
|
|
1088
|
+
starts_1, ends_1, values_list_1 = [], [], []
|
|
1089
|
+
starts_2, ends_2, values_list_2 = [], [], []
|
|
1090
|
+
|
|
1091
|
+
# Process the first set of parameters
|
|
1092
|
+
cur_time = message_start_1
|
|
1093
|
+
for index in range(start_index_1, end_index_1):
|
|
1094
|
+
starts_1.append(cur_time)
|
|
1095
|
+
values_list_1.append(message_values_1[index:index+1])
|
|
1096
|
+
cur_time += period_ns
|
|
1097
|
+
ends_1.append(cur_time)
|
|
1098
|
+
|
|
1099
|
+
# Process the second set of parameters
|
|
1100
|
+
cur_time = message_start_2
|
|
1101
|
+
for index in range(start_index_2, end_index_2):
|
|
1102
|
+
starts_2.append(cur_time)
|
|
1103
|
+
values_list_2.append(message_values_2[index:index+1])
|
|
1104
|
+
cur_time += period_ns
|
|
1105
|
+
ends_2.append(cur_time)
|
|
1106
|
+
|
|
1107
|
+
# Now merge the two sets in order of ascending starts
|
|
1108
|
+
starts, ends, values_list = [], [], []
|
|
1109
|
+
index_1, index_2 = 0, 0
|
|
1110
|
+
|
|
1111
|
+
while index_1 < len(starts_1) and index_2 < len(starts_2):
|
|
1112
|
+
if starts_1[index_1] <= starts_2[index_2]:
|
|
1113
|
+
starts.append(starts_1[index_1])
|
|
1114
|
+
ends.append(ends_1[index_1])
|
|
1115
|
+
values_list.append(values_list_1[index_1])
|
|
1116
|
+
index_1 += 1
|
|
1117
|
+
else:
|
|
1118
|
+
starts.append(starts_2[index_2])
|
|
1119
|
+
ends.append(ends_2[index_2])
|
|
1120
|
+
values_list.append(values_list_2[index_2])
|
|
1121
|
+
index_2 += 1
|
|
1122
|
+
|
|
1123
|
+
# Append remaining elements from the first list
|
|
1124
|
+
while index_1 < len(starts_1):
|
|
1125
|
+
starts.append(starts_1[index_1])
|
|
1126
|
+
ends.append(ends_1[index_1])
|
|
1127
|
+
values_list.append(values_list_1[index_1])
|
|
1128
|
+
index_1 += 1
|
|
1129
|
+
|
|
1130
|
+
# Append remaining elements from the second list
|
|
1131
|
+
while index_2 < len(starts_2):
|
|
1132
|
+
starts.append(starts_2[index_2])
|
|
1133
|
+
ends.append(ends_2[index_2])
|
|
1134
|
+
values_list.append(values_list_2[index_2])
|
|
1135
|
+
index_2 += 1
|
|
1136
|
+
|
|
1137
|
+
return starts, ends, values_list
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def get_headers(sdk, measure_id: int = None, start_time: int = None, end_time: int = None, device_id: int = None,
|
|
1141
|
+
patient_id=None, time_units: str = None, mrn: int = None):
|
|
1142
|
+
"""
|
|
1143
|
+
The method for querying block headers from the dataset, indexed by signal type (measure_id or measure_tag with freq and units),
|
|
1144
|
+
time (start_time_n and end_time_n), and data source (device_id, device_tag, patient_id, or mrn).
|
|
1145
|
+
|
|
1146
|
+
If measure_id is None, measure_tag along with freq and units must not be None, and vice versa.
|
|
1147
|
+
Similarly, if device_id is None, device_tag must not be None, and if patient_id is None, mrn must not be None.
|
|
1148
|
+
|
|
1149
|
+
:param AtriumSDK sdk: The AtriumSDK object.
|
|
1150
|
+
:param int measure_id: The measure identifier. If None, measure_tag must be provided.
|
|
1151
|
+
:param int start_time: The start epoch in nanoseconds of the data you would like to query.
|
|
1152
|
+
:param int end_time: The end epoch in nanoseconds. The end time is not inclusive.
|
|
1153
|
+
:param int device_id: The device identifier. If None, device_tag must be provided.
|
|
1154
|
+
:param int patient_id: The patient identifier. If None, mrn must be provided.
|
|
1155
|
+
"""
|
|
1156
|
+
|
|
1157
|
+
# check that a correct unit type was entered
|
|
1158
|
+
time_units = "ns" if time_units is None else time_units
|
|
1159
|
+
|
|
1160
|
+
if time_units not in time_unit_options.keys():
|
|
1161
|
+
raise ValueError("Invalid time units. Expected one of: %s" % time_unit_options)
|
|
1162
|
+
|
|
1163
|
+
# convert start and end time to nanoseconds
|
|
1164
|
+
start_time = int(start_time * time_unit_options[time_units])
|
|
1165
|
+
end_time = int(end_time * time_unit_options[time_units])
|
|
1166
|
+
|
|
1167
|
+
if patient_id is None and mrn is not None:
|
|
1168
|
+
patient_id = sdk.get_patient_id(mrn)
|
|
1169
|
+
|
|
1170
|
+
# If we don't already have the blocks
|
|
1171
|
+
# Select all blocks from the block_index (sql table) that match params.
|
|
1172
|
+
block_list = sdk.sql_handler.select_blocks(int(measure_id), int(start_time), int(end_time), device_id,
|
|
1173
|
+
patient_id)
|
|
1174
|
+
|
|
1175
|
+
# Concatenate continuous byte intervals to cut down on total number of reads.
|
|
1176
|
+
read_list = condense_byte_read_list(block_list)
|
|
1177
|
+
|
|
1178
|
+
# if no matching block ids
|
|
1179
|
+
if len(read_list) == 0:
|
|
1180
|
+
return [], np.array([]), np.array([])
|
|
1181
|
+
|
|
1182
|
+
# Map file_ids to filenames and return a dictionary.
|
|
1183
|
+
file_id_list = [row[2] for row in read_list]
|
|
1184
|
+
filename_dict = sdk.get_filename_dict(file_id_list)
|
|
1185
|
+
|
|
1186
|
+
# Condense the block list for optimized reading
|
|
1187
|
+
read_list = condense_byte_read_list(block_list)
|
|
1188
|
+
|
|
1189
|
+
# Read the data from the files using the read list
|
|
1190
|
+
encoded_bytes = sdk.file_api.read_file_list(read_list, filename_dict)
|
|
1191
|
+
|
|
1192
|
+
num_bytes_list = [row[5] for row in block_list]
|
|
1193
|
+
byte_start_array = np.cumsum(num_bytes_list, dtype=np.uint64)
|
|
1194
|
+
byte_start_array = np.concatenate([np.array([0], dtype=np.uint64), byte_start_array[:-1]], axis=None)
|
|
1195
|
+
headers = sdk.block.decode_headers(encoded_bytes, byte_start_array)
|
|
1196
|
+
|
|
1197
|
+
return headers
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def delete_unreferenced_tsc_files(sdk):
|
|
1201
|
+
"""
|
|
1202
|
+
The method is for removing unreferenced tsc files. When you write a lot of data with merge=True
|
|
1203
|
+
eventually a lot of unused tsc files will build up since the function is copy on write. This
|
|
1204
|
+
Will use up a lot of space unnecessarily and this function will help you free up that space.
|
|
1205
|
+
|
|
1206
|
+
:param AtriumSDK sdk: The AtriumSDK object.
|
|
1207
|
+
"""
|
|
1208
|
+
# find tsc files in the file_index that have no references to them in the block_index
|
|
1209
|
+
files = sdk.sql_handler.find_unreferenced_tsc_files()
|
|
1210
|
+
|
|
1211
|
+
# if there are no tsc files to remove just return
|
|
1212
|
+
if len(files) == 0:
|
|
1213
|
+
return
|
|
1214
|
+
|
|
1215
|
+
# extract file names from files and make it a set so we can do a set intersection later
|
|
1216
|
+
file_names = {file[1] for file in files}
|
|
1217
|
+
# extract the ids and put them in a tuple so we can remove them from the sql table later
|
|
1218
|
+
file_ids = [(file[0],) for file in files]
|
|
1219
|
+
|
|
1220
|
+
# walk the tsc directory looking for files to delete (walk is a generator for memory efficiency)
|
|
1221
|
+
for root, dir_names, files in sdk.file_api.walk(sdk.file_api.top_level_dir):
|
|
1222
|
+
# remove any dirs that are not digits (device_id or measure_id) so walk doesn't traverse those directories
|
|
1223
|
+
dir_names[:] = [d for d in dir_names if d.isdigit()]
|
|
1224
|
+
|
|
1225
|
+
# check if there is a match between any of the tsc file names to be deleted and files in the current directory
|
|
1226
|
+
matches = set(files) & file_names
|
|
1227
|
+
|
|
1228
|
+
# if you find a match remove the file from disk
|
|
1229
|
+
for m in matches:
|
|
1230
|
+
print(f"Deleting tsc file {m} from disk")
|
|
1231
|
+
sdk.file_api.remove(os.path.join(root, m))
|
|
1232
|
+
|
|
1233
|
+
# free up memory
|
|
1234
|
+
del file_names
|
|
1235
|
+
|
|
1236
|
+
# remove them from the file_index
|
|
1237
|
+
sdk.sql_handler.delete_files_by_ids(file_ids)
|
|
1238
|
+
print("Completed removal of unreferenced tsc files")
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
def get_all_blocks(sdk, measure_id, device_id):
|
|
1242
|
+
query = """
|
|
1243
|
+
SELECT id, measure_id, device_id, file_id, start_byte, num_bytes, start_time_n, end_time_n, num_values
|
|
1244
|
+
FROM block_index
|
|
1245
|
+
WHERE measure_id = ? AND device_id = ?
|
|
1246
|
+
ORDER BY start_time_n ASC;
|
|
1247
|
+
"""
|
|
1248
|
+
|
|
1249
|
+
with sdk.sql_handler.connection() as (conn, cursor):
|
|
1250
|
+
cursor.execute(query, (measure_id, device_id))
|
|
1251
|
+
block_query_result = cursor.fetchall()
|
|
1252
|
+
|
|
1253
|
+
return block_query_result
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
def reencode_dataset(sdk, values_per_block=131072, blocks_per_file=2048, interval_gap_tolerance_nano=0):
|
|
1257
|
+
# WARNING:
|
|
1258
|
+
# This function does not preserve duplicate values. (uses np.unique)
|
|
1259
|
+
original_block_size = sdk.block.block_size
|
|
1260
|
+
sdk.block.block_size = values_per_block
|
|
1261
|
+
|
|
1262
|
+
measures = sdk.get_all_measures()
|
|
1263
|
+
devices = sdk.get_all_devices()
|
|
1264
|
+
|
|
1265
|
+
measure_data = list(measures.items())
|
|
1266
|
+
device_data = list(devices.items())
|
|
1267
|
+
|
|
1268
|
+
for device_id, device_info in tqdm(device_data, desc="Devices"):
|
|
1269
|
+
for measure_id, measure_info in tqdm(measure_data, desc="Measures"):
|
|
1270
|
+
sorted_block_list_by_time = get_all_blocks(sdk, measure_id, device_id)
|
|
1271
|
+
if len(sorted_block_list_by_time) == 0:
|
|
1272
|
+
continue
|
|
1273
|
+
|
|
1274
|
+
file_id_list = list(set([row[3] for row in sorted_block_list_by_time]))
|
|
1275
|
+
filename_dict = sdk.get_filename_dict(file_id_list)
|
|
1276
|
+
|
|
1277
|
+
total_measure_device_interval_array = []
|
|
1278
|
+
|
|
1279
|
+
for block_group in group_sorted_block_list(
|
|
1280
|
+
sorted_block_list_by_time, num_values_per_group=values_per_block * blocks_per_file):
|
|
1281
|
+
|
|
1282
|
+
r_headers, r_times, r_values = sdk.get_data_from_blocks(
|
|
1283
|
+
block_list=block_group,
|
|
1284
|
+
filename_dict=filename_dict,
|
|
1285
|
+
start_time_n=0,
|
|
1286
|
+
end_time_n=2 ** 62,
|
|
1287
|
+
analog=False,
|
|
1288
|
+
time_type="encoded",
|
|
1289
|
+
sort=False,
|
|
1290
|
+
allow_duplicates=True
|
|
1291
|
+
)
|
|
1292
|
+
|
|
1293
|
+
if r_values.size == 0:
|
|
1294
|
+
continue
|
|
1295
|
+
|
|
1296
|
+
# Prepare segments for encode_blocks_from_multiple_segments
|
|
1297
|
+
segments = []
|
|
1298
|
+
block_group_interval_data = []
|
|
1299
|
+
|
|
1300
|
+
# Group data by scale factor, freq, time type
|
|
1301
|
+
for group_headers, group_times, group_values in group_headers_by_scale_factor_freq_time_type(
|
|
1302
|
+
r_headers, r_times, r_values):
|
|
1303
|
+
group_time_type = int(group_headers[0].t_raw_type)
|
|
1304
|
+
group_encoded_time_type = int(group_headers[0].t_encoded_type)
|
|
1305
|
+
|
|
1306
|
+
# Check TSC version to determine if freq_nhz is actually period_ns
|
|
1307
|
+
version_code = group_headers[0].tsc_version_num * 10 + group_headers[0].tsc_version_ext
|
|
1308
|
+
is_version_24_plus = version_code >= 24
|
|
1309
|
+
|
|
1310
|
+
if is_version_24_plus:
|
|
1311
|
+
group_period_ns = int(group_headers[0].freq_nhz) # Actually period_ns for v2.4+
|
|
1312
|
+
group_freq_nhz = None
|
|
1313
|
+
else:
|
|
1314
|
+
group_freq_nhz = int(group_headers[0].freq_nhz)
|
|
1315
|
+
group_period_ns = None
|
|
1316
|
+
|
|
1317
|
+
# Sort and merge the group data
|
|
1318
|
+
if group_time_type == 1:
|
|
1319
|
+
period_ns = group_period_ns if is_version_24_plus else (10 ** 18) // group_freq_nhz
|
|
1320
|
+
|
|
1321
|
+
group_times, sorted_time_indices = np.unique(group_times, return_index=True)
|
|
1322
|
+
group_values = group_values[sorted_time_indices]
|
|
1323
|
+
group_interval_data = get_interval_list_from_ordered_timestamps(group_times, period_ns)
|
|
1324
|
+
group_start_time = int(group_times[0])
|
|
1325
|
+
elif group_time_type == 2:
|
|
1326
|
+
# Merge the gap data into messages (segments)
|
|
1327
|
+
message_starts, message_sizes = reconstruct_messages_multi(
|
|
1328
|
+
group_headers, group_times, freq_nhz=group_freq_nhz, period_ns=group_period_ns)
|
|
1329
|
+
|
|
1330
|
+
# Sort the message data
|
|
1331
|
+
sort_message_time_values(message_starts, message_sizes, group_values)
|
|
1332
|
+
# Revert back to gap array
|
|
1333
|
+
gap_data = create_gap_arr_from_variable_messages(
|
|
1334
|
+
message_starts, message_sizes, freq_nhz=group_freq_nhz, period_ns=group_period_ns)
|
|
1335
|
+
|
|
1336
|
+
group_times = gap_data
|
|
1337
|
+
|
|
1338
|
+
group_interval_data = get_interval_list_from_message_starts_and_sizes(
|
|
1339
|
+
message_starts, message_sizes, freq_nhz=group_freq_nhz, period_ns=group_period_ns)
|
|
1340
|
+
|
|
1341
|
+
group_start_time = int(group_interval_data[0][0])
|
|
1342
|
+
else:
|
|
1343
|
+
raise ValueError("time_type must be 1 or 2")
|
|
1344
|
+
|
|
1345
|
+
block_group_interval_data.append(group_interval_data)
|
|
1346
|
+
|
|
1347
|
+
if np.issubdtype(group_values.dtype, np.integer):
|
|
1348
|
+
raw_value_type = 1
|
|
1349
|
+
encoded_value_type = 3
|
|
1350
|
+
else:
|
|
1351
|
+
raw_value_type = 2
|
|
1352
|
+
encoded_value_type = 2
|
|
1353
|
+
|
|
1354
|
+
# Create segment dictionary
|
|
1355
|
+
segment = {
|
|
1356
|
+
'times': group_times,
|
|
1357
|
+
'values': group_values,
|
|
1358
|
+
'start_ns': group_start_time,
|
|
1359
|
+
'raw_time_type': group_time_type,
|
|
1360
|
+
'encoded_time_type': group_encoded_time_type,
|
|
1361
|
+
'raw_value_type': raw_value_type,
|
|
1362
|
+
'encoded_value_type': encoded_value_type,
|
|
1363
|
+
'scale_m': group_headers[0].scale_m,
|
|
1364
|
+
'scale_b': group_headers[0].scale_b,
|
|
1365
|
+
'freq_nhz': group_freq_nhz,
|
|
1366
|
+
'period_ns': group_period_ns
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
segments.append(segment)
|
|
1370
|
+
|
|
1371
|
+
# Now, call encode_blocks_from_multiple_segments
|
|
1372
|
+
encoded_bytes, encoded_headers, byte_start_array = (
|
|
1373
|
+
sdk.block.encode_blocks_from_multiple_segments(segments))
|
|
1374
|
+
|
|
1375
|
+
# Handle the outputs
|
|
1376
|
+
total_encoded_bytes = encoded_bytes
|
|
1377
|
+
total_encoded_headers = encoded_headers
|
|
1378
|
+
total_byte_start_array = byte_start_array
|
|
1379
|
+
|
|
1380
|
+
# Process the intervals
|
|
1381
|
+
block_group_interval_data = intervals_union_list(block_group_interval_data)
|
|
1382
|
+
|
|
1383
|
+
# Write the encoded bytes to disk
|
|
1384
|
+
filename = sdk.file_api.write_bytes(measure_id, device_id, total_encoded_bytes)
|
|
1385
|
+
|
|
1386
|
+
block_data = []
|
|
1387
|
+
for header_i, header in enumerate(total_encoded_headers):
|
|
1388
|
+
block_data.append({
|
|
1389
|
+
"measure_id": measure_id,
|
|
1390
|
+
"device_id": device_id,
|
|
1391
|
+
"start_byte": int(total_byte_start_array[header_i]),
|
|
1392
|
+
"num_bytes": header.meta_num_bytes + header.t_num_bytes + header.v_num_bytes,
|
|
1393
|
+
"start_time_n": header.start_n,
|
|
1394
|
+
"end_time_n": header.end_n,
|
|
1395
|
+
"num_values": header.num_vals,
|
|
1396
|
+
})
|
|
1397
|
+
|
|
1398
|
+
# Write new and delete old SQL blocks
|
|
1399
|
+
sdk.sql_handler.insert_and_delete_tsc_file_data(
|
|
1400
|
+
filename, block_data, [block[0] for block in block_group])
|
|
1401
|
+
|
|
1402
|
+
total_measure_device_interval_array.append(block_group_interval_data)
|
|
1403
|
+
|
|
1404
|
+
# Combine collected intervals and replace the old intervals with the collected.
|
|
1405
|
+
total_measure_device_interval_array = intervals_union_list(
|
|
1406
|
+
total_measure_device_interval_array, gap_tolerance_nano=interval_gap_tolerance_nano)
|
|
1407
|
+
|
|
1408
|
+
sdk.sql_handler.replace_intervals(measure_id, device_id, total_measure_device_interval_array)
|
|
1409
|
+
|
|
1410
|
+
# Remove TSC files from SQL and disk
|
|
1411
|
+
sdk.sql_handler.delete_files_by_ids(file_id_list)
|
|
1412
|
+
|
|
1413
|
+
for filename in filename_dict.values():
|
|
1414
|
+
file_path = sdk.file_api.to_abs_path(filename, measure_id, device_id)
|
|
1415
|
+
sdk.file_api.remove(file_path)
|
|
1416
|
+
|
|
1417
|
+
sdk.block.block_size = original_block_size
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
def rebuild_intervals_from_existing_blocks(sdk, interval_gap_tolerance_nano=0, max_values_in_ram=1_000_000):
|
|
1421
|
+
"""
|
|
1422
|
+
Rebuilds intervals for all measures and devices based on existing block data.
|
|
1423
|
+
"""
|
|
1424
|
+
measures = sdk.get_all_measures()
|
|
1425
|
+
devices = sdk.get_all_devices()
|
|
1426
|
+
|
|
1427
|
+
measure_data = list(measures.items())
|
|
1428
|
+
device_data = list(devices.items())
|
|
1429
|
+
|
|
1430
|
+
for device_id, device_info in tqdm(device_data, desc="Devices"):
|
|
1431
|
+
for measure_id, measure_info in tqdm(measure_data, desc="Measures"):
|
|
1432
|
+
sorted_block_list_by_time = get_all_blocks(sdk, measure_id, device_id)
|
|
1433
|
+
if len(sorted_block_list_by_time) == 0:
|
|
1434
|
+
continue
|
|
1435
|
+
|
|
1436
|
+
file_id_list = list(set([row[3] for row in sorted_block_list_by_time]))
|
|
1437
|
+
filename_dict = sdk.get_filename_dict(file_id_list)
|
|
1438
|
+
|
|
1439
|
+
total_measure_device_interval_array = []
|
|
1440
|
+
|
|
1441
|
+
# Process all blocks to extract interval information
|
|
1442
|
+
for block_group in group_sorted_block_list(
|
|
1443
|
+
sorted_block_list_by_time, num_values_per_group=max_values_in_ram):
|
|
1444
|
+
|
|
1445
|
+
r_headers, r_times, r_values = sdk.get_data_from_blocks(
|
|
1446
|
+
block_list=block_group,
|
|
1447
|
+
filename_dict=filename_dict,
|
|
1448
|
+
start_time_n=0,
|
|
1449
|
+
end_time_n=2 ** 62,
|
|
1450
|
+
analog=False,
|
|
1451
|
+
time_type="encoded",
|
|
1452
|
+
sort=False,
|
|
1453
|
+
allow_duplicates=True
|
|
1454
|
+
)
|
|
1455
|
+
|
|
1456
|
+
if r_values.size == 0:
|
|
1457
|
+
continue
|
|
1458
|
+
|
|
1459
|
+
block_interval_data = []
|
|
1460
|
+
|
|
1461
|
+
# Group data by scale factor, freq, time type
|
|
1462
|
+
for group_headers, group_times, group_values in group_headers_by_scale_factor_freq_time_type(
|
|
1463
|
+
r_headers, r_times, r_values):
|
|
1464
|
+
group_time_type = int(group_headers[0].t_raw_type)
|
|
1465
|
+
period_ns = int(get_period_ns_from_header(group_headers[0]))
|
|
1466
|
+
|
|
1467
|
+
# Calculate intervals based on time type
|
|
1468
|
+
if group_time_type == 1:
|
|
1469
|
+
group_times, sorted_time_indices = np.unique(group_times, return_index=True)
|
|
1470
|
+
group_interval_data = get_interval_list_from_ordered_timestamps(group_times, period_ns)
|
|
1471
|
+
elif group_time_type == 2:
|
|
1472
|
+
# Merge the gap data into messages (segments)
|
|
1473
|
+
message_starts, message_sizes = reconstruct_messages_multi(
|
|
1474
|
+
group_headers, group_times, period_ns=period_ns)
|
|
1475
|
+
group_interval_data = get_interval_list_from_message_starts_and_sizes(
|
|
1476
|
+
message_starts, message_sizes, period_ns=period_ns)
|
|
1477
|
+
else:
|
|
1478
|
+
raise ValueError("time_type must be 1 or 2")
|
|
1479
|
+
|
|
1480
|
+
block_interval_data.append(group_interval_data)
|
|
1481
|
+
|
|
1482
|
+
# Union intervals within this block
|
|
1483
|
+
block_interval_data = intervals_union_list(block_interval_data)
|
|
1484
|
+
total_measure_device_interval_array.append(block_interval_data)
|
|
1485
|
+
|
|
1486
|
+
# Combine all collected intervals for this measure/device
|
|
1487
|
+
total_measure_device_interval_array = intervals_union_list(
|
|
1488
|
+
total_measure_device_interval_array, gap_tolerance_nano=interval_gap_tolerance_nano)
|
|
1489
|
+
|
|
1490
|
+
# Replace the intervals in the database
|
|
1491
|
+
sdk.sql_handler.replace_intervals(measure_id, device_id, total_measure_device_interval_array)
|
|
1492
|
+
|
|
1493
|
+
def inplace_block_time_fix(
|
|
1494
|
+
sdk,
|
|
1495
|
+
batch_size: int = 500
|
|
1496
|
+
) -> int:
|
|
1497
|
+
"""
|
|
1498
|
+
Scan every block in the SDK’s block_index table, decode it (time_type=1),
|
|
1499
|
+
and if the decoded start/end times differ from what the SQL row reports,
|
|
1500
|
+
update them in batches via SQLHandler.update_block_times.
|
|
1501
|
+
|
|
1502
|
+
:param AtriumSDK sdk: the SDK instance to fix
|
|
1503
|
+
:param int batch_size: how many mismatches to accumulate before flushing
|
|
1504
|
+
:return: total number of blocks updated
|
|
1505
|
+
"""
|
|
1506
|
+
block_rows = []
|
|
1507
|
+
for device_id in sdk.get_all_devices():
|
|
1508
|
+
cur_block_rows = sdk.sql_handler.select_blocks_for_device(device_id)
|
|
1509
|
+
block_rows.extend(cur_block_rows)
|
|
1510
|
+
|
|
1511
|
+
total_rows = len(block_rows)
|
|
1512
|
+
with tqdm(total=total_rows, desc="Fixing Block Times", unit="block") as pbar:
|
|
1513
|
+
for block_group in group_sorted_block_list(
|
|
1514
|
+
block_rows,
|
|
1515
|
+
num_values_per_group=sdk.block.block_size * batch_size,
|
|
1516
|
+
src_sdk_mode=sdk.mode
|
|
1517
|
+
):
|
|
1518
|
+
file_id_list = list({row[3] for row in block_group})
|
|
1519
|
+
filename_dict = sdk.get_filename_dict(file_id_list)
|
|
1520
|
+
read_list = condense_byte_read_list(block_group)
|
|
1521
|
+
encoded_bytes = sdk.file_api.read_file_list(read_list, filename_dict)
|
|
1522
|
+
block_num_bytes = [block[5] for block in block_group]
|
|
1523
|
+
block_num_values = [block[8] for block in block_group]
|
|
1524
|
+
|
|
1525
|
+
encode_group_times, encode_group_values, _ = sdk.block.decode_blocks(
|
|
1526
|
+
encoded_bytes, block_num_bytes, analog=False, time_type=1)
|
|
1527
|
+
|
|
1528
|
+
left = 0
|
|
1529
|
+
right = None
|
|
1530
|
+
to_fix_ids = []
|
|
1531
|
+
to_fix_ranges = []
|
|
1532
|
+
for block in block_group:
|
|
1533
|
+
right = left + block[8]
|
|
1534
|
+
block_times = encode_group_times[left:right]
|
|
1535
|
+
true_start = int(np.min(block_times))
|
|
1536
|
+
true_end = int(np.max(block_times))
|
|
1537
|
+
orig_start = int(block[6])
|
|
1538
|
+
orig_end = int(block[7])
|
|
1539
|
+
blk_id = int(block[0])
|
|
1540
|
+
|
|
1541
|
+
if true_start != orig_start or true_end != orig_end:
|
|
1542
|
+
to_fix_ids.append(blk_id)
|
|
1543
|
+
to_fix_ranges.append((true_start, true_end))
|
|
1544
|
+
left = right
|
|
1545
|
+
|
|
1546
|
+
if to_fix_ids:
|
|
1547
|
+
sdk.sql_handler.update_block_times(to_fix_ids, to_fix_ranges)
|
|
1548
|
+
|
|
1549
|
+
pbar.update(len(block_group))
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def group_sorted_block_list(sorted_block_list, num_values_per_group=8388608, src_sdk_mode=None):
|
|
1553
|
+
next_group = []
|
|
1554
|
+
total_group_size = 0
|
|
1555
|
+
|
|
1556
|
+
for block in sorted_block_list:
|
|
1557
|
+
if total_group_size >= num_values_per_group:
|
|
1558
|
+
yield next_group
|
|
1559
|
+
next_group = []
|
|
1560
|
+
total_group_size = 0
|
|
1561
|
+
|
|
1562
|
+
num_values = block["num_values"] if src_sdk_mode == "api" else block[8]
|
|
1563
|
+
next_group.append(block)
|
|
1564
|
+
total_group_size += num_values
|
|
1565
|
+
|
|
1566
|
+
if total_group_size > 0:
|
|
1567
|
+
yield next_group
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
def group_headers_by_scale_factor_freq_time_type(headers, times_array, values_array):
|
|
1571
|
+
if len(headers) == 0:
|
|
1572
|
+
return
|
|
1573
|
+
|
|
1574
|
+
# Initialize the first header group variables
|
|
1575
|
+
start_idx_vals = 0
|
|
1576
|
+
start_idx_times = 0
|
|
1577
|
+
current_scale_m = headers[0].scale_m
|
|
1578
|
+
current_scale_b = headers[0].scale_b
|
|
1579
|
+
current_t_raw_type = headers[0].t_raw_type
|
|
1580
|
+
current_freq_nhz = headers[0].freq_nhz
|
|
1581
|
+
current_t_encoded_type = headers[0].t_encoded_type
|
|
1582
|
+
current_group = [headers[0]]
|
|
1583
|
+
|
|
1584
|
+
for h in headers[1:]:
|
|
1585
|
+
# Check if header properties match the current group's properties
|
|
1586
|
+
if (h.scale_m == current_scale_m and
|
|
1587
|
+
h.scale_b == current_scale_b and
|
|
1588
|
+
h.t_raw_type == current_t_raw_type and
|
|
1589
|
+
h.freq_nhz == current_freq_nhz and
|
|
1590
|
+
h.t_encoded_type == current_t_encoded_type):
|
|
1591
|
+
# Same grouping factors, add to current group
|
|
1592
|
+
current_group.append(h)
|
|
1593
|
+
else:
|
|
1594
|
+
# Different grouping factors, yield the current group
|
|
1595
|
+
end_idx_vals = start_idx_vals + sum(header.num_vals for header in current_group)
|
|
1596
|
+
if current_t_raw_type == 1:
|
|
1597
|
+
end_idx_times = start_idx_times + sum(header.num_vals for header in current_group)
|
|
1598
|
+
elif current_t_raw_type == 2:
|
|
1599
|
+
end_idx_times = start_idx_times + sum(header.num_gaps * 2 for header in current_group)
|
|
1600
|
+
else:
|
|
1601
|
+
raise ValueError(f"Unknown t_raw_type: {current_t_raw_type}")
|
|
1602
|
+
|
|
1603
|
+
yield (current_group,
|
|
1604
|
+
times_array[start_idx_times:end_idx_times],
|
|
1605
|
+
values_array[start_idx_vals:end_idx_vals])
|
|
1606
|
+
|
|
1607
|
+
# Start a new group
|
|
1608
|
+
start_idx_vals = end_idx_vals
|
|
1609
|
+
start_idx_times = end_idx_times
|
|
1610
|
+
current_scale_m = h.scale_m
|
|
1611
|
+
current_scale_b = h.scale_b
|
|
1612
|
+
current_t_raw_type = h.t_raw_type
|
|
1613
|
+
current_freq_nhz = h.freq_nhz
|
|
1614
|
+
current_t_encoded_type = h.t_encoded_type
|
|
1615
|
+
current_group = [h]
|
|
1616
|
+
|
|
1617
|
+
# After the loop, yield any remaining group
|
|
1618
|
+
if current_group:
|
|
1619
|
+
end_idx_vals = start_idx_vals + sum(header.num_vals for header in current_group)
|
|
1620
|
+
if current_t_raw_type == 1:
|
|
1621
|
+
end_idx_times = start_idx_times + sum(header.num_vals for header in current_group)
|
|
1622
|
+
elif current_t_raw_type == 2:
|
|
1623
|
+
end_idx_times = start_idx_times + sum(header.num_gaps * 2 for header in current_group)
|
|
1624
|
+
else:
|
|
1625
|
+
raise ValueError(f"Unknown t_raw_type: {current_t_raw_type}")
|
|
1626
|
+
|
|
1627
|
+
yield (current_group,
|
|
1628
|
+
times_array[start_idx_times:end_idx_times],
|
|
1629
|
+
values_array[start_idx_vals:end_idx_vals])
|
|
1630
|
+
|
|
1631
|
+
|
|
1632
|
+
def get_interval_list_from_ordered_timestamps(timestamps, period_ns):
|
|
1633
|
+
if timestamps.size == 0:
|
|
1634
|
+
return []
|
|
1635
|
+
|
|
1636
|
+
diffs = np.diff(timestamps)
|
|
1637
|
+
break_indices = np.where(diffs > period_ns)[0]
|
|
1638
|
+
|
|
1639
|
+
# Start indices of each region ([0] + break_indices + 1)
|
|
1640
|
+
start_indices = np.insert(break_indices + 1, 0, 0)
|
|
1641
|
+
# End indices of each region (break_indices + [len(timestamps) - 1])
|
|
1642
|
+
end_indices = np.append(break_indices, timestamps.size - 1)
|
|
1643
|
+
|
|
1644
|
+
start_times = timestamps[start_indices]
|
|
1645
|
+
end_times = timestamps[end_indices] + period_ns
|
|
1646
|
+
|
|
1647
|
+
regions = np.column_stack((start_times, end_times))
|
|
1648
|
+
|
|
1649
|
+
return regions
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def get_interval_list_from_message_starts_and_sizes(message_starts, message_sizes, freq_nhz=None, *, period_ns=None):
|
|
1653
|
+
_validate_freq_period_params(freq_nhz, period_ns)
|
|
1654
|
+
|
|
1655
|
+
if message_starts.size == 0:
|
|
1656
|
+
return np.array([], dtype=np.int64)
|
|
1657
|
+
|
|
1658
|
+
merged_intervals = []
|
|
1659
|
+
current_start = message_starts[0]
|
|
1660
|
+
|
|
1661
|
+
if freq_nhz is not None:
|
|
1662
|
+
current_end = current_start + _message_size_freq_to_duration_ns(int(message_sizes[0]), freq_nhz)
|
|
1663
|
+
else:
|
|
1664
|
+
current_end = current_start + _message_size_period_to_durations_ns(int(message_sizes[0]), period_ns)
|
|
1665
|
+
|
|
1666
|
+
for start, message_size in zip(message_starts[1:], message_sizes[1:]):
|
|
1667
|
+
if freq_nhz is not None:
|
|
1668
|
+
end = start + _message_size_freq_to_duration_ns(int(message_size), freq_nhz)
|
|
1669
|
+
else:
|
|
1670
|
+
end = start + _message_size_period_to_durations_ns(int(message_size), period_ns)
|
|
1671
|
+
|
|
1672
|
+
if start <= current_end:
|
|
1673
|
+
# Overlapping intervals, update the end if needed
|
|
1674
|
+
current_end = max(current_end, end)
|
|
1675
|
+
else:
|
|
1676
|
+
# No overlap, append the current interval and reset
|
|
1677
|
+
merged_intervals.append([current_start, current_end])
|
|
1678
|
+
current_start, current_end = start, end
|
|
1679
|
+
merged_intervals.append([current_start, current_end])
|
|
1680
|
+
|
|
1681
|
+
return np.array(merged_intervals, dtype=np.int64)
|
|
1682
|
+
|
|
1683
|
+
|
|
1684
|
+
def _validate_freq_period_params(freq_nhz, period_ns):
|
|
1685
|
+
if freq_nhz is not None and period_ns is not None:
|
|
1686
|
+
raise ValueError("freq_nhz and period_ns are mutually exclusive")
|
|
1687
|
+
if freq_nhz is None and period_ns is None:
|
|
1688
|
+
raise ValueError("Either freq_nhz or period_ns must be provided")
|