PhaseNet 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PhaseNet-0.1.0.dist-info/LICENSE +674 -0
- PhaseNet-0.1.0.dist-info/METADATA +30 -0
- PhaseNet-0.1.0.dist-info/RECORD +12 -0
- PhaseNet-0.1.0.dist-info/WHEEL +6 -0
- PhaseNet-0.1.0.dist-info/top_level.txt +1 -0
- phasenet/__init__.py +1 -0
- phasenet/data_reader.py +520 -0
- phasenet/detect_peaks.py +175 -0
- phasenet/model.py +467 -0
- phasenet/predict.py +143 -0
- phasenet/train.py +297 -0
- phasenet/util.py +203 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: PhaseNet
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: PhaseNet
|
|
5
|
+
Home-page: UNKNOWN
|
|
6
|
+
Author: Weiqiang Zhu
|
|
7
|
+
Author-email: wayne.weiqiang@gmail.com
|
|
8
|
+
License: GPL-3.0
|
|
9
|
+
Platform: UNKNOWN
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: tensorflow (==2.3)
|
|
20
|
+
Requires-Dist: matplotlib
|
|
21
|
+
Requires-Dist: pandas
|
|
22
|
+
Requires-Dist: tqdm
|
|
23
|
+
Requires-Dist: scipy
|
|
24
|
+
Requires-Dist: obspy (==1.2.2)
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
27
|
+
|
|
28
|
+
UNKNOWN
|
|
29
|
+
|
|
30
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
phasenet/__init__.py,sha256=Pru0BlFBASFCFo7McHdohtKkUtgMPDwbGfyUZlE2_Vw,21
|
|
2
|
+
phasenet/data_reader.py,sha256=yX2oE_nHUY_nn5Lt_x34soTMVXs-gr-IPB8kcJoqVFg,20719
|
|
3
|
+
phasenet/detect_peaks.py,sha256=p_xBQEve6bD87hUv9Oxq8hvVIxEM8eZvljl5TdhnMzE,6626
|
|
4
|
+
phasenet/model.py,sha256=H41hKqvuhpCT-dsx-UFJU_dNhymP2MSq7t1KPgpxB00,19978
|
|
5
|
+
phasenet/predict.py,sha256=2efi2I59V9mNDVLlIxNVdFEWy1oXeuufjSyDxgmvCyQ,6315
|
|
6
|
+
phasenet/train.py,sha256=lO3lIn_GM7GTd7Vp8vwvGr41IXHrIQU5XzpYfQ1aEms,13181
|
|
7
|
+
phasenet/util.py,sha256=vbujd-JBfehSfGSh6GqarfChLgxV14gvI5Zy-TkVZqs,8815
|
|
8
|
+
PhaseNet-0.1.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
9
|
+
PhaseNet-0.1.0.dist-info/METADATA,sha256=OoZBZwFA_VoIPhByTdLHQno7QQ6BUjSryYgdjpckic8,861
|
|
10
|
+
PhaseNet-0.1.0.dist-info/WHEEL,sha256=ADKeyaGyKF5DwBNE0sRE5pvW-bSkFMJfBuhzZ3rceP4,110
|
|
11
|
+
PhaseNet-0.1.0.dist-info/top_level.txt,sha256=bdfMC0KAkW6WaQyU_VcAgWqnbAhs3UOUtvEttbtrxFk,9
|
|
12
|
+
PhaseNet-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
phasenet
|
phasenet/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
phasenet/data_reader.py
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
import tensorflow as tf
|
|
2
|
+
tf.compat.v1.disable_eager_execution()
|
|
3
|
+
import os, threading, logging
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import scipy.interpolate
|
|
7
|
+
pd.options.mode.chained_assignment = None
|
|
8
|
+
import obspy
|
|
9
|
+
from tqdm import tqdm
|
|
10
|
+
from scipy.interpolate import interp1d
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def py_func_decorator(output_types=None, output_shapes=None, name=None):
|
|
14
|
+
def decorator(func):
|
|
15
|
+
def call(*args, **kwargs):
|
|
16
|
+
nonlocal output_shapes
|
|
17
|
+
# flat_output_types = nest.flatten(output_types)
|
|
18
|
+
flat_output_types = tf.nest.flatten(output_types)
|
|
19
|
+
# flat_values = tf.py_func(
|
|
20
|
+
flat_values = tf.numpy_function(
|
|
21
|
+
func,
|
|
22
|
+
inp=args,
|
|
23
|
+
Tout=flat_output_types,
|
|
24
|
+
name=name
|
|
25
|
+
)
|
|
26
|
+
if output_shapes is not None:
|
|
27
|
+
for v, s in zip(flat_values, output_shapes):
|
|
28
|
+
v.set_shape(s)
|
|
29
|
+
# return nest.pack_sequence_as(output_types, flat_values)
|
|
30
|
+
return tf.nest.pack_sequence_as(output_types, flat_values)
|
|
31
|
+
return call
|
|
32
|
+
return decorator
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def dataset_map(iterator, output_types, output_shapes=None, num_parallel_calls=None, name=None, shuffle=False):
|
|
36
|
+
dataset = tf.data.Dataset.range(len(iterator))
|
|
37
|
+
if shuffle:
|
|
38
|
+
dataset = dataset.shuffle(len(iterator))
|
|
39
|
+
@py_func_decorator(output_types, output_shapes, name=name)
|
|
40
|
+
def index_to_entry(idx):
|
|
41
|
+
return iterator[idx]
|
|
42
|
+
return dataset.map(index_to_entry, num_parallel_calls=num_parallel_calls)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def normalize(data, window=3000):
|
|
46
|
+
"""
|
|
47
|
+
data: nt, nch
|
|
48
|
+
"""
|
|
49
|
+
shift = window//2
|
|
50
|
+
nt, nch = data.shape
|
|
51
|
+
|
|
52
|
+
## std in slide windows
|
|
53
|
+
data_pad = np.pad(data, ((window//2, window//2), (0,0)), mode="reflect")
|
|
54
|
+
t = np.arange(0, nt, shift, dtype="int")
|
|
55
|
+
std = np.zeros([len(t)+1, nch])
|
|
56
|
+
mean = np.zeros([len(t)+1, nch])
|
|
57
|
+
for i in range(1, len(std)):
|
|
58
|
+
std[i, :] = np.std(data_pad[i*shift:i*shift+window, :], axis=0)
|
|
59
|
+
mean[i, :] = np.mean(data_pad[i*shift:i*shift+window, :], axis=0)
|
|
60
|
+
|
|
61
|
+
t = np.append(t, nt)
|
|
62
|
+
# std[-1, :] = np.std(data_pad[-window:, :], axis=0)
|
|
63
|
+
# mean[-1, :] = np.mean(data_pad[-window:, :], axis=0)
|
|
64
|
+
std[-1, :], mean[-1, :] = std[-2, :], mean[-2, :]
|
|
65
|
+
std[0, :], mean[0, :] = std[1, :], mean[1, :]
|
|
66
|
+
std[std == 0] = 1
|
|
67
|
+
|
|
68
|
+
## normalize data with interplated std
|
|
69
|
+
t_interp = np.arange(nt, dtype="int")
|
|
70
|
+
std_interp = interp1d(t, std, axis=0, kind="slinear")(t_interp)
|
|
71
|
+
mean_interp = interp1d(t, mean, axis=0, kind="slinear")(t_interp)
|
|
72
|
+
data = (data - mean_interp)/std_interp
|
|
73
|
+
|
|
74
|
+
return data
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def normalize_batch(data, window=3000):
|
|
78
|
+
"""
|
|
79
|
+
data: nsta, nt, nch
|
|
80
|
+
"""
|
|
81
|
+
shift = window//2
|
|
82
|
+
nsta, nt, nch = data.shape
|
|
83
|
+
|
|
84
|
+
## std in slide windows
|
|
85
|
+
data_pad = np.pad(data, ((0,0), (window//2, window//2), (0,0)), mode="reflect")
|
|
86
|
+
t = np.arange(0, nt, shift, dtype="int")
|
|
87
|
+
std = np.zeros([nsta, len(t)+1, nch])
|
|
88
|
+
mean = np.zeros([nsta, len(t)+1, nch])
|
|
89
|
+
for i in range(1, len(t)):
|
|
90
|
+
std[:, i, :] = np.std(data_pad[:, i*shift:i*shift+window, :], axis=1)
|
|
91
|
+
mean[:, i, :] = np.mean(data_pad[:, i*shift:i*shift+window, :], axis=1)
|
|
92
|
+
|
|
93
|
+
t = np.append(t, nt)
|
|
94
|
+
# std[:, -1, :] = np.std(data_pad[:, -window:, :], axis=1)
|
|
95
|
+
# mean[:, -1, :] = np.mean(data_pad[:, -window:, :], axis=1)
|
|
96
|
+
std[:, -1, :], mean[:, -1, :] = std[:, -2, :], mean[:, -2, :]
|
|
97
|
+
std[:, 0, :], mean[:, 0, :] = std[:, 1, :], mean[:, 1, :]
|
|
98
|
+
std[std == 0] = 1
|
|
99
|
+
|
|
100
|
+
# ## normalize data with interplated std
|
|
101
|
+
t_interp = np.arange(nt, dtype="int")
|
|
102
|
+
std_interp = interp1d(t, std, axis=1, kind="slinear")(t_interp)
|
|
103
|
+
mean_interp = interp1d(t, mean, axis=1, kind="slinear")(t_interp)
|
|
104
|
+
data = (data - mean_interp)/std_interp
|
|
105
|
+
|
|
106
|
+
return data
|
|
107
|
+
|
|
108
|
+
class DataConfig():
|
|
109
|
+
|
|
110
|
+
seed = 100
|
|
111
|
+
use_seed = True
|
|
112
|
+
n_channel = 3
|
|
113
|
+
n_class = 3
|
|
114
|
+
sampling_rate = 100
|
|
115
|
+
dt = 1.0/sampling_rate
|
|
116
|
+
X_shape = [3000, 1, n_channel]
|
|
117
|
+
Y_shape = [3000, 1, n_class]
|
|
118
|
+
min_event_gap = 3 * sampling_rate
|
|
119
|
+
dtype = "float32"
|
|
120
|
+
|
|
121
|
+
def __init__(self, **kwargs):
|
|
122
|
+
for k, v in kwargs.items():
|
|
123
|
+
setattr(self, k, v)
|
|
124
|
+
|
|
125
|
+
class DataReader(object):
|
|
126
|
+
def __init__(self,
|
|
127
|
+
data_dir,
|
|
128
|
+
data_list,
|
|
129
|
+
mask_window,
|
|
130
|
+
queue_size,
|
|
131
|
+
coord,
|
|
132
|
+
config=DataConfig()):
|
|
133
|
+
self.config = config
|
|
134
|
+
tmp_list = pd.read_csv(data_list, header=0)
|
|
135
|
+
self.data_list = tmp_list
|
|
136
|
+
self.num_data = len(self.data_list)
|
|
137
|
+
self.data_dir = data_dir
|
|
138
|
+
self.queue_size = queue_size
|
|
139
|
+
self.n_channel = config.n_channel
|
|
140
|
+
self.n_class = config.n_class
|
|
141
|
+
self.X_shape = config.X_shape
|
|
142
|
+
self.Y_shape = config.Y_shape
|
|
143
|
+
self.min_event_gap = config.min_event_gap
|
|
144
|
+
self.mask_window = int(mask_window * config.sampling_rate)
|
|
145
|
+
self.coord = coord
|
|
146
|
+
self.threads = []
|
|
147
|
+
self.buffer = {}
|
|
148
|
+
self.buffer_channels = {}
|
|
149
|
+
self.add_placeholder()
|
|
150
|
+
|
|
151
|
+
def add_placeholder(self):
|
|
152
|
+
self.sample_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=self.config.X_shape)
|
|
153
|
+
self.target_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=self.config.Y_shape)
|
|
154
|
+
self.queue = tf.queue.PaddingFIFOQueue(self.queue_size,
|
|
155
|
+
['float32', 'float32'],
|
|
156
|
+
shapes=[self.config.X_shape, self.config.Y_shape])
|
|
157
|
+
self.enqueue = self.queue.enqueue([self.sample_placeholder, self.target_placeholder])
|
|
158
|
+
|
|
159
|
+
def dequeue(self, num_elements):
|
|
160
|
+
output = self.queue.dequeue_many(num_elements)
|
|
161
|
+
return output
|
|
162
|
+
|
|
163
|
+
def normalize(self, data):
|
|
164
|
+
data -= np.mean(data, axis=0, keepdims=True)
|
|
165
|
+
std_data = np.std(data, axis=0, keepdims=True)
|
|
166
|
+
assert(std_data.shape[-1] == data.shape[-1])
|
|
167
|
+
std_data[std_data == 0] = 1
|
|
168
|
+
data /= std_data
|
|
169
|
+
return data
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def generate_target(self, itp_list, its_list):
|
|
173
|
+
|
|
174
|
+
target = np.zeros(self.Y_shape)
|
|
175
|
+
itp_true = []
|
|
176
|
+
its_true = []
|
|
177
|
+
for itp, its in zip(itp_list, its_list):
|
|
178
|
+
if (itp >= target.shape[0]) or (itp < 0):
|
|
179
|
+
pass
|
|
180
|
+
elif (itp-self.mask_window//2 >= 0) and (itp-self.mask_window//2 < target.shape[0]):
|
|
181
|
+
target[itp-self.mask_window//2:itp+self.mask_window//2, 0, 1] = \
|
|
182
|
+
np.exp(-(np.arange(-self.mask_window//2,self.mask_window//2))**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(itp-self.mask_window//2)]
|
|
183
|
+
itp_true.append(itp)
|
|
184
|
+
elif (itp-self.mask_window//2 < target.shape[0]):
|
|
185
|
+
target[0:itp+self.mask_window//2, 0, 1] = \
|
|
186
|
+
np.exp(-(np.arange(0,itp+self.mask_window//2)-itp)**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(itp-self.mask_window//2)]
|
|
187
|
+
itp_true.append(itp)
|
|
188
|
+
|
|
189
|
+
if (its >= target.shape[0]) or (its < 0):
|
|
190
|
+
pass
|
|
191
|
+
elif (its-self.mask_window//2 >= 0) and (its-self.mask_window//2 < target.shape[0]):
|
|
192
|
+
target[its-self.mask_window//2:its+self.mask_window//2, 0, 2] = \
|
|
193
|
+
np.exp(-(np.arange(-self.mask_window//2,self.mask_window//2))**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(its-self.mask_window//2)]
|
|
194
|
+
its_true.append(its)
|
|
195
|
+
elif (its-self.mask_window//2 < target.shape[0]):
|
|
196
|
+
target[0:its+self.mask_window//2, 0, 2] = \
|
|
197
|
+
np.exp(-(np.arange(0,its+self.mask_window//2)-its)**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(its-self.mask_window//2)]
|
|
198
|
+
its_true.append(its)
|
|
199
|
+
|
|
200
|
+
target[:, :, 0] = 1 - target[:, :, 1] - target[:, :, 2]
|
|
201
|
+
return target, itp_true, its_true
|
|
202
|
+
|
|
203
|
+
def thread_main(self, sess, n_threads=1, start=0):
|
|
204
|
+
stop = False
|
|
205
|
+
while not stop:
|
|
206
|
+
index = list(range(start, self.num_data, n_threads))
|
|
207
|
+
np.random.shuffle(index)
|
|
208
|
+
for i in index:
|
|
209
|
+
fname = os.path.join(self.data_dir, self.data_list.iloc[i]['fname'])
|
|
210
|
+
try:
|
|
211
|
+
if fname not in self.buffer:
|
|
212
|
+
meta = np.load(fname)
|
|
213
|
+
self.buffer[fname] = {'data': meta['data'], 'itp': meta['itp'], 'its': meta['its']}
|
|
214
|
+
meta = self.buffer[fname]
|
|
215
|
+
except:
|
|
216
|
+
logging.error("Failed reading {}".format(fname))
|
|
217
|
+
continue
|
|
218
|
+
|
|
219
|
+
start_tp = meta['itp'].tolist()
|
|
220
|
+
|
|
221
|
+
if self.coord.should_stop():
|
|
222
|
+
stop = True
|
|
223
|
+
break
|
|
224
|
+
|
|
225
|
+
sample = np.zeros(self.X_shape)
|
|
226
|
+
if np.random.random() < 0.95:
|
|
227
|
+
data = np.copy(meta['data'])
|
|
228
|
+
itp = meta['itp']
|
|
229
|
+
its = meta['its']
|
|
230
|
+
start_tp = itp
|
|
231
|
+
|
|
232
|
+
shift = np.random.randint(-(self.X_shape[0]-self.mask_window), min([its-start_tp, self.X_shape[0]])-self.mask_window)
|
|
233
|
+
sample[:, :, :] = data[start_tp+shift:start_tp+self.X_shape[0]+shift, np.newaxis, :]
|
|
234
|
+
itp_list = [itp-start_tp-shift]
|
|
235
|
+
its_list = [its-start_tp-shift]
|
|
236
|
+
else:
|
|
237
|
+
sample[:, :, :] = np.copy(meta['data'][start_tp-self.X_shape[0]:start_tp, np.newaxis, :])
|
|
238
|
+
itp_list = []
|
|
239
|
+
its_list = []
|
|
240
|
+
|
|
241
|
+
sample = self.normalize(sample)
|
|
242
|
+
|
|
243
|
+
if (np.isnan(sample).any() or np.isinf(sample).any() or (not sample.any())):
|
|
244
|
+
continue
|
|
245
|
+
|
|
246
|
+
target, _, _ = self.generate_target(itp_list, its_list)
|
|
247
|
+
|
|
248
|
+
sess.run(self.enqueue, feed_dict={self.sample_placeholder: sample, self.target_placeholder: target})
|
|
249
|
+
return 0
|
|
250
|
+
|
|
251
|
+
def start_threads(self, sess, n_threads=8):
|
|
252
|
+
for i in range(n_threads):
|
|
253
|
+
thread = threading.Thread(target=self.thread_main, args=(sess, n_threads, i))
|
|
254
|
+
thread.daemon = True
|
|
255
|
+
thread.start()
|
|
256
|
+
self.threads.append(thread)
|
|
257
|
+
return self.threads
|
|
258
|
+
|
|
259
|
+
class DataReader_test(DataReader):
|
|
260
|
+
|
|
261
|
+
def add_placeholder(self):
|
|
262
|
+
self.sample_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=None)
|
|
263
|
+
self.target_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=None)
|
|
264
|
+
self.fname_placeholder = tf.compat.v1.placeholder(dtype=tf.string, shape=None)
|
|
265
|
+
self.itp_placeholder = tf.compat.v1.placeholder(dtype=tf.int32, shape=None)
|
|
266
|
+
self.its_placeholder = tf.compat.v1.placeholder(dtype=tf.int32, shape=None)
|
|
267
|
+
self.queue = tf.queue.PaddingFIFOQueue(self.queue_size,
|
|
268
|
+
['float32', 'float32', 'string', 'int32', 'int32'],
|
|
269
|
+
shapes=[self.config.X_shape, self.config.Y_shape, [], [None], [None]])
|
|
270
|
+
self.enqueue = self.queue.enqueue([self.sample_placeholder, self.target_placeholder,
|
|
271
|
+
self.fname_placeholder,
|
|
272
|
+
self.itp_placeholder, self.its_placeholder])
|
|
273
|
+
|
|
274
|
+
def dequeue(self, num_elements):
|
|
275
|
+
output = self.queue.dequeue_up_to(num_elements)
|
|
276
|
+
return output
|
|
277
|
+
|
|
278
|
+
def thread_main(self, sess, n_threads=1, start=0):
|
|
279
|
+
|
|
280
|
+
index = list(range(start, self.num_data, n_threads))
|
|
281
|
+
for i in index:
|
|
282
|
+
fname = self.data_list.iloc[i]['fname']
|
|
283
|
+
fp = os.path.join(self.data_dir, fname)
|
|
284
|
+
try:
|
|
285
|
+
if fp not in self.buffer:
|
|
286
|
+
meta = np.load(fp)
|
|
287
|
+
self.buffer[fp] = {'data': meta['data'], 'itp': meta['itp'], 'its': meta['its']}
|
|
288
|
+
meta = self.buffer[fp]
|
|
289
|
+
except:
|
|
290
|
+
logging.error("Failed reading {}".format(fp))
|
|
291
|
+
continue
|
|
292
|
+
|
|
293
|
+
start_tp = meta['itp'].tolist()
|
|
294
|
+
|
|
295
|
+
if self.coord.should_stop():
|
|
296
|
+
break
|
|
297
|
+
|
|
298
|
+
sample = np.zeros(self.X_shape)
|
|
299
|
+
|
|
300
|
+
np.random.seed(self.config.seed+i)
|
|
301
|
+
shift = np.random.randint(-(self.X_shape[0]-self.mask_window), min([meta['its'].tolist()-start_tp, self.X_shape[0]])-self.mask_window)
|
|
302
|
+
sample[:, :, :] = np.copy(meta['data'][start_tp+shift:start_tp+self.X_shape[0]+shift, np.newaxis, :])
|
|
303
|
+
itp_list = [meta['itp'].tolist()-start_tp-shift]
|
|
304
|
+
its_list = [meta['its'].tolist()-start_tp-shift]
|
|
305
|
+
|
|
306
|
+
sample = self.normalize(sample)
|
|
307
|
+
|
|
308
|
+
if (np.isnan(sample).any() or np.isinf(sample).any() or (not sample.any())):
|
|
309
|
+
continue
|
|
310
|
+
|
|
311
|
+
target, itp_true, its_true = self.generate_target(itp_list, its_list)
|
|
312
|
+
|
|
313
|
+
sess.run(self.enqueue, feed_dict={self.sample_placeholder: sample,
|
|
314
|
+
self.target_placeholder: target,
|
|
315
|
+
self.fname_placeholder: fname,
|
|
316
|
+
self.itp_placeholder: itp_true,
|
|
317
|
+
self.its_placeholder: its_true})
|
|
318
|
+
|
|
319
|
+
return 0
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
class DataReader_pred(DataReader):
|
|
323
|
+
|
|
324
|
+
def __init__(self,
|
|
325
|
+
data_dir,
|
|
326
|
+
data_list):
|
|
327
|
+
|
|
328
|
+
tmp_list = pd.read_csv(data_list, header=0)
|
|
329
|
+
self.data_list = tmp_list
|
|
330
|
+
self.num_data = len(self.data_list)
|
|
331
|
+
self.data_dir = data_dir
|
|
332
|
+
self.dtype = "float32"
|
|
333
|
+
self.X_shape = self.get_data_shape()
|
|
334
|
+
|
|
335
|
+
def get_data_shape(self):
|
|
336
|
+
fname = self.data_list.iloc[0]['fname']
|
|
337
|
+
fp = os.path.join(self.data_dir, fname)
|
|
338
|
+
data = np.load(fp)["data"][:,np.newaxis,:]
|
|
339
|
+
return data.shape
|
|
340
|
+
|
|
341
|
+
def adjust_missingchannels(self, data):
|
|
342
|
+
tmp = np.max(np.abs(data), axis=0, keepdims=True)
|
|
343
|
+
assert(tmp.shape[-1] == data.shape[-1])
|
|
344
|
+
if np.count_nonzero(tmp) > 0:
|
|
345
|
+
data *= data.shape[-1] / np.count_nonzero(tmp)
|
|
346
|
+
return data
|
|
347
|
+
|
|
348
|
+
def __len__(self):
|
|
349
|
+
return self.num_data
|
|
350
|
+
|
|
351
|
+
def __getitem__(self, i):
|
|
352
|
+
fname = self.data_list.iloc[i]['fname']
|
|
353
|
+
fp = os.path.join(self.data_dir, fname)
|
|
354
|
+
|
|
355
|
+
try:
|
|
356
|
+
meta = np.load(fp)
|
|
357
|
+
except:
|
|
358
|
+
logging.error("Failed reading {}".format(fname))
|
|
359
|
+
return (np.zeros(self.X_shape, dtype=self.dtype), fname)
|
|
360
|
+
|
|
361
|
+
shift = 0
|
|
362
|
+
sample = normalize(meta['data'])[:, np.newaxis, :].astype(self.dtype)
|
|
363
|
+
|
|
364
|
+
if not np.array_equal(np.array(sample.shape), np.array(self.X_shape)):
|
|
365
|
+
logging.warning(f"Shape mismatch: {sample.shape} != {self.X_shape} in {fname}")
|
|
366
|
+
tmp = np.zeros(self.X_shape, dtype=self.dtype)
|
|
367
|
+
tmp[:sample.shape[0],0,:sample.shape[2]] = sample[:tmp.shape[0],0,:tmp.shape[2]]
|
|
368
|
+
sample = tmp
|
|
369
|
+
|
|
370
|
+
if np.isnan(sample).any() or np.isinf(sample).any():
|
|
371
|
+
logging.warning(f"Data error: Nan or Inf found in {fname}")
|
|
372
|
+
|
|
373
|
+
sample[np.isnan(sample)] = 0
|
|
374
|
+
sample[np.isinf(sample)] = 0
|
|
375
|
+
|
|
376
|
+
# sample = self.normalize(sample)
|
|
377
|
+
sample = self.adjust_missingchannels(sample)
|
|
378
|
+
|
|
379
|
+
return (sample, fname)
|
|
380
|
+
|
|
381
|
+
def dataset(self, num_parallel=2):
|
|
382
|
+
dataset = dataset_map(self,
|
|
383
|
+
output_types=("float32", "string"),
|
|
384
|
+
output_shapes=(self.X_shape, None),
|
|
385
|
+
num_parallel_calls=num_parallel)
|
|
386
|
+
return dataset
|
|
387
|
+
|
|
388
|
+
class DataReader_mseed(DataReader):
|
|
389
|
+
|
|
390
|
+
def __init__(self,
|
|
391
|
+
data_list,
|
|
392
|
+
stations,
|
|
393
|
+
data_dir=""):
|
|
394
|
+
|
|
395
|
+
self.data_list = pd.read_csv(data_list, header=0)
|
|
396
|
+
self.num_data = len(self.data_list)
|
|
397
|
+
self.data_dir = data_dir
|
|
398
|
+
self.stations = pd.read_csv(stations, delimiter="\t")
|
|
399
|
+
self.dtype = "float32"
|
|
400
|
+
self.X_shape = self.get_data_shape()
|
|
401
|
+
|
|
402
|
+
def get_data_shape(self):
|
|
403
|
+
(data, fname) = self[0]
|
|
404
|
+
return data.shape
|
|
405
|
+
|
|
406
|
+
def read_mseed(self, fp):
|
|
407
|
+
meta = obspy.read(fp)
|
|
408
|
+
meta = meta.detrend("spline", order=2, dspline=5*meta[0].stats.sampling_rate)
|
|
409
|
+
meta = meta.merge(fill_value=0)
|
|
410
|
+
meta = meta.trim(min([st.stats.starttime for st in meta]),
|
|
411
|
+
max([st.stats.endtime for st in meta]),
|
|
412
|
+
pad=True, fill_value=0)
|
|
413
|
+
|
|
414
|
+
if meta[0].stats.sampling_rate != 100:
|
|
415
|
+
logging.warning(f"Sampling rate {meta[0].stats.sampling_rate} != 100 Hz")
|
|
416
|
+
|
|
417
|
+
order = ['3','2','1','E','N','Z']
|
|
418
|
+
order = {key: i for i, key in enumerate(order)}
|
|
419
|
+
|
|
420
|
+
nsta = len(self.stations)
|
|
421
|
+
nt = len(meta[0].data)
|
|
422
|
+
data = np.zeros([nsta, nt, 3], dtype=self.dtype)
|
|
423
|
+
for i in range(nsta):
|
|
424
|
+
sta = self.stations.iloc[i]["station"]
|
|
425
|
+
cmp = sorted(self.stations.iloc[i]["component"].split(","), key=lambda x: order[x])
|
|
426
|
+
for j, c in enumerate(cmp):
|
|
427
|
+
data[i, :, j] = meta.select(id=sta+c)[0].data.astype(self.dtype)
|
|
428
|
+
|
|
429
|
+
return data
|
|
430
|
+
|
|
431
|
+
def __len__(self):
|
|
432
|
+
return self.num_data
|
|
433
|
+
|
|
434
|
+
def __getitem__(self, i):
|
|
435
|
+
|
|
436
|
+
fname = self.data_list.iloc[i]['fname']
|
|
437
|
+
fp = os.path.join(self.data_dir, fname)
|
|
438
|
+
|
|
439
|
+
try:
|
|
440
|
+
data = self.read_mseed(fp)
|
|
441
|
+
except Exception as e:
|
|
442
|
+
logging.error(f"Failed reading {fname}: {e}")
|
|
443
|
+
return (np.zeros(self.X_shape, dtype=self.dtype), fname)
|
|
444
|
+
|
|
445
|
+
sample = normalize_batch(data)[:,:,np.newaxis,:]
|
|
446
|
+
|
|
447
|
+
return (sample.astype(self.dtype), fname)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def dataset(self, num_parallel=2):
|
|
451
|
+
dataset = dataset_map(self,
|
|
452
|
+
output_types=("float32", "string"),
|
|
453
|
+
output_shapes=(self.X_shape, None),
|
|
454
|
+
num_parallel_calls=num_parallel)
|
|
455
|
+
return dataset
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
# def test_DataReader_mseed(data_list, data_dir, stations):
|
|
459
|
+
# print("test_DataReader_mseed:")
|
|
460
|
+
# import timeit
|
|
461
|
+
# data_reader = DataReader_mseed(
|
|
462
|
+
# data_list = data_list,
|
|
463
|
+
# data_dir = data_dir,
|
|
464
|
+
# stations = stations)
|
|
465
|
+
|
|
466
|
+
# start_time = timeit.default_timer()
|
|
467
|
+
# dataset = dataset_map(data_reader,
|
|
468
|
+
# output_types=("float32", "string"),
|
|
469
|
+
# output_shapes=(data_reader.X_shape, None),
|
|
470
|
+
# num_parallel_calls=2)
|
|
471
|
+
# dataset = dataset
|
|
472
|
+
# # dataset = tf.data.Dataset.range(data_reader.num_data)
|
|
473
|
+
# # dataset = dataset.interleave(lambda x: tf.data.Dataset.from_generator(data_reader.generator,
|
|
474
|
+
# # output_types=(data_reader.dtype, "string"),
|
|
475
|
+
# # output_shapes=(data_reader.X_shape, None),
|
|
476
|
+
# # args=(x,)),
|
|
477
|
+
# # cycle_length=data_reader.num_data,
|
|
478
|
+
# # block_length=1,
|
|
479
|
+
# # num_parallel_calls=data_reader.num_data)
|
|
480
|
+
|
|
481
|
+
# sess = tf.compat.v1.Session()
|
|
482
|
+
|
|
483
|
+
# print(len(data_reader))
|
|
484
|
+
# print("-------", tf.data.Dataset.cardinality(dataset))
|
|
485
|
+
# num = 0
|
|
486
|
+
# # for x in tf.compat.v1.data.make_one_shot_iterator(dataset).get_next():
|
|
487
|
+
# # print(num)
|
|
488
|
+
# # num += 1
|
|
489
|
+
# # dum = sess.run(x)
|
|
490
|
+
# # print(dum)
|
|
491
|
+
# x = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next()
|
|
492
|
+
# while True:
|
|
493
|
+
# num+=1
|
|
494
|
+
# print(num)
|
|
495
|
+
# dum = sess.run(x)
|
|
496
|
+
|
|
497
|
+
# print("Tensorflow Dataset:\nexecution time = ", timeit.default_timer() - start_time)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
if __name__ == "__main__":
|
|
501
|
+
pass
|
|
502
|
+
## debug
|
|
503
|
+
# data_reader = DataReader_mseed(
|
|
504
|
+
# data_dir="/data/beroza/zhuwq/Project-PhaseNet-mseed/mseed/",
|
|
505
|
+
# data_list="/data/beroza/zhuwq/Project-PhaseNet-mseed/fname.txt",
|
|
506
|
+
# queue_size=20,
|
|
507
|
+
# coord=None)
|
|
508
|
+
# data_reader.thread_main(None, n_threads=1, start=0)
|
|
509
|
+
# pred_fn(args, data_reader, log_dir=args.output_dir)
|
|
510
|
+
|
|
511
|
+
# test_DataReader_mseed()
|
|
512
|
+
# from glob import glob
|
|
513
|
+
# import os
|
|
514
|
+
# import pandas as pd
|
|
515
|
+
# mseed_dir = "../../quakeflow/mseed"
|
|
516
|
+
# mseed_list = glob(os.path.join(mseed_dir, "*.mseed"), recursive=True)
|
|
517
|
+
# print(f"Number of mseed: {len(mseed_list)}")
|
|
518
|
+
|
|
519
|
+
# stations = pd.read_csv("../../quakeflow/stations.csv", delimiter="\t", index_col="station")
|
|
520
|
+
# test_DataReader_mseed("../../quakeflow/mseed.lst", "../../quakeflow/", "../../quakeflow/stations.csv")
|