PhaseNet 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,30 @@
1
+ Metadata-Version: 2.1
2
+ Name: PhaseNet
3
+ Version: 0.1.0
4
+ Summary: PhaseNet
5
+ Home-page: UNKNOWN
6
+ Author: Weiqiang Zhu
7
+ Author-email: wayne.weiqiang@gmail.com
8
+ License: GPL-3.0
9
+ Platform: UNKNOWN
10
+ Classifier: License :: OSI Approved :: BSD License
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Topic :: Software Development :: Libraries
17
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: tensorflow (==2.3)
20
+ Requires-Dist: matplotlib
21
+ Requires-Dist: pandas
22
+ Requires-Dist: tqdm
23
+ Requires-Dist: scipy
24
+ Requires-Dist: obspy (==1.2.2)
25
+ Provides-Extra: test
26
+ Requires-Dist: pytest ; extra == 'test'
27
+
28
+ UNKNOWN
29
+
30
+
@@ -0,0 +1,12 @@
1
+ phasenet/__init__.py,sha256=Pru0BlFBASFCFo7McHdohtKkUtgMPDwbGfyUZlE2_Vw,21
2
+ phasenet/data_reader.py,sha256=yX2oE_nHUY_nn5Lt_x34soTMVXs-gr-IPB8kcJoqVFg,20719
3
+ phasenet/detect_peaks.py,sha256=p_xBQEve6bD87hUv9Oxq8hvVIxEM8eZvljl5TdhnMzE,6626
4
+ phasenet/model.py,sha256=H41hKqvuhpCT-dsx-UFJU_dNhymP2MSq7t1KPgpxB00,19978
5
+ phasenet/predict.py,sha256=2efi2I59V9mNDVLlIxNVdFEWy1oXeuufjSyDxgmvCyQ,6315
6
+ phasenet/train.py,sha256=lO3lIn_GM7GTd7Vp8vwvGr41IXHrIQU5XzpYfQ1aEms,13181
7
+ phasenet/util.py,sha256=vbujd-JBfehSfGSh6GqarfChLgxV14gvI5Zy-TkVZqs,8815
8
+ PhaseNet-0.1.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
9
+ PhaseNet-0.1.0.dist-info/METADATA,sha256=OoZBZwFA_VoIPhByTdLHQno7QQ6BUjSryYgdjpckic8,861
10
+ PhaseNet-0.1.0.dist-info/WHEEL,sha256=ADKeyaGyKF5DwBNE0sRE5pvW-bSkFMJfBuhzZ3rceP4,110
11
+ PhaseNet-0.1.0.dist-info/top_level.txt,sha256=bdfMC0KAkW6WaQyU_VcAgWqnbAhs3UOUtvEttbtrxFk,9
12
+ PhaseNet-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,6 @@
1
+ Wheel-Version: 1.0
2
+ Generator: bdist_wheel (0.35.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py2-none-any
5
+ Tag: py3-none-any
6
+
@@ -0,0 +1 @@
1
+ phasenet
phasenet/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,520 @@
1
+ import tensorflow as tf
2
+ tf.compat.v1.disable_eager_execution()
3
+ import os, threading, logging
4
+ import numpy as np
5
+ import pandas as pd
6
+ import scipy.interpolate
7
+ pd.options.mode.chained_assignment = None
8
+ import obspy
9
+ from tqdm import tqdm
10
+ from scipy.interpolate import interp1d
11
+
12
+
13
+ def py_func_decorator(output_types=None, output_shapes=None, name=None):
14
+ def decorator(func):
15
+ def call(*args, **kwargs):
16
+ nonlocal output_shapes
17
+ # flat_output_types = nest.flatten(output_types)
18
+ flat_output_types = tf.nest.flatten(output_types)
19
+ # flat_values = tf.py_func(
20
+ flat_values = tf.numpy_function(
21
+ func,
22
+ inp=args,
23
+ Tout=flat_output_types,
24
+ name=name
25
+ )
26
+ if output_shapes is not None:
27
+ for v, s in zip(flat_values, output_shapes):
28
+ v.set_shape(s)
29
+ # return nest.pack_sequence_as(output_types, flat_values)
30
+ return tf.nest.pack_sequence_as(output_types, flat_values)
31
+ return call
32
+ return decorator
33
+
34
+
35
+ def dataset_map(iterator, output_types, output_shapes=None, num_parallel_calls=None, name=None, shuffle=False):
36
+ dataset = tf.data.Dataset.range(len(iterator))
37
+ if shuffle:
38
+ dataset = dataset.shuffle(len(iterator))
39
+ @py_func_decorator(output_types, output_shapes, name=name)
40
+ def index_to_entry(idx):
41
+ return iterator[idx]
42
+ return dataset.map(index_to_entry, num_parallel_calls=num_parallel_calls)
43
+
44
+
45
+ def normalize(data, window=3000):
46
+ """
47
+ data: nt, nch
48
+ """
49
+ shift = window//2
50
+ nt, nch = data.shape
51
+
52
+ ## std in slide windows
53
+ data_pad = np.pad(data, ((window//2, window//2), (0,0)), mode="reflect")
54
+ t = np.arange(0, nt, shift, dtype="int")
55
+ std = np.zeros([len(t)+1, nch])
56
+ mean = np.zeros([len(t)+1, nch])
57
+ for i in range(1, len(std)):
58
+ std[i, :] = np.std(data_pad[i*shift:i*shift+window, :], axis=0)
59
+ mean[i, :] = np.mean(data_pad[i*shift:i*shift+window, :], axis=0)
60
+
61
+ t = np.append(t, nt)
62
+ # std[-1, :] = np.std(data_pad[-window:, :], axis=0)
63
+ # mean[-1, :] = np.mean(data_pad[-window:, :], axis=0)
64
+ std[-1, :], mean[-1, :] = std[-2, :], mean[-2, :]
65
+ std[0, :], mean[0, :] = std[1, :], mean[1, :]
66
+ std[std == 0] = 1
67
+
68
+ ## normalize data with interplated std
69
+ t_interp = np.arange(nt, dtype="int")
70
+ std_interp = interp1d(t, std, axis=0, kind="slinear")(t_interp)
71
+ mean_interp = interp1d(t, mean, axis=0, kind="slinear")(t_interp)
72
+ data = (data - mean_interp)/std_interp
73
+
74
+ return data
75
+
76
+
77
+ def normalize_batch(data, window=3000):
78
+ """
79
+ data: nsta, nt, nch
80
+ """
81
+ shift = window//2
82
+ nsta, nt, nch = data.shape
83
+
84
+ ## std in slide windows
85
+ data_pad = np.pad(data, ((0,0), (window//2, window//2), (0,0)), mode="reflect")
86
+ t = np.arange(0, nt, shift, dtype="int")
87
+ std = np.zeros([nsta, len(t)+1, nch])
88
+ mean = np.zeros([nsta, len(t)+1, nch])
89
+ for i in range(1, len(t)):
90
+ std[:, i, :] = np.std(data_pad[:, i*shift:i*shift+window, :], axis=1)
91
+ mean[:, i, :] = np.mean(data_pad[:, i*shift:i*shift+window, :], axis=1)
92
+
93
+ t = np.append(t, nt)
94
+ # std[:, -1, :] = np.std(data_pad[:, -window:, :], axis=1)
95
+ # mean[:, -1, :] = np.mean(data_pad[:, -window:, :], axis=1)
96
+ std[:, -1, :], mean[:, -1, :] = std[:, -2, :], mean[:, -2, :]
97
+ std[:, 0, :], mean[:, 0, :] = std[:, 1, :], mean[:, 1, :]
98
+ std[std == 0] = 1
99
+
100
+ # ## normalize data with interplated std
101
+ t_interp = np.arange(nt, dtype="int")
102
+ std_interp = interp1d(t, std, axis=1, kind="slinear")(t_interp)
103
+ mean_interp = interp1d(t, mean, axis=1, kind="slinear")(t_interp)
104
+ data = (data - mean_interp)/std_interp
105
+
106
+ return data
107
+
108
+ class DataConfig():
109
+
110
+ seed = 100
111
+ use_seed = True
112
+ n_channel = 3
113
+ n_class = 3
114
+ sampling_rate = 100
115
+ dt = 1.0/sampling_rate
116
+ X_shape = [3000, 1, n_channel]
117
+ Y_shape = [3000, 1, n_class]
118
+ min_event_gap = 3 * sampling_rate
119
+ dtype = "float32"
120
+
121
+ def __init__(self, **kwargs):
122
+ for k, v in kwargs.items():
123
+ setattr(self, k, v)
124
+
125
+ class DataReader(object):
126
+ def __init__(self,
127
+ data_dir,
128
+ data_list,
129
+ mask_window,
130
+ queue_size,
131
+ coord,
132
+ config=DataConfig()):
133
+ self.config = config
134
+ tmp_list = pd.read_csv(data_list, header=0)
135
+ self.data_list = tmp_list
136
+ self.num_data = len(self.data_list)
137
+ self.data_dir = data_dir
138
+ self.queue_size = queue_size
139
+ self.n_channel = config.n_channel
140
+ self.n_class = config.n_class
141
+ self.X_shape = config.X_shape
142
+ self.Y_shape = config.Y_shape
143
+ self.min_event_gap = config.min_event_gap
144
+ self.mask_window = int(mask_window * config.sampling_rate)
145
+ self.coord = coord
146
+ self.threads = []
147
+ self.buffer = {}
148
+ self.buffer_channels = {}
149
+ self.add_placeholder()
150
+
151
+ def add_placeholder(self):
152
+ self.sample_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=self.config.X_shape)
153
+ self.target_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=self.config.Y_shape)
154
+ self.queue = tf.queue.PaddingFIFOQueue(self.queue_size,
155
+ ['float32', 'float32'],
156
+ shapes=[self.config.X_shape, self.config.Y_shape])
157
+ self.enqueue = self.queue.enqueue([self.sample_placeholder, self.target_placeholder])
158
+
159
+ def dequeue(self, num_elements):
160
+ output = self.queue.dequeue_many(num_elements)
161
+ return output
162
+
163
+ def normalize(self, data):
164
+ data -= np.mean(data, axis=0, keepdims=True)
165
+ std_data = np.std(data, axis=0, keepdims=True)
166
+ assert(std_data.shape[-1] == data.shape[-1])
167
+ std_data[std_data == 0] = 1
168
+ data /= std_data
169
+ return data
170
+
171
+
172
+ def generate_target(self, itp_list, its_list):
173
+
174
+ target = np.zeros(self.Y_shape)
175
+ itp_true = []
176
+ its_true = []
177
+ for itp, its in zip(itp_list, its_list):
178
+ if (itp >= target.shape[0]) or (itp < 0):
179
+ pass
180
+ elif (itp-self.mask_window//2 >= 0) and (itp-self.mask_window//2 < target.shape[0]):
181
+ target[itp-self.mask_window//2:itp+self.mask_window//2, 0, 1] = \
182
+ np.exp(-(np.arange(-self.mask_window//2,self.mask_window//2))**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(itp-self.mask_window//2)]
183
+ itp_true.append(itp)
184
+ elif (itp-self.mask_window//2 < target.shape[0]):
185
+ target[0:itp+self.mask_window//2, 0, 1] = \
186
+ np.exp(-(np.arange(0,itp+self.mask_window//2)-itp)**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(itp-self.mask_window//2)]
187
+ itp_true.append(itp)
188
+
189
+ if (its >= target.shape[0]) or (its < 0):
190
+ pass
191
+ elif (its-self.mask_window//2 >= 0) and (its-self.mask_window//2 < target.shape[0]):
192
+ target[its-self.mask_window//2:its+self.mask_window//2, 0, 2] = \
193
+ np.exp(-(np.arange(-self.mask_window//2,self.mask_window//2))**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(its-self.mask_window//2)]
194
+ its_true.append(its)
195
+ elif (its-self.mask_window//2 < target.shape[0]):
196
+ target[0:its+self.mask_window//2, 0, 2] = \
197
+ np.exp(-(np.arange(0,its+self.mask_window//2)-its)**2/(2*(self.mask_window//4)**2))[:target.shape[0]-(its-self.mask_window//2)]
198
+ its_true.append(its)
199
+
200
+ target[:, :, 0] = 1 - target[:, :, 1] - target[:, :, 2]
201
+ return target, itp_true, its_true
202
+
203
+ def thread_main(self, sess, n_threads=1, start=0):
204
+ stop = False
205
+ while not stop:
206
+ index = list(range(start, self.num_data, n_threads))
207
+ np.random.shuffle(index)
208
+ for i in index:
209
+ fname = os.path.join(self.data_dir, self.data_list.iloc[i]['fname'])
210
+ try:
211
+ if fname not in self.buffer:
212
+ meta = np.load(fname)
213
+ self.buffer[fname] = {'data': meta['data'], 'itp': meta['itp'], 'its': meta['its']}
214
+ meta = self.buffer[fname]
215
+ except:
216
+ logging.error("Failed reading {}".format(fname))
217
+ continue
218
+
219
+ start_tp = meta['itp'].tolist()
220
+
221
+ if self.coord.should_stop():
222
+ stop = True
223
+ break
224
+
225
+ sample = np.zeros(self.X_shape)
226
+ if np.random.random() < 0.95:
227
+ data = np.copy(meta['data'])
228
+ itp = meta['itp']
229
+ its = meta['its']
230
+ start_tp = itp
231
+
232
+ shift = np.random.randint(-(self.X_shape[0]-self.mask_window), min([its-start_tp, self.X_shape[0]])-self.mask_window)
233
+ sample[:, :, :] = data[start_tp+shift:start_tp+self.X_shape[0]+shift, np.newaxis, :]
234
+ itp_list = [itp-start_tp-shift]
235
+ its_list = [its-start_tp-shift]
236
+ else:
237
+ sample[:, :, :] = np.copy(meta['data'][start_tp-self.X_shape[0]:start_tp, np.newaxis, :])
238
+ itp_list = []
239
+ its_list = []
240
+
241
+ sample = self.normalize(sample)
242
+
243
+ if (np.isnan(sample).any() or np.isinf(sample).any() or (not sample.any())):
244
+ continue
245
+
246
+ target, _, _ = self.generate_target(itp_list, its_list)
247
+
248
+ sess.run(self.enqueue, feed_dict={self.sample_placeholder: sample, self.target_placeholder: target})
249
+ return 0
250
+
251
+ def start_threads(self, sess, n_threads=8):
252
+ for i in range(n_threads):
253
+ thread = threading.Thread(target=self.thread_main, args=(sess, n_threads, i))
254
+ thread.daemon = True
255
+ thread.start()
256
+ self.threads.append(thread)
257
+ return self.threads
258
+
259
+ class DataReader_test(DataReader):
260
+
261
+ def add_placeholder(self):
262
+ self.sample_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=None)
263
+ self.target_placeholder = tf.compat.v1.placeholder(dtype=tf.float32, shape=None)
264
+ self.fname_placeholder = tf.compat.v1.placeholder(dtype=tf.string, shape=None)
265
+ self.itp_placeholder = tf.compat.v1.placeholder(dtype=tf.int32, shape=None)
266
+ self.its_placeholder = tf.compat.v1.placeholder(dtype=tf.int32, shape=None)
267
+ self.queue = tf.queue.PaddingFIFOQueue(self.queue_size,
268
+ ['float32', 'float32', 'string', 'int32', 'int32'],
269
+ shapes=[self.config.X_shape, self.config.Y_shape, [], [None], [None]])
270
+ self.enqueue = self.queue.enqueue([self.sample_placeholder, self.target_placeholder,
271
+ self.fname_placeholder,
272
+ self.itp_placeholder, self.its_placeholder])
273
+
274
+ def dequeue(self, num_elements):
275
+ output = self.queue.dequeue_up_to(num_elements)
276
+ return output
277
+
278
+ def thread_main(self, sess, n_threads=1, start=0):
279
+
280
+ index = list(range(start, self.num_data, n_threads))
281
+ for i in index:
282
+ fname = self.data_list.iloc[i]['fname']
283
+ fp = os.path.join(self.data_dir, fname)
284
+ try:
285
+ if fp not in self.buffer:
286
+ meta = np.load(fp)
287
+ self.buffer[fp] = {'data': meta['data'], 'itp': meta['itp'], 'its': meta['its']}
288
+ meta = self.buffer[fp]
289
+ except:
290
+ logging.error("Failed reading {}".format(fp))
291
+ continue
292
+
293
+ start_tp = meta['itp'].tolist()
294
+
295
+ if self.coord.should_stop():
296
+ break
297
+
298
+ sample = np.zeros(self.X_shape)
299
+
300
+ np.random.seed(self.config.seed+i)
301
+ shift = np.random.randint(-(self.X_shape[0]-self.mask_window), min([meta['its'].tolist()-start_tp, self.X_shape[0]])-self.mask_window)
302
+ sample[:, :, :] = np.copy(meta['data'][start_tp+shift:start_tp+self.X_shape[0]+shift, np.newaxis, :])
303
+ itp_list = [meta['itp'].tolist()-start_tp-shift]
304
+ its_list = [meta['its'].tolist()-start_tp-shift]
305
+
306
+ sample = self.normalize(sample)
307
+
308
+ if (np.isnan(sample).any() or np.isinf(sample).any() or (not sample.any())):
309
+ continue
310
+
311
+ target, itp_true, its_true = self.generate_target(itp_list, its_list)
312
+
313
+ sess.run(self.enqueue, feed_dict={self.sample_placeholder: sample,
314
+ self.target_placeholder: target,
315
+ self.fname_placeholder: fname,
316
+ self.itp_placeholder: itp_true,
317
+ self.its_placeholder: its_true})
318
+
319
+ return 0
320
+
321
+
322
+ class DataReader_pred(DataReader):
323
+
324
+ def __init__(self,
325
+ data_dir,
326
+ data_list):
327
+
328
+ tmp_list = pd.read_csv(data_list, header=0)
329
+ self.data_list = tmp_list
330
+ self.num_data = len(self.data_list)
331
+ self.data_dir = data_dir
332
+ self.dtype = "float32"
333
+ self.X_shape = self.get_data_shape()
334
+
335
+ def get_data_shape(self):
336
+ fname = self.data_list.iloc[0]['fname']
337
+ fp = os.path.join(self.data_dir, fname)
338
+ data = np.load(fp)["data"][:,np.newaxis,:]
339
+ return data.shape
340
+
341
+ def adjust_missingchannels(self, data):
342
+ tmp = np.max(np.abs(data), axis=0, keepdims=True)
343
+ assert(tmp.shape[-1] == data.shape[-1])
344
+ if np.count_nonzero(tmp) > 0:
345
+ data *= data.shape[-1] / np.count_nonzero(tmp)
346
+ return data
347
+
348
+ def __len__(self):
349
+ return self.num_data
350
+
351
+ def __getitem__(self, i):
352
+ fname = self.data_list.iloc[i]['fname']
353
+ fp = os.path.join(self.data_dir, fname)
354
+
355
+ try:
356
+ meta = np.load(fp)
357
+ except:
358
+ logging.error("Failed reading {}".format(fname))
359
+ return (np.zeros(self.X_shape, dtype=self.dtype), fname)
360
+
361
+ shift = 0
362
+ sample = normalize(meta['data'])[:, np.newaxis, :].astype(self.dtype)
363
+
364
+ if not np.array_equal(np.array(sample.shape), np.array(self.X_shape)):
365
+ logging.warning(f"Shape mismatch: {sample.shape} != {self.X_shape} in {fname}")
366
+ tmp = np.zeros(self.X_shape, dtype=self.dtype)
367
+ tmp[:sample.shape[0],0,:sample.shape[2]] = sample[:tmp.shape[0],0,:tmp.shape[2]]
368
+ sample = tmp
369
+
370
+ if np.isnan(sample).any() or np.isinf(sample).any():
371
+ logging.warning(f"Data error: Nan or Inf found in {fname}")
372
+
373
+ sample[np.isnan(sample)] = 0
374
+ sample[np.isinf(sample)] = 0
375
+
376
+ # sample = self.normalize(sample)
377
+ sample = self.adjust_missingchannels(sample)
378
+
379
+ return (sample, fname)
380
+
381
+ def dataset(self, num_parallel=2):
382
+ dataset = dataset_map(self,
383
+ output_types=("float32", "string"),
384
+ output_shapes=(self.X_shape, None),
385
+ num_parallel_calls=num_parallel)
386
+ return dataset
387
+
388
+ class DataReader_mseed(DataReader):
389
+
390
+ def __init__(self,
391
+ data_list,
392
+ stations,
393
+ data_dir=""):
394
+
395
+ self.data_list = pd.read_csv(data_list, header=0)
396
+ self.num_data = len(self.data_list)
397
+ self.data_dir = data_dir
398
+ self.stations = pd.read_csv(stations, delimiter="\t")
399
+ self.dtype = "float32"
400
+ self.X_shape = self.get_data_shape()
401
+
402
+ def get_data_shape(self):
403
+ (data, fname) = self[0]
404
+ return data.shape
405
+
406
+ def read_mseed(self, fp):
407
+ meta = obspy.read(fp)
408
+ meta = meta.detrend("spline", order=2, dspline=5*meta[0].stats.sampling_rate)
409
+ meta = meta.merge(fill_value=0)
410
+ meta = meta.trim(min([st.stats.starttime for st in meta]),
411
+ max([st.stats.endtime for st in meta]),
412
+ pad=True, fill_value=0)
413
+
414
+ if meta[0].stats.sampling_rate != 100:
415
+ logging.warning(f"Sampling rate {meta[0].stats.sampling_rate} != 100 Hz")
416
+
417
+ order = ['3','2','1','E','N','Z']
418
+ order = {key: i for i, key in enumerate(order)}
419
+
420
+ nsta = len(self.stations)
421
+ nt = len(meta[0].data)
422
+ data = np.zeros([nsta, nt, 3], dtype=self.dtype)
423
+ for i in range(nsta):
424
+ sta = self.stations.iloc[i]["station"]
425
+ cmp = sorted(self.stations.iloc[i]["component"].split(","), key=lambda x: order[x])
426
+ for j, c in enumerate(cmp):
427
+ data[i, :, j] = meta.select(id=sta+c)[0].data.astype(self.dtype)
428
+
429
+ return data
430
+
431
+ def __len__(self):
432
+ return self.num_data
433
+
434
+ def __getitem__(self, i):
435
+
436
+ fname = self.data_list.iloc[i]['fname']
437
+ fp = os.path.join(self.data_dir, fname)
438
+
439
+ try:
440
+ data = self.read_mseed(fp)
441
+ except Exception as e:
442
+ logging.error(f"Failed reading {fname}: {e}")
443
+ return (np.zeros(self.X_shape, dtype=self.dtype), fname)
444
+
445
+ sample = normalize_batch(data)[:,:,np.newaxis,:]
446
+
447
+ return (sample.astype(self.dtype), fname)
448
+
449
+
450
+ def dataset(self, num_parallel=2):
451
+ dataset = dataset_map(self,
452
+ output_types=("float32", "string"),
453
+ output_shapes=(self.X_shape, None),
454
+ num_parallel_calls=num_parallel)
455
+ return dataset
456
+
457
+
458
+ # def test_DataReader_mseed(data_list, data_dir, stations):
459
+ # print("test_DataReader_mseed:")
460
+ # import timeit
461
+ # data_reader = DataReader_mseed(
462
+ # data_list = data_list,
463
+ # data_dir = data_dir,
464
+ # stations = stations)
465
+
466
+ # start_time = timeit.default_timer()
467
+ # dataset = dataset_map(data_reader,
468
+ # output_types=("float32", "string"),
469
+ # output_shapes=(data_reader.X_shape, None),
470
+ # num_parallel_calls=2)
471
+ # dataset = dataset
472
+ # # dataset = tf.data.Dataset.range(data_reader.num_data)
473
+ # # dataset = dataset.interleave(lambda x: tf.data.Dataset.from_generator(data_reader.generator,
474
+ # # output_types=(data_reader.dtype, "string"),
475
+ # # output_shapes=(data_reader.X_shape, None),
476
+ # # args=(x,)),
477
+ # # cycle_length=data_reader.num_data,
478
+ # # block_length=1,
479
+ # # num_parallel_calls=data_reader.num_data)
480
+
481
+ # sess = tf.compat.v1.Session()
482
+
483
+ # print(len(data_reader))
484
+ # print("-------", tf.data.Dataset.cardinality(dataset))
485
+ # num = 0
486
+ # # for x in tf.compat.v1.data.make_one_shot_iterator(dataset).get_next():
487
+ # # print(num)
488
+ # # num += 1
489
+ # # dum = sess.run(x)
490
+ # # print(dum)
491
+ # x = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next()
492
+ # while True:
493
+ # num+=1
494
+ # print(num)
495
+ # dum = sess.run(x)
496
+
497
+ # print("Tensorflow Dataset:\nexecution time = ", timeit.default_timer() - start_time)
498
+
499
+
500
+ if __name__ == "__main__":
501
+ pass
502
+ ## debug
503
+ # data_reader = DataReader_mseed(
504
+ # data_dir="/data/beroza/zhuwq/Project-PhaseNet-mseed/mseed/",
505
+ # data_list="/data/beroza/zhuwq/Project-PhaseNet-mseed/fname.txt",
506
+ # queue_size=20,
507
+ # coord=None)
508
+ # data_reader.thread_main(None, n_threads=1, start=0)
509
+ # pred_fn(args, data_reader, log_dir=args.output_dir)
510
+
511
+ # test_DataReader_mseed()
512
+ # from glob import glob
513
+ # import os
514
+ # import pandas as pd
515
+ # mseed_dir = "../../quakeflow/mseed"
516
+ # mseed_list = glob(os.path.join(mseed_dir, "*.mseed"), recursive=True)
517
+ # print(f"Number of mseed: {len(mseed_list)}")
518
+
519
+ # stations = pd.read_csv("../../quakeflow/stations.csv", delimiter="\t", index_col="station")
520
+ # test_DataReader_mseed("../../quakeflow/mseed.lst", "../../quakeflow/", "../../quakeflow/stations.csv")