das2numpy 1.1.1__tar.gz → 1.2.0__tar.gz

{das2numpy-1.1.1/src/das2numpy.egg-info → das2numpy-1.2.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: das2numpy
-Version: 1.1.1
+Version: 1.2.0
 Summary: A simple and universal package for loading large amounts of distributed acoustic sensing (DAS) data.
 Author-email: Erik Genthe <erik.genthe@desy.de>
-Project-URL: Homepage, https://git.physnet.uni-hamburg.de/wave/das2numpy
+Project-URL: Homepage, https://gitlab.desy.de/wave/tools/das2numpy
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent

{das2numpy-1.1.1 → das2numpy-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "das2numpy"
-version = "1.1.1"
+version = "1.2.0"
 authors = [
   { name="Erik Genthe", email="erik.genthe@desy.de" },
 ]
@@ -20,4 +20,4 @@ dependencies = [
     "numpy", "ffmpeg-python", "h5py", "scipy", "numba",
 ]
 [project.urls]
-Homepage = "https://git.physnet.uni-hamburg.de/wave/das2numpy"
+Homepage = "https://gitlab.desy.de/wave/tools/das2numpy"

{das2numpy-1.1.1 → das2numpy-1.2.0}/src/das2numpy/__init__.py

@@ -26,6 +26,9 @@ def loader(root_path:str, predefined_setup:str, num_worker_threads):
     elif predefined_setup.upper() == "FLAC_200HZ":
         from .setups import flac_200hz
         chunk = flac_200hz.init(root_path, num_worker_threads)
+    elif predefined_setup.upper() == "HDAS":
+        from .setups import hdas
+        chunk = hdas.init(root_path, num_worker_threads)
     elif predefined_setup.upper() == "OPTASENSE":
         from .setups import optasense_b35idefix
         chunk = optasense_b35idefix.init()

{das2numpy-1.1.1 → das2numpy-1.2.0}/src/das2numpy/setups/flac_200hz.py

@@ -90,7 +90,8 @@ def load_file(file_path, file_timestamp, t_start, t_end, t_step, channel_start,
                 + "execute 'module load maxwell ffmpeg'")
     
     # Trim data
-    data = data[:, channel_start:channel_end]
+    if channel_end != -1: # Avoids that last channel is missing when channel_end == -1
+        data = data[:, channel_start:channel_end]
 
     data = data.astype(NP.float32) #This needs to hapen before the binning step!
 

das2numpy-1.1.1/src/das2numpy/setups/silixa_200hz.py → das2numpy-1.2.0/src/das2numpy/setups/hdas.py

@@ -1,28 +1,28 @@
-""" Univsersal setup file for silixa, that detects sampling rate and number of channels by itself. 
-The root directory shall be supplied by the user via an argument
+""" Setup file for hDAS.
 """
 
 import sys as SYS
 from os import path as P
 import datetime as DT
 import numpy as NP
+import h5py as H5PY
 from ..filefinder import FileFinder, to_posix_timestamp_ms
 from ..chunk import Chunk
 from .light_tdms_reader import TdmsReader
 from ..utils import bin
 
-CALIBRATE = True
 
+CALIBRATE = True
 
 
-sample_rate = 200
 
 
 def init(root_path, num_worker_threads):
     assert P.isdir(root_path)
-    file_finder = FileFinder(root_path, ".tdms", filename_to_posix_timestamp)
+    file_finder = FileFinder(root_path, ".h5", filename_to_posix_timestamp)
     assert num_worker_threads >= 1
     multithreaded = num_worker_threads > 1
+    sample_rate = 1000
     return Chunk(
                 file_finder,
                 sample_rate,
@@ -32,43 +32,46 @@ def init(root_path, num_worker_threads):
                 load_file
             )
 
-
 def filename_to_posix_timestamp(file_name:str) -> int:
-    timestamp_str = file_name.split("_UTC_")[1][:19]
-    timestamp_dt = DT.datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S.%f")
+    # Example path: 2026_05_08_15h02m22s_HDAS_StrainRate_ch1.h5
+    # TODO Known bug: The timestamps array in the file has milliseconds! So the milliseonds are missing in the file name! h5['Timestamps']['Timestamps_Data']
+    timestamp_dt = DT.datetime.strptime(file_name.split("s_")[0], "%Y_%m_%d_%Hh%Mm%S")
     timestamp_ms = to_posix_timestamp_ms(timestamp_dt)
     return timestamp_ms
 
 
-
 def load_file(file_path, file_timestamp, t_start, t_end, t_step, channel_start, channel_end, channel_step) -> NP.ndarray:
     """ Loads a single file, trims it. And returns the trimmed data as a numpy array. Downsampling (t_step, channel_step) is also possible!
     """
     
-    with TdmsReader(file_path) as tdms:
-        data = tdms.get_mmap()
-
+    with H5PY.File(file_path, "r") as h5:
+        # Known bug: The file_timestamp is not accurate to milliseconds, because milliseconds are missing in the file name
+        #timestamps_unix_sec = h5['Timestamps']['Timestamps_Data']
+        #print(file_timestamp)
+        #file_timestamp = int(timestamps_unix_sec[0][0] * 1000.0) # Overwrite inaccurate file_timestamp
+        #file_timestamp -= 7200000 # TODO: This is a dirty fix: Internal timestamp is CEST, when hDAS system time is CEST.
+        data = NP.array(h5["StrainRate"]["StrainRate_Data"])
+        #print(file_timestamp)
 
         # Trim data
-        idx_start = 0
+        rel_t_start = 0
         if t_start > file_timestamp: # Check if beginning should be trimmed.
             rel_t_start = t_start - file_timestamp
-            idx_start = int(rel_t_start * sample_rate / 1000.0)
-        idx_end = data.shape[0] 
-        if t_end < file_timestamp + (data.shape[0] * 1000 / sample_rate): # Check if end should be trimmed
+        rel_t_end = -1 
+        if t_end < file_timestamp + data.shape[0]: # Check if end should be trimmed
             rel_t_end = t_end - file_timestamp
-            idx_end = int(rel_t_end * sample_rate / 1000.0) 
-        if idx_start == idx_end:
+        if rel_t_start == rel_t_end:
             return NP.zeros(shape=[0, 0]) # No data should be loaded. Do nothing
-        if file_timestamp + (data.shape[0] * 1000 / sample_rate) <= t_start:
+        if file_timestamp + data.shape[0] <= t_start:
             print("Warning: File does not contain any parts of the requested data.",
                     "This can happen if there are leaks in the data. The corresponding output will be left filled with zeros.\n",
                     f"    Requested range (Posixtimestamps in ms): [{t_start}, {t_end}[\n",
                     f"    Filepath: {file_path}.")
             return NP.zeros(shape=[0, 0])
-        assert idx_end == data.shape[0] or idx_end > idx_start, f"idx_start={idx_start}, idx_end={idx_end}."
-        print(idx_start, idx_end)
-        data = data[idx_start:idx_end, channel_start:channel_end]
+        assert rel_t_end == -1 or rel_t_end > rel_t_start, f"rel_t_start={rel_t_start}, rel_t_end={rel_t_end}."
+        if channel_end == -1: # Avoids that last channel is missing when channel_end == -1
+            channel_end = None
+        data = data[rel_t_start:rel_t_end, channel_start:channel_end]
 
 
         # Downsample data
@@ -86,18 +89,16 @@ def load_file(file_path, file_timestamp, t_start, t_end, t_step, channel_start,
     return data
 
 
+
+
+
 def calibrate(data:NP.ndarray) -> NP.ndarray:
-    """ Convert raw data to strain rate data. 
-    As the resulting values are decimals, the datatype should be float. Otherwise an assertion fails. """
-    #assert data.dtype in (NP.float, NP.float32, NP.float64), f"The data should be floating point. It is {data.dtype}"
+    """ Convert raw data to strain rate data. """
     if data.dtype not in (float, NP.float32, NP.float64):
         NEW_TYPE = NP.float32
         #print("Warning: For calibration the data has to be of type float. Converting from {data.dtype} to {NEW_TYPE}")
         data = data.astype(NEW_TYPE)
-
-    SAMPLE_FREQ = 1000.0 # This remains 1000.0 and not 200 Hz because the original sample rate of the device is relevant here!
-    EICHLAENGE = 10.0
-    factor = 116.0 * 10.0**(-9.0) / 8192.0 * SAMPLE_FREQ / EICHLAENGE
+    factor = 10 * 1e-9
     return data * factor # Result: 1 / s
 
 

{das2numpy-1.1.1 → das2numpy-1.2.0}/src/das2numpy/setups/optasense_b35idefix.py

@@ -29,6 +29,8 @@ def _load_from_h5(file_path, rel_t_start, rel_t_end, t_step, channel_start, chan
     data = file['Acquisition']['Raw[0]']['RawData'] # Data is not loaded into memory at this point! (Lazy evaluation)
 
     # At this point the data gets loaded into memory.
+    if channel_end != -1: # Avoids that last channel is missing when channel_end == -1
+        channel_end = None
     data = data[
             channel_start : channel_end : channel_step, 
             rel_t_start : rel_t_end : t_step

{das2numpy-1.1.1 → das2numpy-1.2.0}/src/das2numpy/setups/silixa.py

@@ -63,6 +63,8 @@ def load_file(file_path, file_timestamp, t_start, t_end, t_step, channel_start,
                     f"    Filepath: {file_path}.")
             return NP.zeros(shape=[0, 0])
         assert rel_t_end == -1 or rel_t_end > rel_t_start, f"rel_t_start={rel_t_start}, rel_t_end={rel_t_end}."
+        if channel_end == -1: # Avoids that last channel is missing when channel_end == -1
+            channel_end = None
         data = data[rel_t_start:rel_t_end, channel_start:channel_end]
 
 

{das2numpy-1.1.1 → das2numpy-1.2.0/src/das2numpy.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: das2numpy
-Version: 1.1.1
+Version: 1.2.0
 Summary: A simple and universal package for loading large amounts of distributed acoustic sensing (DAS) data.
 Author-email: Erik Genthe <erik.genthe@desy.de>
-Project-URL: Homepage, https://git.physnet.uni-hamburg.de/wave/das2numpy
+Project-URL: Homepage, https://gitlab.desy.de/wave/tools/das2numpy
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent

{das2numpy-1.1.1 → das2numpy-1.2.0}/src/das2numpy.egg-info/SOURCES.txt

@@ -13,8 +13,7 @@ src/das2numpy.egg-info/dependency_links.txt
 src/das2numpy.egg-info/requires.txt
 src/das2numpy.egg-info/top_level.txt
 src/das2numpy/setups/flac_200hz.py
+src/das2numpy/setups/hdas.py
 src/das2numpy/setups/light_tdms_reader.py
 src/das2numpy/setups/optasense_b35idefix.py
-src/das2numpy/setups/optasense_b35idefix_fast.py
-src/das2numpy/setups/silixa.py
-src/das2numpy/setups/silixa_200hz.py
+src/das2numpy/setups/silixa.py

{das2numpy-1.1.1 → das2numpy-1.2.0}/src/example.py

@@ -11,6 +11,7 @@ t_end   = datetime(2024, 7, 23, 1, 1, 0)
 channel_start = 0
 channel_end = -1
 #loader = loader("/pnfs/desy.de/m/project/iDAS/raw/2024-DESY/2024-07-23-desy", "SILIXA", 1) # 1000 Hz
+#loader = loader("/pnfs/desy.de/m/project/iDAS/raw/2026-DESY/2026_das_comparison/hdas", "HDAS", 1) # 1000 Hz
 loader = loader("/pnfs/desy.de/m/project/iDAS/work/IDAS_200HZ/", "FLAC_200HZ", 1) # 200 Hz
 data = loader.load_array(t_start, t_end, channel_start, channel_end)
 

das2numpy-1.1.1/src/das2numpy/setups/optasense_b35idefix_fast.py

@@ -1,111 +0,0 @@
-""" Deprecated
-"""
-
-from math import ceil, floor
-import mmap
-from os import path as P
-import numpy as NP
-import h5py as H5PY
-import datetime as DT
-from time import time
-from filefinder import FileFinder, to_posix_timestamp_ms
-from chunk import Chunk
-
-#/wave/seismic-work/markhoff/pilot/data/cache/7wave7seismic-rawdata7OPTA7Disk27DESY-Rec-9-GL8m-Chan10000_2021-05-28T06_01_36+01007DESY-Rec-9-GL8m-Chan10000_2021-05-28T194319Z.h5.bin
-FILE_TIME_SAMPLE_AMOUNT = 60000
-CHANNEL_AMOUNT = 10000
-DATA_ROOT = "/wave/seismic-work/markhoff/pilot/data/cache"
-assert P.isdir(DATA_ROOT)
-
-def _filename_to_posix_timestamp(file_name:str) -> int:
-    return to_posix_timestamp_ms(DT.datetime.strptime(file_name[-25:], "%Y-%m-%dT%H%M%SZ.h5.bin"))
-
-
-
-def _load_from_h5(file_path, rel_t_start, rel_t_end, t_step, channel_start, channel_end, channel_step) -> NP.ndarray:
-    """ Internal helper function """
-    #file_handle = open(file_path, 'rb')
-    #file:H5PY.File = H5PY.File(file_handle, 'r')
-    #data = file['Acquisition']['Raw[0]']['RawData'] # Data is not loaded into memory at this point! (Lazy evaluation)
-#
-    ## At this point the data gets loaded into memory.
-    #data = data[
-    #        channel_start : channel_end : channel_step, 
-    #        rel_t_start : rel_t_end : t_step
-    #]
-
-    DTYPE_SIZE = 4
-    data = None
-    #if channel_step == 1:
-    #    data = NP.fromfile(
-    #        file_path,
-    #        dtype = NP.int32,
-    #        offset = channel_start * FILE_TIME_SAMPLE_AMOUNT * DTYPE_SIZE,
-    #        count = (channel_end-channel_start) * FILE_TIME_SAMPLE_AMOUNT
-    #    )
-    #    data.shape = (channel_end-channel_start, FILE_TIME_SAMPLE_AMOUNT)
-    #    data = data[:, rel_t_start:rel_t_end:t_step]
-    #else:
-    #    data = NP.ndarray(
-    #        shape=(
-    #            ceil((channel_end - channel_start) / channel_step),
-    #            FILE_TIME_SAMPLE_AMOUNT
-    #        ),
-    #        dtype=NP.int32
-    #    )
-    #    file_handle = open(file_path, 'rb')
-    #    data_index = 0
-    #    for channel_index in range(channel_start, channel_end, channel_step):
-    #        file_handle.seek(channel_index * FILE_TIME_SAMPLE_AMOUNT)
-    #        channel_data = NP.frombuffer(file_handle.read(FILE_TIME_SAMPLE_AMOUNT))
-    #        data[data_index] = channel_data
-    #        data_index += 1
-    #    file_handle.close()
-    #    data = data[:, rel_t_start:rel_t_end:t_step]
-
-    data = NP.fromfile(
-        file_path,
-        dtype = NP.int32,
-        offset = channel_start * FILE_TIME_SAMPLE_AMOUNT * DTYPE_SIZE,
-        count = (channel_end-channel_start) * FILE_TIME_SAMPLE_AMOUNT
-    )
-    data.shape = (channel_end-channel_start, FILE_TIME_SAMPLE_AMOUNT)
-    data = data[::channel_step, rel_t_start:rel_t_end:t_step]
-
-    print("Args (channel):", channel_start, channel_end, channel_step)
-    print("Args (time):", rel_t_start, rel_t_end, rel_t_end)
-    print("Fresh after loading: ", data.shape)
-    data = data.transpose() # Extremely efficient :)
-    return data
-
-def _load_from_h5_X(file_path, rel_t_start, rel_t_end, t_step, channel_start, channel_end, channel_step) -> NP.ndarray:
-    DTYPE_SIZE = 4
-    t1 = time()
-    mm = NP.memmap(file_path, dtype=NP.int32, mode='readonly')
-    mm.shape = (CHANNEL_AMOUNT, FILE_TIME_SAMPLE_AMOUNT)
-    t2 = time()
-    data = mm[channel_start:channel_end:channel_step, rel_t_start:rel_t_end:t_step]
-    t3 = time()
-    data = NP.array(data)
-    t4 = time()
-    data = data.transpose() # Extremely efficient :)
-    t5 = time()
-    print("DELTAS", t2-t1, t3-t2, t4-t3, t5-t4)
-    #print("Args (channel):", channel_start, channel_end, channel_step)
-    #print("Args (time):", rel_t_start, rel_t_end, rel_t_end)
-    #print("Fresh after loading: ", data.shape)
-    return data
-
-FILE_FINDER = FileFinder(DATA_ROOT, ".h5.bin", _filename_to_posix_timestamp)
-
-
-def create_chunk():
-    return Chunk( 
-                FILE_FINDER, 
-                CHANNEL_AMOUNT, 
-                FILE_TIME_SAMPLE_AMOUNT, 
-                True, 
-                8,
-                False,
-                _load_from_h5
-            )