dbdreader 0.6.0.dev1__tar.gz → 0.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/PKG-INFO +19 -4
  2. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/README.md +16 -0
  3. dbdreader-0.6.1/dbdreader/_dbdreader.py +294 -0
  4. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/dbdreader.py +68 -9
  5. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/decompress.py +77 -6
  6. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader.egg-info/PKG-INFO +19 -4
  7. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader.egg-info/SOURCES.txt +1 -0
  8. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader.egg-info/requires.txt +0 -1
  9. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/pyproject.toml +5 -4
  10. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/requirements.txt +0 -1
  11. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/COPYING +0 -0
  12. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/COPYRIGHT +0 -0
  13. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/INSTALL.rst +0 -0
  14. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/LICENSE +0 -0
  15. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/MANIFEST.in +0 -0
  16. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/__init__.py +0 -0
  17. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600000.dcd +0 -0
  18. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600000.ebd +0 -0
  19. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600000.ecd +0 -0
  20. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600000.mcg +0 -0
  21. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600000.mlg +0 -0
  22. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600001.dbd +0 -0
  23. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600001.dcd +0 -0
  24. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600001.ebd +0 -0
  25. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/01600001.ecd +0 -0
  26. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/02380107.ecd +0 -0
  27. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/02380108.ecd +0 -0
  28. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/02450133.tcd +0 -0
  29. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/02450137.tcd +0 -0
  30. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-203-00-000.SBD +0 -0
  31. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-203-00-000.TBD +0 -0
  32. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-000.dbd +0 -0
  33. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-000.ebd +0 -0
  34. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-000.sbd +0 -0
  35. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-000.tbd +0 -0
  36. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-001.sbd +0 -0
  37. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-001.tbd +0 -0
  38. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-002.sbd +0 -0
  39. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/amadeus-2014-204-05-002.tbd +0 -0
  40. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/ammonite-2008-028-01-000.mbd +0 -0
  41. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/06a36d4e.cac +0 -0
  42. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/06a36d4e.ccc +0 -0
  43. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/093bd5ed.cac +0 -0
  44. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/0f682cb2.cac +0 -0
  45. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/39b2c12e.cac +0 -0
  46. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/468fd1be.cac +0 -0
  47. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/5fbf5947.cac +0 -0
  48. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/61b1780f.cac +0 -0
  49. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/6f9a08d3.cac +0 -0
  50. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/813b137d.cac +0 -0
  51. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/8d31d0c8.cac +0 -0
  52. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/92610b65.cac +0 -0
  53. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/c4ec741e.cac +0 -0
  54. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/d6f44165.cac +0 -0
  55. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/daad1b20.cac +0 -0
  56. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/daad1b20.ccc +0 -0
  57. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac/dc76ebd5.cac +0 -0
  58. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/cac_missing/d6f44165.cac +0 -0
  59. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/dbd2asc_output.txt +0 -0
  60. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/electa-2023-143-00-050.sbd +0 -0
  61. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/electa-2023-143-00-050.tbd +0 -0
  62. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/empty-2014-204-05-000.dbd +0 -0
  63. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/hal_1002-2024-183-4-4.sbd +0 -0
  64. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/hal_1002-2024-183-4-4.tbd +0 -0
  65. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/hal_1002-2024-183-4-6.tbd +0 -0
  66. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/invalid_encoding-2014-204-05-000.dbd +0 -0
  67. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/sebastian-2014-204-05-000.dbd +0 -0
  68. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/sebastian-2014-204-05-000.ebd +0 -0
  69. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/sebastian-2014-204-05-001.dbd +0 -0
  70. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/sebastian-2014-204-05-001.ebd +0 -0
  71. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/unit_887-2021-321-3-0.sbd +0 -0
  72. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/data/unit_887-2021-321-3-0.tbd +0 -0
  73. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader/scripts.py +0 -0
  74. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader.egg-info/dependency_links.txt +0 -0
  75. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader.egg-info/entry_points.txt +0 -0
  76. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/dbdreader.egg-info/top_level.txt +0 -0
  77. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/examples/read_multiple_dbd_files.py +0 -0
  78. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/examples/read_single_dbd_file.py +0 -0
  79. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/extension/dbdreader.c +0 -0
  80. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/extension/decompress.c +0 -0
  81. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/extension/include/dbdreader.h +0 -0
  82. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/extension/include/decompress.h +0 -0
  83. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/extension/py_dbdreader.c +0 -0
  84. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/lz4/include/lz4.h +0 -0
  85. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/lz4/lz4.c +0 -0
  86. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/setup.cfg +0 -0
  87. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/setup.py +0 -0
  88. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/tests/test_dbdreader.py +0 -0
  89. {dbdreader-0.6.0.dev1 → dbdreader-0.6.1}/tests/test_decompress.py +0 -0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dbdreader
3
- Version: 0.6.0.dev1
3
+ Version: 0.6.1
4
4
  Summary: A python module to access binary data files generated by Teledyne WebbResearch gliders
5
5
  Author-email: Lucas Merckelbach <lucas.merckelbach@hereon.de>
6
- License: GPL-3.0
6
+ License: GPL-3.0-or-later
7
7
  Project-URL: Homepage, https://dbdreader.readthedocs.io/en/latest/
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: Intended Audience :: Science/Research
10
- Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
10
+ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
11
11
  Classifier: Operating System :: POSIX
12
12
  Requires-Python: >=3.10
13
13
  Description-Content-Type: text/markdown
@@ -15,7 +15,6 @@ License-File: LICENSE
15
15
  License-File: COPYING
16
16
  Requires-Dist: lz4>=4.4.5
17
17
  Requires-Dist: numpy>=2.2.6
18
- Requires-Dist: scipy>=1.15.3
19
18
  Dynamic: license-file
20
19
 
21
20
  [![PyPI version](https://badgen.net/pypi/v/dbdreader)](https://pypi.org/project/dbdreader)
@@ -34,6 +33,22 @@ binary files, so that they can be further analysed.
34
33
 
35
34
  ## Change log
36
35
 
36
+ ### Version 6.0.1
37
+
38
+ * Drops dependency on scipy.
39
+
40
+ ### Version 6.0.0
41
+
42
+ * Offers a python only implementation, alongside the C-extension to
43
+ read the binary files. At the expense of processing speed, no
44
+ compiler is required to use this package.
45
+
46
+ * A github workflow is implemented that generates pre-compiled wheels
47
+ for linux and windows platforms for python version 3.10, 3.11, 3.12
48
+ and 3.13.
49
+
50
+ * Some speed improvements of the C-extenstion
51
+
37
52
  ### version 0.5.9
38
53
 
39
54
  * Handles reading errors that can occur when a compressed file turns
@@ -14,6 +14,22 @@ binary files, so that they can be further analysed.
14
14
 
15
15
  ## Change log
16
16
 
17
+ ### Version 6.0.1
18
+
19
+ * Drops dependency on scipy.
20
+
21
+ ### Version 6.0.0
22
+
23
+ * Offers a python only implementation, alongside the C-extension to
24
+ read the binary files. At the expense of processing speed, no
25
+ compiler is required to use this package.
26
+
27
+ * A github workflow is implemented that generates pre-compiled wheels
28
+ for linux and windows platforms for python version 3.10, 3.11, 3.12
29
+ and 3.13.
30
+
31
+ * Some speed improvements of the C-extenstion
32
+
17
33
  ### version 0.5.9
18
34
 
19
35
  * Handles reading errors that can occur when a compressed file turns
@@ -0,0 +1,294 @@
1
+ """
2
+ Pure Python/NumPy replacement for the _dbdreader C extension.
3
+
4
+ Reads Slocum glider binary DBD/SBD/MBD/etc. files. The public API is a
5
+ single function ``get()`` whose signature matches the C extension exactly.
6
+
7
+ Binary format recap
8
+ -------------------
9
+ After the ASCII header there is a "known cycle" (17 bytes) that the glider
10
+ always writes with fixed values so the reader can detect byte order:
11
+
12
+ [0] 's' cycle-tag byte
13
+ [1] <1-byte int>
14
+ [2:4] 0x1234 two-byte uint (little-endian if == 4660 decimal)
15
+ [4:8] 123.456 four-byte float
16
+ [8:16] 123456789... eight-byte double
17
+ [16] 'd' separator before first real cycle
18
+
19
+ Every subsequent cycle looks like:
20
+
21
+ <n_state_bytes> 2-bit-packed state fields (4 fields/byte, MSB first)
22
+ each field: UPDATED=2, SAME=1, NOTSET=0
23
+ <data bytes> concatenated raw values for UPDATED sensors only,
24
+ in ascending sensor-index order
25
+ 'd' (0x64) separator before the next cycle's state bytes
26
+
27
+ Performance strategy
28
+ --------------------
29
+ Two-pass approach to minimise Python overhead:
30
+
31
+ Pass 1 – cheap Python loop over cycles to locate cycle boundaries.
32
+ Uses a pre-built per-byte-position lookup table so each cycle
33
+ costs only n_state_bytes Python ops (not n_sensors).
34
+
35
+ Pass 2 – batch numpy operations over ALL cycles at once:
36
+ • extract state bytes for every cycle in one fancy-index read
37
+ • decode fields and compute chunk offsets with vectorised ops
38
+ • read sensor values for wanted sensors with vectorised byte extraction
39
+
40
+ This replaces the original approach of ~8 numpy array allocations per cycle
41
+ inside a Python loop, which dominated runtime for files with many cycles.
42
+ """
43
+
44
+ import struct
45
+ import numpy as np
46
+
47
+
48
+ # ── constants (mirror dbdreader.h) ───────────────────────────────────────────
49
+ FILLVALUE = 1e9
50
+ UPDATED = 2
51
+ SAME = 1
52
+ NOTSET = 0
53
+ _FMT_CHAR = {2: 'h', 4: 'f', 8: 'd'} # byte-size → struct char
54
+
55
+ # Shift amounts to extract 4 × 2-bit fields from one byte (MSB first)
56
+ _SHIFTS = np.array([6, 4, 2, 0], dtype=np.int32)
57
+
58
+
59
+ # ── helpers ──────────────────────────────────────────────────────────────────
60
+
61
+ def _read_file(filename: str) -> bytes:
62
+ """Return raw decompressed bytes for *filename* (handles .?cd/.?cg files)."""
63
+ from dbdreader.decompress import is_compressed, Decompressor
64
+ if is_compressed(filename):
65
+ with Decompressor(filename) as d:
66
+ return d.decompress()
67
+ with open(filename, 'rb') as fh:
68
+ return fh.read()
69
+
70
+
71
+ def _insert_ti(vi_list: list, ti: int):
72
+ """
73
+ Insert *ti* into sorted *vi_list* before the first element > ti,
74
+ replicating the C code in get_variable().
75
+
76
+ Returns (vit, nti) where vit is the extended list and nti is the index
77
+ of ti within vit.
78
+ """
79
+ vit = []
80
+ i = 0
81
+ while i < len(vi_list):
82
+ if vi_list[i] > ti:
83
+ break
84
+ vit.append(vi_list[i])
85
+ i += 1
86
+ vit.append(ti)
87
+ nti = i
88
+ for j in range(i, len(vi_list)):
89
+ vit.append(vi_list[j])
90
+ return vit, nti
91
+
92
+
93
+ def _build_chunk_lut(n_state_bytes: int, n_sensors: int, bs_list: list) -> list:
94
+ """
95
+ Build a lookup table: lut[byte_pos][byte_value] = total data bytes
96
+ contributed to the chunk by the four sensors encoded in that state byte.
97
+
98
+ Used in Pass 1 to compute chunksize per cycle in O(n_state_bytes) ops.
99
+ """
100
+ byte_vals = np.arange(256, dtype=np.int32)
101
+ lut = np.zeros((n_state_bytes, 256), dtype=np.int32)
102
+ for bpos in range(n_state_bytes):
103
+ for slot in range(4):
104
+ sidx = bpos * 4 + slot
105
+ if sidx >= n_sensors:
106
+ break
107
+ shift = 6 - slot * 2 # 6, 4, 2, 0
108
+ states = (byte_vals >> shift) & 3 # (256,)
109
+ lut[bpos] += (states == UPDATED) * bs_list[sidx]
110
+ return lut.tolist() # nested Python list for fast indexing
111
+
112
+
113
+ # ── public API ───────────────────────────────────────────────────────────────
114
+
115
+ def get(n_state_bytes, n_sensors, bin_offset, byte_sizes,
116
+ filename, ti, vi, return_nans, skip_initial_line, max_values_to_read):
117
+ """
118
+ Read one or more sensor time-series from a glider binary data file.
119
+
120
+ Parameters match the C extension exactly:
121
+
122
+ n_state_bytes : int – number of state bytes per cycle
123
+ n_sensors : int – total sensors in the file
124
+ bin_offset : int – byte offset to the start of binary data
125
+ byte_sizes : tuple – byte size for each of the n_sensors sensors
126
+ filename : str – path to the data file (compressed or not)
127
+ ti : int – sensor index of the time variable
128
+ vi : tuple – sorted sensor indices to retrieve
129
+ return_nans : int – 1 → include NOTSET slots as FILLVALUE rows
130
+ skip_initial_line : int – 1 → discard first data cycle
131
+ max_values_to_read : int – stop after this many rows (0 = unlimited)
132
+
133
+ Returns
134
+ -------
135
+ (error_no, result) where error_no is 0 on success and result is a
136
+ list of 2*nv lists:
137
+ [t_0, t_1, …, t_{nv-1}, v_0, v_1, …, v_{nv-1}]
138
+ where each t_i / v_i is a Python list of floats.
139
+ """
140
+
141
+ try:
142
+ data = _read_file(filename)
143
+ except FileNotFoundError:
144
+ return 2, [] # ERROR_FILE_NOT_FOUND
145
+ except Exception:
146
+ return 1, [] # ERROR_UNEXPECTED_END_OF_FILE
147
+ nv = len(vi)
148
+ bs_list = list(byte_sizes)
149
+ vit, nti = _insert_ti(list(vi), ti)
150
+ nvt = len(vit) # = nv + 1
151
+
152
+ # ── byte-order detection ─────────────────────────────────────────────────
153
+ endian = '<' if struct.unpack_from('<H', data, bin_offset + 2)[0] == 4660 else '>'
154
+
155
+ # ── numpy array of byte sizes (used in vectorised pass 2) ────────────────
156
+ bs_arr = np.array(bs_list, dtype=np.int32) # (n_sensors,)
157
+ vit_arr = np.array(vit, dtype=np.intp) # (nvt,)
158
+
159
+ # ── chunk-size lookup table (for pass 1) ─────────────────────────────────
160
+ chunk_lut = _build_chunk_lut(n_state_bytes, n_sensors, bs_list)
161
+
162
+ # ── PASS 1: locate cycle boundaries ──────────────────────────────────────
163
+ # For each cycle record (state_bytes + data_chunk + separator):
164
+ # * record the file position of the state bytes
165
+ # * compute the data-chunk size using the lookup table
166
+ # Cost: O(n_cycles × n_state_bytes) pure-Python ops — cheap.
167
+
168
+ min_offset_value = -2 if return_nans else -1
169
+ file_size = len(data)
170
+ pos = bin_offset + 17 # skip known cycle
171
+ write_data = not bool(skip_initial_line)
172
+
173
+ state_positions = [] # file pos of state bytes for each cycle
174
+ chunk_sizes = [] # data-chunk byte count for each cycle
175
+ write_flags = [] # whether to emit output for each cycle
176
+
177
+ while pos < file_size:
178
+ if pos + n_state_bytes > file_size:
179
+ break
180
+
181
+ # Chunksize via LUT: O(n_state_bytes) lookups
182
+ chunksize = 0
183
+ for bpos in range(n_state_bytes):
184
+ chunksize += chunk_lut[bpos][data[pos + bpos]]
185
+
186
+ state_positions.append(pos)
187
+ chunk_sizes.append(chunksize)
188
+ write_flags.append(write_data)
189
+
190
+ pos += n_state_bytes + chunksize + 1
191
+ write_data = True
192
+
193
+ n_cycles = len(state_positions)
194
+ if n_cycles == 0:
195
+ return 0, [[] for _ in range(nv)] + [[] for _ in range(nv)]
196
+
197
+ # ── PASS 2: vectorised processing of all cycles at once ──────────────────
198
+ data_u8 = np.frombuffer(data, dtype=np.uint8)
199
+ sp_arr = np.array(state_positions, dtype=np.intp) # (n_cycles,)
200
+ cstart = sp_arr + n_state_bytes # chunk start positions
201
+
202
+ # Extract state bytes for ALL cycles: shape (n_cycles, n_state_bytes)
203
+ sb_idx = sp_arr[:, np.newaxis] + np.arange(n_state_bytes, dtype=np.intp)
204
+ all_sb = data_u8[sb_idx] # uint8, (n_cycles, n_state_bytes)
205
+
206
+ # Decode 2-bit fields: (n_cycles, n_state_bytes, 4) → (n_cycles, n_sensors)
207
+ all_fields = ((all_sb[:, :, np.newaxis] >> _SHIFTS) & 3) # int, (n_cycles, n_state_bytes, 4)
208
+ all_fields = all_fields.reshape(n_cycles, n_state_bytes * 4)[:, :n_sensors]
209
+
210
+ # Compute byte offsets within the data chunk for every sensor in every cycle.
211
+ # UPDATED → cumulative offset; SAME → -1; NOTSET → -2.
212
+ upd_mask = (all_fields == UPDATED) # bool, (n_cycles, n_sensors)
213
+ w_bs = np.where(upd_mask, bs_arr[np.newaxis, :], 0) # int32, (n_cycles, n_sensors)
214
+ cum_bs = np.cumsum(w_bs, axis=1) # int32, (n_cycles, n_sensors)
215
+ all_off = np.where(upd_mask, cum_bs - w_bs, np.int32(-1)) # (n_cycles, n_sensors)
216
+ all_off = np.where(all_fields == NOTSET, np.int32(-2), all_off)
217
+
218
+ # Offsets for only the wanted sensors: (n_cycles, nvt)
219
+ wanted_off = all_off[:, vit_arr] # int32, (n_cycles, nvt)
220
+
221
+ # ── helper: read all values for one column of wanted_off ─────────────────
222
+ def _read_col(col: int) -> np.ndarray:
223
+ """
224
+ Return float64 array (n_cycles,) for wanted sensor at column *col*.
225
+ UPDATED cycles → actual data value.
226
+ SAME cycles → carry-forward of last UPDATED value (numpy ffill).
227
+ NOTSET cycles → NaN (or FILLVALUE if return_nans).
228
+ """
229
+ sidx = vit[col]
230
+ bs = bs_list[sidx]
231
+ offsets = wanted_off[:, col] # int32, (n_cycles,)
232
+ vals = np.full(n_cycles, np.nan, dtype=np.float64)
233
+
234
+ upd = np.where(offsets >= 0)[0] # indices of UPDATED cycles
235
+ if upd.size:
236
+ abs_pos = cstart[upd] + offsets[upd].astype(np.intp)
237
+
238
+ if bs == 1:
239
+ raw = data_u8[abs_pos].view(np.int8).astype(np.float64)
240
+ vals[upd] = raw
241
+ else:
242
+ # Gather bs consecutive bytes for each updated cycle,
243
+ # reinterpret as the sensor's native type.
244
+ bidx = abs_pos[:, np.newaxis] + np.arange(bs, dtype=np.intp)
245
+ raw = data_u8[bidx].tobytes() # flat byte buffer
246
+ dt = np.dtype(endian + _FMT_CHAR[bs])
247
+ vals[upd] = np.frombuffer(raw, dtype=dt)
248
+
249
+ # Forward-fill: propagate each UPDATED value into subsequent SAME slots.
250
+ # Classic numpy trick: build an index array that "sticks" at the last
251
+ # non-NaN position, then use it to index vals.
252
+ has_val = ~np.isnan(vals)
253
+ ff_idx = np.where(has_val, np.arange(n_cycles, dtype=np.intp), np.intp(0))
254
+ np.maximum.accumulate(ff_idx, out=ff_idx)
255
+ vals = vals[ff_idx]
256
+
257
+ # NOTSET cycles: mark as FILLVALUE (if return_nans) or leave as
258
+ # carried-forward value (which will be excluded by the mask below).
259
+ if return_nans:
260
+ vals[offsets == np.int32(-2)] = FILLVALUE
261
+
262
+ return vals
263
+
264
+ # ── read time and sensor values ───────────────────────────────────────────
265
+ t_vals = _read_col(nti) # (n_cycles,) float64
266
+ write_arr = np.array(write_flags, dtype=bool)
267
+
268
+ result_t = [None] * nv
269
+ result_v = [None] * nv
270
+
271
+ for col in range(nvt):
272
+ if col == nti:
273
+ continue
274
+ j = col - (1 if col > nti else 0) # output index in result_t/v
275
+
276
+ v_vals = _read_col(col)
277
+ v_off = wanted_off[:, col]
278
+
279
+ # Include this cycle in output if:
280
+ # • write_flags says so (skip_initial_line handling)
281
+ # • sensor has a valid offset (UPDATED or SAME; also NOTSET if return_nans)
282
+ include = (v_off >= min_offset_value)
283
+ mask = write_arr & include
284
+
285
+ if max_values_to_read > 0:
286
+ keep = np.where(mask)[0]
287
+ if keep.size > max_values_to_read:
288
+ mask = np.zeros(n_cycles, dtype=bool)
289
+ mask[keep[:max_values_to_read]] = True
290
+
291
+ result_t[j] = t_vals[mask].tolist()
292
+ result_v[j] = v_vals[mask].tolist()
293
+
294
+ return 0, [result_t[j] for j in range(nv)] + [result_v[j] for j in range(nv)]
@@ -4,7 +4,6 @@ import os
4
4
  import struct
5
5
  import time
6
6
  import numpy
7
- from scipy.interpolate import interp1d as si_interp1d
8
7
  import glob
9
8
  import sys
10
9
  import re
@@ -183,6 +182,20 @@ DBD_ERROR_DECOMPRESSION_ERROR = 15
183
182
 
184
183
 
185
184
  class DbdError(Exception):
185
+ '''Exception raised for all dbdreader errors.
186
+
187
+ Parameters
188
+ ----------
189
+ value: int, optional
190
+ one of the DBD_ERROR_* constants defined in this module,
191
+ identifying the kind of error that occurred.
192
+ mesg: str or None, optional
193
+ additional, free-form message appended to the standard
194
+ message associated with ``value``.
195
+ data: object or None, optional
196
+ optional payload with extra information about the error, for
197
+ example a :class:`DbdError.MissingCacheFileData` namedtuple.
198
+ '''
186
199
  MissingCacheFileData = namedtuple('MissingCacheFileData',
187
200
  'missing_cache_files cache_dir')
188
201
 
@@ -294,6 +307,32 @@ def heading_interpolating_function_factory(t, v):
294
307
  return lambda _t: numpy.arctan2(yi(_t), xi(_t)) % (2*numpy.pi)
295
308
 
296
309
 
310
+ def _default_interp1d_factory(x, y):
311
+ '''Default linear interpolating function factory used by get_sync()
312
+ and get_CTD_sync().
313
+
314
+ Equivalent to scipy.interpolate.interp1d(x, y, bounds_error=False,
315
+ fill_value=numpy.nan), implemented with numpy.interp instead so that
316
+ dbdreader does not require scipy as a runtime dependency. Callers that
317
+ want other interpolation schemes (e.g. cubic splines) can still pass
318
+ their own interpolating_function_factory, importing scipy themselves
319
+ if needed.
320
+
321
+ numpy.interp requires x to be sorted (non-decreasing); scipy's interp1d
322
+ sorts internally by default (assume_sorted=False), so we do the same
323
+ here rather than assume the caller's x is already ordered.
324
+ '''
325
+ if len(x) < 1:
326
+ raise ValueError("Interpolation requires at least one data point.")
327
+ x = numpy.asarray(x)
328
+ y = numpy.asarray(y)
329
+ if numpy.any(numpy.diff(x) < 0):
330
+ idx = numpy.argsort(x)
331
+ x = x[idx]
332
+ y = y[idx]
333
+ return partial(numpy.interp, xp=x, fp=y, left=numpy.nan, right=numpy.nan)
334
+
335
+
297
336
  class DBDCache(object):
298
337
 
299
338
  '''DBDCache manager
@@ -420,6 +459,8 @@ class DBDPatternSelect(object):
420
459
  ----------
421
460
  date_format : str, optional
422
461
  date format used to interpret date strings.
462
+ cacheDir : str or None, optional
463
+ path to CAC file cache directory. If None, the default path is used.
423
464
 
424
465
  Note
425
466
  ----
@@ -442,9 +483,6 @@ class DBDPatternSelect(object):
442
483
  date_format: str
443
484
  format to interpret date strings. Example "%H %d %m %Y"
444
485
 
445
- cachedDir: str or None, optional
446
- path to CAC file cache directory. If None, the default path is used.
447
-
448
486
  '''
449
487
  self.date_format=date_format
450
488
 
@@ -638,6 +676,13 @@ class DBDHeader(object):
638
676
 
639
677
  @property
640
678
  def factored(self):
679
+ ''' Value of the sensor_list_factored header field, or None if
680
+ the header has not been read yet.
681
+
682
+ Returns
683
+ -------
684
+ int or None
685
+ '''
641
686
  try:
642
687
  r = self.info['sensor_list_factored']
643
688
  except KeyError:
@@ -686,9 +731,23 @@ class DBDHeader(object):
686
731
  self.info['parameter_list'] = all_parameter_names
687
732
  return parameter
688
733
 
689
- # private methods
690
734
  def parse(self,line):
691
- # parses a binary datastream. So, first decode it to ascii.
735
+ ''' Parses a single header line.
736
+
737
+ Decodes a binary header line and, if it names a recognised
738
+ header keyword, stores the corresponding value in ``self.info``.
739
+
740
+ Parameters
741
+ ----------
742
+ line : bytes
743
+ single header line, as read from the binary file.
744
+
745
+ Returns
746
+ -------
747
+ str
748
+ the keyword parsed from the line. If the keyword is
749
+ recognised, its value is also stored in ``self.info``.
750
+ '''
692
751
  words=line.decode('ascii').rstrip().split(":")
693
752
  param=words[0]
694
753
  if param in self.keywords.keys():
@@ -709,7 +768,7 @@ class DBD(object):
709
768
  filename: str
710
769
  dbd filename
711
770
 
712
- cachedDir: str or None, optional
771
+ cacheDir: str or None, optional
713
772
  path to CAC file cache directory. If None, the default path is used.
714
773
 
715
774
  skip_initial_line : bool, default: True
@@ -1292,7 +1351,7 @@ class MultiDBD(object):
1292
1351
  missions: list of str
1293
1352
  List of missions names that should be considered only.
1294
1353
 
1295
- maxfiles: int
1354
+ max_files: int
1296
1355
  maximum number of files to be read, where
1297
1356
  >0: the first n files are read
1298
1357
  <0: the last n files are read.
@@ -1597,7 +1656,7 @@ class MultiDBD(object):
1597
1656
  tv = self.get(*parameters, decimalLatLon=decimalLatLon, discardBadLatLon=discardBadLatLon,
1598
1657
  return_nans=False)
1599
1658
 
1600
- default_interpolating_function_factory = partial(si_interp1d, bounds_error=False, fill_value=numpy.nan)
1659
+ default_interpolating_function_factory = _default_interp1d_factory
1601
1660
 
1602
1661
  t = tv[0][0]
1603
1662
  r = []
@@ -163,6 +163,19 @@ def decompress_file(filename):
163
163
  return FileDecompressor().decompress(filename)
164
164
 
165
165
  def is_compressed(filename):
166
+ ''' Checks whether a filename indicates an lz4-compressed glider data file.
167
+
168
+ Parameters
169
+ ----------
170
+ filename : str
171
+ filename to check
172
+
173
+ Returns
174
+ -------
175
+ bool
176
+ True if the file extension indicates a compressed
177
+ [demnst]bd/[demnst]cg file, False otherwise.
178
+ '''
166
179
  [basename,ext]=os.path.splitext(filename)
167
180
  # all compressed [demnst]bd files end in [demnst]cd
168
181
  return bool(re_match("[demnst]c[dg]$", ext))
@@ -172,21 +185,45 @@ class BytesIORW:
172
185
  ''' Helper class implementing a BytesIO buffer that can be written to and read from.
173
186
 
174
187
  Note that the methods write() and readline() are implemented only.
188
+
189
+ Parameters
190
+ ----------
191
+ source : iterator of bytes
192
+ iterator (typically a decompressed-blocks generator) that
193
+ readline() draws from once the buffer is exhausted.
175
194
  '''
176
195
  def __init__(self, source):
177
196
  self.bytesIO = ioBytesIO()
178
197
  self.pointer_start = 0
179
198
  self.pointer_end = 0
180
199
  self.source = source
181
-
200
+
182
201
  def write(self, b):
202
+ ''' Writes bytes to the buffer.
203
+
204
+ Parameters
205
+ ----------
206
+ b : bytes
207
+ data to append to the buffer. The read pointer is reset to
208
+ the start of the newly written data.
209
+ '''
183
210
  self.pointer_start = self.bytesIO.tell()
184
211
  self.bytesIO.write(b)
185
212
  self.pointer_end = self.bytesIO.tell()
186
213
  self.bytesIO.seek(self.pointer_start)
187
214
  self.is_exhausted = False
188
-
215
+
189
216
  def readline(self):
217
+ ''' Reads a single line from the buffer.
218
+
219
+ If the buffer is exhausted, the next block is pulled from
220
+ ``source`` and appended before returning.
221
+
222
+ Returns
223
+ -------
224
+ bytes
225
+ a single line, or an empty bytes object if source is exhausted.
226
+ '''
190
227
  line = self.bytesIO.readline()
191
228
  pointer = self.bytesIO.tell()
192
229
  if pointer == self.pointer_end:
@@ -207,8 +244,14 @@ class CompressedFile:
207
244
 
208
245
  The main reason for the class is to be able to read the header of a compressed
209
246
  glider data file.
247
+
248
+ Parameters
249
+ ----------
250
+ filename : str
251
+ name of the compressed file to read. This class is designed to
252
+ be used within a context manager, which opens the file.
210
253
  '''
211
-
254
+
212
255
 
213
256
  def __init__(self, filename):
214
257
  self.filename = filename
@@ -221,15 +264,29 @@ class CompressedFile:
221
264
  self.bytesIO = BytesIORW(source)
222
265
  return self
223
266
 
224
-
267
+
225
268
  def __exit__(self, *p, **kwds):
226
269
  self.fp.close()
227
270
 
228
271
  def readline(self):
272
+ ''' Reads and decompresses a single line from the file.
273
+
274
+ Returns
275
+ -------
276
+ bytes
277
+ a single decompressed line, or an empty bytes object at end of file.
278
+ '''
229
279
  line = self.bytesIO.readline()
230
280
  return line
231
-
281
+
232
282
  def readlines(self):
283
+ ''' Generator that reads and decompresses the file line by line.
284
+
285
+ Yields
286
+ ------
287
+ bytes
288
+ a single decompressed line.
289
+ '''
233
290
  while True:
234
291
  line = self.readline()
235
292
  if not line:
@@ -237,11 +294,25 @@ class CompressedFile:
237
294
  yield line
238
295
 
239
296
  def seek(self, offset):
297
+ ''' Sets the read position in the (decompressed, buffered) data.
298
+
299
+ Parameters
300
+ ----------
301
+ offset : int
302
+ absolute byte position to seek to.
303
+ '''
240
304
  return self.bytesIO.bytesIO.seek(offset)
241
305
 
242
306
  def tell(self):
307
+ ''' Returns the current read position in the (decompressed, buffered) data.
308
+
309
+ Returns
310
+ -------
311
+ int
312
+ '''
243
313
  return self.bytesIO.bytesIO.tell()
244
-
314
+
245
315
 
246
316
  def close(self):
317
+ ''' Closes the underlying compressed file. '''
247
318
  self.fp.close()
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dbdreader
3
- Version: 0.6.0.dev1
3
+ Version: 0.6.1
4
4
  Summary: A python module to access binary data files generated by Teledyne WebbResearch gliders
5
5
  Author-email: Lucas Merckelbach <lucas.merckelbach@hereon.de>
6
- License: GPL-3.0
6
+ License: GPL-3.0-or-later
7
7
  Project-URL: Homepage, https://dbdreader.readthedocs.io/en/latest/
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: Intended Audience :: Science/Research
10
- Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
10
+ Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
11
11
  Classifier: Operating System :: POSIX
12
12
  Requires-Python: >=3.10
13
13
  Description-Content-Type: text/markdown
@@ -15,7 +15,6 @@ License-File: LICENSE
15
15
  License-File: COPYING
16
16
  Requires-Dist: lz4>=4.4.5
17
17
  Requires-Dist: numpy>=2.2.6
18
- Requires-Dist: scipy>=1.15.3
19
18
  Dynamic: license-file
20
19
 
21
20
  [![PyPI version](https://badgen.net/pypi/v/dbdreader)](https://pypi.org/project/dbdreader)
@@ -34,6 +33,22 @@ binary files, so that they can be further analysed.
34
33
 
35
34
  ## Change log
36
35
 
36
+ ### Version 6.0.1
37
+
38
+ * Drops dependency on scipy.
39
+
40
+ ### Version 6.0.0
41
+
42
+ * Offers a python only implementation, alongside the C-extension to
43
+ read the binary files. At the expense of processing speed, no
44
+ compiler is required to use this package.
45
+
46
+ * A github workflow is implemented that generates pre-compiled wheels
47
+ for linux and windows platforms for python version 3.10, 3.11, 3.12
48
+ and 3.13.
49
+
50
+ * Some speed improvements of the C-extenstion
51
+
37
52
  ### version 0.5.9
38
53
 
39
54
  * Handles reading errors that can occur when a compressed file turns
@@ -8,6 +8,7 @@ pyproject.toml
8
8
  requirements.txt
9
9
  setup.py
10
10
  dbdreader/__init__.py
11
+ dbdreader/_dbdreader.py
11
12
  dbdreader/dbdreader.py
12
13
  dbdreader/decompress.py
13
14
  dbdreader/scripts.py
@@ -1,3 +1,2 @@
1
1
  lz4>=4.4.5
2
2
  numpy>=2.2.6
3
- scipy>=1.15.3
@@ -1,23 +1,22 @@
1
1
  [project]
2
2
  name = "dbdreader"
3
- version = "0.6.0dev1"
3
+ version = "0.6.1"
4
4
  description = "A python module to access binary data files generated by Teledyne WebbResearch gliders"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
7
7
  authors = [
8
8
  {name = "Lucas Merckelbach", email = "lucas.merckelbach@hereon.de"}
9
9
  ]
10
- license = {text = "GPL-3.0"}
10
+ license = {text = "GPL-3.0-or-later"}
11
11
  classifiers = [
12
12
  "Programming Language :: Python :: 3",
13
13
  "Intended Audience :: Science/Research",
14
- "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
14
+ "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
15
15
  "Operating System :: POSIX",
16
16
  ]
17
17
  dependencies = [
18
18
  "lz4>=4.4.5",
19
19
  "numpy>=2.2.6",
20
- "scipy>=1.15.3",
21
20
  ]
22
21
 
23
22
 
@@ -32,7 +31,9 @@ Homepage = "https://dbdreader.readthedocs.io/en/latest/"
32
31
  dev = [
33
32
  "black>=26.3.1",
34
33
  "ipython>=8.39.0",
34
+ "myst-parser>=4.0.1",
35
35
  "pytest>=9.0.2",
36
+ "sphinx-rtd-theme>=3.1.0",
36
37
  ]
37
38
 
38
39
 
@@ -1,5 +1,4 @@
1
1
  numpy
2
- scipy
3
2
  pytest
4
3
  lz4
5
4
 
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes