oceanstream 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream/__init__.py +0 -0
- oceanstream/cli/__init__.py +3 -0
- oceanstream/cli/main.py +267 -0
- oceanstream/denoise/__init__.py +2 -0
- oceanstream/denoise/apply_masks.py +49 -0
- oceanstream/denoise/applying_masks_handler.py +173 -0
- oceanstream/denoise/background_noise_remover.py +97 -0
- oceanstream/denoise/create_masks.py +87 -0
- oceanstream/denoise/noise_masks.py +398 -0
- oceanstream/denoise/types.py +73 -0
- oceanstream/echodata/__init__.py +5 -0
- oceanstream/echodata/ensure_time_continuity.py +75 -0
- oceanstream/echodata/mvbs_computation.py +77 -0
- oceanstream/echodata/raw_handler.py +577 -0
- oceanstream/echodata/sv_computation.py +118 -0
- oceanstream/echodata/sv_dataset_extension.py +126 -0
- oceanstream/echodata/sv_interpolation.py +260 -0
- oceanstream/echodata/target_strength_computation.py +109 -0
- oceanstream/exports/__init__.py +2 -0
- oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
- oceanstream/exports/csv/csv_export_from_raw.py +92 -0
- oceanstream/exports/csv/csv_export_nasc.py +135 -0
- oceanstream/exports/frequency_differencing_handler.py +210 -0
- oceanstream/exports/nasc.py +23 -0
- oceanstream/exports/nasc_computation.py +62 -0
- oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
- oceanstream/exports/shoals/shoal_process.py +224 -0
- oceanstream/exports/shoals/shoals_handler.py +35 -0
- oceanstream/plot/__init__.py +2 -0
- oceanstream/plot/echogram.py +542 -0
- oceanstream/process/__init__.py +21 -0
- oceanstream/process/combine_zarr.py +90 -0
- oceanstream/process/file_processor.py +151 -0
- oceanstream/process/folder_processor.py +246 -0
- oceanstream/process/process.py +77 -0
- oceanstream/process/processed_data_io.py +140 -0
- oceanstream/settings/__init__.py +1 -0
- oceanstream/settings/config.py +17 -0
- oceanstream/utils.py +101 -0
- oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
- oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
- oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
- oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
- oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
- oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import scipy.ndimage as nd_img
|
|
4
|
+
import xarray as xr
|
|
5
|
+
from echopype.mask.api import apply_mask
|
|
6
|
+
from haversine import haversine
|
|
7
|
+
from pandas import DataFrame
|
|
8
|
+
|
|
9
|
+
from oceanstream.exports.nasc_computation import compute_per_dataset_nasc
|
|
10
|
+
from oceanstream.utils import tfc
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def split_shoal_mask(Sv: xr.Dataset):
|
|
14
|
+
"""
|
|
15
|
+
Given a Sv dataset with an existing shoal mask, generates a list of
|
|
16
|
+
individual masks for each shoal
|
|
17
|
+
|
|
18
|
+
Parameters:
|
|
19
|
+
- Sv: xr.Dataset - Sv dataset with an existing shoal mask
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
- [xr.DataArray]: list of individual shoal masks
|
|
23
|
+
|
|
24
|
+
Example:
|
|
25
|
+
>>> split_shoal_mask(Sv)
|
|
26
|
+
"""
|
|
27
|
+
all_shoals = Sv["mask_shoal"]
|
|
28
|
+
labelled = nd_img.label(all_shoals)
|
|
29
|
+
la = labelled[0]
|
|
30
|
+
ls = labelled[1]
|
|
31
|
+
shoals = []
|
|
32
|
+
for i in range(1, ls + 1):
|
|
33
|
+
shoal = xr.DataArray(
|
|
34
|
+
data=(la == i),
|
|
35
|
+
dims=all_shoals.dims,
|
|
36
|
+
coords=all_shoals.coords,
|
|
37
|
+
)
|
|
38
|
+
shoal.attrs["label"] = i
|
|
39
|
+
shoal_dict = process_single_shoal(Sv, shoal)
|
|
40
|
+
shoals.append(shoal_dict)
|
|
41
|
+
return shoals
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def process_single_shoal(Sv: xr.Dataset, mask: xr.DataArray):
|
|
45
|
+
"""
|
|
46
|
+
Given a Sv dataset and a shoal mask containing a single shoal, returns a
|
|
47
|
+
dataframe containing shoal metadata for each channel
|
|
48
|
+
|
|
49
|
+
Parameters:
|
|
50
|
+
- Sv: xr.Dataset - Sv dataset
|
|
51
|
+
- mask: xr.DataArray - single-shoal mask
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
- [pd.Dataframe]: a list of single-row-per-channel dictionaries containing shoal metadata
|
|
55
|
+
|
|
56
|
+
Example:
|
|
57
|
+
>>> process_single_shoal(Sv, mask)
|
|
58
|
+
"""
|
|
59
|
+
channels = Sv["channel"]
|
|
60
|
+
results = [process_single_shoal_channel(Sv, mask, c) for c in channels]
|
|
61
|
+
results = [r for r in results if r is not None]
|
|
62
|
+
if results == []:
|
|
63
|
+
return None
|
|
64
|
+
return results
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def process_single_shoal_channel(Sv: xr.Dataset, mask: xr.DataArray, channel: str):
|
|
68
|
+
"""
|
|
69
|
+
Given a Sv dataset, a shoal mask containing a single shoal and the desired channel,
|
|
70
|
+
returns a dataframe containing shoal metadata for that specific channel
|
|
71
|
+
|
|
72
|
+
Parameters:
|
|
73
|
+
- Sv: xr.Dataset - Sv dataset
|
|
74
|
+
- mask: xr.DataArray - single-shoal mask
|
|
75
|
+
- channel: str - channel
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
- dict: a dictionary containing shoal metadata
|
|
79
|
+
|
|
80
|
+
Example:
|
|
81
|
+
>>> process_single_shoal_channel(Sv, mask, channel)
|
|
82
|
+
"""
|
|
83
|
+
mc = mask.sel(channel=channel)
|
|
84
|
+
Sv_sel = Sv.sel(channel=channel)
|
|
85
|
+
Sv_masked = apply_mask(Sv_sel.copy(deep=True), mc)
|
|
86
|
+
|
|
87
|
+
# create a dataset that has the mask instead of Sv
|
|
88
|
+
md = Sv_masked.copy(deep=True)
|
|
89
|
+
md["Sv"] = ~md["Sv"].isnull()
|
|
90
|
+
|
|
91
|
+
Sv_mean = Sv_masked.Sv.mean(skipna=True).values.item()
|
|
92
|
+
frequency = Sv_masked.frequency_nominal.values.item()
|
|
93
|
+
area = mc.sum().values.item()
|
|
94
|
+
if area == 0:
|
|
95
|
+
return None
|
|
96
|
+
filename = Path(md.source_filenames.values.item()).stem
|
|
97
|
+
label = mc.attrs["label"]
|
|
98
|
+
|
|
99
|
+
ping_true = md["Sv"].any(dim="range_sample")
|
|
100
|
+
range_true = md["Sv"].any(dim="ping_time")
|
|
101
|
+
subset_md = md.sel(ping_time=ping_true, range_sample=range_true)
|
|
102
|
+
|
|
103
|
+
# TODO figure out exactly how the shiny tool does plotting
|
|
104
|
+
start_time = subset_md.ping_time.min().values # .item()
|
|
105
|
+
end_time = subset_md.ping_time.max().values # .item()
|
|
106
|
+
start_range = subset_md.range_sample.min().values.item()
|
|
107
|
+
end_range = subset_md.range_sample.max().values.item()
|
|
108
|
+
|
|
109
|
+
bbox_0 = (mc["range_sample"] == start_range).argmax(dim="range_sample").item()
|
|
110
|
+
bbox_2 = (mc["range_sample"] == end_range).argmax(dim="range_sample").item()
|
|
111
|
+
bbox_1 = (mc["ping_time"] == start_time).argmax(
|
|
112
|
+
dim="ping_time"
|
|
113
|
+
).item() * 2 # for historical compatibility
|
|
114
|
+
bbox_3 = (mc["ping_time"] == end_time).argmax(dim="ping_time").item() * 2
|
|
115
|
+
centroid_0 = (bbox_0 + bbox_2) / 2
|
|
116
|
+
centroid_1 = (bbox_1 + bbox_3) / 2
|
|
117
|
+
|
|
118
|
+
npings = len(subset_md.ping_time)
|
|
119
|
+
nsamples = len(subset_md.range_sample)
|
|
120
|
+
mean_range = subset_md.range_sample.mean().values
|
|
121
|
+
|
|
122
|
+
start_lat = subset_md.latitude[0].item()
|
|
123
|
+
end_lat = subset_md.latitude[-1].item()
|
|
124
|
+
start_lon = subset_md.longitude[0].item()
|
|
125
|
+
end_lon = subset_md.longitude[-1].item()
|
|
126
|
+
length_meters = haversine((start_lat, start_lon), (end_lat, end_lon), "m")
|
|
127
|
+
|
|
128
|
+
nasc_list = compute_per_dataset_nasc(Sv)
|
|
129
|
+
nasc = nasc_list["NASC_dataset"]["NASC"].sel(channel=channel).item()
|
|
130
|
+
|
|
131
|
+
return_dict = {
|
|
132
|
+
"label": label,
|
|
133
|
+
"frequency": frequency,
|
|
134
|
+
"filename": filename,
|
|
135
|
+
"area": area,
|
|
136
|
+
"bbox.0": bbox_0,
|
|
137
|
+
"bbox.1": bbox_1,
|
|
138
|
+
"bbox.2": bbox_2,
|
|
139
|
+
"bbox.3": bbox_3,
|
|
140
|
+
"centroid.0": centroid_0,
|
|
141
|
+
"centroid.1": centroid_1,
|
|
142
|
+
"Sv_mean": Sv_mean,
|
|
143
|
+
"npings": npings,
|
|
144
|
+
"nsamples": nsamples,
|
|
145
|
+
"corrected_length": length_meters,
|
|
146
|
+
"mean_range": mean_range,
|
|
147
|
+
"start_range": start_range,
|
|
148
|
+
"end_range": end_range,
|
|
149
|
+
"start_time": start_time,
|
|
150
|
+
"end_time": end_time,
|
|
151
|
+
"start_lat": start_lat,
|
|
152
|
+
"end_lat": end_lat,
|
|
153
|
+
"start_lon": start_lon,
|
|
154
|
+
"end_lon": end_lon,
|
|
155
|
+
"nasc": nasc,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return return_dict
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def process_shoals(Sv: xr.Dataset):
|
|
162
|
+
"""
|
|
163
|
+
Given a Sv dataset with an existing shoal mask, extracts shoal metadata
|
|
164
|
+
|
|
165
|
+
Parameters:
|
|
166
|
+
- Sv: xr.Dataset - Sv dataset with an existing shoal mask
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
- [dict]: list of individual dicts for each shoal/channel combination
|
|
170
|
+
|
|
171
|
+
Example:
|
|
172
|
+
>>> process_shoals(Sv)
|
|
173
|
+
"""
|
|
174
|
+
if 0 in tfc(Sv["mask_shoal"]):
|
|
175
|
+
return_dict = {
|
|
176
|
+
"label": None,
|
|
177
|
+
"frequency": None,
|
|
178
|
+
"filename": Path(Sv.source_filenames.values.item()).stem,
|
|
179
|
+
"area": None,
|
|
180
|
+
"bbox.0": None,
|
|
181
|
+
"bbox.1": None,
|
|
182
|
+
"bbox.2": None,
|
|
183
|
+
"bbox.3": None,
|
|
184
|
+
"centroid.0": None,
|
|
185
|
+
"centroid.1": None,
|
|
186
|
+
"Sv_mean": None,
|
|
187
|
+
"npings": None,
|
|
188
|
+
"nsamples": None,
|
|
189
|
+
"corrected_length": None,
|
|
190
|
+
"mean_range": None,
|
|
191
|
+
"start_range": None,
|
|
192
|
+
"end_range": None,
|
|
193
|
+
"start_time": None,
|
|
194
|
+
"end_time": None,
|
|
195
|
+
"start_lat": None,
|
|
196
|
+
"end_lat": None,
|
|
197
|
+
"start_lon": None,
|
|
198
|
+
"end_lon": None,
|
|
199
|
+
"nasc": None,
|
|
200
|
+
}
|
|
201
|
+
return [return_dict]
|
|
202
|
+
# masks = split_shoal_mask(Sv)
|
|
203
|
+
# dicts = [process_single_shoal(Sv, mask) for mask in masks]
|
|
204
|
+
dicts = split_shoal_mask(Sv)
|
|
205
|
+
results = [item for sublist in dicts for item in sublist]
|
|
206
|
+
results = [r for r in results if r is not None]
|
|
207
|
+
return results
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def write_shoals_to_csv(shoal_list: [], filename):
|
|
211
|
+
"""
|
|
212
|
+
Given a shoal list and a filename, it exports the list as csv to the filename
|
|
213
|
+
|
|
214
|
+
Parameters:
|
|
215
|
+
- shoal_list: []
|
|
216
|
+
the list of dicts containing the shoal metadata
|
|
217
|
+
- filename: str
|
|
218
|
+
The file name to use.
|
|
219
|
+
|
|
220
|
+
Returns:
|
|
221
|
+
- None
|
|
222
|
+
"""
|
|
223
|
+
df = DataFrame(shoal_list)
|
|
224
|
+
df.to_csv(filename, index=False)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import xarray as xr
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
from xarray import Dataset
|
|
5
|
+
|
|
6
|
+
from .shoal_detection_handler import attach_shoal_mask_to_ds
|
|
7
|
+
from .shoal_process import process_shoals, write_shoals_to_csv
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_shoals_list(ds: xr.Dataset, config) -> tuple[Any, dict, Dataset]:
|
|
11
|
+
parameters = config["shoals"]["parameters"]
|
|
12
|
+
method = config["shoals"]["method"]
|
|
13
|
+
|
|
14
|
+
shoal_dataset = attach_shoal_mask_to_ds(ds, parameters=parameters, method=method)
|
|
15
|
+
shoal_list = process_shoals(shoal_dataset)
|
|
16
|
+
|
|
17
|
+
return shoal_list, profiling_info, shoal_dataset
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def write_csv(ds: xr.Dataset, config):
|
|
21
|
+
shoal_list = None
|
|
22
|
+
shoal_dataset = None
|
|
23
|
+
|
|
24
|
+
if config["shoals"]["enabled"]:
|
|
25
|
+
shoal_list, shoal_dataset = get_shoals_list(ds, config)
|
|
26
|
+
|
|
27
|
+
if config["export_csv"]:
|
|
28
|
+
write_shoals_to_csv(
|
|
29
|
+
shoal_list,
|
|
30
|
+
os.path.join(
|
|
31
|
+
config["output_folder"], config["raw_path"].stem + "_fish_schools.csv"
|
|
32
|
+
),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
return shoal_list, shoal_dataset
|