oceanstream 0.0.14.dev0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream-0.1.1/PKG-INFO +270 -0
- oceanstream-0.1.1/oceanstream/README.md +214 -0
- oceanstream-0.1.1/oceanstream/TODO.md +515 -0
- oceanstream-0.1.1/oceanstream/__init__.py +111 -0
- oceanstream-0.1.1/oceanstream/adcp/__init__.py +7 -0
- oceanstream-0.1.1/oceanstream/adcp/processor.py +63 -0
- oceanstream-0.1.1/oceanstream/app.py +64 -0
- oceanstream-0.1.1/oceanstream/cli.py +2167 -0
- oceanstream-0.1.1/oceanstream/config/__init__.py +1 -0
- oceanstream-0.1.1/oceanstream/config/settings.py +190 -0
- oceanstream-0.1.1/oceanstream/configuration.py +260 -0
- oceanstream-0.1.1/oceanstream/echodata/__init__.py +299 -0
- oceanstream-0.1.1/oceanstream/echodata/calibrate/__init__.py +23 -0
- oceanstream-0.1.1/oceanstream/echodata/calibrate/calibration.py +266 -0
- oceanstream-0.1.1/oceanstream/echodata/calibrate/saildrone.py +275 -0
- oceanstream-0.1.1/oceanstream/echodata/compute/__init__.py +27 -0
- oceanstream-0.1.1/oceanstream/echodata/compute/mvbs.py +125 -0
- oceanstream-0.1.1/oceanstream/echodata/compute/nasc.py +224 -0
- oceanstream-0.1.1/oceanstream/echodata/compute/sv.py +471 -0
- oceanstream-0.1.1/oceanstream/echodata/concat.py +397 -0
- oceanstream-0.1.1/oceanstream/echodata/config.py +573 -0
- oceanstream-0.1.1/oceanstream/echodata/consolidate/__init__.py +17 -0
- oceanstream-0.1.1/oceanstream/echodata/consolidate/depth.py +259 -0
- oceanstream-0.1.1/oceanstream/echodata/convert.py +244 -0
- oceanstream-0.1.1/oceanstream/echodata/denoise/__init__.py +38 -0
- oceanstream-0.1.1/oceanstream/echodata/denoise/attenuation.py +112 -0
- oceanstream-0.1.1/oceanstream/echodata/denoise/background_noise.py +218 -0
- oceanstream-0.1.1/oceanstream/echodata/denoise/denoise.py +499 -0
- oceanstream-0.1.1/oceanstream/echodata/denoise/impulse_noise.py +158 -0
- oceanstream-0.1.1/oceanstream/echodata/denoise/transient_noise.py +108 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/__init__.py +80 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/absorption.py +234 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/blended.py +350 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/copernicus.py +298 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/enrich.py +804 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/geoparquet.py +849 -0
- oceanstream-0.1.1/oceanstream/echodata/environment/sound_speed.py +287 -0
- oceanstream-0.1.1/oceanstream/echodata/plot/__init__.py +33 -0
- oceanstream-0.1.1/oceanstream/echodata/plot/echogram.py +789 -0
- oceanstream-0.1.1/oceanstream/echodata/processor.py +1094 -0
- oceanstream-0.1.1/oceanstream/echodata/seabed/__init__.py +36 -0
- oceanstream-0.1.1/oceanstream/echodata/seabed/detection.py +730 -0
- oceanstream-0.1.1/oceanstream/echodata/stac/__init__.py +40 -0
- oceanstream-0.1.1/oceanstream/echodata/stac/echodata_emit.py +820 -0
- oceanstream-0.1.1/oceanstream/echodata/stac/segments.py +529 -0
- oceanstream-0.1.1/oceanstream/echodata/storage.py +400 -0
- oceanstream-0.1.1/oceanstream/geotrack/README.md +440 -0
- oceanstream-0.1.1/oceanstream/geotrack/__init__.py +8 -0
- oceanstream-0.1.1/oceanstream/geotrack/binning.py +90 -0
- oceanstream-0.1.1/oceanstream/geotrack/campaign.py +337 -0
- oceanstream-0.1.1/oceanstream/geotrack/csv_reader.py +225 -0
- oceanstream-0.1.1/oceanstream/geotrack/deduplication.py +186 -0
- oceanstream-0.1.1/oceanstream/geotrack/geoparquet_writer.py +180 -0
- oceanstream-0.1.1/oceanstream/geotrack/interpolation.py +240 -0
- oceanstream-0.1.1/oceanstream/geotrack/metadata.py +147 -0
- oceanstream-0.1.1/oceanstream/geotrack/processor.py +2316 -0
- oceanstream-0.1.1/oceanstream/geotrack/raster/__init__.py +9 -0
- oceanstream-0.1.1/oceanstream/geotrack/raster/heatmap.py +405 -0
- oceanstream-0.1.1/oceanstream/geotrack/report.py +596 -0
- oceanstream-0.1.1/oceanstream/geotrack/tiling/__init__.py +13 -0
- oceanstream-0.1.1/oceanstream/geotrack/tiling/pmtiles.py +701 -0
- oceanstream-0.1.1/oceanstream/geotrack/validation.py +170 -0
- oceanstream-0.1.1/oceanstream/multibeam/__init__.py +7 -0
- oceanstream-0.1.1/oceanstream/multibeam/processor.py +63 -0
- oceanstream-0.1.1/oceanstream/providers/__init__.py +3 -0
- oceanstream-0.1.1/oceanstream/providers/base.py +45 -0
- oceanstream-0.1.1/oceanstream/providers/factory.py +22 -0
- oceanstream-0.1.1/oceanstream/providers/models.py +18 -0
- oceanstream-0.1.1/oceanstream/providers/r2r/__init__.py +11 -0
- oceanstream-0.1.1/oceanstream/providers/r2r/r2r.py +330 -0
- oceanstream-0.1.1/oceanstream/providers/r2r/r2r_archive.py +90 -0
- oceanstream-0.1.1/oceanstream/providers/r2r/r2r_metadata.py +113 -0
- oceanstream-0.1.1/oceanstream/providers/r2r_archive.py +138 -0
- oceanstream-0.1.1/oceanstream/providers/r2r_metadata.py +135 -0
- oceanstream-0.1.1/oceanstream/providers/saildrone.py +270 -0
- oceanstream-0.1.1/oceanstream/semantic/__init__.py +3 -0
- oceanstream-0.1.1/oceanstream/semantic/data/README.md +87 -0
- oceanstream-0.1.1/oceanstream/semantic/data/cf-standard-names.json +131 -0
- oceanstream-0.1.1/oceanstream/semantic/data/saildrone-aliases.json +293 -0
- oceanstream-0.1.1/oceanstream/semantic/semantic.py +184 -0
- oceanstream-0.1.1/oceanstream/sensors/README.md +291 -0
- oceanstream-0.1.1/oceanstream/sensors/__init__.py +15 -0
- oceanstream-0.1.1/oceanstream/sensors/catalogue.py +193 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/README.md +90 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/airmar-150wx/README.md +40 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/airmar-150wx/sensor.json +42 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/apogee-si111/README.md +34 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/apogee-si111/sensor.json +21 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/gnss-navigation/sensor.json +35 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/imu-navigation/README.md +38 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/imu-navigation/sensor.json +41 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/kipp-zonen-cmp/README.md +35 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/kipp-zonen-cmp/sensor.json +23 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/lci90i-winch/README.md +60 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/lci90i-winch/sensor.json +41 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/licor-li190r/README.md +34 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/licor-li190r/sensor.json +21 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/sbe-911plus/README.md +66 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/sbe-911plus/sensor.json +107 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/sbe37-odo/README.md +37 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/sbe37-odo/sensor.json +32 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/thermistor-0.5m/README.md +34 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/thermistor-0.5m/sensor.json +21 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/valeport-minisvs/README.md +64 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/valeport-minisvs/sensor.json +29 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/wave-imu/README.md +33 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/wave-imu/sensor.json +18 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-eco-flntu/README.md +53 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-eco-flntu/sensor.json +24 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-flbbcd/README.md +31 -0
- oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-flbbcd/sensor.json +20 -0
- oceanstream-0.1.1/oceanstream/sensors/loader.py +72 -0
- oceanstream-0.1.1/oceanstream/sensors/processor_base.py +77 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/__init__.py +80 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/example_sensor.py +52 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/nmea_gnss.py +432 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/r2r_ctd.py +778 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/r2r_fluorometer.py +149 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/r2r_ssv.py +119 -0
- oceanstream-0.1.1/oceanstream/sensors/processors/r2r_winch.py +263 -0
- oceanstream-0.1.1/oceanstream/sensors/saildrone.py +90 -0
- oceanstream-0.1.1/oceanstream/stac/__init__.py +3 -0
- oceanstream-0.1.1/oceanstream/stac/emit.py +313 -0
- oceanstream-0.1.1/oceanstream/storage/__init__.py +5 -0
- oceanstream-0.1.1/oceanstream/storage/azure_blob.py +23 -0
- oceanstream-0.1.1/oceanstream/storage/config.py +267 -0
- oceanstream-0.1.1/oceanstream/storage/crypto.py +143 -0
- oceanstream-0.1.1/oceanstream/storage/filesystem.py +366 -0
- oceanstream-0.1.1/oceanstream/storage/local.py +11 -0
- oceanstream-0.1.1/oceanstream/storage/manager.py +326 -0
- oceanstream-0.1.1/oceanstream/storage/providers.py +275 -0
- oceanstream-0.1.1/pyproject.toml +131 -0
- oceanstream-0.0.14.dev0/.flake8 +0 -29
- oceanstream-0.0.14.dev0/.gitignore +0 -178
- oceanstream-0.0.14.dev0/CODE_OF_CONDUCT.md +0 -128
- oceanstream-0.0.14.dev0/LICENSE +0 -21
- oceanstream-0.0.14.dev0/MANIFEST.in +0 -9
- oceanstream-0.0.14.dev0/PKG-INFO +0 -179
- oceanstream-0.0.14.dev0/README.md +0 -152
- oceanstream-0.0.14.dev0/notebooks/README.md +0 -1
- oceanstream-0.0.14.dev0/oceanstream/__init__.py +0 -5
- oceanstream-0.0.14.dev0/oceanstream/cli/__init__.py +0 -3
- oceanstream-0.0.14.dev0/oceanstream/cli/main.py +0 -363
- oceanstream-0.0.14.dev0/oceanstream/cli/requirements.txt +0 -1
- oceanstream-0.0.14.dev0/oceanstream/convert/__init__.py +0 -5
- oceanstream-0.0.14.dev0/oceanstream/convert/convert_raw_file.py +0 -58
- oceanstream-0.0.14.dev0/oceanstream/core.py +0 -174
- oceanstream-0.0.14.dev0/oceanstream/denoise/__init__.py +0 -2
- oceanstream-0.0.14.dev0/oceanstream/denoise/apply_masks.py +0 -49
- oceanstream-0.0.14.dev0/oceanstream/denoise/applying_masks_handler.py +0 -173
- oceanstream-0.0.14.dev0/oceanstream/denoise/background_noise_remover.py +0 -97
- oceanstream-0.0.14.dev0/oceanstream/denoise/create_masks.py +0 -87
- oceanstream-0.0.14.dev0/oceanstream/denoise/noise_masks.py +0 -398
- oceanstream-0.0.14.dev0/oceanstream/denoise/requirements.txt +0 -0
- oceanstream-0.0.14.dev0/oceanstream/denoise/types.py +0 -73
- oceanstream-0.0.14.dev0/oceanstream/echodata/__init__.py +0 -5
- oceanstream-0.0.14.dev0/oceanstream/echodata/dataset.py +0 -586
- oceanstream-0.0.14.dev0/oceanstream/echodata/ensure_time_continuity.py +0 -75
- oceanstream-0.0.14.dev0/oceanstream/echodata/mvbs_computation.py +0 -77
- oceanstream-0.0.14.dev0/oceanstream/echodata/raw_handler.py +0 -577
- oceanstream-0.0.14.dev0/oceanstream/echodata/requirements.txt +0 -0
- oceanstream-0.0.14.dev0/oceanstream/echodata/sv_computation.py +0 -125
- oceanstream-0.0.14.dev0/oceanstream/echodata/sv_dataset_extension.py +0 -125
- oceanstream-0.0.14.dev0/oceanstream/echodata/sv_interpolation.py +0 -259
- oceanstream-0.0.14.dev0/oceanstream/echodata/target_strength_computation.py +0 -109
- oceanstream-0.0.14.dev0/oceanstream/exports/__init__.py +0 -6
- oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +0 -183
- oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_raw.py +0 -92
- oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_nasc.py +0 -135
- oceanstream-0.0.14.dev0/oceanstream/exports/frequency_differencing_handler.py +0 -210
- oceanstream-0.0.14.dev0/oceanstream/exports/nasc.py +0 -23
- oceanstream-0.0.14.dev0/oceanstream/exports/nasc_computation.py +0 -62
- oceanstream-0.0.14.dev0/oceanstream/exports/requirements.txt +0 -4
- oceanstream-0.0.14.dev0/oceanstream/exports/shoals/shoal_detection_handler.py +0 -109
- oceanstream-0.0.14.dev0/oceanstream/exports/shoals/shoal_process.py +0 -224
- oceanstream-0.0.14.dev0/oceanstream/exports/shoals/shoals_handler.py +0 -35
- oceanstream-0.0.14.dev0/oceanstream/plot/__init__.py +0 -2
- oceanstream-0.0.14.dev0/oceanstream/plot/echogram.py +0 -579
- oceanstream-0.0.14.dev0/oceanstream/plot/requirements.txt +0 -2
- oceanstream-0.0.14.dev0/oceanstream/process/__init__.py +0 -23
- oceanstream-0.0.14.dev0/oceanstream/process/aws/__init__.py +0 -9
- oceanstream-0.0.14.dev0/oceanstream/process/aws/s3.py +0 -94
- oceanstream-0.0.14.dev0/oceanstream/process/azure/__init__.py +0 -8
- oceanstream-0.0.14.dev0/oceanstream/process/azure/azure_processor.py +0 -66
- oceanstream-0.0.14.dev0/oceanstream/process/azure/blob_storage.py +0 -85
- oceanstream-0.0.14.dev0/oceanstream/process/cloud.py +0 -21
- oceanstream-0.0.14.dev0/oceanstream/process/combine_zarr.py +0 -89
- oceanstream-0.0.14.dev0/oceanstream/process/file_processor.py +0 -237
- oceanstream-0.0.14.dev0/oceanstream/process/file_processor_class.py +0 -217
- oceanstream-0.0.14.dev0/oceanstream/process/folder_processor.py +0 -395
- oceanstream-0.0.14.dev0/oceanstream/process/process.py +0 -81
- oceanstream-0.0.14.dev0/oceanstream/process/processed_data_io.py +0 -140
- oceanstream-0.0.14.dev0/oceanstream/process/requirements.txt +0 -0
- oceanstream-0.0.14.dev0/oceanstream/process/utils.py +0 -226
- oceanstream-0.0.14.dev0/oceanstream/settings/__init__.py +0 -1
- oceanstream-0.0.14.dev0/oceanstream/settings/config.py +0 -17
- oceanstream-0.0.14.dev0/oceanstream/settings/defaults.config.json +0 -120
- oceanstream-0.0.14.dev0/oceanstream/utils.py +0 -101
- oceanstream-0.0.14.dev0/oceanstream.egg-info/PKG-INFO +0 -179
- oceanstream-0.0.14.dev0/oceanstream.egg-info/SOURCES.txt +0 -78
- oceanstream-0.0.14.dev0/oceanstream.egg-info/dependency_links.txt +0 -1
- oceanstream-0.0.14.dev0/oceanstream.egg-info/entry_points.txt +0 -2
- oceanstream-0.0.14.dev0/oceanstream.egg-info/requires.txt +0 -31
- oceanstream-0.0.14.dev0/oceanstream.egg-info/top_level.txt +0 -1
- oceanstream-0.0.14.dev0/pyproject.toml +0 -8
- oceanstream-0.0.14.dev0/requirements-dev.txt +0 -52
- oceanstream-0.0.14.dev0/requirements-local.txt +0 -4
- oceanstream-0.0.14.dev0/requirements.txt +0 -10
- oceanstream-0.0.14.dev0/setup.cfg +0 -60
- oceanstream-0.0.14.dev0/setup.py +0 -19
- /oceanstream-0.0.14.dev0/.gitattributes → /oceanstream-0.1.1/LICENSE +0 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oceanstream
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Process oceanographic measurements from USVs and write partitioned GeoParquet datasets with geospatial metadata.
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: oceanography,geoparquet,stac,saildrone,geospatial,usv,marine-data
|
|
8
|
+
Author: Andrei Neacsu
|
|
9
|
+
Author-email: andrei@oceanstream.io
|
|
10
|
+
Requires-Python: >=3.11,<3.13
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Oceanography
|
|
21
|
+
Provides-Extra: adcp
|
|
22
|
+
Provides-Extra: all
|
|
23
|
+
Provides-Extra: echodata
|
|
24
|
+
Provides-Extra: echodata-viz
|
|
25
|
+
Provides-Extra: geo
|
|
26
|
+
Provides-Extra: geotrack
|
|
27
|
+
Provides-Extra: multibeam
|
|
28
|
+
Requires-Dist: adlfs (>=2024.4.0) ; extra == "echodata" or extra == "all"
|
|
29
|
+
Requires-Dist: azure-storage-blob (>=12.19)
|
|
30
|
+
Requires-Dist: bottleneck (==1.5.0) ; extra == "echodata" or extra == "all"
|
|
31
|
+
Requires-Dist: copernicusmarine (>=1.0.0) ; extra == "echodata" or extra == "all"
|
|
32
|
+
Requires-Dist: dask[complete] (==2024.9.0) ; extra == "echodata" or extra == "all"
|
|
33
|
+
Requires-Dist: datashader (>=0.16) ; extra == "echodata-viz" or extra == "all"
|
|
34
|
+
Requires-Dist: distributed (==2024.9.0) ; extra == "echodata" or extra == "all"
|
|
35
|
+
Requires-Dist: fsspec (==2025.3.2) ; extra == "echodata" or extra == "all"
|
|
36
|
+
Requires-Dist: geopandas (>=0.14) ; extra == "geotrack" or extra == "all" or extra == "geo"
|
|
37
|
+
Requires-Dist: gsw (>=3.6.0) ; extra == "echodata" or extra == "all"
|
|
38
|
+
Requires-Dist: hvplot (>=0.9) ; extra == "echodata-viz" or extra == "all"
|
|
39
|
+
Requires-Dist: matplotlib (>=3.7) ; extra == "echodata-viz" or extra == "all"
|
|
40
|
+
Requires-Dist: netcdf4 (>=1.6) ; extra == "echodata" or extra == "all"
|
|
41
|
+
Requires-Dist: numcodecs (==0.14.1) ; extra == "echodata" or extra == "all"
|
|
42
|
+
Requires-Dist: numpy (==1.26.4) ; extra == "echodata" or extra == "all"
|
|
43
|
+
Requires-Dist: openpyxl (>=3.0) ; extra == "echodata" or extra == "all"
|
|
44
|
+
Requires-Dist: pandas (>=2.1)
|
|
45
|
+
Requires-Dist: pyarrow (>=14)
|
|
46
|
+
Requires-Dist: pynmea2 (>=1.18) ; extra == "geotrack" or extra == "all" or extra == "geo"
|
|
47
|
+
Requires-Dist: rich (>=13.7)
|
|
48
|
+
Requires-Dist: seabirdscientific (>=2.7) ; extra == "geotrack" or extra == "all" or extra == "geo"
|
|
49
|
+
Requires-Dist: shapely (>=2.0) ; extra == "geotrack" or extra == "all" or extra == "geo"
|
|
50
|
+
Requires-Dist: tqdm (>=4.66)
|
|
51
|
+
Requires-Dist: typer (>=0.12)
|
|
52
|
+
Requires-Dist: xarray (==2024.11.0) ; extra == "echodata" or extra == "all"
|
|
53
|
+
Requires-Dist: zarr (>=2.16) ; extra == "echodata" or extra == "all"
|
|
54
|
+
Project-URL: Homepage, https://oceanstream.io
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
# Oceanstream
|
|
58
|
+
|
|
59
|
+
This project processes oceanographic measurements collected from Unmanned Surface Vehicles (USVs). The pipeline reads CSV files from a specified directory, consolidates the data, and stores it in a GeoParquet format partitioned by latitude and longitude bins. Optionally, it can upload the resulting GeoParquet files to Azure Blob Storage.
|
|
60
|
+
|
|
61
|
+
## Project Structure
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
oceanstream
|
|
65
|
+
├── src
|
|
66
|
+
│ ├── app.py # Main entry point for the application
|
|
67
|
+
│ ├── cli.py # Command-line interface for running the pipeline
|
|
68
|
+
│ ├── config # Configuration settings
|
|
69
|
+
│ │ ├── __init__.py
|
|
70
|
+
│ │ └── settings.py
|
|
71
|
+
│ ├── pipeline # Data processing pipeline
|
|
72
|
+
│ │ ├── __init__.py
|
|
73
|
+
│ │ ├── csv_reader.py # Functions for reading CSV files
|
|
74
|
+
│ │ ├── binning.py # Functions for data partitioning
|
|
75
|
+
│ │ └── geoparquet_writer.py # Functions for writing geoparquet files
|
|
76
|
+
│ ├── storage # Storage handling
|
|
77
|
+
│ │ ├── __init__.py
|
|
78
|
+
│ │ ├── local.py # Local storage functions
|
|
79
|
+
│ │ └── azure_blob.py # Azure Blob Storage functions
|
|
80
|
+
│ └── types # Data models and types
|
|
81
|
+
│ ├── __init__.py
|
|
82
|
+
│ └── models.py
|
|
83
|
+
├── data
|
|
84
|
+
│ └── raw_data # Directory for raw CSV data
|
|
85
|
+
│ └── .gitkeep
|
|
86
|
+
├── tests # Unit tests for the application
|
|
87
|
+
│ ├── __init__.py
|
|
88
|
+
│ └── test_pipeline.py
|
|
89
|
+
├── .env.example # Template for environment variables
|
|
90
|
+
├── .gitignore # Git ignore file
|
|
91
|
+
├── .python-version # Python version specification
|
|
92
|
+
├── pyproject.toml # Project dependencies and configuration
|
|
93
|
+
└── README.md # Project documentation
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Setup Instructions
|
|
97
|
+
|
|
98
|
+
1. **Clone the Repository**:
|
|
99
|
+
```bash
|
|
100
|
+
git clone <repository-url>
|
|
101
|
+
cd oceanstream
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
2. **Create a Virtual Environment**:
|
|
105
|
+
```bash
|
|
106
|
+
python3.12 -m venv venv
|
|
107
|
+
source venv/bin/activate # On Windows use `venv\Scripts\activate`
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
3. **Install Dependencies**:
|
|
111
|
+
|
|
112
|
+
Oceanstream is organized into optional processing modules. Install only what you need:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
# Install core + geotrack processing (GPS/navigation data → GeoParquet)
|
|
116
|
+
pip install -e ".[geotrack]"
|
|
117
|
+
|
|
118
|
+
# Install core + echodata processing (echosounder data → Zarr)
|
|
119
|
+
pip install -e ".[echodata]"
|
|
120
|
+
|
|
121
|
+
# Install all processing modules
|
|
122
|
+
pip install -e ".[all]"
|
|
123
|
+
|
|
124
|
+
# Install for development (includes all modules + dev tools)
|
|
125
|
+
pip install -e ".[all]" -r requirements-dev.txt
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
**Available extras:**
|
|
129
|
+
- `geotrack` - GPS/navigation track processing (pandas, geopandas, shapely)
|
|
130
|
+
- `echodata` - Echosounder data processing (echopype, xarray, zarr, netcdf4)
|
|
131
|
+
- `multibeam` - Multibeam sonar processing (planned)
|
|
132
|
+
- `adcp` - ADCP current profiler processing (planned)
|
|
133
|
+
- `all` - All processing modules
|
|
134
|
+
- `geo` - Legacy alias for `geotrack`
|
|
135
|
+
|
|
136
|
+
4. **Configure Environment Variables**:
|
|
137
|
+
Copy `.env.example` to `.env` and fill in the necessary configuration values.
|
|
138
|
+
|
|
139
|
+
5. **Run Processing Commands**:
|
|
140
|
+
|
|
141
|
+
Oceanstream provides separate commands for each data type:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
# Process geotrack data (CSV → GeoParquet)
|
|
145
|
+
oceanstream process geotrack --input-dir raw_data --output-dir out/geoparquet -v
|
|
146
|
+
|
|
147
|
+
# Process echosounder data (planned - requires echodata extra)
|
|
148
|
+
oceanstream process echodata --input-dir raw_echodata --output-dir out/echodata -v
|
|
149
|
+
|
|
150
|
+
# Process multibeam data (planned - requires multibeam extra)
|
|
151
|
+
oceanstream process multibeam --input-dir raw_multibeam --output-dir out/multibeam -v
|
|
152
|
+
|
|
153
|
+
# Process ADCP data (planned - requires adcp extra)
|
|
154
|
+
oceanstream process adcp --input-dir raw_adcp --output-dir out/adcp -v
|
|
155
|
+
|
|
156
|
+
# List available data providers
|
|
157
|
+
oceanstream providers
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
All processing commands support `--provider` flag to specify the data source:
|
|
161
|
+
```bash
|
|
162
|
+
oceanstream process --provider saildrone geotrack --input-dir data -v
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Usage
|
|
166
|
+
|
|
167
|
+
### Geotrack Processing (GPS/Navigation Data)
|
|
168
|
+
|
|
169
|
+
CLI usage examples:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
# Process sample fixture data bundled with tests
|
|
173
|
+
oceanstream process geotrack --input-dir oceanstream/tests/data/raw_data --output-dir out/geoparquet -v
|
|
174
|
+
|
|
175
|
+
# Process your raw_data directory at repo root (default input-dir is ./raw_data)
|
|
176
|
+
oceanstream process geotrack --output-dir out/geoparquet -v
|
|
177
|
+
|
|
178
|
+
# Dry run to see what would be processed
|
|
179
|
+
oceanstream process geotrack --input-dir raw_data --dry-run -v
|
|
180
|
+
|
|
181
|
+
# List available columns in the data
|
|
182
|
+
oceanstream process geotrack --input-dir raw_data --list-columns
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
The CLI reads CSVs, auto-derives coarse 5° bins, and writes a partitioned GeoParquet dataset with metadata. Use `-v` for progress logs.
|
|
186
|
+
|
|
187
|
+
### Processing Modules
|
|
188
|
+
|
|
189
|
+
Oceanstream is organized into separate processing modules:
|
|
190
|
+
|
|
191
|
+
- **`oceanstream.geotrack`** - Process GPS/navigation track data into GeoParquet format
|
|
192
|
+
- **`oceanstream.echodata`** - Process echosounder data (EK60/EK80) into Zarr (coming soon)
|
|
193
|
+
- **`oceanstream.multibeam`** - Process multibeam sonar data (coming soon)
|
|
194
|
+
- **`oceanstream.adcp`** - Process ADCP current profiler data (coming soon)
|
|
195
|
+
|
|
196
|
+
Each module can be installed independently using pip extras (see Installation section).
|
|
197
|
+
|
|
198
|
+
## Using OceanStream Data in GIS Tools
|
|
199
|
+
|
|
200
|
+
OceanStream generates cloud-optimized GeoParquet files designed to work seamlessly with modern GIS tools and data analysis frameworks. Our output includes:
|
|
201
|
+
|
|
202
|
+
- **GeoParquet**: Columnar format with embedded geometry and spatial partitioning
|
|
203
|
+
- **STAC Metadata**: Standard catalog format for discovery and integration
|
|
204
|
+
- **PMTiles** (optional): Vector tiles for web-based visualization
|
|
205
|
+
|
|
206
|
+
### Comprehensive GIS Integration Guides
|
|
207
|
+
|
|
208
|
+
We provide detailed integration guides for popular GIS tools and frameworks:
|
|
209
|
+
|
|
210
|
+
**Desktop GIS:**
|
|
211
|
+
- [QGIS](../docs/gis-integration/qgis.md) - Open-source desktop GIS
|
|
212
|
+
- [ArcGIS Pro](../docs/gis-integration/arcgis-pro.md) - Professional ESRI platform
|
|
213
|
+
|
|
214
|
+
**Data Analysis:**
|
|
215
|
+
- [DuckDB](../docs/gis-integration/duckdb.md) - Fast in-process SQL analytics
|
|
216
|
+
- [GeoPandas](../docs/gis-integration/geopandas.md) - Python spatial data analysis
|
|
217
|
+
|
|
218
|
+
**Web GIS** (coming soon):
|
|
219
|
+
- Leaflet + PMTiles
|
|
220
|
+
- Mapbox GL JS
|
|
221
|
+
- STAC Browser
|
|
222
|
+
|
|
223
|
+
**See [GIS Integration Documentation](../docs/gis-integration/) for complete guides with:**
|
|
224
|
+
- Installation instructions
|
|
225
|
+
- Step-by-step usage examples
|
|
226
|
+
- Code samples and workflows
|
|
227
|
+
- Performance optimization tips
|
|
228
|
+
- Troubleshooting guides
|
|
229
|
+
|
|
230
|
+
### Quick Start Examples
|
|
231
|
+
|
|
232
|
+
**Load in QGIS:**
|
|
233
|
+
```bash
|
|
234
|
+
# Generate data
|
|
235
|
+
oceanstream process geotrack --input-source ./data/sample.csv --output-dir ./output
|
|
236
|
+
|
|
237
|
+
# Open QGIS and drag-and-drop .parquet files from:
|
|
238
|
+
# output/campaign_id/lat_bin=X/lon_bin=Y/*.parquet
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
**Query with DuckDB:**
|
|
242
|
+
```sql
|
|
243
|
+
INSTALL spatial;
|
|
244
|
+
LOAD spatial;
|
|
245
|
+
|
|
246
|
+
SELECT time, latitude, longitude, temperature_sea_water
|
|
247
|
+
FROM read_parquet('output/campaign_id/**/*.parquet')
|
|
248
|
+
WHERE lat_bin = 30 AND lon_bin = -120
|
|
249
|
+
LIMIT 10;
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
**Analyze with GeoPandas:**
|
|
253
|
+
```python
|
|
254
|
+
import geopandas as gpd
|
|
255
|
+
|
|
256
|
+
# Read all spatial partitions
|
|
257
|
+
gdf = gpd.read_parquet('output/campaign_id/')
|
|
258
|
+
|
|
259
|
+
# Filter and analyze
|
|
260
|
+
warm_water = gdf[gdf['temperature_sea_water'] > 25]
|
|
261
|
+
print(f"Found {len(warm_water)} warm water measurements")
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
## Contributing
|
|
265
|
+
|
|
266
|
+
Contributions are welcome! Please open an issue or submit a pull request for any enhancements or bug fixes.
|
|
267
|
+
|
|
268
|
+
## License
|
|
269
|
+
|
|
270
|
+
This project is licensed under the MIT License. See the LICENSE file for details.
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# Oceanstream
|
|
2
|
+
|
|
3
|
+
This project processes oceanographic measurements collected from Unmanned Surface Vehicles (USVs). The pipeline reads CSV files from a specified directory, consolidates the data, and stores it in a GeoParquet format partitioned by latitude and longitude bins. Optionally, it can upload the resulting GeoParquet files to Azure Blob Storage.
|
|
4
|
+
|
|
5
|
+
## Project Structure
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
oceanstream
|
|
9
|
+
├── src
|
|
10
|
+
│ ├── app.py # Main entry point for the application
|
|
11
|
+
│ ├── cli.py # Command-line interface for running the pipeline
|
|
12
|
+
│ ├── config # Configuration settings
|
|
13
|
+
│ │ ├── __init__.py
|
|
14
|
+
│ │ └── settings.py
|
|
15
|
+
│ ├── pipeline # Data processing pipeline
|
|
16
|
+
│ │ ├── __init__.py
|
|
17
|
+
│ │ ├── csv_reader.py # Functions for reading CSV files
|
|
18
|
+
│ │ ├── binning.py # Functions for data partitioning
|
|
19
|
+
│ │ └── geoparquet_writer.py # Functions for writing geoparquet files
|
|
20
|
+
│ ├── storage # Storage handling
|
|
21
|
+
│ │ ├── __init__.py
|
|
22
|
+
│ │ ├── local.py # Local storage functions
|
|
23
|
+
│ │ └── azure_blob.py # Azure Blob Storage functions
|
|
24
|
+
│ └── types # Data models and types
|
|
25
|
+
│ ├── __init__.py
|
|
26
|
+
│ └── models.py
|
|
27
|
+
├── data
|
|
28
|
+
│ └── raw_data # Directory for raw CSV data
|
|
29
|
+
│ └── .gitkeep
|
|
30
|
+
├── tests # Unit tests for the application
|
|
31
|
+
│ ├── __init__.py
|
|
32
|
+
│ └── test_pipeline.py
|
|
33
|
+
├── .env.example # Template for environment variables
|
|
34
|
+
├── .gitignore # Git ignore file
|
|
35
|
+
├── .python-version # Python version specification
|
|
36
|
+
├── pyproject.toml # Project dependencies and configuration
|
|
37
|
+
└── README.md # Project documentation
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Setup Instructions
|
|
41
|
+
|
|
42
|
+
1. **Clone the Repository**:
|
|
43
|
+
```bash
|
|
44
|
+
git clone <repository-url>
|
|
45
|
+
cd oceanstream
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
2. **Create a Virtual Environment**:
|
|
49
|
+
```bash
|
|
50
|
+
python3.12 -m venv venv
|
|
51
|
+
source venv/bin/activate # On Windows use `venv\Scripts\activate`
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
3. **Install Dependencies**:
|
|
55
|
+
|
|
56
|
+
Oceanstream is organized into optional processing modules. Install only what you need:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# Install core + geotrack processing (GPS/navigation data → GeoParquet)
|
|
60
|
+
pip install -e ".[geotrack]"
|
|
61
|
+
|
|
62
|
+
# Install core + echodata processing (echosounder data → Zarr)
|
|
63
|
+
pip install -e ".[echodata]"
|
|
64
|
+
|
|
65
|
+
# Install all processing modules
|
|
66
|
+
pip install -e ".[all]"
|
|
67
|
+
|
|
68
|
+
# Install for development (includes all modules + dev tools)
|
|
69
|
+
pip install -e ".[all]" -r requirements-dev.txt
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
**Available extras:**
|
|
73
|
+
- `geotrack` - GPS/navigation track processing (pandas, geopandas, shapely)
|
|
74
|
+
- `echodata` - Echosounder data processing (echopype, xarray, zarr, netcdf4)
|
|
75
|
+
- `multibeam` - Multibeam sonar processing (planned)
|
|
76
|
+
- `adcp` - ADCP current profiler processing (planned)
|
|
77
|
+
- `all` - All processing modules
|
|
78
|
+
- `geo` - Legacy alias for `geotrack`
|
|
79
|
+
|
|
80
|
+
4. **Configure Environment Variables**:
|
|
81
|
+
Copy `.env.example` to `.env` and fill in the necessary configuration values.
|
|
82
|
+
|
|
83
|
+
5. **Run Processing Commands**:
|
|
84
|
+
|
|
85
|
+
Oceanstream provides separate commands for each data type:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
# Process geotrack data (CSV → GeoParquet)
|
|
89
|
+
oceanstream process geotrack --input-dir raw_data --output-dir out/geoparquet -v
|
|
90
|
+
|
|
91
|
+
# Process echosounder data (planned - requires echodata extra)
|
|
92
|
+
oceanstream process echodata --input-dir raw_echodata --output-dir out/echodata -v
|
|
93
|
+
|
|
94
|
+
# Process multibeam data (planned - requires multibeam extra)
|
|
95
|
+
oceanstream process multibeam --input-dir raw_multibeam --output-dir out/multibeam -v
|
|
96
|
+
|
|
97
|
+
# Process ADCP data (planned - requires adcp extra)
|
|
98
|
+
oceanstream process adcp --input-dir raw_adcp --output-dir out/adcp -v
|
|
99
|
+
|
|
100
|
+
# List available data providers
|
|
101
|
+
oceanstream providers
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
All processing commands support `--provider` flag to specify the data source:
|
|
105
|
+
```bash
|
|
106
|
+
oceanstream process --provider saildrone geotrack --input-dir data -v
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Usage
|
|
110
|
+
|
|
111
|
+
### Geotrack Processing (GPS/Navigation Data)
|
|
112
|
+
|
|
113
|
+
CLI usage examples:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
# Process sample fixture data bundled with tests
|
|
117
|
+
oceanstream process geotrack --input-dir oceanstream/tests/data/raw_data --output-dir out/geoparquet -v
|
|
118
|
+
|
|
119
|
+
# Process your raw_data directory at repo root (default input-dir is ./raw_data)
|
|
120
|
+
oceanstream process geotrack --output-dir out/geoparquet -v
|
|
121
|
+
|
|
122
|
+
# Dry run to see what would be processed
|
|
123
|
+
oceanstream process geotrack --input-dir raw_data --dry-run -v
|
|
124
|
+
|
|
125
|
+
# List available columns in the data
|
|
126
|
+
oceanstream process geotrack --input-dir raw_data --list-columns
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The CLI reads CSVs, auto-derives coarse 5° bins, and writes a partitioned GeoParquet dataset with metadata. Use `-v` for progress logs.
|
|
130
|
+
|
|
131
|
+
### Processing Modules
|
|
132
|
+
|
|
133
|
+
Oceanstream is organized into separate processing modules:
|
|
134
|
+
|
|
135
|
+
- **`oceanstream.geotrack`** - Process GPS/navigation track data into GeoParquet format
|
|
136
|
+
- **`oceanstream.echodata`** - Process echosounder data (EK60/EK80) into Zarr (coming soon)
|
|
137
|
+
- **`oceanstream.multibeam`** - Process multibeam sonar data (coming soon)
|
|
138
|
+
- **`oceanstream.adcp`** - Process ADCP current profiler data (coming soon)
|
|
139
|
+
|
|
140
|
+
Each module can be installed independently using pip extras (see Installation section).
|
|
141
|
+
|
|
142
|
+
## Using OceanStream Data in GIS Tools
|
|
143
|
+
|
|
144
|
+
OceanStream generates cloud-optimized GeoParquet files designed to work seamlessly with modern GIS tools and data analysis frameworks. Our output includes:
|
|
145
|
+
|
|
146
|
+
- **GeoParquet**: Columnar format with embedded geometry and spatial partitioning
|
|
147
|
+
- **STAC Metadata**: Standard catalog format for discovery and integration
|
|
148
|
+
- **PMTiles** (optional): Vector tiles for web-based visualization
|
|
149
|
+
|
|
150
|
+
### Comprehensive GIS Integration Guides
|
|
151
|
+
|
|
152
|
+
We provide detailed integration guides for popular GIS tools and frameworks:
|
|
153
|
+
|
|
154
|
+
**Desktop GIS:**
|
|
155
|
+
- [QGIS](../docs/gis-integration/qgis.md) - Open-source desktop GIS
|
|
156
|
+
- [ArcGIS Pro](../docs/gis-integration/arcgis-pro.md) - Professional ESRI platform
|
|
157
|
+
|
|
158
|
+
**Data Analysis:**
|
|
159
|
+
- [DuckDB](../docs/gis-integration/duckdb.md) - Fast in-process SQL analytics
|
|
160
|
+
- [GeoPandas](../docs/gis-integration/geopandas.md) - Python spatial data analysis
|
|
161
|
+
|
|
162
|
+
**Web GIS** (coming soon):
|
|
163
|
+
- Leaflet + PMTiles
|
|
164
|
+
- Mapbox GL JS
|
|
165
|
+
- STAC Browser
|
|
166
|
+
|
|
167
|
+
**See [GIS Integration Documentation](../docs/gis-integration/) for complete guides with:**
|
|
168
|
+
- Installation instructions
|
|
169
|
+
- Step-by-step usage examples
|
|
170
|
+
- Code samples and workflows
|
|
171
|
+
- Performance optimization tips
|
|
172
|
+
- Troubleshooting guides
|
|
173
|
+
|
|
174
|
+
### Quick Start Examples
|
|
175
|
+
|
|
176
|
+
**Load in QGIS:**
|
|
177
|
+
```bash
|
|
178
|
+
# Generate data
|
|
179
|
+
oceanstream process geotrack --input-source ./data/sample.csv --output-dir ./output
|
|
180
|
+
|
|
181
|
+
# Open QGIS and drag-and-drop .parquet files from:
|
|
182
|
+
# output/campaign_id/lat_bin=X/lon_bin=Y/*.parquet
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
**Query with DuckDB:**
|
|
186
|
+
```sql
|
|
187
|
+
INSTALL spatial;
|
|
188
|
+
LOAD spatial;
|
|
189
|
+
|
|
190
|
+
SELECT time, latitude, longitude, temperature_sea_water
|
|
191
|
+
FROM read_parquet('output/campaign_id/**/*.parquet')
|
|
192
|
+
WHERE lat_bin = 30 AND lon_bin = -120
|
|
193
|
+
LIMIT 10;
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
**Analyze with GeoPandas:**
|
|
197
|
+
```python
|
|
198
|
+
import geopandas as gpd
|
|
199
|
+
|
|
200
|
+
# Read all spatial partitions
|
|
201
|
+
gdf = gpd.read_parquet('output/campaign_id/')
|
|
202
|
+
|
|
203
|
+
# Filter and analyze
|
|
204
|
+
warm_water = gdf[gdf['temperature_sea_water'] > 25]
|
|
205
|
+
print(f"Found {len(warm_water)} warm water measurements")
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## Contributing
|
|
209
|
+
|
|
210
|
+
Contributions are welcome! Please open an issue or submit a pull request for any enhancements or bug fixes.
|
|
211
|
+
|
|
212
|
+
## License
|
|
213
|
+
|
|
214
|
+
This project is licensed under the MIT License. See the LICENSE file for details.
|