oceanstream 0.0.14.dev0__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. oceanstream-0.1.1/PKG-INFO +270 -0
  2. oceanstream-0.1.1/oceanstream/README.md +214 -0
  3. oceanstream-0.1.1/oceanstream/TODO.md +515 -0
  4. oceanstream-0.1.1/oceanstream/__init__.py +111 -0
  5. oceanstream-0.1.1/oceanstream/adcp/__init__.py +7 -0
  6. oceanstream-0.1.1/oceanstream/adcp/processor.py +63 -0
  7. oceanstream-0.1.1/oceanstream/app.py +64 -0
  8. oceanstream-0.1.1/oceanstream/cli.py +2167 -0
  9. oceanstream-0.1.1/oceanstream/config/__init__.py +1 -0
  10. oceanstream-0.1.1/oceanstream/config/settings.py +190 -0
  11. oceanstream-0.1.1/oceanstream/configuration.py +260 -0
  12. oceanstream-0.1.1/oceanstream/echodata/__init__.py +299 -0
  13. oceanstream-0.1.1/oceanstream/echodata/calibrate/__init__.py +23 -0
  14. oceanstream-0.1.1/oceanstream/echodata/calibrate/calibration.py +266 -0
  15. oceanstream-0.1.1/oceanstream/echodata/calibrate/saildrone.py +275 -0
  16. oceanstream-0.1.1/oceanstream/echodata/compute/__init__.py +27 -0
  17. oceanstream-0.1.1/oceanstream/echodata/compute/mvbs.py +125 -0
  18. oceanstream-0.1.1/oceanstream/echodata/compute/nasc.py +224 -0
  19. oceanstream-0.1.1/oceanstream/echodata/compute/sv.py +471 -0
  20. oceanstream-0.1.1/oceanstream/echodata/concat.py +397 -0
  21. oceanstream-0.1.1/oceanstream/echodata/config.py +573 -0
  22. oceanstream-0.1.1/oceanstream/echodata/consolidate/__init__.py +17 -0
  23. oceanstream-0.1.1/oceanstream/echodata/consolidate/depth.py +259 -0
  24. oceanstream-0.1.1/oceanstream/echodata/convert.py +244 -0
  25. oceanstream-0.1.1/oceanstream/echodata/denoise/__init__.py +38 -0
  26. oceanstream-0.1.1/oceanstream/echodata/denoise/attenuation.py +112 -0
  27. oceanstream-0.1.1/oceanstream/echodata/denoise/background_noise.py +218 -0
  28. oceanstream-0.1.1/oceanstream/echodata/denoise/denoise.py +499 -0
  29. oceanstream-0.1.1/oceanstream/echodata/denoise/impulse_noise.py +158 -0
  30. oceanstream-0.1.1/oceanstream/echodata/denoise/transient_noise.py +108 -0
  31. oceanstream-0.1.1/oceanstream/echodata/environment/__init__.py +80 -0
  32. oceanstream-0.1.1/oceanstream/echodata/environment/absorption.py +234 -0
  33. oceanstream-0.1.1/oceanstream/echodata/environment/blended.py +350 -0
  34. oceanstream-0.1.1/oceanstream/echodata/environment/copernicus.py +298 -0
  35. oceanstream-0.1.1/oceanstream/echodata/environment/enrich.py +804 -0
  36. oceanstream-0.1.1/oceanstream/echodata/environment/geoparquet.py +849 -0
  37. oceanstream-0.1.1/oceanstream/echodata/environment/sound_speed.py +287 -0
  38. oceanstream-0.1.1/oceanstream/echodata/plot/__init__.py +33 -0
  39. oceanstream-0.1.1/oceanstream/echodata/plot/echogram.py +789 -0
  40. oceanstream-0.1.1/oceanstream/echodata/processor.py +1094 -0
  41. oceanstream-0.1.1/oceanstream/echodata/seabed/__init__.py +36 -0
  42. oceanstream-0.1.1/oceanstream/echodata/seabed/detection.py +730 -0
  43. oceanstream-0.1.1/oceanstream/echodata/stac/__init__.py +40 -0
  44. oceanstream-0.1.1/oceanstream/echodata/stac/echodata_emit.py +820 -0
  45. oceanstream-0.1.1/oceanstream/echodata/stac/segments.py +529 -0
  46. oceanstream-0.1.1/oceanstream/echodata/storage.py +400 -0
  47. oceanstream-0.1.1/oceanstream/geotrack/README.md +440 -0
  48. oceanstream-0.1.1/oceanstream/geotrack/__init__.py +8 -0
  49. oceanstream-0.1.1/oceanstream/geotrack/binning.py +90 -0
  50. oceanstream-0.1.1/oceanstream/geotrack/campaign.py +337 -0
  51. oceanstream-0.1.1/oceanstream/geotrack/csv_reader.py +225 -0
  52. oceanstream-0.1.1/oceanstream/geotrack/deduplication.py +186 -0
  53. oceanstream-0.1.1/oceanstream/geotrack/geoparquet_writer.py +180 -0
  54. oceanstream-0.1.1/oceanstream/geotrack/interpolation.py +240 -0
  55. oceanstream-0.1.1/oceanstream/geotrack/metadata.py +147 -0
  56. oceanstream-0.1.1/oceanstream/geotrack/processor.py +2316 -0
  57. oceanstream-0.1.1/oceanstream/geotrack/raster/__init__.py +9 -0
  58. oceanstream-0.1.1/oceanstream/geotrack/raster/heatmap.py +405 -0
  59. oceanstream-0.1.1/oceanstream/geotrack/report.py +596 -0
  60. oceanstream-0.1.1/oceanstream/geotrack/tiling/__init__.py +13 -0
  61. oceanstream-0.1.1/oceanstream/geotrack/tiling/pmtiles.py +701 -0
  62. oceanstream-0.1.1/oceanstream/geotrack/validation.py +170 -0
  63. oceanstream-0.1.1/oceanstream/multibeam/__init__.py +7 -0
  64. oceanstream-0.1.1/oceanstream/multibeam/processor.py +63 -0
  65. oceanstream-0.1.1/oceanstream/providers/__init__.py +3 -0
  66. oceanstream-0.1.1/oceanstream/providers/base.py +45 -0
  67. oceanstream-0.1.1/oceanstream/providers/factory.py +22 -0
  68. oceanstream-0.1.1/oceanstream/providers/models.py +18 -0
  69. oceanstream-0.1.1/oceanstream/providers/r2r/__init__.py +11 -0
  70. oceanstream-0.1.1/oceanstream/providers/r2r/r2r.py +330 -0
  71. oceanstream-0.1.1/oceanstream/providers/r2r/r2r_archive.py +90 -0
  72. oceanstream-0.1.1/oceanstream/providers/r2r/r2r_metadata.py +113 -0
  73. oceanstream-0.1.1/oceanstream/providers/r2r_archive.py +138 -0
  74. oceanstream-0.1.1/oceanstream/providers/r2r_metadata.py +135 -0
  75. oceanstream-0.1.1/oceanstream/providers/saildrone.py +270 -0
  76. oceanstream-0.1.1/oceanstream/semantic/__init__.py +3 -0
  77. oceanstream-0.1.1/oceanstream/semantic/data/README.md +87 -0
  78. oceanstream-0.1.1/oceanstream/semantic/data/cf-standard-names.json +131 -0
  79. oceanstream-0.1.1/oceanstream/semantic/data/saildrone-aliases.json +293 -0
  80. oceanstream-0.1.1/oceanstream/semantic/semantic.py +184 -0
  81. oceanstream-0.1.1/oceanstream/sensors/README.md +291 -0
  82. oceanstream-0.1.1/oceanstream/sensors/__init__.py +15 -0
  83. oceanstream-0.1.1/oceanstream/sensors/catalogue.py +193 -0
  84. oceanstream-0.1.1/oceanstream/sensors/definitions/README.md +90 -0
  85. oceanstream-0.1.1/oceanstream/sensors/definitions/airmar-150wx/README.md +40 -0
  86. oceanstream-0.1.1/oceanstream/sensors/definitions/airmar-150wx/sensor.json +42 -0
  87. oceanstream-0.1.1/oceanstream/sensors/definitions/apogee-si111/README.md +34 -0
  88. oceanstream-0.1.1/oceanstream/sensors/definitions/apogee-si111/sensor.json +21 -0
  89. oceanstream-0.1.1/oceanstream/sensors/definitions/gnss-navigation/sensor.json +35 -0
  90. oceanstream-0.1.1/oceanstream/sensors/definitions/imu-navigation/README.md +38 -0
  91. oceanstream-0.1.1/oceanstream/sensors/definitions/imu-navigation/sensor.json +41 -0
  92. oceanstream-0.1.1/oceanstream/sensors/definitions/kipp-zonen-cmp/README.md +35 -0
  93. oceanstream-0.1.1/oceanstream/sensors/definitions/kipp-zonen-cmp/sensor.json +23 -0
  94. oceanstream-0.1.1/oceanstream/sensors/definitions/lci90i-winch/README.md +60 -0
  95. oceanstream-0.1.1/oceanstream/sensors/definitions/lci90i-winch/sensor.json +41 -0
  96. oceanstream-0.1.1/oceanstream/sensors/definitions/licor-li190r/README.md +34 -0
  97. oceanstream-0.1.1/oceanstream/sensors/definitions/licor-li190r/sensor.json +21 -0
  98. oceanstream-0.1.1/oceanstream/sensors/definitions/sbe-911plus/README.md +66 -0
  99. oceanstream-0.1.1/oceanstream/sensors/definitions/sbe-911plus/sensor.json +107 -0
  100. oceanstream-0.1.1/oceanstream/sensors/definitions/sbe37-odo/README.md +37 -0
  101. oceanstream-0.1.1/oceanstream/sensors/definitions/sbe37-odo/sensor.json +32 -0
  102. oceanstream-0.1.1/oceanstream/sensors/definitions/thermistor-0.5m/README.md +34 -0
  103. oceanstream-0.1.1/oceanstream/sensors/definitions/thermistor-0.5m/sensor.json +21 -0
  104. oceanstream-0.1.1/oceanstream/sensors/definitions/valeport-minisvs/README.md +64 -0
  105. oceanstream-0.1.1/oceanstream/sensors/definitions/valeport-minisvs/sensor.json +29 -0
  106. oceanstream-0.1.1/oceanstream/sensors/definitions/wave-imu/README.md +33 -0
  107. oceanstream-0.1.1/oceanstream/sensors/definitions/wave-imu/sensor.json +18 -0
  108. oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-eco-flntu/README.md +53 -0
  109. oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-eco-flntu/sensor.json +24 -0
  110. oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-flbbcd/README.md +31 -0
  111. oceanstream-0.1.1/oceanstream/sensors/definitions/wetlabs-flbbcd/sensor.json +20 -0
  112. oceanstream-0.1.1/oceanstream/sensors/loader.py +72 -0
  113. oceanstream-0.1.1/oceanstream/sensors/processor_base.py +77 -0
  114. oceanstream-0.1.1/oceanstream/sensors/processors/__init__.py +80 -0
  115. oceanstream-0.1.1/oceanstream/sensors/processors/example_sensor.py +52 -0
  116. oceanstream-0.1.1/oceanstream/sensors/processors/nmea_gnss.py +432 -0
  117. oceanstream-0.1.1/oceanstream/sensors/processors/r2r_ctd.py +778 -0
  118. oceanstream-0.1.1/oceanstream/sensors/processors/r2r_fluorometer.py +149 -0
  119. oceanstream-0.1.1/oceanstream/sensors/processors/r2r_ssv.py +119 -0
  120. oceanstream-0.1.1/oceanstream/sensors/processors/r2r_winch.py +263 -0
  121. oceanstream-0.1.1/oceanstream/sensors/saildrone.py +90 -0
  122. oceanstream-0.1.1/oceanstream/stac/__init__.py +3 -0
  123. oceanstream-0.1.1/oceanstream/stac/emit.py +313 -0
  124. oceanstream-0.1.1/oceanstream/storage/__init__.py +5 -0
  125. oceanstream-0.1.1/oceanstream/storage/azure_blob.py +23 -0
  126. oceanstream-0.1.1/oceanstream/storage/config.py +267 -0
  127. oceanstream-0.1.1/oceanstream/storage/crypto.py +143 -0
  128. oceanstream-0.1.1/oceanstream/storage/filesystem.py +366 -0
  129. oceanstream-0.1.1/oceanstream/storage/local.py +11 -0
  130. oceanstream-0.1.1/oceanstream/storage/manager.py +326 -0
  131. oceanstream-0.1.1/oceanstream/storage/providers.py +275 -0
  132. oceanstream-0.1.1/pyproject.toml +131 -0
  133. oceanstream-0.0.14.dev0/.flake8 +0 -29
  134. oceanstream-0.0.14.dev0/.gitignore +0 -178
  135. oceanstream-0.0.14.dev0/CODE_OF_CONDUCT.md +0 -128
  136. oceanstream-0.0.14.dev0/LICENSE +0 -21
  137. oceanstream-0.0.14.dev0/MANIFEST.in +0 -9
  138. oceanstream-0.0.14.dev0/PKG-INFO +0 -179
  139. oceanstream-0.0.14.dev0/README.md +0 -152
  140. oceanstream-0.0.14.dev0/notebooks/README.md +0 -1
  141. oceanstream-0.0.14.dev0/oceanstream/__init__.py +0 -5
  142. oceanstream-0.0.14.dev0/oceanstream/cli/__init__.py +0 -3
  143. oceanstream-0.0.14.dev0/oceanstream/cli/main.py +0 -363
  144. oceanstream-0.0.14.dev0/oceanstream/cli/requirements.txt +0 -1
  145. oceanstream-0.0.14.dev0/oceanstream/convert/__init__.py +0 -5
  146. oceanstream-0.0.14.dev0/oceanstream/convert/convert_raw_file.py +0 -58
  147. oceanstream-0.0.14.dev0/oceanstream/core.py +0 -174
  148. oceanstream-0.0.14.dev0/oceanstream/denoise/__init__.py +0 -2
  149. oceanstream-0.0.14.dev0/oceanstream/denoise/apply_masks.py +0 -49
  150. oceanstream-0.0.14.dev0/oceanstream/denoise/applying_masks_handler.py +0 -173
  151. oceanstream-0.0.14.dev0/oceanstream/denoise/background_noise_remover.py +0 -97
  152. oceanstream-0.0.14.dev0/oceanstream/denoise/create_masks.py +0 -87
  153. oceanstream-0.0.14.dev0/oceanstream/denoise/noise_masks.py +0 -398
  154. oceanstream-0.0.14.dev0/oceanstream/denoise/requirements.txt +0 -0
  155. oceanstream-0.0.14.dev0/oceanstream/denoise/types.py +0 -73
  156. oceanstream-0.0.14.dev0/oceanstream/echodata/__init__.py +0 -5
  157. oceanstream-0.0.14.dev0/oceanstream/echodata/dataset.py +0 -586
  158. oceanstream-0.0.14.dev0/oceanstream/echodata/ensure_time_continuity.py +0 -75
  159. oceanstream-0.0.14.dev0/oceanstream/echodata/mvbs_computation.py +0 -77
  160. oceanstream-0.0.14.dev0/oceanstream/echodata/raw_handler.py +0 -577
  161. oceanstream-0.0.14.dev0/oceanstream/echodata/requirements.txt +0 -0
  162. oceanstream-0.0.14.dev0/oceanstream/echodata/sv_computation.py +0 -125
  163. oceanstream-0.0.14.dev0/oceanstream/echodata/sv_dataset_extension.py +0 -125
  164. oceanstream-0.0.14.dev0/oceanstream/echodata/sv_interpolation.py +0 -259
  165. oceanstream-0.0.14.dev0/oceanstream/echodata/target_strength_computation.py +0 -109
  166. oceanstream-0.0.14.dev0/oceanstream/exports/__init__.py +0 -6
  167. oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +0 -183
  168. oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_raw.py +0 -92
  169. oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_nasc.py +0 -135
  170. oceanstream-0.0.14.dev0/oceanstream/exports/frequency_differencing_handler.py +0 -210
  171. oceanstream-0.0.14.dev0/oceanstream/exports/nasc.py +0 -23
  172. oceanstream-0.0.14.dev0/oceanstream/exports/nasc_computation.py +0 -62
  173. oceanstream-0.0.14.dev0/oceanstream/exports/requirements.txt +0 -4
  174. oceanstream-0.0.14.dev0/oceanstream/exports/shoals/shoal_detection_handler.py +0 -109
  175. oceanstream-0.0.14.dev0/oceanstream/exports/shoals/shoal_process.py +0 -224
  176. oceanstream-0.0.14.dev0/oceanstream/exports/shoals/shoals_handler.py +0 -35
  177. oceanstream-0.0.14.dev0/oceanstream/plot/__init__.py +0 -2
  178. oceanstream-0.0.14.dev0/oceanstream/plot/echogram.py +0 -579
  179. oceanstream-0.0.14.dev0/oceanstream/plot/requirements.txt +0 -2
  180. oceanstream-0.0.14.dev0/oceanstream/process/__init__.py +0 -23
  181. oceanstream-0.0.14.dev0/oceanstream/process/aws/__init__.py +0 -9
  182. oceanstream-0.0.14.dev0/oceanstream/process/aws/s3.py +0 -94
  183. oceanstream-0.0.14.dev0/oceanstream/process/azure/__init__.py +0 -8
  184. oceanstream-0.0.14.dev0/oceanstream/process/azure/azure_processor.py +0 -66
  185. oceanstream-0.0.14.dev0/oceanstream/process/azure/blob_storage.py +0 -85
  186. oceanstream-0.0.14.dev0/oceanstream/process/cloud.py +0 -21
  187. oceanstream-0.0.14.dev0/oceanstream/process/combine_zarr.py +0 -89
  188. oceanstream-0.0.14.dev0/oceanstream/process/file_processor.py +0 -237
  189. oceanstream-0.0.14.dev0/oceanstream/process/file_processor_class.py +0 -217
  190. oceanstream-0.0.14.dev0/oceanstream/process/folder_processor.py +0 -395
  191. oceanstream-0.0.14.dev0/oceanstream/process/process.py +0 -81
  192. oceanstream-0.0.14.dev0/oceanstream/process/processed_data_io.py +0 -140
  193. oceanstream-0.0.14.dev0/oceanstream/process/requirements.txt +0 -0
  194. oceanstream-0.0.14.dev0/oceanstream/process/utils.py +0 -226
  195. oceanstream-0.0.14.dev0/oceanstream/settings/__init__.py +0 -1
  196. oceanstream-0.0.14.dev0/oceanstream/settings/config.py +0 -17
  197. oceanstream-0.0.14.dev0/oceanstream/settings/defaults.config.json +0 -120
  198. oceanstream-0.0.14.dev0/oceanstream/utils.py +0 -101
  199. oceanstream-0.0.14.dev0/oceanstream.egg-info/PKG-INFO +0 -179
  200. oceanstream-0.0.14.dev0/oceanstream.egg-info/SOURCES.txt +0 -78
  201. oceanstream-0.0.14.dev0/oceanstream.egg-info/dependency_links.txt +0 -1
  202. oceanstream-0.0.14.dev0/oceanstream.egg-info/entry_points.txt +0 -2
  203. oceanstream-0.0.14.dev0/oceanstream.egg-info/requires.txt +0 -31
  204. oceanstream-0.0.14.dev0/oceanstream.egg-info/top_level.txt +0 -1
  205. oceanstream-0.0.14.dev0/pyproject.toml +0 -8
  206. oceanstream-0.0.14.dev0/requirements-dev.txt +0 -52
  207. oceanstream-0.0.14.dev0/requirements-local.txt +0 -4
  208. oceanstream-0.0.14.dev0/requirements.txt +0 -10
  209. oceanstream-0.0.14.dev0/setup.cfg +0 -60
  210. oceanstream-0.0.14.dev0/setup.py +0 -19
  211. /oceanstream-0.0.14.dev0/.gitattributes → /oceanstream-0.1.1/LICENSE +0 -0
@@ -0,0 +1,270 @@
1
+ Metadata-Version: 2.4
2
+ Name: oceanstream
3
+ Version: 0.1.1
4
+ Summary: Process oceanographic measurements from USVs and write partitioned GeoParquet datasets with geospatial metadata.
5
+ License: MIT
6
+ License-File: LICENSE
7
+ Keywords: oceanography,geoparquet,stac,saildrone,geospatial,usv,marine-data
8
+ Author: Andrei Neacsu
9
+ Author-email: andrei@oceanstream.io
10
+ Requires-Python: >=3.11,<3.13
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: GIS
20
+ Classifier: Topic :: Scientific/Engineering :: Oceanography
21
+ Provides-Extra: adcp
22
+ Provides-Extra: all
23
+ Provides-Extra: echodata
24
+ Provides-Extra: echodata-viz
25
+ Provides-Extra: geo
26
+ Provides-Extra: geotrack
27
+ Provides-Extra: multibeam
28
+ Requires-Dist: adlfs (>=2024.4.0) ; extra == "echodata" or extra == "all"
29
+ Requires-Dist: azure-storage-blob (>=12.19)
30
+ Requires-Dist: bottleneck (==1.5.0) ; extra == "echodata" or extra == "all"
31
+ Requires-Dist: copernicusmarine (>=1.0.0) ; extra == "echodata" or extra == "all"
32
+ Requires-Dist: dask[complete] (==2024.9.0) ; extra == "echodata" or extra == "all"
33
+ Requires-Dist: datashader (>=0.16) ; extra == "echodata-viz" or extra == "all"
34
+ Requires-Dist: distributed (==2024.9.0) ; extra == "echodata" or extra == "all"
35
+ Requires-Dist: fsspec (==2025.3.2) ; extra == "echodata" or extra == "all"
36
+ Requires-Dist: geopandas (>=0.14) ; extra == "geotrack" or extra == "all" or extra == "geo"
37
+ Requires-Dist: gsw (>=3.6.0) ; extra == "echodata" or extra == "all"
38
+ Requires-Dist: hvplot (>=0.9) ; extra == "echodata-viz" or extra == "all"
39
+ Requires-Dist: matplotlib (>=3.7) ; extra == "echodata-viz" or extra == "all"
40
+ Requires-Dist: netcdf4 (>=1.6) ; extra == "echodata" or extra == "all"
41
+ Requires-Dist: numcodecs (==0.14.1) ; extra == "echodata" or extra == "all"
42
+ Requires-Dist: numpy (==1.26.4) ; extra == "echodata" or extra == "all"
43
+ Requires-Dist: openpyxl (>=3.0) ; extra == "echodata" or extra == "all"
44
+ Requires-Dist: pandas (>=2.1)
45
+ Requires-Dist: pyarrow (>=14)
46
+ Requires-Dist: pynmea2 (>=1.18) ; extra == "geotrack" or extra == "all" or extra == "geo"
47
+ Requires-Dist: rich (>=13.7)
48
+ Requires-Dist: seabirdscientific (>=2.7) ; extra == "geotrack" or extra == "all" or extra == "geo"
49
+ Requires-Dist: shapely (>=2.0) ; extra == "geotrack" or extra == "all" or extra == "geo"
50
+ Requires-Dist: tqdm (>=4.66)
51
+ Requires-Dist: typer (>=0.12)
52
+ Requires-Dist: xarray (==2024.11.0) ; extra == "echodata" or extra == "all"
53
+ Requires-Dist: zarr (>=2.16) ; extra == "echodata" or extra == "all"
54
+ Project-URL: Homepage, https://oceanstream.io
55
+ Description-Content-Type: text/markdown
56
+
57
+ # Oceanstream
58
+
59
+ This project processes oceanographic measurements collected from Unmanned Surface Vehicles (USVs). The pipeline reads CSV files from a specified directory, consolidates the data, and stores it in a GeoParquet format partitioned by latitude and longitude bins. Optionally, it can upload the resulting GeoParquet files to Azure Blob Storage.
60
+
61
+ ## Project Structure
62
+
63
+ ```
64
+ oceanstream
65
+ ├── src
66
+ │ ├── app.py # Main entry point for the application
67
+ │ ├── cli.py # Command-line interface for running the pipeline
68
+ │ ├── config # Configuration settings
69
+ │ │ ├── __init__.py
70
+ │ │ └── settings.py
71
+ │ ├── pipeline # Data processing pipeline
72
+ │ │ ├── __init__.py
73
+ │ │ ├── csv_reader.py # Functions for reading CSV files
74
+ │ │ ├── binning.py # Functions for data partitioning
75
+ │ │ └── geoparquet_writer.py # Functions for writing geoparquet files
76
+ │ ├── storage # Storage handling
77
+ │ │ ├── __init__.py
78
+ │ │ ├── local.py # Local storage functions
79
+ │ │ └── azure_blob.py # Azure Blob Storage functions
80
+ │ └── types # Data models and types
81
+ │ ├── __init__.py
82
+ │ └── models.py
83
+ ├── data
84
+ │ └── raw_data # Directory for raw CSV data
85
+ │ └── .gitkeep
86
+ ├── tests # Unit tests for the application
87
+ │ ├── __init__.py
88
+ │ └── test_pipeline.py
89
+ ├── .env.example # Template for environment variables
90
+ ├── .gitignore # Git ignore file
91
+ ├── .python-version # Python version specification
92
+ ├── pyproject.toml # Project dependencies and configuration
93
+ └── README.md # Project documentation
94
+ ```
95
+
96
+ ## Setup Instructions
97
+
98
+ 1. **Clone the Repository**:
99
+ ```bash
100
+ git clone <repository-url>
101
+ cd oceanstream
102
+ ```
103
+
104
+ 2. **Create a Virtual Environment**:
105
+ ```bash
106
+ python3.12 -m venv venv
107
+ source venv/bin/activate # On Windows use `venv\Scripts\activate`
108
+ ```
109
+
110
+ 3. **Install Dependencies**:
111
+
112
+ Oceanstream is organized into optional processing modules. Install only what you need:
113
+
114
+ ```bash
115
+ # Install core + geotrack processing (GPS/navigation data → GeoParquet)
116
+ pip install -e ".[geotrack]"
117
+
118
+ # Install core + echodata processing (echosounder data → Zarr)
119
+ pip install -e ".[echodata]"
120
+
121
+ # Install all processing modules
122
+ pip install -e ".[all]"
123
+
124
+ # Install for development (includes all modules + dev tools)
125
+ pip install -e ".[all]" -r requirements-dev.txt
126
+ ```
127
+
128
+ **Available extras:**
129
+ - `geotrack` - GPS/navigation track processing (pandas, geopandas, shapely)
130
+ - `echodata` - Echosounder data processing (echopype, xarray, zarr, netcdf4)
131
+ - `multibeam` - Multibeam sonar processing (planned)
132
+ - `adcp` - ADCP current profiler processing (planned)
133
+ - `all` - All processing modules
134
+ - `geo` - Legacy alias for `geotrack`
135
+
136
+ 4. **Configure Environment Variables**:
137
+ Copy `.env.example` to `.env` and fill in the necessary configuration values.
138
+
139
+ 5. **Run Processing Commands**:
140
+
141
+ Oceanstream provides separate commands for each data type:
142
+
143
+ ```bash
144
+ # Process geotrack data (CSV → GeoParquet)
145
+ oceanstream process geotrack --input-dir raw_data --output-dir out/geoparquet -v
146
+
147
+ # Process echosounder data (planned - requires echodata extra)
148
+ oceanstream process echodata --input-dir raw_echodata --output-dir out/echodata -v
149
+
150
+ # Process multibeam data (planned - requires multibeam extra)
151
+ oceanstream process multibeam --input-dir raw_multibeam --output-dir out/multibeam -v
152
+
153
+ # Process ADCP data (planned - requires adcp extra)
154
+ oceanstream process adcp --input-dir raw_adcp --output-dir out/adcp -v
155
+
156
+ # List available data providers
157
+ oceanstream providers
158
+ ```
159
+
160
+ All processing commands support `--provider` flag to specify the data source:
161
+ ```bash
162
+ oceanstream process --provider saildrone geotrack --input-dir data -v
163
+ ```
164
+
165
+ ## Usage
166
+
167
+ ### Geotrack Processing (GPS/Navigation Data)
168
+
169
+ CLI usage examples:
170
+
171
+ ```bash
172
+ # Process sample fixture data bundled with tests
173
+ oceanstream process geotrack --input-dir oceanstream/tests/data/raw_data --output-dir out/geoparquet -v
174
+
175
+ # Process your raw_data directory at repo root (default input-dir is ./raw_data)
176
+ oceanstream process geotrack --output-dir out/geoparquet -v
177
+
178
+ # Dry run to see what would be processed
179
+ oceanstream process geotrack --input-dir raw_data --dry-run -v
180
+
181
+ # List available columns in the data
182
+ oceanstream process geotrack --input-dir raw_data --list-columns
183
+ ```
184
+
185
+ The CLI reads CSVs, auto-derives coarse 5° bins, and writes a partitioned GeoParquet dataset with metadata. Use `-v` for progress logs.
186
+
187
+ ### Processing Modules
188
+
189
+ Oceanstream is organized into separate processing modules:
190
+
191
+ - **`oceanstream.geotrack`** - Process GPS/navigation track data into GeoParquet format
192
+ - **`oceanstream.echodata`** - Process echosounder data (EK60/EK80) into Zarr (coming soon)
193
+ - **`oceanstream.multibeam`** - Process multibeam sonar data (coming soon)
194
+ - **`oceanstream.adcp`** - Process ADCP current profiler data (coming soon)
195
+
196
+ Each module can be installed independently using pip extras (see Installation section).
197
+
198
+ ## Using OceanStream Data in GIS Tools
199
+
200
+ OceanStream generates cloud-optimized GeoParquet files designed to work seamlessly with modern GIS tools and data analysis frameworks. Our output includes:
201
+
202
+ - **GeoParquet**: Columnar format with embedded geometry and spatial partitioning
203
+ - **STAC Metadata**: Standard catalog format for discovery and integration
204
+ - **PMTiles** (optional): Vector tiles for web-based visualization
205
+
206
+ ### Comprehensive GIS Integration Guides
207
+
208
+ We provide detailed integration guides for popular GIS tools and frameworks:
209
+
210
+ **Desktop GIS:**
211
+ - [QGIS](../docs/gis-integration/qgis.md) - Open-source desktop GIS
212
+ - [ArcGIS Pro](../docs/gis-integration/arcgis-pro.md) - Professional ESRI platform
213
+
214
+ **Data Analysis:**
215
+ - [DuckDB](../docs/gis-integration/duckdb.md) - Fast in-process SQL analytics
216
+ - [GeoPandas](../docs/gis-integration/geopandas.md) - Python spatial data analysis
217
+
218
+ **Web GIS** (coming soon):
219
+ - Leaflet + PMTiles
220
+ - Mapbox GL JS
221
+ - STAC Browser
222
+
223
+ **See [GIS Integration Documentation](../docs/gis-integration/) for complete guides with:**
224
+ - Installation instructions
225
+ - Step-by-step usage examples
226
+ - Code samples and workflows
227
+ - Performance optimization tips
228
+ - Troubleshooting guides
229
+
230
+ ### Quick Start Examples
231
+
232
+ **Load in QGIS:**
233
+ ```bash
234
+ # Generate data
235
+ oceanstream process geotrack --input-source ./data/sample.csv --output-dir ./output
236
+
237
+ # Open QGIS and drag-and-drop .parquet files from:
238
+ # output/campaign_id/lat_bin=X/lon_bin=Y/*.parquet
239
+ ```
240
+
241
+ **Query with DuckDB:**
242
+ ```sql
243
+ INSTALL spatial;
244
+ LOAD spatial;
245
+
246
+ SELECT time, latitude, longitude, temperature_sea_water
247
+ FROM read_parquet('output/campaign_id/**/*.parquet')
248
+ WHERE lat_bin = 30 AND lon_bin = -120
249
+ LIMIT 10;
250
+ ```
251
+
252
+ **Analyze with GeoPandas:**
253
+ ```python
254
+ import geopandas as gpd
255
+
256
+ # Read all spatial partitions
257
+ gdf = gpd.read_parquet('output/campaign_id/')
258
+
259
+ # Filter and analyze
260
+ warm_water = gdf[gdf['temperature_sea_water'] > 25]
261
+ print(f"Found {len(warm_water)} warm water measurements")
262
+ ```
263
+
264
+ ## Contributing
265
+
266
+ Contributions are welcome! Please open an issue or submit a pull request for any enhancements or bug fixes.
267
+
268
+ ## License
269
+
270
+ This project is licensed under the MIT License. See the LICENSE file for details.
@@ -0,0 +1,214 @@
1
+ # Oceanstream
2
+
3
+ This project processes oceanographic measurements collected from Unmanned Surface Vehicles (USVs). The pipeline reads CSV files from a specified directory, consolidates the data, and stores it in a GeoParquet format partitioned by latitude and longitude bins. Optionally, it can upload the resulting GeoParquet files to Azure Blob Storage.
4
+
5
+ ## Project Structure
6
+
7
+ ```
8
+ oceanstream
9
+ ├── src
10
+ │ ├── app.py # Main entry point for the application
11
+ │ ├── cli.py # Command-line interface for running the pipeline
12
+ │ ├── config # Configuration settings
13
+ │ │ ├── __init__.py
14
+ │ │ └── settings.py
15
+ │ ├── pipeline # Data processing pipeline
16
+ │ │ ├── __init__.py
17
+ │ │ ├── csv_reader.py # Functions for reading CSV files
18
+ │ │ ├── binning.py # Functions for data partitioning
19
+ │ │ └── geoparquet_writer.py # Functions for writing geoparquet files
20
+ │ ├── storage # Storage handling
21
+ │ │ ├── __init__.py
22
+ │ │ ├── local.py # Local storage functions
23
+ │ │ └── azure_blob.py # Azure Blob Storage functions
24
+ │ └── types # Data models and types
25
+ │ ├── __init__.py
26
+ │ └── models.py
27
+ ├── data
28
+ │ └── raw_data # Directory for raw CSV data
29
+ │ └── .gitkeep
30
+ ├── tests # Unit tests for the application
31
+ │ ├── __init__.py
32
+ │ └── test_pipeline.py
33
+ ├── .env.example # Template for environment variables
34
+ ├── .gitignore # Git ignore file
35
+ ├── .python-version # Python version specification
36
+ ├── pyproject.toml # Project dependencies and configuration
37
+ └── README.md # Project documentation
38
+ ```
39
+
40
+ ## Setup Instructions
41
+
42
+ 1. **Clone the Repository**:
43
+ ```bash
44
+ git clone <repository-url>
45
+ cd oceanstream
46
+ ```
47
+
48
+ 2. **Create a Virtual Environment**:
49
+ ```bash
50
+ python3.12 -m venv venv
51
+ source venv/bin/activate # On Windows use `venv\Scripts\activate`
52
+ ```
53
+
54
+ 3. **Install Dependencies**:
55
+
56
+ Oceanstream is organized into optional processing modules. Install only what you need:
57
+
58
+ ```bash
59
+ # Install core + geotrack processing (GPS/navigation data → GeoParquet)
60
+ pip install -e ".[geotrack]"
61
+
62
+ # Install core + echodata processing (echosounder data → Zarr)
63
+ pip install -e ".[echodata]"
64
+
65
+ # Install all processing modules
66
+ pip install -e ".[all]"
67
+
68
+ # Install for development (includes all modules + dev tools)
69
+ pip install -e ".[all]" -r requirements-dev.txt
70
+ ```
71
+
72
+ **Available extras:**
73
+ - `geotrack` - GPS/navigation track processing (pandas, geopandas, shapely)
74
+ - `echodata` - Echosounder data processing (echopype, xarray, zarr, netcdf4)
75
+ - `multibeam` - Multibeam sonar processing (planned)
76
+ - `adcp` - ADCP current profiler processing (planned)
77
+ - `all` - All processing modules
78
+ - `geo` - Legacy alias for `geotrack`
79
+
80
+ 4. **Configure Environment Variables**:
81
+ Copy `.env.example` to `.env` and fill in the necessary configuration values.
82
+
83
+ 5. **Run Processing Commands**:
84
+
85
+ Oceanstream provides separate commands for each data type:
86
+
87
+ ```bash
88
+ # Process geotrack data (CSV → GeoParquet)
89
+ oceanstream process geotrack --input-dir raw_data --output-dir out/geoparquet -v
90
+
91
+ # Process echosounder data (planned - requires echodata extra)
92
+ oceanstream process echodata --input-dir raw_echodata --output-dir out/echodata -v
93
+
94
+ # Process multibeam data (planned - requires multibeam extra)
95
+ oceanstream process multibeam --input-dir raw_multibeam --output-dir out/multibeam -v
96
+
97
+ # Process ADCP data (planned - requires adcp extra)
98
+ oceanstream process adcp --input-dir raw_adcp --output-dir out/adcp -v
99
+
100
+ # List available data providers
101
+ oceanstream providers
102
+ ```
103
+
104
+ All processing commands support `--provider` flag to specify the data source:
105
+ ```bash
106
+ oceanstream process --provider saildrone geotrack --input-dir data -v
107
+ ```
108
+
109
+ ## Usage
110
+
111
+ ### Geotrack Processing (GPS/Navigation Data)
112
+
113
+ CLI usage examples:
114
+
115
+ ```bash
116
+ # Process sample fixture data bundled with tests
117
+ oceanstream process geotrack --input-dir oceanstream/tests/data/raw_data --output-dir out/geoparquet -v
118
+
119
+ # Process your raw_data directory at repo root (default input-dir is ./raw_data)
120
+ oceanstream process geotrack --output-dir out/geoparquet -v
121
+
122
+ # Dry run to see what would be processed
123
+ oceanstream process geotrack --input-dir raw_data --dry-run -v
124
+
125
+ # List available columns in the data
126
+ oceanstream process geotrack --input-dir raw_data --list-columns
127
+ ```
128
+
129
+ The CLI reads CSVs, auto-derives coarse 5° bins, and writes a partitioned GeoParquet dataset with metadata. Use `-v` for progress logs.
130
+
131
+ ### Processing Modules
132
+
133
+ Oceanstream is organized into separate processing modules:
134
+
135
+ - **`oceanstream.geotrack`** - Process GPS/navigation track data into GeoParquet format
136
+ - **`oceanstream.echodata`** - Process echosounder data (EK60/EK80) into Zarr (coming soon)
137
+ - **`oceanstream.multibeam`** - Process multibeam sonar data (coming soon)
138
+ - **`oceanstream.adcp`** - Process ADCP current profiler data (coming soon)
139
+
140
+ Each module can be installed independently using pip extras (see Installation section).
141
+
142
+ ## Using OceanStream Data in GIS Tools
143
+
144
+ OceanStream generates cloud-optimized GeoParquet files designed to work seamlessly with modern GIS tools and data analysis frameworks. Our output includes:
145
+
146
+ - **GeoParquet**: Columnar format with embedded geometry and spatial partitioning
147
+ - **STAC Metadata**: Standard catalog format for discovery and integration
148
+ - **PMTiles** (optional): Vector tiles for web-based visualization
149
+
150
+ ### Comprehensive GIS Integration Guides
151
+
152
+ We provide detailed integration guides for popular GIS tools and frameworks:
153
+
154
+ **Desktop GIS:**
155
+ - [QGIS](../docs/gis-integration/qgis.md) - Open-source desktop GIS
156
+ - [ArcGIS Pro](../docs/gis-integration/arcgis-pro.md) - Professional ESRI platform
157
+
158
+ **Data Analysis:**
159
+ - [DuckDB](../docs/gis-integration/duckdb.md) - Fast in-process SQL analytics
160
+ - [GeoPandas](../docs/gis-integration/geopandas.md) - Python spatial data analysis
161
+
162
+ **Web GIS** (coming soon):
163
+ - Leaflet + PMTiles
164
+ - Mapbox GL JS
165
+ - STAC Browser
166
+
167
+ **See [GIS Integration Documentation](../docs/gis-integration/) for complete guides with:**
168
+ - Installation instructions
169
+ - Step-by-step usage examples
170
+ - Code samples and workflows
171
+ - Performance optimization tips
172
+ - Troubleshooting guides
173
+
174
+ ### Quick Start Examples
175
+
176
+ **Load in QGIS:**
177
+ ```bash
178
+ # Generate data
179
+ oceanstream process geotrack --input-source ./data/sample.csv --output-dir ./output
180
+
181
+ # Open QGIS and drag-and-drop .parquet files from:
182
+ # output/campaign_id/lat_bin=X/lon_bin=Y/*.parquet
183
+ ```
184
+
185
+ **Query with DuckDB:**
186
+ ```sql
187
+ INSTALL spatial;
188
+ LOAD spatial;
189
+
190
+ SELECT time, latitude, longitude, temperature_sea_water
191
+ FROM read_parquet('output/campaign_id/**/*.parquet')
192
+ WHERE lat_bin = 30 AND lon_bin = -120
193
+ LIMIT 10;
194
+ ```
195
+
196
+ **Analyze with GeoPandas:**
197
+ ```python
198
+ import geopandas as gpd
199
+
200
+ # Read all spatial partitions
201
+ gdf = gpd.read_parquet('output/campaign_id/')
202
+
203
+ # Filter and analyze
204
+ warm_water = gdf[gdf['temperature_sea_water'] > 25]
205
+ print(f"Found {len(warm_water)} warm water measurements")
206
+ ```
207
+
208
+ ## Contributing
209
+
210
+ Contributions are welcome! Please open an issue or submit a pull request for any enhancements or bug fixes.
211
+
212
+ ## License
213
+
214
+ This project is licensed under the MIT License. See the LICENSE file for details.