shipgrav 1.0.3__tar.gz → 1.0.5__tar.gz

{shipgrav-1.0.3 → shipgrav-1.0.5}/.github/test_conda_env.yml

@@ -9,3 +9,4 @@ dependencies:
   - tomli
   - pyyaml
   - coverage
+  - tqdm

{shipgrav-1.0.3 → shipgrav-1.0.5}/.github/workflows/test.yml

@@ -1,11 +1,12 @@
 name: tests
 on:
   push:
+    branches:
+      - main
   #workflow_dispatch:
   pull_request:
     branches:
-      # only branches from forks which have the form 'user:branch-name'
-      - '**:**'
+      - main
   schedule:
     - cron:  '42 0 * * 6'
 jobs:

{shipgrav-1.0.3 → shipgrav-1.0.5}/CHANGELOG

@@ -1,3 +1,29 @@
+v1.0.5
+
+2024-11-20  Hannah F Mark  <hmark@whoi.edu>
+
+    * corrections to dependencies lists in docs
+    * bug fixes for example scripts
+
+v1.0.4
+
+2024-11-01  Hannah F Mark  <hmark@whoi.edu>
+
+    * update dependencies and docs
+    * add a flag to turn progress bars on/off so tests are cleaner
+    * add more tests for better coverage, fix bugs discovered by tests
+    * format example scripts for jupytext, update docs and optional dependencies
+    * update example scripts to use pooch for download, delete data_download script
+    * add progress bars for things (mostly file reads) and tqdm dependency
+
+2024-10-29  Hannah F Mark  <hmark@whoi.edu>
+
+    * clean/format code with isort and flake8
+    * update triggers for test workflow: run on push and on PR
+    * use abs path for example files so tests can be run from outside tests/ dir
+
+v1.0.3
+
 2024-10-25  Hannah F Mark  <hmark@whoi.edu>
 
     * update pkg_resources (deprecated from 3.13) to importlib.resources for test suite
@@ -13,6 +39,14 @@
     * fix indexing in one part of RMBA example script for future deprecation of chained assignment
     * add detail to contributor guidelines
 
+v1.0.2
+
+2024-09-17  Hannah F Mark  <hmark@whoi.edu>
+
+    * update docs and optional dependencies
+
+v1.0.1
+
 2024-07-15  Hannah F Mark  <hmark@whoi.edu>
 
     * PEP8 reformat, typos

{shipgrav-1.0.3 → shipgrav-1.0.5}/PKG-INFO

@@ -1,10 +1,9 @@
 Metadata-Version: 2.3
 Name: shipgrav
-Version: 1.0.3
+Version: 1.0.5
 Summary: Functions for marine gravity data processing and reduction
 Author-email: "Hannah F. Mark" <hmark@whoi.edu>
 Maintainer-email: "Hannah F. Mark" <hmark@whoi.edu>
-License-File: LICENSE
 Keywords: UNOLS,gravimetry,marine gravity
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
@@ -15,13 +14,18 @@ Requires-Dist: pyyaml
 Requires-Dist: scipy
 Requires-Dist: statsmodels
 Requires-Dist: tomli
+Requires-Dist: tqdm
 Provides-Extra: examples
 Requires-Dist: geographiclib; extra == 'examples'
+Requires-Dist: jupyterlab; extra == 'examples'
+Requires-Dist: jupytext; extra == 'examples'
 Requires-Dist: matplotlib; extra == 'examples'
+Requires-Dist: pooch; extra == 'examples'
 Description-Content-Type: text/markdown
 
 # shipgrav
 [![build status](https://github.com/PFPE/shipgrav/workflows/tests/badge.svg)](https://github.com/PFPE/shipgrav/actions)
+[![codecov](https://codecov.io/gh/PFPE/shipgrav/branch/main/graph/badge.svg)](https://codecov.io/gh/PFPE/shipgrav)
 
 shipgrav is a Python package designed for reading, processing, and reducing marine gravity data from UNOLS ships. It is created and maintained by PFPE for the marine gravimetry community. The shipgrav repository also contains scripts with example workflows for gravity data processing and reduction.
 
@@ -32,9 +36,10 @@ scipy\
 pandas 2.0+\
 statsmodels\
 tomli\
-pyyaml
+pyyaml\
+tqdm
 
-To run the example scripts, you will also need matplotlib.
+To run the example scripts, you will also need matplotlib, geographiclib, and pooch. To run the example scripts in jupyter, you will also need jupyterlab and jupytext.
 
 ## Installation
 shipgrav can be installed from PyPI using `pip`. Detailed instructions are in the [documentation](https://shipgrav.readthedocs.io/).

{shipgrav-1.0.3 → shipgrav-1.0.5}/docs/requirements.txt

@@ -8,3 +8,5 @@ scipy
 tomli
 statsmodels
 pyyaml
+tqdm
+pooch

{shipgrav-1.0.3 → shipgrav-1.0.5}/example-scripts/RMBA_calc.py

@@ -1,26 +1,13 @@
-import numpy as np
-from pandas import read_csv
-import shipgrav.grav as sgg
-import matplotlib.pyplot as plt
-from geographiclib.geodesic import Geodesic
-from scipy.signal import firwin, filtfilt
-from scipy.interpolate import interp1d
-from glob import glob
-import os
-import sys
-
-########################################################################
-# Example script for calculating the mantle Bouger anomaly (MBA) and
-# residual MBA (RMBA), and estimating crustal thickness variations, using
-# data from a Ride transit.
-
+# %% [markdown]
+# ### Example script for calculating the mantle Bouger anomaly (MBA) and residual MBA (RMBA), and estimating crustal thickness variations, using data from a Ride transit.
+#
 # This script works with a file from the interactive line picker script
-# that is located in
-# data/Ride/SR2312/gravimeter/DGS/line-segments/
-
+# that is provided in a zenodo repository at https://doi.org/10.5281/zenodo.12733929
+# The file is automatically downloaded by this script using pooch
+#
 # It is hard-coded to run on a specific example file but could be
 # adjusted (see below)
-
+#
 # Tthe RMBA thermal correction is highly sensitive to edge effects,
 # Here we will demo the process of embedding the line segment
 # in a longer line pulled from global grids: FAA, bathymetry,
@@ -28,50 +15,74 @@ import sys
 # For a cruise with multibeam and magnetics (for bathymetry and age)
 # both of those could be embedded alongside FAA. For this cruise,
 # everything but the FAA is entirely pulled from global grids.
-
+#
 # Since this is an example running on a known input file, we have pre-
 # tracked the coordinates in the following global grids:
-# Bathymetry:
+#
+# #### Bathymetry:
+#
 # The GEBCO_2014 Grid, version 20141103, http://www.gebco.net
-# FAA:
+#
+# #### FAA:
+#
 # Sandwell, D. T., R. D. Müller, W. H. F. Smith, E. Garcia, R. Francis (2014).
 # New global marine gravity model from CryoSat-2 and Jason-1 reveals buried
 # tectonic structure, Science 346(6205), pp. 65-67,
 # doi: 10.1126/science.1258213
-# Sediment thickness:
+#
+# #### Sediment thickness:
+#
 # Divins, D.L. (2003). Total Sediment Thickness of the World's Oceans &
 # Marginal Seas, NOAA National Geophysical Data Center, Boulder, CO.
-# Plate age:
+#
+# #### Plate age:
+#
 # Seton, M., Müller, R. D., Zahirovic, S., Williams, S., Wright, N., Cannon, J.,
 # Whittaker, J., Matthews, K., McGirr, R., (2020). A global dataset of
 # present-day oceanic crustal age and seafloor spreading parameters,
 # Geochemistry, Geophysics, Geosystems, doi: 10.1029/2020GC009214
+#
 # and
+#
 # Müller, R. D., Zahirovic, S., Williams, S. E., Cannon, J., Seton, M.,
 # Bower, D. J., Tetley, M. G., Heine, C., Le Breton, E., Liu, S., Russell, S. H. J.,
 # Yang, T., Leonard, J., and Gurnis, M. (2019). A global plate model
 # including lithospheric deformation along major rifts and orogens since the
 # Triassic. Tectonics, 38, doi: 10.1029/2018TC005462
 # (age grids accessed from earthbyte.org)
-
+#
 # To run this example using a different input file, you will need
 # to supply paths to grids of bathymetry, FAA, sediment thickness,
 # and plate age; and make sure that units match what the script expects.
-########################################################################
 
+# %% 
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pooch
+import shipgrav.grav as sgg
+from geographiclib.geodesic import Geodesic
+from pandas import read_csv
+from scipy.interpolate import interp1d
+from scipy.signal import filtfilt, firwin
+from tqdm import tqdm
+
+# %%
+# read the data
 ship = 'Ride'
 cruise = 'SR2312'
-root = 'data/'
-seg_path = os.path.join(root, ship, cruise, 'gravimeter/DGS/line-segments')
-# seg_files = np.sort(glob(os.path.join(seg_path,'segment*.dat')))
-# we want to use one particular file that is saved here, which is:
-seg_file = os.path.join(seg_path, 'example_segment_s3_330234-428488.dat')
 
-# read the data
-data = read_csv(seg_file, parse_dates=[16,])  # fully offshore Newport, OR
+seg_file = dgs_files = pooch.retrieve(url="https://zenodo.org/records/12733929/files/data.zip", 
+        known_hash="md5:83b0411926c0fef9d7ccb2515bb27cc0", progressbar=True, 
+        processor=pooch.Unzip(
+            members=['data/Ride/SR2312/gravimeter/DGS/line-segments/example_segment_s3_330234-428488.dat']))
+
+data = read_csv(seg_file[0], parse_dates=[16,])  # fully offshore Newport, OR
 # this segment is a straight-ish line; it has a lot of stations along it
 # where the ship was more or less stationary but that's ok
 
+# %%
 # set up coordinates for extending the ends of the line
 wgs = Geodesic.WGS84  # object for calculating line extensions etc
 faz01 = wgs.Inverse(data.iloc[0]['lat_new'], data.iloc[0]['lon_new'],
@@ -86,16 +97,17 @@ if l_seg < l_ext:  # but if it's a short segment, just use the segment length on
     l_ext = l_seg
 # calculate distances btwn points to get avg spacing for extensions
 xrng = np.zeros(len(data))
-for i in range(1, len(data)):
+for i in tqdm(range(1, len(data)),desc='calculating ranges'):
     xrng[i] = wgs.Inverse(data.iloc[0]['lat_new'], data.iloc[0]['lon_new'],
                           data.iloc[i]['lat_new'], data.iloc[i]['lon_new'])['s12']
 dx_avg = np.mean(np.diff(xrng))
 n_ext = int(l_ext/dx_avg)
 
+# %%
 # get the actual coordinate points for each extended side
 frnt_ext = np.zeros((n_ext, 2))
 back_ext = np.zeros((n_ext, 2))
-for i in range(n_ext):
+for i in tqdm(range(n_ext),desc='calculating extensions'):
     frnt_pt = wgs.Direct(data.iloc[0]['lat_new'],
                          data.iloc[0]['lon_new'], faz10, dx_avg*(i+1))
     back_pt = wgs.Direct(data.iloc[-1]['lat_new'],
@@ -110,6 +122,8 @@ a_lon = np.hstack(
     [frnt_ext[:, 0][::-1], data['lon_new'].values, back_ext[:, 0]])
 a_lat = np.hstack(
     [frnt_ext[:, 1][::-1], data['lat_new'].values, back_ext[:, 1]])
+
+# %%
 ########################################################################
 # this would be the point where you would track through gridfiles
 # if using a different line than the example file.
@@ -125,13 +139,21 @@ a_lat = np.hstack(
 # age_grd = os.path.expanduser('~/Data/Muller_agegrids/age.2020.1.GeeK2007.1m.nc')
 # os.system('gmt grdtrack temp.ll -G%s -G%s -G%s -G%s > tracked.llm' % (dep_grd, faa_grd, sed_grd, age_grd))
 ########################################################################
+
+# %%
 # since we've already pre-tracked the example line, we'll just load in the
 # info here
-track = read_csv(os.path.join(seg_path, 'tracked.llm'), sep='\t',
+
+track_file = dgs_files = pooch.retrieve(url="https://zenodo.org/records/12733929/files/data.zip", 
+        known_hash="md5:83b0411926c0fef9d7ccb2515bb27cc0", progressbar=True, 
+        processor=pooch.Unzip(
+            members=['data/Ride/SR2312/gravimeter/DGS/line-segments/tracked.llm']))
+track = read_csv(track_file[0], sep='\t',
                  names=['lon', 'lat', 'dep', 'faa', 'sed', 'age'])
 # NOTE that if you are working with data from near a coast, line extensions
 # might end up on land. Check for NaN values in your tracked file.
 
+# %%
 # filter FAA from the segment since Sandwell grid is very long-wavelength
 sampling = 1
 taps = 2*240
@@ -141,6 +163,7 @@ wn = freq/nyquist
 B = firwin(taps, wn, window='blackman')  # approx equivalent to matlab fir1
 ffaa = filtfilt(B, 1, data['faa'])
 
+# %%
 # embed our FAA in the tracked line (everything else remains tracked)
 track.loc[n_ext+taps:n_ext+taps+len(ffaa[taps:-taps])-1,'faa'] = ffaa[taps:-taps]
 # NOTE you may want to check your embedded FAA for large jumps at the ends
@@ -148,9 +171,10 @@ track.loc[n_ext+taps:n_ext+taps+len(ffaa[taps:-taps])-1,'faa'] = ffaa[taps:-taps
 # the jumps aren't *too* big* and also we don't care that much about the
 # results of this example, but for real analyses, you should care!
 
+# %%
 # interpolate everything to an even X spacing (after recalculating total distance)
 xpts_line = np.zeros(len(track))
-for i in range(1, len(track)):
+for i in tqdm(range(1, len(track)),desc='interpolating to even spacing'):
     xpts_line[i] = wgs.Inverse(track.iloc[0]['lat'], track.iloc[0]['lon'],
                                track.iloc[i]['lat'], track.iloc[i]['lon'])['s12']
 # NOTE be careful with the number of points here
@@ -169,6 +193,7 @@ age_int = fage(xobs)
 fsed = interp1d(xpts_line, track['sed'].values)
 sed_int = fsed(xobs)
 
+# %%
 # calculate topography, sediment, and crust corrections (for MBA)
 # and thermal correction (for RMBA)
 # sed density approx based on Hamilton 1978, doi: 10.1121/1.381747
@@ -189,6 +214,7 @@ anom_c = sgg.grav1d_padded(xobs, dep_int-sed_int-dz_c, 0, drho[2])  # crust
 
 MBA = FAA_int - anom_w - anom_s - anom_c  # mantle Bouger anomaly
 
+# %%
 # correct for plate cooling (RMBA)
 corrs = np.zeros((len(temps)-1, len(xobs)))  # calculate gravity due to temps
 for i in range(len(temps)-1):  # loop over a set of isotherms defined above
@@ -196,9 +222,11 @@ for i in range(len(temps)-1):  # loop over a set of isotherms defined above
     corrs[i, :] = sgg.grav1d_padded(
         xobs, dep_int-sed_int-ziso-dz_c, 0, drho[i+3])
 
+# %%
 # RMBA is MBA minus summed thermal correction
 RMBA = MBA - np.sum(corrs, axis=0)
 
+# %%
 # estimate crustal thickness
 nx = len(xobs)
 ny = 1  # we're only working with a line, but the function works for a 2D region
@@ -215,6 +243,7 @@ wsmall = 5
 cthick, rev_grav = sgg.crustal_thickness_2D(RMBA, nx=nx, ny=ny, dx=dx, dy=dy,
                                             zdown=zdown, rho=rho, wlarge=wlarge, wsmall=wsmall, back=True)
 
+# %%
 # make some plots to see what we got out of this
 plt.figure()
 plt.plot(xobs/1e3, np.real(rev_grav), label='recovered RMBA')
@@ -230,3 +259,5 @@ plt.xlabel('distance along line [km]')
 plt.ylabel('crustal thickness variation [km]')
 
 plt.show()
+
+# %%

{shipgrav-1.0.3 → shipgrav-1.0.5}/example-scripts/dgs_bgm_comp.py

@@ -1,33 +1,44 @@
-import numpy as np
-import shipgrav.grav as sgg
-import shipgrav.nav as sgn
-import shipgrav.io as sgi
-import tomli as tm
-from glob import glob
-from scipy.interpolate import interp1d
-from scipy.signal import firwin, filtfilt
-import matplotlib.pyplot as plt
-import os
-import sys
-
-########################################################################
-# Example script for reading and lightly processing DGS laptop data
-# from a Thompson cruise, and comparing to BGM serial files.
+# %% [markdown]
+# ### Example script for reading and lightly processing DGS laptop data from a Thompson cruise, and comparing to BGM serial files.
+#
+# Data files are downloaded by the script using pooch
 #
 # Read DGS and navigation files
+#
 # Read BGM files
+#
 # Correct for meter bias with info from shipgrav
+#
 # Compare BGM and DGS outputs
+#
 # Use timestamps to sync more accurate nav with the gravity data.
+#
 # Calculate FAA (free air anomaly)
+#
 # Plot
+#
 # satellite data tracked from v32.1 Global Gravity grid, which
 # includes data from SIO, NOAA, and NGA.
+#
 # Reference: Sandwell et al. (2014) New global marine gravity model
 # from CryoSat-2 and Jason-1 reveals buried tectonic struture.
 # Science 346(6205), DOI: 10.1126/science.1258213
-########################################################################
 
+# %%
+import os
+from glob import glob
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pooch
+import shipgrav.grav as sgg
+import shipgrav.io as sgi
+import tomli as tm
+from scipy.interpolate import interp1d
+from scipy.signal import filtfilt, firwin
+
+
+# %%
 # set some general metadata
 ship = 'Thompson'
 cruise = 'TN400'        # this is used for filepaths
@@ -39,26 +50,39 @@ with open('../shipgrav/database.toml', 'rb') as f:
 nav_tag = info['nav-talkers'][ship]
 biases = info['bias-values'][ship]
 
-# set up file paths, get lists of input files
-root = 'data/'
-dgs_path = os.path.join(root, ship, cruise, 'gravimeter/DGS')
-# we only have serial BGM
-bgm_path = os.path.join(root, ship, cruise, 'gravimeter/BGM3/serial')
-nav_path = os.path.join(root, ship, cruise, 'NAV')
-dgs_files = np.sort(glob(os.path.join(dgs_path, 'AT1M-Grav-PROC_*.Raw')))
-bgm_files = np.sort(glob(os.path.join(bgm_path, 'BGM3-GRAV-RAW*.Raw')))
-nav_files = np.sort(glob(os.path.join(nav_path, 'POSMV*%s*.Raw' % nav_tag)))
-
+# %%
+# get files: DGS laptop from zenodo, BGM serial and nav from R2R
+dgs_files = pooch.retrieve(url="https://zenodo.org/records/12733929/files/data.zip", 
+        known_hash="md5:83b0411926c0fef9d7ccb2515bb27cc0", progressbar=True, 
+        processor=pooch.Unzip(
+            members=['data/Thompson/TN400/gravimeter/DGS/AT1M-Grav-PROC_20220314-000001.Raw',
+                    'data/Thompson/TN400/gravimeter/DGS/AT1M-Grav-PROC_20220313-000001.Raw']))
+
+bgm_files = pooch.retrieve(url="https://service.rvdata.us/data/cruise/TN400/fileset/151470",
+        known_hash='b0ccc52e60334284dd79573e27e050eead77f5ac1d59f510385c7e05a8474bcf',progressbar=True,
+        processor=pooch.Untar(
+            members=['TN400/151470/data/BGM3-GRAV-RAW_20220313-000001.Raw',
+                    'TN400/151470/data/BGM3-GRAV-RAW_20220314-000001.Raw']))
+
+nav_files = pooch.retrieve(url="https://service.rvdata.us/data/cruise/TN400/fileset/151457",
+        known_hash="76e66365c41d393510bb7ab9637305296282e9041415c1343faa171af28abf85",progressbar=True,
+        processor=pooch.Untar(
+            members=['TN400/151457/data/POSMV-V5-INGGA-RAW_20220313-000001.Raw',
+                    'TN400/151457/data/POSMV-V5-INGGA-RAW_20220314-000001.Raw']))
+
+# %%
 # read and sort the nav data
 gps_nav = sgi.read_nav(ship, nav_files)
 gps_nav.sort_values('time_sec', inplace=True)
 gps_nav.reset_index(inplace=True, drop=True)
 
+# %%
 # we happen to know that there are some weird nav dropouts in this dataset
 # so clean them up here
 bad_inds = np.where(np.diff(gps_nav['lon']) > 1)[0]
 gps_nav.drop(bad_inds, axis=0, inplace=True)
 
+# %%
 # read and sort the DGS laptop data
 dgs_data = sgi.read_dgs_laptop(dgs_files, ship)
 dgs_data.sort_values('date_time', inplace=True)
@@ -67,11 +91,15 @@ dgs_data['tsec'] = [e.timestamp()
                     for e in dgs_data['date_time']]  # get posix timestamps
 dgs_data['grav'] = dgs_data['rgrav'] + biases['dgs']
 
+# %%
 # read and sort the BGM data
 bgm_data = sgi.read_bgm_raw(bgm_files, ship)
+bgm_data.sort_values('date_time', inplace=True)
+bgm_data.reset_index(inplace=True, drop=True)
 bgm_data['tsec'] = [e.timestamp() for e in bgm_data['date_time']]
 bgm_data['grav'] = bgm_data['rgrav'] + biases['bgm']
 
+# %%
 # sync data geographic coordinates to nav by interpolating with timestamps
 # (interpolators use posix timestamps, not datetimes)
 gps_lon_int = interp1d(gps_nav['time_sec'].values, gps_nav['lon'].values,
@@ -84,6 +112,7 @@ dgs_data['lat_new'] = gps_lat_int(dgs_data['tsec'].values)
 bgm_data['lon_new'] = gps_lon_int(bgm_data['tsec'].values)
 bgm_data['lat_new'] = gps_lat_int(bgm_data['tsec'].values)
 
+# %%
 # calculate corrections for FAA for DGS and BGM
 for df in [dgs_data, bgm_data]:
     ellipsoid_ht = np.zeros(len(df))  # we are working at sea level
@@ -97,6 +126,7 @@ for df in [dgs_data, bgm_data]:
     df['faa'] = df['grav'] - lat_corr + eotvos_corr + tide_corr
     df['full_field'] = df['grav'] + eotvos_corr + tide_corr
 
+# %%
 # apply a lowpass filter
 taps = 2*240
 freq = 1./240
@@ -108,10 +138,16 @@ B = firwin(taps, wn, window='blackman')  # approx equivalent to matlab fir1
 dfaa = filtfilt(B, 1, dgs_data['faa'])
 bfaa = filtfilt(B, 1, bgm_data['faa'])
 
+# %%
 # load satellite data for comparison
-sat_grav = np.loadtxt(os.path.join(root, ship, cruise, 'sandwell_tracked.llg'), usecols=(3,),
+sat_path = pooch.retrieve(url="https://zenodo.org/records/12733929/files/data.zip", 
+        known_hash="md5:83b0411926c0fef9d7ccb2515bb27cc0", progressbar=True, 
+        processor=pooch.Unzip(
+            members=['data/Thompson/TN400/sandwell_tracked.llg']))
+sat_grav = np.loadtxt(sat_path[0], usecols=(3,),
                       delimiter=',', skiprows=1)
 
+# %%
 # plot this data and satellite data (trim edge effects from filtering)
 plt.figure(figsize=(11, 4.8))
 plt.plot(dgs_data.iloc[taps:-taps//2]['date_time'],
@@ -125,3 +161,5 @@ plt.ylabel('Free air anomaly [mGal]')
 plt.legend(fontsize=8)
 plt.tight_layout()
 plt.show()
+
+# %%

{shipgrav-1.0.3 → shipgrav-1.0.5}/example-scripts/dgs_ccp_calc.py

@@ -1,27 +1,35 @@
-import numpy as np
-import shipgrav.io as sgi
-import shipgrav.grav as sgg
-import shipgrav.nav as sgn
-import tomli as tm
-from scipy.signal import firwin, filtfilt
-import matplotlib.pyplot as plt
-from glob import glob
-import os
-import sys
-
-########################################################################
-# Example script using data from TN400 to illustrate how cross-
-# coupling coefficients are calculated and applied.
+# %% [markdown]
+# ### Example script using data from TN400 to illustrate how cross-coupling coefficients are calculated and applied.
+#
+# Data files are downloaded by the script using pooch
 #
 # Read DGS data files
 # (not bothering with navigation syncing; see dgs_bgm_comp.py)
+#
 # Calculate FAA
+#
 # Calculate various kinematic parameters
+#
 # Fit for cross-coupling coefficients
+#
 # Correct for cross-coupling
+#
 # Plot with and without corrections
-########################################################################
 
+# %%
+import os
+from glob import glob
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pooch
+import shipgrav.grav as sgg
+import shipgrav.io as sgi
+import shipgrav.nav as sgn
+import tomli as tm
+from scipy.signal import filtfilt, firwin
+
+# %%
 # set some general metadata
 ship = 'Thompson'
 cruise = 'TN400'
@@ -32,11 +40,16 @@ with open('../shipgrav/database.toml', 'rb') as f:
 # nav_tag = info['nav-talkers'][ship]  # could use this for nav file read
 bias_dgs = info['bias-values'][ship]['dgs']
 
-# set up file paths, get lists of input files
-root = 'data/'
-dgs_path = os.path.join(root, ship, cruise, 'gravimeter/DGS')
-dgs_files = np.sort(glob(os.path.join(dgs_path, 'AT1M*.Raw')))
+# %%
+# get the files from Zenodo: TN400 DGS laptop data
+# the archive download includes some things not used for this example; those are not unpacked
+dgs_files = pooch.retrieve(url="https://zenodo.org/records/12733929/files/data.zip", 
+        known_hash="md5:83b0411926c0fef9d7ccb2515bb27cc0", progressbar=True, 
+        processor=pooch.Unzip(
+            members=['data/Thompson/TN400/gravimeter/DGS/AT1M-Grav-PROC_20220314-000001.Raw',
+                    'data/Thompson/TN400/gravimeter/DGS/AT1M-Grav-PROC_20220313-000001.Raw']))
 
+# %%
 # read and sort the gravimeter data
 dgs_data = sgi.read_dgs_laptop(dgs_files, ship)
 dgs_data.sort_values('date_time', inplace=True)
@@ -45,9 +58,11 @@ dgs_data['tsec'] = [e.timestamp()
                     for e in dgs_data['date_time']]  # get posix timestamps
 dgs_data['grav'] = dgs_data['rgrav'] + bias_dgs
 
+# %%
 # trim some bits that we happen to know are bad
 dgs_data = dgs_data.iloc[1000:]
 
+# %%
 # calculate corrections for FAA
 ellipsoid_ht = np.zeros(len(dgs_data))  # we are working at sea level
 lat_corr = sgg.wgs_grav(dgs_data['lat']) + \
@@ -60,6 +75,7 @@ tide_corr = sgg.longman_tide_prediction(
 dgs_data['faa'] = dgs_data['grav'] - lat_corr + eotvos_corr + tide_corr
 dgs_data['full_field'] = dgs_data['grav'] + eotvos_corr + tide_corr
 
+# %%
 # calculate kinematic variables and corrections for tilt correction
 # (maybe not strictly necessary? depends who you ask)
 gps_vn, gps_ve = sgn.latlon_to_EN(
@@ -77,6 +93,7 @@ vel[np.isnan(vel)] = 0
 acc_cross, acc_long = sgn.rotate_acceleration_EN_to_cl(
     crse, gps_eacc, gps_nacc)  # gps-derived cross and long accel
 
+# %%
 # tilt correction
 up_vecs = sgg.up_vecs(1/sampling, lat_corr, acc_cross,
                       acc_long, 0, 240, 0.7071, 240, 0.7071)
@@ -85,6 +102,7 @@ cross_in_plat = acc_cross*up_vecs[1, :]
 long_in_plat = acc_long*up_vecs[0, :]
 level_error = lat_corr - igf_in_plat - long_in_plat + cross_in_plat
 
+# %%
 # calculate cross-coupling coefficients
 _, model = sgg.calc_cross_coupling_coefficients(dgs_data['faa'].values, dgs_data['vcc'].values, dgs_data['ve'].values,
                                                 dgs_data['al'].values, dgs_data['ax'].values, level_error.values)
@@ -93,6 +111,7 @@ print('cross coupling parameters from fit:')
 print('ve %.3f, vcc %.3f, al %.3f, ax %.3f, lev %.3f' % (model.params.ve,
       model.params.vcc, model.params.al, model.params.ax, model.params.lev))
 
+# %%
 # apply cross-coupling correction and plot the (filtered) FAA
 dgs_data['faa_ccp'] = dgs_data['faa'] + model.params.ve*dgs_data['ve'] + \
     model.params.vcc*dgs_data['vcc'] + \
@@ -108,6 +127,8 @@ B = firwin(taps, wn, window='blackman')  # approx equivalent to matlab fir1
 
 ffaa = filtfilt(B, 1, dgs_data['faa'])
 cfaa = filtfilt(B, 1, dgs_data['faa_ccp'])
+
+# %%
 plt.figure(figsize=(11, 4.8))
 plt.plot(dgs_data.iloc[taps:-taps//2]['date_time'],
          ffaa[taps:-taps//2], label='no ccp')
@@ -118,3 +139,5 @@ plt.ylabel('Free air anomaly [mGal]')
 plt.legend(fontsize=8)
 plt.tight_layout()
 plt.show()
+
+# %%