pypromice 1.3.6__py3-none-any.whl → 1.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pypromice might be problematic. Click here for more details.

Files changed (53) hide show
  1. pypromice/postprocess/bufr_to_csv.py +15 -3
  2. pypromice/postprocess/bufr_utilities.py +91 -18
  3. pypromice/postprocess/create_bufr_files.py +178 -0
  4. pypromice/postprocess/get_bufr.py +248 -397
  5. pypromice/postprocess/make_metadata_csv.py +214 -0
  6. pypromice/postprocess/real_time_utilities.py +41 -11
  7. pypromice/process/L0toL1.py +12 -5
  8. pypromice/process/L1toL2.py +69 -14
  9. pypromice/process/L2toL3.py +1034 -186
  10. pypromice/process/aws.py +139 -808
  11. pypromice/process/get_l2.py +90 -0
  12. pypromice/process/get_l2tol3.py +111 -0
  13. pypromice/process/join_l2.py +112 -0
  14. pypromice/process/join_l3.py +551 -120
  15. pypromice/process/load.py +161 -0
  16. pypromice/process/resample.py +147 -0
  17. pypromice/process/utilities.py +68 -0
  18. pypromice/process/write.py +503 -0
  19. pypromice/qc/github_data_issues.py +10 -16
  20. pypromice/qc/persistence.py +52 -30
  21. pypromice/resources/__init__.py +28 -0
  22. pypromice/{process/metadata.csv → resources/file_attributes.csv} +0 -2
  23. pypromice/resources/variable_aliases_GC-Net.csv +78 -0
  24. pypromice/resources/variables.csv +106 -0
  25. pypromice/station_configuration.py +118 -0
  26. pypromice/tx/get_l0tx.py +7 -4
  27. pypromice/tx/payload_formats.csv +1 -0
  28. pypromice/tx/tx.py +27 -6
  29. pypromice/utilities/__init__.py +0 -0
  30. pypromice/utilities/git.py +62 -0
  31. {pypromice-1.3.6.dist-info → pypromice-1.4.1.dist-info}/METADATA +4 -4
  32. pypromice-1.4.1.dist-info/RECORD +53 -0
  33. {pypromice-1.3.6.dist-info → pypromice-1.4.1.dist-info}/WHEEL +1 -1
  34. pypromice-1.4.1.dist-info/entry_points.txt +13 -0
  35. pypromice/postprocess/station_configurations.toml +0 -762
  36. pypromice/process/get_l3.py +0 -46
  37. pypromice/process/variables.csv +0 -92
  38. pypromice/qc/persistence_test.py +0 -150
  39. pypromice/test/test_config1.toml +0 -69
  40. pypromice/test/test_config2.toml +0 -54
  41. pypromice/test/test_email +0 -75
  42. pypromice/test/test_payload_formats.csv +0 -4
  43. pypromice/test/test_payload_types.csv +0 -7
  44. pypromice/test/test_percentile.py +0 -229
  45. pypromice/test/test_raw1.txt +0 -4468
  46. pypromice/test/test_raw_DataTable2.txt +0 -11167
  47. pypromice/test/test_raw_SlimTableMem1.txt +0 -1155
  48. pypromice/test/test_raw_transmitted1.txt +0 -15411
  49. pypromice/test/test_raw_transmitted2.txt +0 -28
  50. pypromice-1.3.6.dist-info/RECORD +0 -53
  51. pypromice-1.3.6.dist-info/entry_points.txt +0 -8
  52. {pypromice-1.3.6.dist-info → pypromice-1.4.1.dist-info}/LICENSE.txt +0 -0
  53. {pypromice-1.3.6.dist-info → pypromice-1.4.1.dist-info}/top_level.txt +0 -0
@@ -1,46 +0,0 @@
1
- #!/usr/bin/env python
2
- import logging, os, sys, unittest
3
- from argparse import ArgumentParser
4
- from pypromice.process.aws import AWS
5
-
6
- def parse_arguments_l3():
7
- parser = ArgumentParser(description="AWS L3 processor")
8
-
9
- parser.add_argument('-c', '--config_file', type=str, required=True,
10
- help='Path to config (TOML) file')
11
- parser.add_argument('-i', '--inpath', default='data', type=str, required=True,
12
- help='Path to input data')
13
- parser.add_argument('-o', '--outpath', default=None, type=str, required=False,
14
- help='Path where to write output')
15
- parser.add_argument('-v', '--variables', default=None, type=str,
16
- required=False, help='File path to variables look-up table')
17
- parser.add_argument('-m', '--metadata', default=None, type=str,
18
- required=False, help='File path to metadata')
19
- args = parser.parse_args()
20
- return args
21
-
22
- def get_l3():
23
- args = parse_arguments_l3()
24
-
25
- logging.basicConfig(
26
- format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
27
- level=logging.INFO,
28
- stream=sys.stdout,
29
- )
30
-
31
- station_name = args.config_file.split('/')[-1].split('.')[0]
32
- station_path = os.path.join(args.inpath, station_name)
33
-
34
- if os.path.exists(station_path):
35
- aws = AWS(args.config_file, station_path, args.variables, args.metadata)
36
- else:
37
- aws = AWS(args.config_file, args.inpath, args.variables, args.metadata)
38
-
39
- aws.process()
40
-
41
- if args.outpath is not None:
42
- aws.write(args.outpath)
43
-
44
- if __name__ == "__main__":
45
- get_l3()
46
-
@@ -1,92 +0,0 @@
1
- field,standard_name,long_name,units,lo,hi,OOL,station_type,data_type,max_decimals,coverage_content_type,coordinates,instantaneous_hourly,comment
2
- time,time,Time,yyyy-mm-dd HH:MM:SS,,,,all,all,,physicalMeasurement,time lat lon alt,,
3
- rec,record,Record,-,,,,all,,0,referenceInformation,time lat lon alt,,L0 only
4
- p_u,air_pressure,Air pressure (upper boom),hPa,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False,
5
- p_l,air_pressure,Air pressure (lower boom),hPa,650,1100,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
6
- t_u,air_temperature,Air temperature (upper boom),degrees_C,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False,
7
- t_l,air_temperature,Air temperature (lower boom),degrees_C,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l ,two-boom,all,4,physicalMeasurement,time lat lon alt,False,PT100 temperature at boom
8
- rh_u,relative_humidity,Relative humidity (upper boom),%,0,100,rh_u_cor,all,all,4,physicalMeasurement,time lat lon alt,False,
9
- rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,0,150,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False,
10
- qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,0,100,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
11
- rh_l,relative_humidity,Relative humidity (lower boom),%,0,100,rh_l_cor,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
12
- rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,0,150,dshf_l dlhf_l qh_l,two-boom,all,4,modelResult,time lat lon alt,False,
13
- qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,0,100,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
14
- wspd_u,wind_speed,Wind speed (upper boom),m s-1,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,all,4,physicalMeasurement,time lat lon alt,False,
15
- wspd_l,wind_speed,Wind speed (lower boom),m s-1,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,all,4,physicalMeasurement,time lat lon alt,False,
16
- wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,1,360,wspd_x_u wspd_y_u,all,all,4,physicalMeasurement,time lat lon alt,False,
17
- wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,,,,one-boom,,4,qualityInformation,time lat lon alt,False,L0 only
18
- wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,1,360,wspd_x_l wspd_y_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
19
- wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,-100,100,wdir_u wspd_u,all,all,4,modelResult,time lat lon alt,False,L0 only
20
- wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,-100,100,wdir_u wspd_u,all,all,4,modelResult,time lat lon alt,False,L0 only
21
- wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,-100,100,wdir_l wspd_l,two-boom,all,4,modelResult,time lat lon alt,False,L0 only
22
- wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,-100,100,wdir_l wspd_l,two-boom,all,4,modelResult,time lat lon alt,False,L0 only
23
- dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,-10,1500,albedo dsr_cor usr_cor,all,all,4,physicalMeasurement,time lat lon alt,False,"Actually radiation_at_sensor, not flux. Units 1E-5 V. Engineering units."
24
- dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
25
- usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,-10,1000,albedo dsr_cor usr_cor,all,all,4,physicalMeasurement,time lat lon alt,False,
26
- usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,0,1000,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
27
- albedo,surface_albedo,Albedo,-,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
28
- dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,50,500,albedo dsr_cor usr_cor cc t_surf,all,all,4,physicalMeasurement,time lat lon alt,False,
29
- ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,50,500,t_surf,all,all,4,physicalMeasurement,time lat lon alt,False,
30
- cc,cloud_area_fraction,Cloud cover,%,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
31
- t_surf,surface_temperature,Surface temperature,C,-80,40,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
32
- dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
33
- dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,,,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
34
- dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
35
- dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,,,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
36
- z_boom_u,distance_to_surface_from_boom,Upper boom height,m,0.3,10,dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,True,
37
- z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,,,,all,,4,qualityInformation,time lat lon alt,True,L0 only
38
- z_boom_l,distance_to_surface_from_boom,Lower boom height,m,0.3,5,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,True,
39
- z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,,,,two-boom,,4,qualityInformation,time lat lon alt,True,L0 only
40
- z_stake,distance_to_surface_from_stake_assembly,Stake height,m,0.3,8,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,HeightStakes(m)
41
- z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,,,,one-boom,,4,qualityInformation,time lat lon alt,True,L0 only
42
- z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,0,30,z_pt_cor,one-boom,all,4,physicalMeasurement,time lat lon alt,False,DepthPressureTransducer(m)
43
- z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,0,30,,one-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later)
44
- precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,0,,precip_u_cor precip_u_rate,all,all,4,physicalMeasurement,time lat lon alt,True,Without wind/undercatch correction
45
- precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,0,,,all,all,4,modelResult,time lat lon alt,True,With wind/undercatch correction
46
- precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,0,,,all,all,4,modelResult,time lat lon alt,True,L0 only
47
- precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,0,,precip_l_cor precip_l_rate,two-boom,all,4,physicalMeasurement,time lat lon alt,True,Without wind/undercatch correction
48
- precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,0,,,two-boom,all,4,modelResult,time lat lon alt,True,With wind/undercatch correction
49
- precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,0,,,two-boom,all,4,modelResult,time lat lon alt,True,L0 only
50
- t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,t1 is installed @ 1 m depth
51
- t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,
52
- t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,
53
- t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,
54
- t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,
55
- t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,
56
- t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,
57
- t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,t8 is installed @ 10 m depth
58
- t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
59
- t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
60
- t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False,
61
- tilt_x,platform_view_angle_x,Tilt to east,degrees,-30,30,dsr_cor usr_cor albedo,all,all,4,physicalMeasurement,time lat lon alt,False,
62
- tilt_y,platform_view_angle_y,Tilt to north,degrees,-30,30,dsr_cor usr_cor albedo,all,all,4,physicalMeasurement,time lat lon alt,False,
63
- rot,platform_azimuth_angle,Station rotation from true North,degrees,0,360,,all,all,2,physicalMeasurement,time lat lon alt,False,v4 addition
64
- gps_lat,gps_latitude,Latitude,degrees_north,50,83,,all,all,6,coordinate,time lat lon alt,True,
65
- gps_lon,gps_longitude,Longitude,degrees_east,5,70,,all,all,6,coordinate,time lat lon alt,True,
66
- gps_alt,gps_altitude,Altitude,m,0,3000,,all,all,2,coordinate,time lat lon alt,True,
67
- gps_time,gps_time,GPS time,s,0,240000,,all,all,,coordinate,time lat lon alt,True,
68
- gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,,,,one-boom,all,,physicalMeasurement,time lat lon alt,True,
69
- gps_geounit,gps_geounit,GeoUnit,-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only
70
- gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,,,,all,all,2,qualityInformation,time lat lon alt,True,NMEA: Horizontal dilution of precision
71
- gps_numsat,gps_numsat,GPS number of satellites,-,,,,,all,0,qualityInformation,time lat lon alt,True,L0 only
72
- gps_q,gps_q,Quality,-,,,,,all,,qualityInformation,time lat lon alt,True,L0 only
73
- lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True,
74
- lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True,
75
- alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True,
76
- batt_v,battery_voltage,Battery voltage,V,0,30,,all,all,2,physicalMeasurement,time lat lon alt,True,
77
- batt_v_ini,,,-,0,30,,,all,2,physicalMeasurement,time lat lon alt,True,L0 only
78
- batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,0,30,,,all,2,physicalMeasurement,time lat lon alt,True,L0 only
79
- fan_dc_u,fan_current,Fan current (upper boom),mA,0,200,,all,all,2,physicalMeasurement,time lat lon alt,True,
80
- fan_dc_l,fan_current,Fan current (lower boom),mA,0,200,,two-boom,all,2,physicalMeasurement,time lat lon alt,True,
81
- freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,0,10000,precip_u,,all,,physicalMeasurement,time lat lon alt,True,L0 only
82
- t_log,temperature_of_logger,Logger temperature,degrees_C,-80,40,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,LoggerTemperature(C)
83
- t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,-80,40,t_surf dlr ulr,all,all,4,physicalMeasurement,time lat lon alt,False,
84
- p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,-350,100,,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
85
- t_i,air_temperature,Air temperature (instantaneous),degrees_C,-80,40,,all,all,4,physicalMeasurement,time lat lon alt,True,"PT100 temperature at boom, for meteorological observations"
86
- rh_i,relative_humidity,Relative humidity (instantaneous),%,0,150,rh_i_cor,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
87
- rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,0,100,,all,all,4,modelResult,time lat lon alt,True,For meteorological observations
88
- wspd_i,wind_speed,Wind speed (instantaneous),m s-1,0,100,wdir_i wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
89
- wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,1,360,wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations
90
- wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations
91
- wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations
92
- msg_i,message,Message string (instantaneous),-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only
@@ -1,150 +0,0 @@
1
- import unittest
2
-
3
- import numpy as np
4
- import numpy.testing
5
- import pandas as pd
6
-
7
- from pypromice.qc.persistence import find_persistent_regions
8
-
9
-
10
- class PersistenceQATestCase(unittest.TestCase):
11
- def test_1_hour_persistent(self):
12
- self._test_1_hour_repeat(10)
13
-
14
- def test_1_hour_second_index(self):
15
- self._test_1_hour_repeat(0)
16
-
17
- def test_1_hour_last_index(self):
18
- self._test_1_hour_repeat(-2)
19
-
20
- def _test_1_hour_repeat(self, index: int):
21
- self.assertTrue(index < -1 or index >= 0)
22
- time_range = pd.date_range(
23
- start="2023-01-26", end="2023-01-27", freq="h", tz="utc", inclusive="left"
24
- )
25
- input_series = pd.Series(index=time_range, data=np.arange(0, len(time_range)))
26
- input_series[index + 1] = input_series[index]
27
- min_repeats = 1
28
- expected_output = input_series.map(lambda _: False)
29
- expected_output[index + 1] = True
30
-
31
- persistent_mask = find_persistent_regions(
32
- input_series, min_repeats=min_repeats, max_diff=0.001
33
- )
34
-
35
- pd.testing.assert_series_equal(expected_output, persistent_mask, check_names=False)
36
-
37
- def test_no_persistent_period(self):
38
- time_range = pd.date_range(
39
- start="2023-01-26", end="2023-01-27", freq="h", tz="utc", inclusive="left"
40
- )
41
- input_series = pd.Series(index=time_range, data=np.arange(0, len(time_range)))
42
- min_repeats = 1
43
- expected_output = input_series.map(lambda _: False)
44
-
45
- persistent_mask = find_persistent_regions(
46
- input_series, min_repeats=min_repeats, max_diff=0.001
47
- )
48
-
49
- pd.testing.assert_series_equal(expected_output, persistent_mask, check_names=False)
50
-
51
- def test_persistent_period_longer_than_period_threshold(self):
52
- time_range = pd.date_range(
53
- start="2023-01-26", end="2023-01-28", freq="h", tz="utc", inclusive="left"
54
- )
55
- index_start = 23
56
- index_end = 33
57
- min_repeats = 4
58
- expected_filter_start = 27
59
- expected_filter_end = 33
60
- input_series = pd.Series(index=time_range, data=np.arange(0, len(time_range)))
61
- input_series[index_start:index_end] = input_series[index_start]
62
- expected_output = input_series.map(lambda _: False)
63
- expected_output[expected_filter_start:expected_filter_end] = True
64
-
65
- persistent_mask = find_persistent_regions(
66
- input_series, min_repeats=min_repeats, max_diff=0.001
67
- )
68
-
69
- pd.testing.assert_series_equal(expected_output, persistent_mask, check_names=False)
70
-
71
- def test_period_threshold_longer_than_persistent_period(self):
72
- time_range = pd.date_range(
73
- start="2023-01-26", end="2023-01-28", freq="h", tz="utc", inclusive="left"
74
- )
75
- index_start = 23
76
- index_end = 27
77
- min_repeats = 10
78
- input_series = pd.Series(index=time_range, data=np.arange(0, len(time_range)))
79
- input_series[index_start:index_end] = input_series[index_start]
80
- expected_output = input_series.map(lambda _: False)
81
-
82
- persistent_mask = find_persistent_regions(
83
- input_series, min_repeats=min_repeats, max_diff=0.001
84
- )
85
-
86
- pd.testing.assert_series_equal(expected_output, persistent_mask, check_names=False)
87
-
88
- def test_persistent_period_at_the_end(self):
89
- time_range = pd.date_range(
90
- start="2023-01-26", end="2023-01-28", freq="h", tz="utc", inclusive="left"
91
- )
92
- index_start = 23
93
- min_repeats = 4
94
- expected_filter_start = 27
95
- input_series = pd.Series(index=time_range, data=np.arange(0, len(time_range)))
96
- input_series[index_start:] = input_series[index_start]
97
- expected_output = input_series.map(lambda _: False)
98
- expected_output[expected_filter_start:] = True
99
-
100
- persistent_mask = find_persistent_regions(
101
- input_series, min_repeats=min_repeats, max_diff=0.001
102
- )
103
-
104
- pd.testing.assert_series_equal(expected_output, persistent_mask, check_names=False)
105
-
106
- def test_dont_filter_nan_values(self):
107
- time_range = pd.date_range(
108
- start="2023-01-26", end="2023-01-27", freq="h", tz="utc", inclusive="left"
109
- )
110
- input_series = pd.Series(
111
- index=time_range, data=np.zeros_like(time_range, dtype="float")
112
- )
113
- min_repeats = 4
114
- input_series[:] = np.nan
115
- input_series[9] = -11
116
- input_series[10:12] = -10
117
- input_series[15] = -9
118
- # There are >=4 repeats if the nan values are forward filled. [10:15] == -10
119
- # The output mask shouldn't filter nan values.
120
- expected_output = input_series.map(lambda _: False)
121
-
122
- persistent_mask = find_persistent_regions(
123
- input_series, min_repeats=min_repeats, max_diff=0.001
124
- )
125
-
126
- pd.testing.assert_series_equal(expected_output, persistent_mask, check_names=False)
127
-
128
- def test_series_with_nan_values_between_persistent_values(self):
129
- time_range = pd.date_range(
130
- start="2023-01-26", end="2023-01-27", freq="h", tz="utc", inclusive="left"
131
- )
132
- values = np.zeros_like(time_range, dtype="float")
133
- values[:] = np.nan
134
- values[9] = -11
135
- values[16] = -11
136
- values[17] = -9
137
- series = pd.Series(index=time_range, data=values)
138
- expected_mask = np.zeros_like(values, dtype="bool")
139
- period = 4
140
- # The value and index 16 is the same as 9 which is longer than period
141
- # Note: The station region mask shall not filter nan values
142
- expected_mask[16] = True
143
-
144
- output_mask = find_persistent_regions(series, min_repeats=period, max_diff=0.01)
145
-
146
- np.testing.assert_equal(expected_mask, output_mask)
147
-
148
-
149
- if __name__ == "__main__":
150
- unittest.main()
@@ -1,69 +0,0 @@
1
- station_id = 'TEST1'
2
- logger_type = 'CR1000X'
3
- nodata = ['-999', 'NAN'] # if one is a string, all must be strings
4
- number_of_booms = 1 #1-boom = promice, 2-boom = gc-net
5
- latitude = 79.91
6
- longitude = 24.09
7
-
8
- ['test_raw1.txt']
9
- format = 'raw'
10
- skiprows = 4
11
- hygroclip_t_offset = 40
12
- dsr_eng_coef = 14.01
13
- usr_eng_coef = 12.72
14
- dlr_eng_coef = 11.08
15
- ulr_eng_coef = 11.42
16
- pt_z_coef = 0.39571
17
- pt_z_p_coef = 1022.5
18
- pt_z_factor = 2.5
19
- pt_antifreeze = 50
20
- boom_azimuth = 0
21
- columns = ['time','rec','SKIP_3','p_u','t_u','SKIP_6','rh_u','wspd_u','wdir_u','wdir_std_u',
22
- 'dsr','usr','dlr','ulr','t_rad','z_boom_u','z_boom_q_u','z_stake','z_stake_q','z_pt',
23
- 't_i_1','t_i_2','t_i_3','t_i_4','t_i_5','t_i_6','t_i_7',
24
- 't_i_8','tilt_x','tilt_y','gps_time','gps_lat','gps_lon',
25
- 'gps_alt','gps_geoid','SKIP_36','gps_q','gps_numsat','gps_hdop',
26
- 't_log','fan_dc_u','batt_v_ini','batt_v']
27
-
28
- ['test_raw_SlimTableMem1.txt']
29
- format = 'STM'
30
- skiprows = 4
31
- latitude = 79.83
32
- longitude = 25.17
33
- hygroclip_t_offset = 40
34
- dsr_eng_coef = 12.55
35
- usr_eng_coef = 12.52
36
- dlr_eng_coef = 13.73
37
- ulr_eng_coef = 10.43
38
- pt_z_coef = 0.4369
39
- pt_z_p_coef = 1005.9
40
- pt_z_factor = 2.5
41
- pt_antifreeze = 50
42
- boom_azimuth = 0
43
- columns = ['time','rec','min_y','p_u','t_u','SKIP_6','rh_u','wspd_u','wdir_u',
44
- 'wd_std_u','dsr','usr','dlr','ulr','t_rad','z_boom_u',
45
- 'z_boom_q_u','z_stake','z_stake_q','z_pt','t_i_1','t_i_2',
46
- 't_i_3','t_i_4','t_i_5','t_i_6','t_i_7','t_i_8','tilt_x',
47
- 'tilt_y','gps_time','gps_lat','gps_lon','gps_alt',
48
- 'gps_geoid','SKIP_36','gps_q','gps_numsats','gps_hdop',
49
- 't_log','fan_dc_u','batt_v_ss','batt_v']
50
-
51
- ['test_raw_transmitted1.txt']
52
- format = 'TX'
53
- skiprows = 0
54
- hygroclip_t_offset = 0
55
- dsr_eng_coef = 12.2
56
- usr_eng_coef = 11.15
57
- dlr_eng_coef = 10.28
58
- ulr_eng_coef = 8.24
59
- pt_z_coef = 0.3999
60
- pt_z_p_coef = 982.4
61
- pt_z_factor = 2.5
62
- pt_antifreeze = 50
63
- boom_azimuth = 0
64
- columns = ['time','rec','p_u','t_u','SKIP_5','rh_u','wspd_u','wdir_u','dsr',
65
- 'usr','dlr','ulr','t_rad','z_boom_u','z_stake','z_pt',
66
- 't_i_1','t_i_2','t_i_3','t_i_4','t_i_5','t_i_6','t_i_7',
67
- 't_i_8','tilt_x','tilt_y','gps_time','gps_lat','gps_lon',
68
- 'gps_alt','gps_hdop','fan_dc_u','batt_v']
69
-
@@ -1,54 +0,0 @@
1
- station_id = 'TEST2'
2
- logger_type = 'CR1000X'
3
- nodata = ['-999', 'NAN'] # if one is a string, all must be strings
4
- number_of_booms = 2
5
-
6
- ['test_raw_transmitted2.txt']
7
- format = 'TX'
8
- skiprows = 0
9
- latitude = 55.688513
10
- longitude = 12.582191
11
- hygroclip_t_offset = 0
12
- dsr_eng_coef = 13.34
13
- usr_eng_coef = 11.90
14
- dlr_eng_coef = 9.63
15
- ulr_eng_coef = 9.72
16
- # pt_z_coef =
17
- # pt_z_p_coef =
18
- # pt_z_factor =
19
- # pt_antifreeze =
20
- tilt_y_factor = -1 # New inclinometer, y direction is opposite than the old PROMICE v2 inclinometer.
21
- boom_azimuth = 0 # Only relevant for PROMICE v2 stations
22
- columns = ['time','rec','p_l','p_u','t_l','t_u','rh_l','rh_u',
23
- 'wspd_l','wdir_l','wspd_u','wdir_u','dsr',
24
- 'usr','dlr','ulr','t_rad','z_boom_l','z_boom_u',
25
- 't_i_1','t_i_2','t_i_3','t_i_4','t_i_5','t_i_6','t_i_7',
26
- 't_i_8','t_i_9','t_i_10','t_i_11','tilt_x','tilt_y','rot',
27
- 'precip_l','precip_u','gps_time','gps_lat','gps_lon',
28
- 'gps_alt','gps_hdop','fan_dc_l','fan_dc_u','batt_v', 'p_i',
29
- 't_i','rh_i','wspd_i','wdir_i','msg_i']
30
-
31
-
32
- ['test_raw_DataTable2.txt']
33
- format = 'raw'
34
- skiprows = 4
35
- latitude = 55.688513
36
- longitude = 12.582191
37
- hygroclip_t_offset = 0
38
- dsr_eng_coef = 13.34
39
- usr_eng_coef = 11.90
40
- dlr_eng_coef = 9.63
41
- ulr_eng_coef = 9.72
42
- # pt_z_coef =
43
- # pt_z_p_coef =
44
- # pt_z_factor =
45
- # pt_antifreeze =
46
- tilt_y_factor = -1 # New inclinometer, y direction is opposite than the old PROMICE v2 inclinometer.
47
- boom_azimuth = 0 # Only relevant for PROMICE v2 stations
48
- columns = ['time','rec','p_l','p_u','t_l','rh_l','wspd_l','wdir_l',
49
- 't_u','rh_u','wspd_u','wdir_u','dsr', 'usr', 'dlr',
50
- 'ulr','t_rad','z_boom_l','z_boom_u','t_i_1','t_i_2','t_i_3','t_i_4',
51
- 't_i_5','t_i_6','t_i_7','t_i_8','t_i_9','t_i_10','t_i_11','tilt_x',
52
- 'tilt_y','rot','precip_l','SKIP_35','precip_u','SKIP_37',
53
- 'gps_time','gps_lat','gps_lon','gps_alt','gps_geoid','gps_geounit',
54
- 'gps_q','gps_numsat','gps_hdop','fan_dc_l','fan_dc_u','batt_v','t_log']
pypromice/test/test_email DELETED
@@ -1,75 +0,0 @@
1
- Delivered-To: geus.aws@gmail.com
2
- Received: by 2002:a50:fe91:0:b0:43b:5235:2b05 with SMTP id d17csp1741072edt;
3
- Mon, 25 Jul 2022 03:01:42 -0700 (PDT)
4
- X-Google-Smtp-Source: AGRyM1v1eQ1QVtVNUZEGRV6Ddi6Ah8lgtju0dNwaMATCWnhQHanPZ5Q8+SOZ7hUTRPbhXCinmMn/
5
- X-Received: by 2002:a05:6871:295:b0:10d:c587:d2c4 with SMTP id i21-20020a056871029500b0010dc587d2c4mr5977417oae.122.1658743302589;
6
- Mon, 25 Jul 2022 03:01:42 -0700 (PDT)
7
- ARC-Seal: i=1; a=rsa-sha256; t=1658743302; cv=none;
8
- d=google.com; s=arc-20160816;
9
- b=AqG/qXwM8Qjcgz4aw96KJGU/z4e7vC7yRPw0NVAIjc6MJ85H6F58GY89JNLDsVMkOw
10
- a0vZz/pWaz5o8wB5yZ+BBIdf2ImHzqYjUfqewvXS2xrt7hyE4BAyI4r40sJNmQoQzccF
11
- CZNHuWntgZw8ntW0nN3tX3ma26pbIBai1hr8QywAyluOugvDuMP7ckUysYqld4GhX4fW
12
- qBa0nprF7AC8gcoFpt0DQybj8tB/CVpXTPqnRhgvxtja1K24rdYwIcoizJBtMQw7VlM7
13
- mPOo8i7CSYJS7x7TFTgVPg4Ae85yCj6vYlvTICEWBkS389IDIZK5k8E10GN8GKQUN5DU
14
- wQug==
15
- ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
16
- h=message-id:mime-version:subject:to:from:reply-to:date;
17
- bh=OOCiGWkYYU7jainW/gAZNi1LX19KmanlKArRM9kkONE=;
18
- b=wRVS363YfkwDXW/cPlPFyMDqxf1Km7kyijJvgkYralUMAXll/qdx9JoIu3YbNk218S
19
- QK9bCGlRwNR2m9Do4e1K5D+BQOPTe120MFmlZa0HoyqkWhjBVNW/N81OawlNNkeeN+Y4
20
- QllhEC6yS6EhU2li95mu45g7X/sJrhJ1uPM4Bcl98Hf+wfXw5ZJbZbkNrc8zwmCtgyW6
21
- mnzq0M0QvhKXCXK6+YAUIANsPtxO1CfiFfnZggoMr5RT5+hPZw6OEn1aHgoDwsI3RLlG
22
- zpHuIDAfgo65ecGsLDQIgWt0RggVipvT6mKBifub9wnbGgnj9smWVPoItQj1Fl3ya8+S
23
- L9og==
24
- ARC-Authentication-Results: i=1; mx.google.com;
25
- spf=pass (google.com: domain of sbdservice@sbd.iridium.com designates 12.47.179.11 as permitted sender) smtp.mailfrom=sbdservice@sbd.iridium.com
26
- Return-Path: <sbdservice@sbd.iridium.com>
27
- Received: from istsbdmailsv05.sbd.iridium.com (outbound.sbd.iridium.com. [12.47.179.11])
28
- by mx.google.com with ESMTPS id u42-20020a056870702a00b000f333413e16si9950741oae.95.2022.07.25.03.01.42
29
- for <geus.aws@gmail.com>
30
- (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
31
- Mon, 25 Jul 2022 03:01:42 -0700 (PDT)
32
- Received-SPF: pass (google.com: domain of sbdservice@sbd.iridium.com designates 12.47.179.11 as permitted sender) client-ip=12.47.179.11;
33
- Authentication-Results: mx.google.com;
34
- spf=pass (google.com: domain of sbdservice@sbd.iridium.com designates 12.47.179.11 as permitted sender) smtp.mailfrom=sbdservice@sbd.iridium.com
35
- Received: from pps.filterd (istsbdmailsv05.sbd.iridium.com [127.0.0.1])
36
- by istsbdmailsv05.sbd.iridium.com (8.16.1.2/8.16.1.2) with SMTP id 26P9tBt9007175
37
- for <geus.aws@gmail.com>; Mon, 25 Jul 2022 10:01:41 GMT
38
- Date: Mon, 25 Jul 2022 10:01:41 GMT
39
- Received: from sbd.iridium.com (istsbdspp02.sbd.iridium.com [10.11.1.156])
40
- by istsbdmailsv05.sbd.iridium.com with ESMTP id 3hg8tfh9fm-1
41
- for <geus.aws@gmail.com>; Mon, 25 Jul 2022 10:01:41 +0000
42
- Reply-To: <sbdservice@sbd.iridium.com>
43
- From: sbdservice@sbd.iridium.com
44
- To: geus.aws@gmail.com
45
- Subject: SBD Msg From Unit: 300234061165160
46
- MIME-Version: 1.0
47
- Content-Type: multipart/mixed;boundary="SBD.Boundary.605592468"
48
- Message-ID: <3hg8tfh9fm-1@istsbdmailsv05.sbd.iridium.com>
49
- X-TUID: F2mWOvnGwvQ6
50
-
51
- SBM Message
52
- --SBD.Boundary.605592468
53
- Content-Type: text/plain;charset=US-ASCII
54
- Content-Disposition: inline
55
- Content-Transfer-Encoding: 7bit
56
-
57
- MOMSN: 36820
58
- MTMSN: 0
59
- Time of Session (UTC): Mon Jul 25 10:01:41 2022
60
- Session Status: 00 - Transfer OK
61
- Message Size (bytes): 27
62
-
63
- Unit Location: Lat = 67.00865 Long = -47.07785
64
- CEPradius = 5
65
-
66
-
67
-
68
- --SBD.Boundary.605592468
69
- Content-Type: application/x-zip-compressed; name="SBMmessage.sbd"
70
- Content-Disposition: attachment; filename="300234061165160_036820.sbd"
71
- Content-Transfer-Encoding: base64
72
-
73
- Hj0/zCBjJ5LMLj8/CwAAAABVYk+CxDOmM4h7
74
- --SBD.Boundary.605592468--
75
-
@@ -1,4 +0,0 @@
1
- type,expected_values,format,description,flags,notes
2
- 30,12,tfffffffffff,Promice 2015 summer message,,Test 1
3
- 32,12,tfffffffffff,Promice 2015 winter message,,Test 2
4
- 75,38,tfffffffffffFFfffgneffffffffffffffffff,THE VERSION-3!,don’t use,Test 3
@@ -1,7 +0,0 @@
1
- type_letter,bytes
2
- f,2
3
- l,4
4
- t,4
5
- g,4
6
- n,4
7
- e,4