pypromice 1.4.2__tar.gz → 1.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pypromice might be problematic. Click here for more details.
- {pypromice-1.4.2/src/pypromice.egg-info → pypromice-1.4.3}/PKG-INFO +1 -1
- {pypromice-1.4.2 → pypromice-1.4.3}/setup.py +1 -1
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/tx/tx.py +4 -4
- {pypromice-1.4.2 → pypromice-1.4.3/src/pypromice.egg-info}/PKG-INFO +1 -1
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice.egg-info/SOURCES.txt +0 -1
- pypromice-1.4.2/src/pypromice/tx/get_watsontx.py +0 -147
- {pypromice-1.4.2 → pypromice-1.4.3}/LICENSE.txt +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/MANIFEST.in +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/README.md +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/setup.cfg +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/get/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/get/get.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/get/get_promice_data.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/bufr_to_csv.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/bufr_utilities.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/create_bufr_files.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/get_bufr.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/make_metadata_csv.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/positions_seed.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/postprocess/real_time_utilities.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/L0toL1.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/L1toL2.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/L2toL3.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/aws.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/get_l2.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/get_l2tol3.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/join_l2.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/join_l3.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/load.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/resample.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/utilities.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/value_clipping.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/process/write.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/github_data_issues.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/percentiles/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/percentiles/compute_thresholds.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/percentiles/outlier_detector.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/percentiles/thresholds.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/qc/persistence.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/resources/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/resources/file_attributes.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/resources/variable_aliases_GC-Net.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/resources/variables.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/station_configuration.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/tx/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/tx/get_l0tx.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/tx/get_msg.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/tx/payload_formats.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/tx/payload_types.csv +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/utilities/__init__.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice/utilities/git.py +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice.egg-info/dependency_links.txt +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice.egg-info/entry_points.txt +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice.egg-info/requires.txt +0 -0
- {pypromice-1.4.2 → pypromice-1.4.3}/src/pypromice.egg-info/top_level.txt +0 -0
|
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
5
5
|
|
|
6
6
|
setuptools.setup(
|
|
7
7
|
name="pypromice",
|
|
8
|
-
version="1.4.
|
|
8
|
+
version="1.4.3",
|
|
9
9
|
author="GEUS Glaciology and Climate",
|
|
10
10
|
description="PROMICE/GC-Net data processing toolbox",
|
|
11
11
|
long_description=long_description,
|
|
@@ -894,10 +894,10 @@ def sortLines(in_file, out_file, replace_unsorted=True): #
|
|
|
894
894
|
# out_f.write(headers)
|
|
895
895
|
out_f.writelines(unique_lines)
|
|
896
896
|
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
897
|
+
# Replace input file with new sorted file
|
|
898
|
+
if replace_unsorted:
|
|
899
|
+
os.remove(in_file)
|
|
900
|
+
os.rename(out_file, in_file)
|
|
901
901
|
|
|
902
902
|
def addTail(in_file, out_dir, aws_name, header_names='', lines_limit=100):
|
|
903
903
|
'''Generate tails file from L0tx file
|
|
@@ -49,7 +49,6 @@ src/pypromice/resources/variables.csv
|
|
|
49
49
|
src/pypromice/tx/__init__.py
|
|
50
50
|
src/pypromice/tx/get_l0tx.py
|
|
51
51
|
src/pypromice/tx/get_msg.py
|
|
52
|
-
src/pypromice/tx/get_watsontx.py
|
|
53
52
|
src/pypromice/tx/payload_formats.csv
|
|
54
53
|
src/pypromice/tx/payload_types.csv
|
|
55
54
|
src/pypromice/tx/tx.py
|
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
|
-
"""
|
|
4
|
-
Created on Fri Jul 22 16:20:09 2022
|
|
5
|
-
|
|
6
|
-
Script to get L0tx transmission messages from the Watson River station using
|
|
7
|
-
the tx module
|
|
8
|
-
|
|
9
|
-
@author: Penelope How, pho@geus.dk
|
|
10
|
-
"""
|
|
11
|
-
from argparse import ArgumentParser
|
|
12
|
-
|
|
13
|
-
from configparser import ConfigParser
|
|
14
|
-
import os, imaplib, email, re
|
|
15
|
-
from glob import glob
|
|
16
|
-
from datetime import datetime
|
|
17
|
-
|
|
18
|
-
from pypromice.tx import getMail, L0tx, sortLines
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def parse_arguments_watson():
|
|
22
|
-
parser = ArgumentParser(description="AWS L0 transmission fetcher for Watson River measurements")
|
|
23
|
-
parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file')
|
|
24
|
-
parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file')
|
|
25
|
-
parser.add_argument('-o', '--outpath', default=None, type=str, required=False, help='Path where to write output (if given)')
|
|
26
|
-
parser.add_argument('-f', '--formats', default=None, type=str, required=False, help='Path to Payload format .csv file')
|
|
27
|
-
parser.add_argument('-t', '--types', default=None, type=str, required=False, help='Path to Payload type .csv file')
|
|
28
|
-
parser.add_argument('-u', '--uid', default=None, type=str, required=True, help='Last AWS uid .ini file')
|
|
29
|
-
args = parser.parse_args()
|
|
30
|
-
return args
|
|
31
|
-
|
|
32
|
-
#------------------------------------------------------------------------------
|
|
33
|
-
def get_watsontx():
|
|
34
|
-
"""Executed from the command line"""
|
|
35
|
-
args = parse_arguments_watson()
|
|
36
|
-
|
|
37
|
-
# Set payload formatter paths
|
|
38
|
-
formatter_file = args.formats
|
|
39
|
-
type_file = args.types
|
|
40
|
-
|
|
41
|
-
# Set credential paths
|
|
42
|
-
accounts_file = args.account
|
|
43
|
-
credentials_file = args.password
|
|
44
|
-
|
|
45
|
-
# Set last aws uid path
|
|
46
|
-
# last_uid = 1000000
|
|
47
|
-
uid_file = args.uid
|
|
48
|
-
|
|
49
|
-
# Set last aws uid path
|
|
50
|
-
with open(uid_file, 'r') as last_uid_f:
|
|
51
|
-
last_uid = int(last_uid_f.readline())
|
|
52
|
-
|
|
53
|
-
# Set output file directory
|
|
54
|
-
out_dir = args.outpath
|
|
55
|
-
if not os.path.exists(out_dir):
|
|
56
|
-
os.mkdir(out_dir)
|
|
57
|
-
|
|
58
|
-
#------------------------------------------------------------------------------
|
|
59
|
-
|
|
60
|
-
# Define accounts and credentials ini file paths
|
|
61
|
-
accounts_ini = ConfigParser()
|
|
62
|
-
accounts_ini.read_file(open(accounts_file))
|
|
63
|
-
accounts_ini.read(credentials_file)
|
|
64
|
-
|
|
65
|
-
# Get credentials
|
|
66
|
-
account = accounts_ini.get('aws', 'account')
|
|
67
|
-
server = accounts_ini.get('aws', 'server')
|
|
68
|
-
port = accounts_ini.getint('aws', 'port')
|
|
69
|
-
password = accounts_ini.get('aws', 'password')
|
|
70
|
-
if not password:
|
|
71
|
-
password = input('password for AWS email account: ')
|
|
72
|
-
print('AWS data from server %s, account %s' %(server, account))
|
|
73
|
-
|
|
74
|
-
#------------------------------------------------------------------------------
|
|
75
|
-
|
|
76
|
-
# Log in to email server
|
|
77
|
-
mail_server = imaplib.IMAP4_SSL(server, port)
|
|
78
|
-
typ, accountDetails = mail_server.login(account, password)
|
|
79
|
-
if typ != 'OK':
|
|
80
|
-
print('Not able to sign in!')
|
|
81
|
-
raise
|
|
82
|
-
|
|
83
|
-
# Grab new emails
|
|
84
|
-
result, data = mail_server.select(mailbox='"[Gmail]/All Mail"',
|
|
85
|
-
readonly=True)
|
|
86
|
-
print('mailbox contains %s messages' %data[0])
|
|
87
|
-
|
|
88
|
-
#------------------------------------------------------------------------------
|
|
89
|
-
|
|
90
|
-
# Get L0tx datalines from email transmissions
|
|
91
|
-
for uid, mail in getMail(mail_server, last_uid=last_uid):
|
|
92
|
-
message = email.message_from_string(mail)
|
|
93
|
-
try:
|
|
94
|
-
name = str(message.get_all('subject')[0])
|
|
95
|
-
d = datetime.strptime(message.get_all('date')[0],
|
|
96
|
-
'%a, %d %b %Y %H:%M:%S %z')
|
|
97
|
-
except:
|
|
98
|
-
name=None
|
|
99
|
-
d=None
|
|
100
|
-
|
|
101
|
-
if name and ('Watson' in name or 'GIOS' in name):
|
|
102
|
-
print(f'Watson/GIOS station message, {d.strftime("%Y-%m-%d %H:%M:%S")}')
|
|
103
|
-
|
|
104
|
-
l0 = L0tx(message, formatter_file, type_file,
|
|
105
|
-
sender_name=['emailrelay@konectgds.com','sbdservice'])
|
|
106
|
-
|
|
107
|
-
if l0.msg:
|
|
108
|
-
content, attachment = l0.getEmailBody()
|
|
109
|
-
attachment_name = str(attachment.get_filename())
|
|
110
|
-
out_fn = re.sub(r'\d*\.dat$', '', attachment_name) + '.txt'
|
|
111
|
-
out_path = os.sep.join((out_dir, out_fn))
|
|
112
|
-
|
|
113
|
-
print(f'Writing to {out_fn}')
|
|
114
|
-
print(l0.msg)
|
|
115
|
-
|
|
116
|
-
with open(out_path, mode='a') as out_f:
|
|
117
|
-
out_f.write(l0.msg + '\n')
|
|
118
|
-
|
|
119
|
-
#------------------------------------------------------------------------------
|
|
120
|
-
|
|
121
|
-
# Sort L0tx files and add tails
|
|
122
|
-
for f in glob(out_dir+'/*.txt'):
|
|
123
|
-
|
|
124
|
-
# Sort lines in L0tx file and remove duplicates
|
|
125
|
-
in_dirn, in_fn = os.path.split(f)
|
|
126
|
-
out_fn = 'sorted_' + in_fn
|
|
127
|
-
out_pn = os.sep.join((in_dirn, out_fn))
|
|
128
|
-
sortLines(f, out_pn)
|
|
129
|
-
|
|
130
|
-
# Close mail server if open
|
|
131
|
-
if 'mail_server' in locals():
|
|
132
|
-
print(f'\nClosing {account}')
|
|
133
|
-
mail_server.close()
|
|
134
|
-
resp = mail_server.logout()
|
|
135
|
-
assert resp[0].upper() == 'BYE'
|
|
136
|
-
|
|
137
|
-
# Write last aws uid to ini file
|
|
138
|
-
try:
|
|
139
|
-
with open(uid_file, 'w') as last_uid_f:
|
|
140
|
-
last_uid_f.write(uid)
|
|
141
|
-
except:
|
|
142
|
-
print(f'Could not write last uid {uid} to {uid_file}')
|
|
143
|
-
|
|
144
|
-
print('Finished')
|
|
145
|
-
|
|
146
|
-
if __name__ == "__main__":
|
|
147
|
-
get_watsontx()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|