alfrd 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alfrd/__init__.py +4 -0
- alfrd/lib.py +221 -0
- alfrd/util.py +177 -0
- alfrd-0.0.2.dist-info/LICENSE +29 -0
- alfrd-0.0.2.dist-info/METADATA +225 -0
- alfrd-0.0.2.dist-info/RECORD +9 -0
- alfrd-0.0.2.dist-info/WHEEL +5 -0
- alfrd-0.0.2.dist-info/entry_points.txt +2 -0
- alfrd-0.0.2.dist-info/top_level.txt +1 -0
alfrd/__init__.py
ADDED
alfrd/lib.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import gspread
|
|
2
|
+
from google.oauth2.service_account import Credentials
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import warnings
|
|
7
|
+
from alfrd import c
|
|
8
|
+
from gspread_formatting import ConditionalFormatRule, GridRange, BooleanCondition, BooleanRule, CellFormat, Color, get_conditional_format_rules
|
|
9
|
+
|
|
10
|
+
class GSC:
|
|
11
|
+
"""
|
|
12
|
+
Creates instance of google Google Spreadsheet Credential to open and update a worksheet
|
|
13
|
+
"""
|
|
14
|
+
def __init__(self, sid='', url='', key=f"{Path().home()}/.alfred/credentials.json", wid=0, wname=''):
|
|
15
|
+
"""
|
|
16
|
+
if sid is empty, uses url to get the spreadsheet id
|
|
17
|
+
"""
|
|
18
|
+
self.sid = sid
|
|
19
|
+
self.url = url
|
|
20
|
+
self.key = key
|
|
21
|
+
self.wid = wid
|
|
22
|
+
self.wname = wname
|
|
23
|
+
self.authorized = False
|
|
24
|
+
self.scopes = ["https://www.googleapis.com/auth/spreadsheets"]
|
|
25
|
+
self.creds = Credentials.from_service_account_file(key, scopes=self.scopes)
|
|
26
|
+
|
|
27
|
+
def auth(self):
|
|
28
|
+
self.client = gspread.authorize(self.creds)
|
|
29
|
+
self.authorized = True
|
|
30
|
+
|
|
31
|
+
def open(self):
|
|
32
|
+
if not self.authorized: self.auth()
|
|
33
|
+
if not self.sid:
|
|
34
|
+
regex = "([\w-]){44}"
|
|
35
|
+
sid_match = re.search(regex,self.url)
|
|
36
|
+
self.sid = str(sid_match.group())
|
|
37
|
+
|
|
38
|
+
self.spreadsheet = self.client.open_by_key(self.sid)
|
|
39
|
+
self.sheet = self.spreadsheet.get_worksheet(self.w) if not self.wname else self.spreadsheet.worksheet(self.wname)
|
|
40
|
+
self.df = pd.DataFrame(self.sheet.get_all_records(numericise_ignore=['all']))
|
|
41
|
+
print(f"{c['g']}Success!{c['x']}")
|
|
42
|
+
return self.df
|
|
43
|
+
|
|
44
|
+
def update(self, dataframe):
|
|
45
|
+
self.sheet.update([dataframe.columns.values.tolist()] + dataframe.values.tolist())
|
|
46
|
+
print(f"{c['g']}Updated!{c['x']}")
|
|
47
|
+
|
|
48
|
+
class LogFrame:
|
|
49
|
+
"""
|
|
50
|
+
Input
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
:gsc: Google Spreadsheet Credentials instance
|
|
54
|
+
:primary_value: the unique identifier of the row corrosponding to the primary_colname
|
|
55
|
+
:primary_colname: primary column name for unique identifier
|
|
56
|
+
:registered: keeps count of success and failed script runs in a tuple (count_success, count_failed)
|
|
57
|
+
|
|
58
|
+
"""
|
|
59
|
+
def __init__(self, gsc , primary_value='', primary_colname='FILE_NAME'):
|
|
60
|
+
self.gsc = gsc
|
|
61
|
+
self.df_sheet = self.gsc.df
|
|
62
|
+
self.primary_value = primary_value
|
|
63
|
+
self.primary_colname = primary_colname
|
|
64
|
+
self.working_col = ''
|
|
65
|
+
|
|
66
|
+
self.registered = 0,0 # (count_success, count_failed)
|
|
67
|
+
self.color ={'g': Color(red=0.56,green=0.77,blue=0.49),
|
|
68
|
+
'r': Color(red=0.8784314,green=0.4,blue=0.4),
|
|
69
|
+
'rh': Color(red=0.71,green=0.13,blue=0.0),
|
|
70
|
+
'rl': Color(red=0.98,green=0.63,blue=0.57),
|
|
71
|
+
'gl': Color(red=0.42,green=0.86,blue=0.31),
|
|
72
|
+
'gh': Color(red=0.42,green=0.60,blue=0.42)}
|
|
73
|
+
|
|
74
|
+
def col_data(self, colname='', data='', count=0, force=False, chk_colname=''):
|
|
75
|
+
"""
|
|
76
|
+
program to change the column data of dataframe
|
|
77
|
+
checks if primary_value exist and then change the value for empty cell with the given data.
|
|
78
|
+
|
|
79
|
+
set primary_colname - primary column name to find unique values corrosponding to the primary_value e.g fitsfile name
|
|
80
|
+
|
|
81
|
+
Input
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
:df_sheet: pandas dataframe to work on
|
|
85
|
+
:primary_value: the unique identifier of the row corrosponding to the primary_colname
|
|
86
|
+
:colname: column name to alter data
|
|
87
|
+
:data: data to fill in the corrosponding column for the corrosponding identifier row
|
|
88
|
+
:count: iterative count
|
|
89
|
+
:force: to force change the data in the column even if the cell is not empty.
|
|
90
|
+
:primary_colname: primary column name for unique identifier
|
|
91
|
+
:chk_colname: column name to check value for, if this is given no data is changed.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
---
|
|
95
|
+
|
|
96
|
+
count if chk_column is empty;
|
|
97
|
+
|
|
98
|
+
(count, column_value) corrosponding to the `chk_colname`
|
|
99
|
+
"""
|
|
100
|
+
colname = self.working_col if not colname else colname
|
|
101
|
+
if not self.primary_value : print(f"{c['y']}No primary value given{c['x']}")
|
|
102
|
+
primary_col_values = self.df_sheet[self.primary_colname].str.strip()
|
|
103
|
+
if (not force) or (primary_col_values.isin([self.primary_value]).any() and not self.df_sheet.loc[primary_col_values==self.primary_value,colname].count()):
|
|
104
|
+
|
|
105
|
+
if chk_colname :
|
|
106
|
+
if self.df_sheet.loc[primary_col_values==self.primary_value,chk_colname].count():
|
|
107
|
+
val = str(self.df_sheet.loc[primary_col_values==self.primary_value,chk_colname].values[0]).strip()
|
|
108
|
+
count+=1
|
|
109
|
+
return count, val
|
|
110
|
+
else:
|
|
111
|
+
return count, ''
|
|
112
|
+
else:
|
|
113
|
+
count+=1
|
|
114
|
+
self.df_sheet.loc[primary_col_values==self.primary_value,colname] = data
|
|
115
|
+
else:
|
|
116
|
+
print("not updating", self.primary_value, f"{self.df_sheet.loc[primary_col_values==self.primary_value,colname].values}")
|
|
117
|
+
return count
|
|
118
|
+
|
|
119
|
+
def isval_unique(self, colname=''):
|
|
120
|
+
"""
|
|
121
|
+
checks if the colname has unique values and returns boolean
|
|
122
|
+
"""
|
|
123
|
+
colname = self.working_col if not colname else colname
|
|
124
|
+
_, colv = self.col_data(colname='', data='', count=0, chk_colname=colname)
|
|
125
|
+
c = self.df_sheet[colname].value_counts()[colv]
|
|
126
|
+
r = False if int(c)!=1 else True
|
|
127
|
+
return r
|
|
128
|
+
|
|
129
|
+
def get_value(self, colname=''):
|
|
130
|
+
"""
|
|
131
|
+
gives the cell value for the colname returns string value
|
|
132
|
+
"""
|
|
133
|
+
colname = self.working_col if not colname else colname
|
|
134
|
+
_, colv = self.col_data(colname='', data='', count=0, chk_colname=colname)
|
|
135
|
+
r = str(colv).strip()
|
|
136
|
+
return r
|
|
137
|
+
|
|
138
|
+
def isvalue(self, value, colname=''):
|
|
139
|
+
"""
|
|
140
|
+
Returns boolean by matching value in cell
|
|
141
|
+
"""
|
|
142
|
+
v = self.get_value(colname)
|
|
143
|
+
return str(value) == str(v)
|
|
144
|
+
|
|
145
|
+
def put_value(self, value, colname='', count=0):
|
|
146
|
+
colname = self.working_col if not colname else colname
|
|
147
|
+
count = self.col_data(colname=colname, data=value, count=count)
|
|
148
|
+
return count
|
|
149
|
+
|
|
150
|
+
def update_sheet(self, count, failed, comment_col='Comment4', csvfile = 'df_sheet.csv'):
|
|
151
|
+
"""
|
|
152
|
+
updates the google sheet if there is atleast one new count/failed count for the update
|
|
153
|
+
"""
|
|
154
|
+
try:
|
|
155
|
+
if count - self.registered[0] or failed - self.registered[1]:
|
|
156
|
+
self.gsc.update(self.df_sheet)
|
|
157
|
+
self.registered = count, failed
|
|
158
|
+
else:
|
|
159
|
+
print('skipped')
|
|
160
|
+
except Exception as e:
|
|
161
|
+
print(f'failed to update on google sheet: {e}')
|
|
162
|
+
failed = self.col_data(colname=comment_col, data=f'failed:{e}', count=failed)
|
|
163
|
+
self.df_sheet.to_csv(csvfile)
|
|
164
|
+
|
|
165
|
+
def create_conditional_format(self, range, c='g', valtype='timeinmin', custom_clr=None):
|
|
166
|
+
clr = self.color[c] if not custom_clr else custom_clr
|
|
167
|
+
rule ={
|
|
168
|
+
'timeinmin' : ConditionalFormatRule(
|
|
169
|
+
ranges=[GridRange.from_a1_range(f'{range}', self.gsc.sheet)],
|
|
170
|
+
booleanRule=BooleanRule(
|
|
171
|
+
condition=BooleanCondition(
|
|
172
|
+
type='CUSTOM_FORMULA',values=[f'=AND(ISNUMBER(SEARCH("m", {range})), ISNUMBER(SEARCH("s", {range})))']),
|
|
173
|
+
format=CellFormat(backgroundColor=clr,
|
|
174
|
+
))),
|
|
175
|
+
'True' : ConditionalFormatRule(
|
|
176
|
+
ranges=[GridRange.from_a1_range(f'{range}', self.gsc.sheet)],
|
|
177
|
+
booleanRule=BooleanRule(
|
|
178
|
+
condition=BooleanCondition(
|
|
179
|
+
type='TEXT_CONTAINS',values=[f'True']),
|
|
180
|
+
format=CellFormat(backgroundColor=clr,
|
|
181
|
+
))),
|
|
182
|
+
'False' : ConditionalFormatRule(
|
|
183
|
+
ranges=[GridRange.from_a1_range(f'{range}', self.gsc.sheet)],
|
|
184
|
+
booleanRule=BooleanRule(
|
|
185
|
+
condition=BooleanCondition(
|
|
186
|
+
type='TEXT_CONTAINS',values=[f'False']),
|
|
187
|
+
format=CellFormat(backgroundColor=clr,
|
|
188
|
+
))),
|
|
189
|
+
'fail' : ConditionalFormatRule(
|
|
190
|
+
ranges=[GridRange.from_a1_range(f'{range}', self.gsc.sheet)],
|
|
191
|
+
booleanRule=BooleanRule(
|
|
192
|
+
condition=BooleanCondition(
|
|
193
|
+
type='TEXT_CONTAINS',values=[f'fail']),
|
|
194
|
+
format=CellFormat(backgroundColor=clr,
|
|
195
|
+
))),
|
|
196
|
+
}
|
|
197
|
+
return rule[valtype]
|
|
198
|
+
|
|
199
|
+
def create_rule(self, range, type='TEXT_CONTAINS', value='True', c='g', custom_clr=None):
|
|
200
|
+
clr = self.color[c] if not custom_clr else custom_clr
|
|
201
|
+
return ConditionalFormatRule(
|
|
202
|
+
ranges=[GridRange.from_a1_range(f'{range}', self.gsc.sheet)],
|
|
203
|
+
booleanRule=BooleanRule(
|
|
204
|
+
condition=BooleanCondition(
|
|
205
|
+
type=type, values=[value]),
|
|
206
|
+
format=CellFormat(backgroundColor=clr,
|
|
207
|
+
)))
|
|
208
|
+
|
|
209
|
+
def create_color(self, r=0.56,g=0.77,b=0.49):
|
|
210
|
+
return Color(red=r,green=g,blue=b)
|
|
211
|
+
|
|
212
|
+
def add_conditional_format(self, *new_rules):
|
|
213
|
+
rules = get_conditional_format_rules(self.gsc.sheet)
|
|
214
|
+
for rule in new_rules:
|
|
215
|
+
rules.append(rule)
|
|
216
|
+
rules.save()
|
|
217
|
+
|
|
218
|
+
def clear_conditional_format(self,):
|
|
219
|
+
rules = get_conditional_format_rules(self.gsc.sheet)
|
|
220
|
+
rules.clear()
|
|
221
|
+
rules.save()
|
alfrd/util.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import os
|
|
4
|
+
import subprocess, glob, shutil, time
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
|
|
7
|
+
def read_inputfile(folder,inputfile='.inp'):
|
|
8
|
+
"""Read the input file and return a dictionary with the parameters.
|
|
9
|
+
|
|
10
|
+
Returns
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
(params, files, input_folder)
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
params = defaultdict(list)
|
|
17
|
+
input_folder= None
|
|
18
|
+
# params['array_type']='generic'
|
|
19
|
+
|
|
20
|
+
files=glob.glob(f'{folder}/*{inputfile}',recursive=True)
|
|
21
|
+
if not files: files=glob.glob(f'{folder}/*/*{inputfile}',recursive=False)
|
|
22
|
+
if not files: files=glob.glob(f'{folder}/*/*/*{inputfile}',recursive=False)
|
|
23
|
+
if files:
|
|
24
|
+
input_folder = str(Path(files[-1]).parent) + '/'
|
|
25
|
+
for filepath in files:
|
|
26
|
+
if '.inp' in filepath:
|
|
27
|
+
with open(filepath,'r') as f:
|
|
28
|
+
pr=f.read().splitlines()
|
|
29
|
+
for p in pr:
|
|
30
|
+
if '#' in p:
|
|
31
|
+
continue
|
|
32
|
+
elif '=' in p:
|
|
33
|
+
k,v=p.split('=')
|
|
34
|
+
try:
|
|
35
|
+
v=int(v)
|
|
36
|
+
except:
|
|
37
|
+
try:
|
|
38
|
+
v=float(v)
|
|
39
|
+
except:
|
|
40
|
+
v=str(v).strip()
|
|
41
|
+
v = v.lower() == 'true' if (any(boolv == v.lower() for boolv in ['true', 'false'])) else v
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# elif '//' in v or ';' in v: v=re.split('\\|;', str(v))
|
|
45
|
+
params[k.strip()]=v
|
|
46
|
+
return params, files, input_folder
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def find_size(fitsfile):
|
|
50
|
+
size = np.round(Path(fitsfile).stat().st_size/(1024*1024),2)
|
|
51
|
+
if size >= 1024.0 :
|
|
52
|
+
size = size/1024
|
|
53
|
+
size = f"{np.round(size, 2)} GB"
|
|
54
|
+
else:
|
|
55
|
+
size = f"{np.round(size, 2)} MB"
|
|
56
|
+
return size
|
|
57
|
+
|
|
58
|
+
def find_project(fitsfile):
|
|
59
|
+
project = str(Path(fitsfile).parent.name)
|
|
60
|
+
return str(project)
|
|
61
|
+
|
|
62
|
+
def build_path(filepath):
|
|
63
|
+
"""
|
|
64
|
+
This builds new file path by adding _{n+1} if there exists one with similar name,
|
|
65
|
+
"""
|
|
66
|
+
opt = filepath
|
|
67
|
+
if Path(filepath).exists():
|
|
68
|
+
numb = 1
|
|
69
|
+
while Path(filepath).exists():
|
|
70
|
+
filepath = "{0}_{2}{1}".format(
|
|
71
|
+
*(Path(opt).parent / Path(opt).stem, Path(opt).suffix,numb))
|
|
72
|
+
try :
|
|
73
|
+
if Path(filepath).exists():
|
|
74
|
+
numb += 1
|
|
75
|
+
except:
|
|
76
|
+
pass
|
|
77
|
+
return filepath
|
|
78
|
+
|
|
79
|
+
def symlink_bywd(wd, fitsfile, create=True):
|
|
80
|
+
rawsymlink = f"{wd}/raw/{Path(fitsfile).name}"
|
|
81
|
+
Path(f"{wd}/raw").mkdir(parents=True, exist_ok=True)
|
|
82
|
+
if create : os.symlink(fitsfile, rawsymlink)
|
|
83
|
+
return rawsymlink
|
|
84
|
+
|
|
85
|
+
def dir_for_project(fitsfile, tdir='/data/avi/reductions/100test/', ifolder='/data/avi/gh/picard/src/picard/input_template/', create=True, splitted=False):
|
|
86
|
+
"""
|
|
87
|
+
looks for wd using project name
|
|
88
|
+
"""
|
|
89
|
+
new = True
|
|
90
|
+
segment = find_project(fitsfile)
|
|
91
|
+
wd = f"{tdir}{segment}/wd"
|
|
92
|
+
wd_ifolder = None
|
|
93
|
+
lookfile = fitsfile if not splitted else f"{str(Path(fitsfile).stem)}_split{Path(fitsfile).suffix}"
|
|
94
|
+
if create:
|
|
95
|
+
if not Path(f"{wd}/raw/{Path(lookfile).name}").exists():
|
|
96
|
+
wd = build_path(wd)
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
rawsymlink = symlink_bywd(wd, fitsfile)
|
|
100
|
+
|
|
101
|
+
wd_ifolder = f'{wd}/input_template/'
|
|
102
|
+
if not Path(wd_ifolder).exists():shutil.copytree(ifolder,wd_ifolder)
|
|
103
|
+
except Exception as e:
|
|
104
|
+
print(f"exists? : {segment} : {e}")
|
|
105
|
+
new = False
|
|
106
|
+
|
|
107
|
+
new = False
|
|
108
|
+
else:
|
|
109
|
+
possible_file = glob.glob(f'{wd}*/raw/{Path(lookfile).name}')
|
|
110
|
+
|
|
111
|
+
if len(possible_file):
|
|
112
|
+
wd_ifolder = Path(possible_file[0]).parent.parent / "input_template"
|
|
113
|
+
new = False
|
|
114
|
+
return wd_ifolder, new
|
|
115
|
+
|
|
116
|
+
def del_extra_wdfolder(wd_ifolder, fitsfile):
|
|
117
|
+
"""
|
|
118
|
+
useful when for a project name there are more folders created by mistakes,
|
|
119
|
+
e.g wd_1/ wd_2/ instead of just wd/
|
|
120
|
+
|
|
121
|
+
Note: assumes wd_1, wd_2 etc are created by mistake
|
|
122
|
+
"""
|
|
123
|
+
if 'wd_' in str(wd_ifolder):
|
|
124
|
+
subprocess.run(['rm','-rf', str(Path(wd_ifolder).parent.parent)])
|
|
125
|
+
wd_ifolder, new= dir_for_project(fitsfile)
|
|
126
|
+
print('created',wd_ifolder)
|
|
127
|
+
|
|
128
|
+
def latest_file(path: Path, pattern: str = "*"):
|
|
129
|
+
"""
|
|
130
|
+
to get the last file that was generated, this can be useful for getting any new logfiles.
|
|
131
|
+
"""
|
|
132
|
+
files = path.glob(pattern)
|
|
133
|
+
lastf = Path('')
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
lastf = max(files, key=lambda x: x.stat().st_ctime)
|
|
137
|
+
finally:
|
|
138
|
+
return lastf
|
|
139
|
+
|
|
140
|
+
def timeinmin(td):
|
|
141
|
+
"""
|
|
142
|
+
convert timdedelta in XXmYYs format
|
|
143
|
+
"""
|
|
144
|
+
tdm,tds = 0,0
|
|
145
|
+
if td>=60:
|
|
146
|
+
tdm = td//60
|
|
147
|
+
tds = td%60
|
|
148
|
+
else:
|
|
149
|
+
tds = td
|
|
150
|
+
ret_time = f"{int(tdm)}m{np.round(tds,1)}s"
|
|
151
|
+
|
|
152
|
+
return ret_time
|
|
153
|
+
|
|
154
|
+
def del_fl(ifolder, count, fl='*ms*', rm=False):
|
|
155
|
+
"""
|
|
156
|
+
delete files from the input folder
|
|
157
|
+
"""
|
|
158
|
+
filefound = glob.glob(f"{ifolder}/../{fl}")
|
|
159
|
+
if len(filefound):
|
|
160
|
+
cmd = ['rm','-rf']
|
|
161
|
+
cmd.extend(str(Path(ifolder.parent) / " ".join(filefound)).split(' '))
|
|
162
|
+
print(cmd)
|
|
163
|
+
count+=1
|
|
164
|
+
if rm: subprocess.run(cmd)
|
|
165
|
+
return count
|
|
166
|
+
|
|
167
|
+
def build_logpath(wd_ifolder):
|
|
168
|
+
"""
|
|
169
|
+
builds rpicard and casa style log files for the current time
|
|
170
|
+
"""
|
|
171
|
+
thisdate = time.strftime('%F-%T', time.gmtime())
|
|
172
|
+
thisdate = thisdate.replace(':', '_')
|
|
173
|
+
|
|
174
|
+
errlogf = Path(wd_ifolder).parent / f'mpi_and_err.out_{thisdate}'
|
|
175
|
+
casalogf = Path(wd_ifolder).parent / f'casa.log_{thisdate}'
|
|
176
|
+
|
|
177
|
+
return str(errlogf), str(casalogf)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024, Avinash Kumar
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: alfrd
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Automated Logical FRamework for script execution Dynamically(ALFRD)
|
|
5
|
+
Home-page: https://github.com/avialxee/alfrd/
|
|
6
|
+
Author: Avinash Kumar
|
|
7
|
+
Author-email: avialxee@gmail.com
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/avialxee/alfrd/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: google-api-python-client
|
|
18
|
+
Requires-Dist: google-auth-httplib2
|
|
19
|
+
Requires-Dist: google-auth-oauthlib
|
|
20
|
+
Requires-Dist: gspread
|
|
21
|
+
Requires-Dist: gspread-formatting
|
|
22
|
+
Requires-Dist: pandas
|
|
23
|
+
Requires-Dist: numpy
|
|
24
|
+
Requires-Dist: psutil
|
|
25
|
+
Requires-Dist: protobuf ==3.19.6
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest >=3.7 ; extra == 'dev'
|
|
28
|
+
|
|
29
|
+
# ALFRD : Automated Logical FRamework for executing scripts Dynamically
|
|
30
|
+
|
|
31
|
+
This program is written for the [SMILE project](smilescience.info) supported by the ERC starting grant, particularly with following purposes in mind:
|
|
32
|
+
|
|
33
|
+
- Communicate with google spreadsheets and update progress periodically when required.
|
|
34
|
+
- Run pipeline based on the spreadsheet progress/requirements.
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
### 1. Create API credentials on Google Cloud
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
A similar guide is available [here](https://www.cybrosys.com/blog/how-to-use-gspread-python-api)
|
|
41
|
+
|
|
42
|
+
NOTE: In order to successfully create a google console project a billing detail is usually required. But the sheets API service is available for free, refer [here](https://developers.google.com/sheets/api/limits)
|
|
43
|
+
|
|
44
|
+
- step 1:
|
|
45
|
+
Go to https://console.cloud.google.com/
|
|
46
|
+
|
|
47
|
+
- step 2:
|
|
48
|
+
click on the drop-down to create a new project
|
|
49
|
+
- can choose organization or leave on default
|
|
50
|
+
|
|
51
|
+
- step 3
|
|
52
|
+
Search in the top bar (or press / ) and type : "Google sheets api"
|
|
53
|
+
|
|
54
|
+
- step 4
|
|
55
|
+
In the search results - under Marketplace select the first result which should be the same : Google sheets api
|
|
56
|
+
|
|
57
|
+
- step 5
|
|
58
|
+
Enable the service In the product details page
|
|
59
|
+
|
|
60
|
+
- step 6
|
|
61
|
+
Now select create credentials > Application Data
|
|
62
|
+
|
|
63
|
+
- step 7
|
|
64
|
+
Create an account name
|
|
65
|
+
Select create and continue
|
|
66
|
+
|
|
67
|
+
- step 8
|
|
68
|
+
Search and select "Editor" in role > Continue
|
|
69
|
+
|
|
70
|
+
- step 9
|
|
71
|
+
Skip next optional step
|
|
72
|
+
Select Done
|
|
73
|
+
|
|
74
|
+
- step 10
|
|
75
|
+
The Credentials are successfully created
|
|
76
|
+
Select "Credentials" on the left menu
|
|
77
|
+
|
|
78
|
+
- step 11
|
|
79
|
+
select/edit account that was just created
|
|
80
|
+
also copy the email address that is shown
|
|
81
|
+
|
|
82
|
+
- step 12
|
|
83
|
+
Select keys tab > Add keys > Create New Keys > JSON > save
|
|
84
|
+
|
|
85
|
+
- step 13
|
|
86
|
+
Go to the Google spreadsheet and "share" the sheet to the email address that was copied, as Editor.
|
|
87
|
+
|
|
88
|
+
### 2. Installing
|
|
89
|
+
|
|
90
|
+
- Download [ALFRD](https://github.com/avialxee/alfrd) and unzip / Or
|
|
91
|
+
`git clone https://github.com/avialxee/alfrd`
|
|
92
|
+
|
|
93
|
+
- install alfrd
|
|
94
|
+
```
|
|
95
|
+
cd alfrd/
|
|
96
|
+
pip install .
|
|
97
|
+
```
|
|
98
|
+
this should install alfrd and all the dependencies automatically.
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
### 3. Using ALFRD
|
|
102
|
+
|
|
103
|
+
##### Example 1 : Initializing and creating instance
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from alfrd.lib import GSC, LogFrame
|
|
107
|
+
from alfrd.util import timeinmin, read_inputfile
|
|
108
|
+
|
|
109
|
+
url='https://spreadsheet/link'
|
|
110
|
+
worksheet='worksheet-name'
|
|
111
|
+
|
|
112
|
+
gsc = GSC(url=url, wname=worksheet, key='path/to/json/file') # default path for key = home/usr/.alfred
|
|
113
|
+
_ = gsc.open()
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
##### Example 2: Inititalize the framework
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
lf = LogFrame(gsc)
|
|
120
|
+
lf.primary_colname = 'FILE_NAME'
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
##### Example 3: Run - Iterate for each row
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from pathlib import Path
|
|
127
|
+
import time, subprocess
|
|
128
|
+
import glob
|
|
129
|
+
|
|
130
|
+
count,failed=0,0
|
|
131
|
+
allfiles=[]
|
|
132
|
+
folder_for_fits ='path/to/allfits'
|
|
133
|
+
|
|
134
|
+
allfiles.extend(glob.glob(f"{folder_for_fits}*/*fits"))
|
|
135
|
+
|
|
136
|
+
for fitsfile in allfiles:
|
|
137
|
+
fitsfile_name = Path(fitsfile).name
|
|
138
|
+
lf.primary_value = fitsfile_name
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
##### Example 4: using column logic and adding values on success/failures
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
if lf.isvalue(value='True', colname='TSYS') and lf.isval_unique('Project'):
|
|
145
|
+
|
|
146
|
+
lf.working_col = 'Comment1' # working column
|
|
147
|
+
print(lf.get_value())
|
|
148
|
+
|
|
149
|
+
result = 'test'
|
|
150
|
+
if result :
|
|
151
|
+
count = lf.put_value(result, count=count)
|
|
152
|
+
else:
|
|
153
|
+
failed = lf.put_value('failed: logfile_path', count=failed)
|
|
154
|
+
|
|
155
|
+
print(lf.get_value())
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
##### Example 5: Script execution
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
def run_picard(cmd):
|
|
162
|
+
t0 = time.time()
|
|
163
|
+
subprocess.run(cmd)
|
|
164
|
+
t1 = time.time()
|
|
165
|
+
td = timeinmin(t1-t0)
|
|
166
|
+
return td
|
|
167
|
+
|
|
168
|
+
def scripted_picard_1(wd_ifolder, count, failed, col='fits to ms'):
|
|
169
|
+
"""
|
|
170
|
+
converts fitsfile to ms file; checks if conversion was successful; skips if file exists; logs in the spreadsheet;
|
|
171
|
+
"""
|
|
172
|
+
td=0
|
|
173
|
+
cmd=["picard",'-n','10',"-l","e",'--input',wd_ifolder]
|
|
174
|
+
|
|
175
|
+
params, files, input_folder = read_inputfile(wd_ifolder, "observation.inp")
|
|
176
|
+
|
|
177
|
+
if not Path(f"{wd_ifolder}").exists():
|
|
178
|
+
print('input folder missing..')
|
|
179
|
+
elif not Path(f"{wd_ifolder}/../{params['ms_name']}").exists():
|
|
180
|
+
td = run_picard(cmd)
|
|
181
|
+
count = lf.put_value(colname=col, value=td, count=count)
|
|
182
|
+
elif Path(f"{wd_ifolder}/../{params['ms_name']}").exists():
|
|
183
|
+
print('skipped')
|
|
184
|
+
|
|
185
|
+
if not Path(f"{wd_ifolder}/../{params['ms_name']}").exists():
|
|
186
|
+
count-=1
|
|
187
|
+
failed = lf.put_value(colname=col, value='failed',count=failed)
|
|
188
|
+
return td, count, failed
|
|
189
|
+
|
|
190
|
+
if lf.isvalue(value='True', colname='TSYS') and lf.isval_unique('Project'):
|
|
191
|
+
lf.working_col = 'Comment1'
|
|
192
|
+
ttaken, count, failed = scripted_picard_1(wd_ifolder='path/to/wd/input_template', count=count, failed=failed )
|
|
193
|
+
|
|
194
|
+
print(lf.get_value(colname='fits to ms'))
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
##### Example 6: Update the sheet
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
lf.update_sheet(count=count, failed=failed, csvfile='df_sheet.csv') # if updating the sheet fails, a copy of the dataframe is saved locally at the csvfile path.
|
|
201
|
+
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
##### Example 7: Conditional Formatting
|
|
206
|
+
need to run only once.
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
r1 = lf.create_rule(range='G2:G105', type='TEXT_CONTAINS' ,value='False', c='r') # check if TSYS == False --> background color = red
|
|
210
|
+
r2 = lf.create_rule(range='G2:G105', type="TEXT_CONTAINS", value='True', custom_clr=lf.create_color(0.56, 0.77, 0.49))
|
|
211
|
+
r3 = lf.create_conditional_format(range='F2:F105', c='g', valtype='timeinmin') # check if value in XXmYYs format --> background color = green
|
|
212
|
+
|
|
213
|
+
r4 = lf.create_conditional_format(range='F2:F105', c='r', valtype='fail') # check if value contains fail --> background color = red
|
|
214
|
+
r4 = lf.create_rule(range='F2:F105', type='TEXT_CONTAINS', c='r', value='fail') # similar to above
|
|
215
|
+
print(r1,r2,r3,r4)
|
|
216
|
+
lf.add_conditional_format(r1, r2, r3, r4)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### Attribution
|
|
220
|
+
|
|
221
|
+
When using ALFRD, please add a link to this repository in a footnote.
|
|
222
|
+
|
|
223
|
+
### Acknowledgement
|
|
224
|
+
|
|
225
|
+
ALFRD was developed within the "Search for Milli-Lenses" (SMILE) project. SMILE has received funding from the European Research Council (ERC) under the HORIZON ERC Grants 2021 programme (grant agreement No. 101040021).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
alfrd/__init__.py,sha256=SJZVVn7e9_tMdwHpDndI9GSGy1MSHLP1mgwol5iU9qU,130
|
|
2
|
+
alfrd/lib.py,sha256=TnZ5GRudvY-M_oVQt42Ha624xNPkEwQpi7jyCefzCLY,9704
|
|
3
|
+
alfrd/util.py,sha256=c0pTwaS6jvUl5TtSY0d6jtGkVTRfwANdtr6gmmCO5ec,5848
|
|
4
|
+
alfrd-0.0.2.dist-info/LICENSE,sha256=2f_9BzkG3cCAEASv3Cg3PKxK5GNALO-9CCNyOqG9YIQ,1520
|
|
5
|
+
alfrd-0.0.2.dist-info/METADATA,sha256=N68Z3k0uPiy7J2DYbtQOCIckyUVIilBfZV6cRFIeInM,7216
|
|
6
|
+
alfrd-0.0.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
7
|
+
alfrd-0.0.2.dist-info/entry_points.txt,sha256=u7BNh1wsMktmaH00vD-oPQ-4yRy5KK7YjWcWttIddRw,36
|
|
8
|
+
alfrd-0.0.2.dist-info/top_level.txt,sha256=PPfCpx9_2y7jc7hnn3BQwXEJXyylBRq54LVpy8Fqn7k,6
|
|
9
|
+
alfrd-0.0.2.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
alfrd
|