corebridge 0.2.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corebridge/__init__.py +1 -0
- corebridge/_modidx.py +64 -0
- corebridge/aicorebridge.py +278 -0
- corebridge/core.py +288 -0
- corebridge/rscript.py +312 -0
- corebridge-0.2.24.dist-info/LICENSE +201 -0
- corebridge-0.2.24.dist-info/METADATA +207 -0
- corebridge-0.2.24.dist-info/RECORD +11 -0
- corebridge-0.2.24.dist-info/WHEEL +5 -0
- corebridge-0.2.24.dist-info/entry_points.txt +2 -0
- corebridge-0.2.24.dist-info/top_level.txt +1 -0
corebridge/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.24"
|
corebridge/_modidx.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Autogenerated by nbdev
|
|
2
|
+
|
|
3
|
+
d = { 'settings': { 'branch': 'main',
|
|
4
|
+
'doc_baseurl': '/corebridge',
|
|
5
|
+
'doc_host': 'https://fenke.github.io',
|
|
6
|
+
'git_url': 'https://github.com/fenke/corebridge',
|
|
7
|
+
'lib_path': 'corebridge'},
|
|
8
|
+
'syms': { 'corebridge.aicorebridge': { 'corebridge.aicorebridge.AICoreModule': ( 'aicorebridge.html#aicoremodule',
|
|
9
|
+
'corebridge/aicorebridge.py'),
|
|
10
|
+
'corebridge.aicorebridge.AICoreModule.__init__': ( 'aicorebridge.html#aicoremodule.__init__',
|
|
11
|
+
'corebridge/aicorebridge.py'),
|
|
12
|
+
'corebridge.aicorebridge.AICoreModule._init_processor': ( 'aicorebridge.html#aicoremodule._init_processor',
|
|
13
|
+
'corebridge/aicorebridge.py'),
|
|
14
|
+
'corebridge.aicorebridge.AICoreModule.call_processor': ( 'aicorebridge.html#aicoremodule.call_processor',
|
|
15
|
+
'corebridge/aicorebridge.py'),
|
|
16
|
+
'corebridge.aicorebridge.AICoreModule.get_call_data': ( 'aicorebridge.html#aicoremodule.get_call_data',
|
|
17
|
+
'corebridge/aicorebridge.py'),
|
|
18
|
+
'corebridge.aicorebridge.AICoreModule.get_callargs': ( 'aicorebridge.html#aicoremodule.get_callargs',
|
|
19
|
+
'corebridge/aicorebridge.py'),
|
|
20
|
+
'corebridge.aicorebridge.AICoreModule.infer': ( 'aicorebridge.html#aicoremodule.infer',
|
|
21
|
+
'corebridge/aicorebridge.py'),
|
|
22
|
+
'corebridge.aicorebridge.AICoreModule.init_annotated_param': ( 'aicorebridge.html#aicoremodule.init_annotated_param',
|
|
23
|
+
'corebridge/aicorebridge.py'),
|
|
24
|
+
'corebridge.aicorebridge.build_historic_args': ( 'aicorebridge.html#build_historic_args',
|
|
25
|
+
'corebridge/aicorebridge.py')},
|
|
26
|
+
'corebridge.core': { 'corebridge.core.init_console_logging': ('core.html#init_console_logging', 'corebridge/core.py'),
|
|
27
|
+
'corebridge.core.pop_nan_values': ('core.html#pop_nan_values', 'corebridge/core.py'),
|
|
28
|
+
'corebridge.core.set_time_index_zone': ('core.html#set_time_index_zone', 'corebridge/core.py'),
|
|
29
|
+
'corebridge.core.timeseries_dataframe': ('core.html#timeseries_dataframe', 'corebridge/core.py'),
|
|
30
|
+
'corebridge.core.timeseries_dataframe_from_datadict': ( 'core.html#timeseries_dataframe_from_datadict',
|
|
31
|
+
'corebridge/core.py'),
|
|
32
|
+
'corebridge.core.timeseries_dataframe_resample': ( 'core.html#timeseries_dataframe_resample',
|
|
33
|
+
'corebridge/core.py'),
|
|
34
|
+
'corebridge.core.timeseries_dataframe_to_datadict': ( 'core.html#timeseries_dataframe_to_datadict',
|
|
35
|
+
'corebridge/core.py')},
|
|
36
|
+
'corebridge.rscript': { 'corebridge.rscript.calc_hash_from_data_files': ( 'rscriptbridge.html#calc_hash_from_data_files',
|
|
37
|
+
'corebridge/rscript.py'),
|
|
38
|
+
'corebridge.rscript.calc_hash_from_files': ( 'rscriptbridge.html#calc_hash_from_files',
|
|
39
|
+
'corebridge/rscript.py'),
|
|
40
|
+
'corebridge.rscript.calc_hash_from_flowobject': ( 'rscriptbridge.html#calc_hash_from_flowobject',
|
|
41
|
+
'corebridge/rscript.py'),
|
|
42
|
+
'corebridge.rscript.calc_hash_from_input_files': ( 'rscriptbridge.html#calc_hash_from_input_files',
|
|
43
|
+
'corebridge/rscript.py'),
|
|
44
|
+
'corebridge.rscript.check_rscript_lib': ( 'rscriptbridge.html#check_rscript_lib',
|
|
45
|
+
'corebridge/rscript.py'),
|
|
46
|
+
'corebridge.rscript.check_rscript_libs': ( 'rscriptbridge.html#check_rscript_libs',
|
|
47
|
+
'corebridge/rscript.py'),
|
|
48
|
+
'corebridge.rscript.check_script_inputs': ( 'rscriptbridge.html#check_script_inputs',
|
|
49
|
+
'corebridge/rscript.py'),
|
|
50
|
+
'corebridge.rscript.check_script_output': ( 'rscriptbridge.html#check_script_output',
|
|
51
|
+
'corebridge/rscript.py'),
|
|
52
|
+
'corebridge.rscript.generate_checksum_file': ( 'rscriptbridge.html#generate_checksum_file',
|
|
53
|
+
'corebridge/rscript.py'),
|
|
54
|
+
'corebridge.rscript.get_asset_path': ('rscriptbridge.html#get_asset_path', 'corebridge/rscript.py'),
|
|
55
|
+
'corebridge.rscript.get_rscript_env': ('rscriptbridge.html#get_rscript_env', 'corebridge/rscript.py'),
|
|
56
|
+
'corebridge.rscript.get_rscript_libpath': ( 'rscriptbridge.html#get_rscript_libpath',
|
|
57
|
+
'corebridge/rscript.py'),
|
|
58
|
+
'corebridge.rscript.get_save_path': ('rscriptbridge.html#get_save_path', 'corebridge/rscript.py'),
|
|
59
|
+
'corebridge.rscript.install_R_package_wait': ( 'rscriptbridge.html#install_r_package_wait',
|
|
60
|
+
'corebridge/rscript.py'),
|
|
61
|
+
'corebridge.rscript.run_rscript_nowait': ( 'rscriptbridge.html#run_rscript_nowait',
|
|
62
|
+
'corebridge/rscript.py'),
|
|
63
|
+
'corebridge.rscript.run_rscript_wait': ( 'rscriptbridge.html#run_rscript_wait',
|
|
64
|
+
'corebridge/rscript.py')}}}
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_aicorebridge.ipynb.
|
|
2
|
+
|
|
3
|
+
# %% auto 0
|
|
4
|
+
__all__ = ['syslog', 'annotated_arg_builders', 'build_historic_args', 'AICoreModule']
|
|
5
|
+
|
|
6
|
+
# %% ../nbs/01_aicorebridge.ipynb 4
|
|
7
|
+
import typing
|
|
8
|
+
import logging
|
|
9
|
+
import traceback
|
|
10
|
+
import inspect
|
|
11
|
+
import datetime
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import pandas, pandas as pd
|
|
15
|
+
import numpy, numpy as np
|
|
16
|
+
|
|
17
|
+
from dateutil import parser
|
|
18
|
+
from fastcore.basics import patch_to, patch
|
|
19
|
+
from .core import *
|
|
20
|
+
from . import __version__
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# %% ../nbs/01_aicorebridge.ipynb 5
|
|
24
|
+
syslog = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
# %% ../nbs/01_aicorebridge.ipynb 6
|
|
27
|
+
try:
|
|
28
|
+
print(f"Loading {__name__} {__version__} from {__file__}")
|
|
29
|
+
except:
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
# %% ../nbs/01_aicorebridge.ipynb 8
|
|
33
|
+
def build_historic_args(data:pd.DataFrame, history:dict|list) -> dict:
|
|
34
|
+
"""Create a timeseries DataFrame from historic data defined in `history`.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
data : pd.DataFrame
|
|
39
|
+
The input time-series DataFrame.
|
|
40
|
+
history : dict or list of dicts
|
|
41
|
+
Historic data definition, each item in the list is a dictionary with a startDate key to set the start of a section of historic data in the result and a column-value pair for each of the columns in the
|
|
42
|
+
|
|
43
|
+
Returns
|
|
44
|
+
-------
|
|
45
|
+
historic_data : dict
|
|
46
|
+
Historic data in dictionary format where keys are column names and values are the historic values as numpy array.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
if not history:
|
|
50
|
+
return {}
|
|
51
|
+
|
|
52
|
+
if isinstance(history, dict):
|
|
53
|
+
return history
|
|
54
|
+
|
|
55
|
+
if not isinstance(history, list):
|
|
56
|
+
return {}
|
|
57
|
+
|
|
58
|
+
if isinstance(data, pd.DataFrame):
|
|
59
|
+
dates = data.index.astype(np.int64).astype(np.float64) / 1e9
|
|
60
|
+
dates = dates.to_numpy()
|
|
61
|
+
elif data.dtype.names is not None:
|
|
62
|
+
dates = data.view(dtype=np.float64).reshape(data.shape[0],len(data.dtype))[:,0]
|
|
63
|
+
else:
|
|
64
|
+
dates = data[:,0]
|
|
65
|
+
dates = dates.astype(np.int64)
|
|
66
|
+
|
|
67
|
+
columns = list(set([K for I in history for K in I.keys() if K != 'startDate']))
|
|
68
|
+
column_data = {K:np.full(len(dates), np.nan, dtype=np.float64) for K in columns}
|
|
69
|
+
|
|
70
|
+
for I in history:
|
|
71
|
+
date = parser.parse(str((I.pop('startDate','2000-01-01T00:00:00+00:00')))).timestamp()
|
|
72
|
+
mask = np.greater_equal(dates, date)
|
|
73
|
+
for K,V in I.items():
|
|
74
|
+
column_data[K][mask] = V
|
|
75
|
+
|
|
76
|
+
return column_data
|
|
77
|
+
#return pd.DataFrame(column_data, index=data.index)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# %% ../nbs/01_aicorebridge.ipynb 12
|
|
81
|
+
class AICoreModule(): pass
|
|
82
|
+
|
|
83
|
+
# %% ../nbs/01_aicorebridge.ipynb 13
|
|
84
|
+
@patch
|
|
85
|
+
def __init__(self:AICoreModule,
|
|
86
|
+
processor:typing.Callable, # data processing function
|
|
87
|
+
save_dir:str, # path where the module can keep files
|
|
88
|
+
assets_dir:str,
|
|
89
|
+
*args, **kwargs):
|
|
90
|
+
|
|
91
|
+
self.init_time = datetime.datetime.now(datetime.UTC)
|
|
92
|
+
self._init_processor(processor)
|
|
93
|
+
|
|
94
|
+
self.init_args = args
|
|
95
|
+
self.init_kwargs = dict(
|
|
96
|
+
**kwargs,
|
|
97
|
+
assets_dir=assets_dir,
|
|
98
|
+
save_dir=save_dir
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# %% ../nbs/01_aicorebridge.ipynb 14
|
|
104
|
+
@patch
|
|
105
|
+
def _init_processor(
|
|
106
|
+
self:AICoreModule,
|
|
107
|
+
processor:typing.Callable):
|
|
108
|
+
"""Initializes processor related variables on self"""
|
|
109
|
+
|
|
110
|
+
self.processor = processor
|
|
111
|
+
self.processor_signature = inspect.signature(self.processor)
|
|
112
|
+
self.processor_params = dict(self.processor_signature.parameters)
|
|
113
|
+
self.return_param = self.processor_params.pop('return', None)
|
|
114
|
+
self.data_param, *self.call_params = list(self.processor_params.keys())
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# %% ../nbs/01_aicorebridge.ipynb 15
|
|
118
|
+
# can be overloaded
|
|
119
|
+
@patch
|
|
120
|
+
def call_processor(self:AICoreModule, calldata, **callargs):
|
|
121
|
+
return self.processor(calldata, **callargs)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# %% ../nbs/01_aicorebridge.ipynb 17
|
|
125
|
+
@patch
|
|
126
|
+
def infer(self:AICoreModule, data:dict, *_, **kwargs):
|
|
127
|
+
try:
|
|
128
|
+
|
|
129
|
+
msg=[
|
|
130
|
+
f"Startup time: {self.init_time.isoformat()}",
|
|
131
|
+
f"{self.processor.__name__}({self.processor_signature})",
|
|
132
|
+
f"init_args: {self.init_args}, init_kwargs: {self.init_kwargs}",
|
|
133
|
+
]
|
|
134
|
+
|
|
135
|
+
# Pickup params, pop those that are not intended for the processor
|
|
136
|
+
lastSeen = kwargs.pop('lastSeen', False)
|
|
137
|
+
recordformat = kwargs.pop('format', "records").lower()
|
|
138
|
+
timezone = kwargs.get('timezone', 'UTC')
|
|
139
|
+
msg.append(f"lastSeen: {lastSeen}, recordformat: {recordformat}, timezone: {timezone}")
|
|
140
|
+
|
|
141
|
+
samplerPeriod = kwargs.pop('samplerPeriod', self.init_kwargs.get('samplerPeriod','h'))
|
|
142
|
+
samplerMethod = kwargs.pop('samplerMethod', self.init_kwargs.get('samplerMethod',None))
|
|
143
|
+
reversed = kwargs.pop('reversed', False)
|
|
144
|
+
|
|
145
|
+
calldata = self.get_call_data(
|
|
146
|
+
data,
|
|
147
|
+
recordformat=recordformat,
|
|
148
|
+
timezone=timezone)
|
|
149
|
+
|
|
150
|
+
msg.append(f"calldata shape: {calldata.shape}")
|
|
151
|
+
|
|
152
|
+
history = build_historic_args(calldata, kwargs.pop('history', {}))
|
|
153
|
+
|
|
154
|
+
callargs = self.get_callargs(kwargs, history)
|
|
155
|
+
|
|
156
|
+
for arg, val in callargs.items():
|
|
157
|
+
msg.append(f"{arg}: {val}")
|
|
158
|
+
|
|
159
|
+
result = timeseries_dataframe(
|
|
160
|
+
self.call_processor(
|
|
161
|
+
calldata,
|
|
162
|
+
**callargs),
|
|
163
|
+
timezone=timezone)
|
|
164
|
+
|
|
165
|
+
msg.append(f"result shape: {result.shape}")
|
|
166
|
+
|
|
167
|
+
if samplerMethod:
|
|
168
|
+
msg.append(f"Sampler: {samplerMethod}, period: {samplerPeriod}")
|
|
169
|
+
result = timeseries_dataframe_resample(result, samplerPeriod, samplerMethod)
|
|
170
|
+
|
|
171
|
+
msg.append(f"return-data shape: {result.shape}")
|
|
172
|
+
|
|
173
|
+
if reversed:
|
|
174
|
+
result = result[::-1]
|
|
175
|
+
|
|
176
|
+
return {
|
|
177
|
+
'msg':msg,
|
|
178
|
+
'data': timeseries_dataframe_to_datadict(
|
|
179
|
+
result if not lastSeen else result[-1:],
|
|
180
|
+
recordformat=recordformat,
|
|
181
|
+
timezone=timezone,
|
|
182
|
+
popNaN=True)
|
|
183
|
+
}
|
|
184
|
+
except Exception as err:
|
|
185
|
+
msg.append(''.join(traceback.format_exception(None, err, err.__traceback__)))
|
|
186
|
+
syslog.exception(f"Exception {str(err)} in infer()")
|
|
187
|
+
return {
|
|
188
|
+
'msg': f"Unexpected {err=}, {type(err)=}",
|
|
189
|
+
'data': []
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# %% ../nbs/01_aicorebridge.ipynb 19
|
|
194
|
+
# Specialized types for initializing annotated parameters
|
|
195
|
+
# Add types by adding a tuple with the type name and a builder function
|
|
196
|
+
annotated_arg_builders = {
|
|
197
|
+
str(B[0]):B[1] for B in [
|
|
198
|
+
(numpy.ndarray, lambda X: numpy.array(X, dtype=X.dtype))
|
|
199
|
+
]
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
# %% ../nbs/01_aicorebridge.ipynb 20
|
|
203
|
+
@patch
|
|
204
|
+
def init_annotated_param(self:AICoreModule, param_name, value):
|
|
205
|
+
"""
|
|
206
|
+
Initialize argument for the processor call
|
|
207
|
+
|
|
208
|
+
param_name: name of the parameter to be initialized
|
|
209
|
+
value: value of the parameter read from infer data to be used for initialization
|
|
210
|
+
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
annotation = self.processor_signature.parameters[param_name].annotation
|
|
214
|
+
|
|
215
|
+
# try to convert value to one of the types in the annotation
|
|
216
|
+
for T in typing.get_args(annotation):
|
|
217
|
+
try:
|
|
218
|
+
builder = annotated_arg_builders.get(str(T), T)
|
|
219
|
+
return builder(value)
|
|
220
|
+
except TypeError as err:
|
|
221
|
+
continue
|
|
222
|
+
try:
|
|
223
|
+
return annotation(value)
|
|
224
|
+
except TypeError as err:
|
|
225
|
+
syslog.exception(f"Exception {str(err)} in fallback conversion to {annotation} of {type(value)}")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# %% ../nbs/01_aicorebridge.ipynb 21
|
|
230
|
+
@patch
|
|
231
|
+
def get_callargs(self:AICoreModule, kwargs, history):
|
|
232
|
+
"Get arguments for the processor call"
|
|
233
|
+
|
|
234
|
+
# Remove null / None values
|
|
235
|
+
kwargs = {k:v for k,v in kwargs.items() if v is not None}
|
|
236
|
+
|
|
237
|
+
return {
|
|
238
|
+
K:self.init_annotated_param(
|
|
239
|
+
K,
|
|
240
|
+
history.get(
|
|
241
|
+
K,
|
|
242
|
+
self.init_kwargs.get(
|
|
243
|
+
K,
|
|
244
|
+
kwargs.get(
|
|
245
|
+
K,
|
|
246
|
+
self.processor_signature.parameters[K].default
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
for K in self.call_params
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
# %% ../nbs/01_aicorebridge.ipynb 25
|
|
256
|
+
@patch
|
|
257
|
+
def get_call_data(
|
|
258
|
+
self:AICoreModule,
|
|
259
|
+
data:dict,
|
|
260
|
+
recordformat='records',
|
|
261
|
+
timezone='UTC'):
|
|
262
|
+
|
|
263
|
+
"Convert data to the processor signature"
|
|
264
|
+
|
|
265
|
+
df = set_time_index_zone(timeseries_dataframe_from_datadict(
|
|
266
|
+
data, ['datetimeMeasure', 'time'], recordformat), timezone)
|
|
267
|
+
|
|
268
|
+
df.sort_index(inplace=True)
|
|
269
|
+
|
|
270
|
+
if self.processor_params[self.data_param].annotation == pd.DataFrame:
|
|
271
|
+
return df
|
|
272
|
+
elif len(df.columns) > 1:
|
|
273
|
+
df.index = (df.index - datetime.datetime(1970,1,1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(seconds=1)
|
|
274
|
+
return df.to_records(index=True)
|
|
275
|
+
else:
|
|
276
|
+
df.index = (df.index - datetime.datetime(1970,1,1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(seconds=1)
|
|
277
|
+
return df.reset_index().to_numpy()
|
|
278
|
+
|
corebridge/core.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
|
|
2
|
+
|
|
3
|
+
# %% auto 0
|
|
4
|
+
__all__ = ['ResamplerMethods', 'ReSamplerPeriods', 'init_console_logging', 'set_time_index_zone', 'timeseries_dataframe',
|
|
5
|
+
'timeseries_dataframe_from_datadict', 'pop_nan_values', 'timeseries_dataframe_to_datadict',
|
|
6
|
+
'timeseries_dataframe_resample']
|
|
7
|
+
|
|
8
|
+
# %% ../nbs/00_core.ipynb 3
|
|
9
|
+
import typing, logging
|
|
10
|
+
import os
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
# %% ../nbs/00_core.ipynb 4
|
|
15
|
+
def init_console_logging(name=None, level=logging.INFO):
|
|
16
|
+
'''Setup none-blocking stream handler for sending loggin to the console.'''
|
|
17
|
+
|
|
18
|
+
# Only if no handlers defined.
|
|
19
|
+
if not logging.getLogger(name).handlers:
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger()
|
|
22
|
+
logger.setLevel(level)
|
|
23
|
+
|
|
24
|
+
console = logging.StreamHandler()
|
|
25
|
+
console.setLevel(level)
|
|
26
|
+
|
|
27
|
+
# set a format which is simpler for console use
|
|
28
|
+
formatter = logging.Formatter("%(asctime)s %(levelname)s\t%(process)d\t%(name)s\t%(filename)s\t%(lineno)d\t%(message)s", datefmt='%Y-%m-%dT%H:%M:%S%z')
|
|
29
|
+
|
|
30
|
+
# tell the handler to use this format
|
|
31
|
+
console.setFormatter(formatter)
|
|
32
|
+
|
|
33
|
+
# add the handler to the root logger
|
|
34
|
+
logger.addHandler(console)
|
|
35
|
+
return logger
|
|
36
|
+
else:
|
|
37
|
+
logging.getLogger(name).info(f'There already is a logger installed for {name}.')
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# %% ../nbs/00_core.ipynb 5
|
|
41
|
+
try:
|
|
42
|
+
logging.getLogger(__name__).info(f"Loading {__name__} from {__file__}")
|
|
43
|
+
except:
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
# %% ../nbs/00_core.ipynb 8
|
|
47
|
+
def set_time_index_zone(df:pd.DataFrame, timezone):
|
|
48
|
+
"""
|
|
49
|
+
Sets the time zone of the index of a pandas DataFrame.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
df (pd.DataFrame): The DataFrame whose index time zone is to be set.
|
|
53
|
+
timezone (str): The desired time zone.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
pd.DataFrame: The DataFrame with its index time zone set to the specified time zone.
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
None
|
|
60
|
+
|
|
61
|
+
Examples:
|
|
62
|
+
>>> df = pd.DataFrame({'A': [1, 2, 3]}, index=pd.DatetimeIndex(['2022-01-01', '2022-01-02', '2022-01-03']))
|
|
63
|
+
>>> set_time_index_zone(df, 'Europe/Berlin')
|
|
64
|
+
A
|
|
65
|
+
2022-01-01 1
|
|
66
|
+
2022-01-02 2
|
|
67
|
+
2022-01-03 3
|
|
68
|
+
DatetimeIndex: 3 entries, 2022-01-01 01:00:00+01:00 to 2022-01-03 01:00:00+01:00
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
if isinstance(df.index, pd.DatetimeIndex):
|
|
72
|
+
df.index.name = 'time'
|
|
73
|
+
if not hasattr(df.index, 'tz') or not df.index.tz or not df.index.tz:
|
|
74
|
+
df.index = df.index.tz_localize('UTC').tz_convert(timezone)
|
|
75
|
+
elif str(df.index.tz) != timezone:
|
|
76
|
+
df.index = df.index.tz_convert(timezone)
|
|
77
|
+
|
|
78
|
+
return df
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# %% ../nbs/00_core.ipynb 12
|
|
82
|
+
def timeseries_dataframe(
|
|
83
|
+
data:typing.Union[pd.DataFrame, pd.Series, dict, np.ndarray, np.recarray],
|
|
84
|
+
timezone='UTC',
|
|
85
|
+
columnnames=None):
|
|
86
|
+
|
|
87
|
+
"""Convert various tabular data formats to timeseries DataFrame
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
data (Union[pd.DataFrame, pd.Series, dict, np.ndarray, np.recarray]): The input data to be converted.
|
|
91
|
+
timezone (str, optional): The timezone to set for the index of the DataFrame. Defaults to 'UTC'.
|
|
92
|
+
columnnames (Optional[List[str]]): The column names to use for the DataFrame. Defaults to None.
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
pd.DataFrame: The converted timeseries DataFrame with the index set to the specified timezone.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
if isinstance(data, pd.DataFrame):
|
|
99
|
+
df = data
|
|
100
|
+
|
|
101
|
+
elif isinstance(data, pd.Series):
|
|
102
|
+
df = pd.DataFrame(data)
|
|
103
|
+
|
|
104
|
+
elif isinstance(data, dict):
|
|
105
|
+
# assume a dict/mapping of individual arrays representing timeseries
|
|
106
|
+
df = pd.DataFrame({
|
|
107
|
+
C:pd.Series(data=A[:,1], index=pd.DatetimeIndex(A[:,0]*1e9)) if isinstance(A, np.ndarray) else A
|
|
108
|
+
for C,A in data.items()
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
elif data.dtype.names is not None:
|
|
112
|
+
# structured or recarray, we use column names from the recarray
|
|
113
|
+
df = pd.DataFrame(
|
|
114
|
+
data=data.view(dtype=np.float64).reshape(data.shape[0],len(data.dtype))[:,range(1,len(data.dtype))],
|
|
115
|
+
index=pd.DatetimeIndex(data.view(dtype=np.float64).reshape(data.shape[0],len(data.dtype))[:,0] * 1e9),
|
|
116
|
+
columns=data.dtype.names[1:]
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
else:
|
|
120
|
+
if data.shape[0] > 0:
|
|
121
|
+
# column names, either 'value' if there is only one column, or
|
|
122
|
+
# value_0, value_1 .... value_nn when more the one column is present
|
|
123
|
+
if data.shape[1]>2:
|
|
124
|
+
columns=[f"value_{str(i+1)}" for i in range(data.shape[1]-1)] if not columnnames else [f"{str(i)}" for i in columnnames[1:]]
|
|
125
|
+
else:
|
|
126
|
+
columns=['value']
|
|
127
|
+
|
|
128
|
+
df = pd.DataFrame(
|
|
129
|
+
data=data[:, 1:],
|
|
130
|
+
index=pd.DatetimeIndex(data[:,0]*1e9),
|
|
131
|
+
columns=columns
|
|
132
|
+
)
|
|
133
|
+
else:
|
|
134
|
+
return pd.DataFrame()
|
|
135
|
+
|
|
136
|
+
return set_time_index_zone(df, timezone)
|
|
137
|
+
|
|
138
|
+
# %% ../nbs/00_core.ipynb 15
|
|
139
|
+
def timeseries_dataframe_from_datadict(
|
|
140
|
+
data:dict,
|
|
141
|
+
timecolumns=None,
|
|
142
|
+
recordformat='records'):
|
|
143
|
+
|
|
144
|
+
"""
|
|
145
|
+
Converts a data dict into a pandas DataFrame based on the specified record format.
|
|
146
|
+
Parameters:
|
|
147
|
+
- data: A dictionary containing the data to convert.
|
|
148
|
+
- timecolumns: A list of column names to be treated as time columns.
|
|
149
|
+
- recordformat: A string specifying the format of the data records ('records', 'table', 'split', 'index', 'tight').
|
|
150
|
+
Returns:
|
|
151
|
+
- df: A pandas DataFrame with a DatetimeIndex representing the converted data.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
orient = recordformat.lower()
|
|
155
|
+
assert orient in ['records', 'table', 'split', 'index', 'tight']
|
|
156
|
+
assert timecolumns, 'No time columns specified'
|
|
157
|
+
|
|
158
|
+
if orient == 'records':
|
|
159
|
+
# data is a structured ndarray, sequence of tuples or dicts, or DataFrame
|
|
160
|
+
df = pd.DataFrame.from_records(data)
|
|
161
|
+
time_columns_in_df = [C for C in df.columns if C in timecolumns]
|
|
162
|
+
if not time_columns_in_df:
|
|
163
|
+
syslog.error(f"No column in records {df.columns} matches specification in time columns {timecolumns}, assuming first column is time")
|
|
164
|
+
time_column = df.columns[0]
|
|
165
|
+
else:
|
|
166
|
+
time_column = time_columns_in_df[0]
|
|
167
|
+
|
|
168
|
+
elif orient == 'table':
|
|
169
|
+
# data is in pandas table format
|
|
170
|
+
time_column = data['schema']['primaryKey'][0]
|
|
171
|
+
df = pd.DataFrame.from_dict(data['data']).set_index(data['schema']['primaryKey'])
|
|
172
|
+
df.index.name = 'time'
|
|
173
|
+
else:
|
|
174
|
+
# data is formatted according to 'orient' parameter (pandas)
|
|
175
|
+
df = pd.DataFrame.from_dict(data, orient=orient)
|
|
176
|
+
time_column = df.index.name
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
df.columns = list(df.columns)
|
|
180
|
+
df[time_column] = pd.to_datetime(df[time_column],utc=True,format='ISO8601')
|
|
181
|
+
df.set_index(time_column, inplace=True)
|
|
182
|
+
df.index = pd.DatetimeIndex(df.index).round('ms')
|
|
183
|
+
|
|
184
|
+
df.index.name = 'time'
|
|
185
|
+
|
|
186
|
+
return df
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
# %% ../nbs/00_core.ipynb 18
|
|
190
|
+
def pop_nan_values(data):
|
|
191
|
+
"""
|
|
192
|
+
Recursively pop keys with nan values from dict or lists with dicts.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
data (Union[list, dict]): The data to be processed.
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
Union[list, dict]: The processed data with keys with nan values removed.
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
if isinstance(data, list):
|
|
202
|
+
return [pop_nan_values(v) for v in data if pd.notnull([v]).any()]
|
|
203
|
+
elif isinstance(data, dict):
|
|
204
|
+
return {k:pop_nan_values(v) for k, v in data.items() if pd.notnull([v]).any()}
|
|
205
|
+
else:
|
|
206
|
+
return data
|
|
207
|
+
|
|
208
|
+
# %% ../nbs/00_core.ipynb 19
|
|
209
|
+
def timeseries_dataframe_to_datadict(
|
|
210
|
+
data:typing.Union[pd.DataFrame, pd.Series, dict],
|
|
211
|
+
recordformat:str='records',
|
|
212
|
+
timezone:str='UTC',
|
|
213
|
+
popNaN:bool=False):
|
|
214
|
+
|
|
215
|
+
"""
|
|
216
|
+
Convert a timeseries DataFrame or Series into a dictionary representation.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
data (Union[pd.DataFrame, pd.Series, dict]): The input data to be converted. It can be a pandas DataFrame, Series, or a dictionary.
|
|
220
|
+
recordformat (str, optional): The format of the output records. Defaults to 'records'.
|
|
221
|
+
timezone (str, optional): The timezone to use for the DataFrame index. Defaults to 'UTC'.
|
|
222
|
+
popNaN (bool, optional): Whether to remove NaN values from the output dictionary. Defaults to False.
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
Union[dict, list]: The converted dictionary representation of the input data. If `popNaN` is True, it returns a dictionary with NaN values removed. Otherwise, it returns a dictionary or a list of dictionaries depending on the `recordformat` parameter.
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
orient = recordformat.lower()
|
|
229
|
+
|
|
230
|
+
normalized_data = timeseries_dataframe(data, timezone=timezone)
|
|
231
|
+
if isinstance(normalized_data.index, pd.DatetimeIndex):
|
|
232
|
+
normalized_data.index = normalized_data.index.map(lambda x: x.isoformat())
|
|
233
|
+
|
|
234
|
+
if orient == 'records':
|
|
235
|
+
records = normalized_data.reset_index().to_dict(orient='records')
|
|
236
|
+
else:
|
|
237
|
+
records = normalized_data.to_dict(orient=orient)
|
|
238
|
+
|
|
239
|
+
if popNaN and normalized_data.isna().any(axis=None):
|
|
240
|
+
return pop_nan_values(records)
|
|
241
|
+
|
|
242
|
+
return records
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
# %% ../nbs/00_core.ipynb 26
|
|
246
|
+
#def interpolate_timeseries(sampler, period, method_args):
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
ResamplerMethods = dict(
|
|
250
|
+
count=lambda R: R.count(),
|
|
251
|
+
median=lambda R: R.median(),
|
|
252
|
+
mean=lambda R: R.mean(),
|
|
253
|
+
min=lambda R: R.min(),
|
|
254
|
+
max=lambda R: R.max(),
|
|
255
|
+
sum=lambda R: R.sum(),
|
|
256
|
+
std=lambda R: R.std(),
|
|
257
|
+
var=lambda R: R.var(),
|
|
258
|
+
nearest=lambda R: R.nearest(),
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
ReSamplerPeriods = dict(
|
|
262
|
+
H='h', T='min', S='sec', L='ms', U='us', N='ns'
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
def timeseries_dataframe_resample(df:pd.DataFrame, period:str, method:str):
|
|
266
|
+
"""
|
|
267
|
+
Resamples a time-series DataFrame on the specified period and method.
|
|
268
|
+
|
|
269
|
+
Parameters:
|
|
270
|
+
df (pd.DataFrame): The input time-series DataFrame.
|
|
271
|
+
period (str): The resampling period.
|
|
272
|
+
method (str): The resampling method. Can be a string of multiple methods separated by ';'.
|
|
273
|
+
method_args (dict, optional): Additional arguments for the resampling method.
|
|
274
|
+
|
|
275
|
+
Returns:
|
|
276
|
+
pd.DataFrame: The resampled DataFrame.
|
|
277
|
+
"""
|
|
278
|
+
sampler = df.resample(ReSamplerPeriods.get(period, str(period)))
|
|
279
|
+
|
|
280
|
+
dataframes = [df]
|
|
281
|
+
for M in str(method).split(';'):
|
|
282
|
+
sdf = ResamplerMethods.get(M)(sampler)
|
|
283
|
+
sdf.columns = [f"{C}_{M}" for C in df.columns]
|
|
284
|
+
dataframes.append(sdf)
|
|
285
|
+
|
|
286
|
+
return pd.concat(dataframes, axis=1, join='outer')
|
|
287
|
+
|
|
288
|
+
|