corebridge 0.2.24__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
corebridge/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.2.24"
corebridge/_modidx.py ADDED
@@ -0,0 +1,64 @@
1
+ # Autogenerated by nbdev
2
+
3
+ d = { 'settings': { 'branch': 'main',
4
+ 'doc_baseurl': '/corebridge',
5
+ 'doc_host': 'https://fenke.github.io',
6
+ 'git_url': 'https://github.com/fenke/corebridge',
7
+ 'lib_path': 'corebridge'},
8
+ 'syms': { 'corebridge.aicorebridge': { 'corebridge.aicorebridge.AICoreModule': ( 'aicorebridge.html#aicoremodule',
9
+ 'corebridge/aicorebridge.py'),
10
+ 'corebridge.aicorebridge.AICoreModule.__init__': ( 'aicorebridge.html#aicoremodule.__init__',
11
+ 'corebridge/aicorebridge.py'),
12
+ 'corebridge.aicorebridge.AICoreModule._init_processor': ( 'aicorebridge.html#aicoremodule._init_processor',
13
+ 'corebridge/aicorebridge.py'),
14
+ 'corebridge.aicorebridge.AICoreModule.call_processor': ( 'aicorebridge.html#aicoremodule.call_processor',
15
+ 'corebridge/aicorebridge.py'),
16
+ 'corebridge.aicorebridge.AICoreModule.get_call_data': ( 'aicorebridge.html#aicoremodule.get_call_data',
17
+ 'corebridge/aicorebridge.py'),
18
+ 'corebridge.aicorebridge.AICoreModule.get_callargs': ( 'aicorebridge.html#aicoremodule.get_callargs',
19
+ 'corebridge/aicorebridge.py'),
20
+ 'corebridge.aicorebridge.AICoreModule.infer': ( 'aicorebridge.html#aicoremodule.infer',
21
+ 'corebridge/aicorebridge.py'),
22
+ 'corebridge.aicorebridge.AICoreModule.init_annotated_param': ( 'aicorebridge.html#aicoremodule.init_annotated_param',
23
+ 'corebridge/aicorebridge.py'),
24
+ 'corebridge.aicorebridge.build_historic_args': ( 'aicorebridge.html#build_historic_args',
25
+ 'corebridge/aicorebridge.py')},
26
+ 'corebridge.core': { 'corebridge.core.init_console_logging': ('core.html#init_console_logging', 'corebridge/core.py'),
27
+ 'corebridge.core.pop_nan_values': ('core.html#pop_nan_values', 'corebridge/core.py'),
28
+ 'corebridge.core.set_time_index_zone': ('core.html#set_time_index_zone', 'corebridge/core.py'),
29
+ 'corebridge.core.timeseries_dataframe': ('core.html#timeseries_dataframe', 'corebridge/core.py'),
30
+ 'corebridge.core.timeseries_dataframe_from_datadict': ( 'core.html#timeseries_dataframe_from_datadict',
31
+ 'corebridge/core.py'),
32
+ 'corebridge.core.timeseries_dataframe_resample': ( 'core.html#timeseries_dataframe_resample',
33
+ 'corebridge/core.py'),
34
+ 'corebridge.core.timeseries_dataframe_to_datadict': ( 'core.html#timeseries_dataframe_to_datadict',
35
+ 'corebridge/core.py')},
36
+ 'corebridge.rscript': { 'corebridge.rscript.calc_hash_from_data_files': ( 'rscriptbridge.html#calc_hash_from_data_files',
37
+ 'corebridge/rscript.py'),
38
+ 'corebridge.rscript.calc_hash_from_files': ( 'rscriptbridge.html#calc_hash_from_files',
39
+ 'corebridge/rscript.py'),
40
+ 'corebridge.rscript.calc_hash_from_flowobject': ( 'rscriptbridge.html#calc_hash_from_flowobject',
41
+ 'corebridge/rscript.py'),
42
+ 'corebridge.rscript.calc_hash_from_input_files': ( 'rscriptbridge.html#calc_hash_from_input_files',
43
+ 'corebridge/rscript.py'),
44
+ 'corebridge.rscript.check_rscript_lib': ( 'rscriptbridge.html#check_rscript_lib',
45
+ 'corebridge/rscript.py'),
46
+ 'corebridge.rscript.check_rscript_libs': ( 'rscriptbridge.html#check_rscript_libs',
47
+ 'corebridge/rscript.py'),
48
+ 'corebridge.rscript.check_script_inputs': ( 'rscriptbridge.html#check_script_inputs',
49
+ 'corebridge/rscript.py'),
50
+ 'corebridge.rscript.check_script_output': ( 'rscriptbridge.html#check_script_output',
51
+ 'corebridge/rscript.py'),
52
+ 'corebridge.rscript.generate_checksum_file': ( 'rscriptbridge.html#generate_checksum_file',
53
+ 'corebridge/rscript.py'),
54
+ 'corebridge.rscript.get_asset_path': ('rscriptbridge.html#get_asset_path', 'corebridge/rscript.py'),
55
+ 'corebridge.rscript.get_rscript_env': ('rscriptbridge.html#get_rscript_env', 'corebridge/rscript.py'),
56
+ 'corebridge.rscript.get_rscript_libpath': ( 'rscriptbridge.html#get_rscript_libpath',
57
+ 'corebridge/rscript.py'),
58
+ 'corebridge.rscript.get_save_path': ('rscriptbridge.html#get_save_path', 'corebridge/rscript.py'),
59
+ 'corebridge.rscript.install_R_package_wait': ( 'rscriptbridge.html#install_r_package_wait',
60
+ 'corebridge/rscript.py'),
61
+ 'corebridge.rscript.run_rscript_nowait': ( 'rscriptbridge.html#run_rscript_nowait',
62
+ 'corebridge/rscript.py'),
63
+ 'corebridge.rscript.run_rscript_wait': ( 'rscriptbridge.html#run_rscript_wait',
64
+ 'corebridge/rscript.py')}}}
@@ -0,0 +1,278 @@
1
+ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_aicorebridge.ipynb.
2
+
3
+ # %% auto 0
4
+ __all__ = ['syslog', 'annotated_arg_builders', 'build_historic_args', 'AICoreModule']
5
+
6
+ # %% ../nbs/01_aicorebridge.ipynb 4
7
+ import typing
8
+ import logging
9
+ import traceback
10
+ import inspect
11
+ import datetime
12
+ import json
13
+ import os
14
+ import pandas, pandas as pd
15
+ import numpy, numpy as np
16
+
17
+ from dateutil import parser
18
+ from fastcore.basics import patch_to, patch
19
+ from .core import *
20
+ from . import __version__
21
+
22
+
23
+ # %% ../nbs/01_aicorebridge.ipynb 5
24
+ syslog = logging.getLogger(__name__)
25
+
26
+ # %% ../nbs/01_aicorebridge.ipynb 6
27
+ try:
28
+ print(f"Loading {__name__} {__version__} from {__file__}")
29
+ except:
30
+ pass
31
+
32
+ # %% ../nbs/01_aicorebridge.ipynb 8
33
+ def build_historic_args(data:pd.DataFrame, history:dict|list) -> dict:
34
+ """Create a timeseries DataFrame from historic data defined in `history`.
35
+
36
+ Parameters
37
+ ----------
38
+ data : pd.DataFrame
39
+ The input time-series DataFrame.
40
+ history : dict or list of dicts
41
+ Historic data definition, each item in the list is a dictionary with a startDate key to set the start of a section of historic data in the result and a column-value pair for each of the columns in the
42
+
43
+ Returns
44
+ -------
45
+ historic_data : dict
46
+ Historic data in dictionary format where keys are column names and values are the historic values as numpy array.
47
+ """
48
+
49
+ if not history:
50
+ return {}
51
+
52
+ if isinstance(history, dict):
53
+ return history
54
+
55
+ if not isinstance(history, list):
56
+ return {}
57
+
58
+ if isinstance(data, pd.DataFrame):
59
+ dates = data.index.astype(np.int64).astype(np.float64) / 1e9
60
+ dates = dates.to_numpy()
61
+ elif data.dtype.names is not None:
62
+ dates = data.view(dtype=np.float64).reshape(data.shape[0],len(data.dtype))[:,0]
63
+ else:
64
+ dates = data[:,0]
65
+ dates = dates.astype(np.int64)
66
+
67
+ columns = list(set([K for I in history for K in I.keys() if K != 'startDate']))
68
+ column_data = {K:np.full(len(dates), np.nan, dtype=np.float64) for K in columns}
69
+
70
+ for I in history:
71
+ date = parser.parse(str((I.pop('startDate','2000-01-01T00:00:00+00:00')))).timestamp()
72
+ mask = np.greater_equal(dates, date)
73
+ for K,V in I.items():
74
+ column_data[K][mask] = V
75
+
76
+ return column_data
77
+ #return pd.DataFrame(column_data, index=data.index)
78
+
79
+
80
+ # %% ../nbs/01_aicorebridge.ipynb 12
81
+ class AICoreModule(): pass
82
+
83
+ # %% ../nbs/01_aicorebridge.ipynb 13
84
+ @patch
85
+ def __init__(self:AICoreModule,
86
+ processor:typing.Callable, # data processing function
87
+ save_dir:str, # path where the module can keep files
88
+ assets_dir:str,
89
+ *args, **kwargs):
90
+
91
+ self.init_time = datetime.datetime.now(datetime.UTC)
92
+ self._init_processor(processor)
93
+
94
+ self.init_args = args
95
+ self.init_kwargs = dict(
96
+ **kwargs,
97
+ assets_dir=assets_dir,
98
+ save_dir=save_dir
99
+ )
100
+
101
+
102
+
103
+ # %% ../nbs/01_aicorebridge.ipynb 14
104
+ @patch
105
+ def _init_processor(
106
+ self:AICoreModule,
107
+ processor:typing.Callable):
108
+ """Initializes processor related variables on self"""
109
+
110
+ self.processor = processor
111
+ self.processor_signature = inspect.signature(self.processor)
112
+ self.processor_params = dict(self.processor_signature.parameters)
113
+ self.return_param = self.processor_params.pop('return', None)
114
+ self.data_param, *self.call_params = list(self.processor_params.keys())
115
+
116
+
117
+ # %% ../nbs/01_aicorebridge.ipynb 15
118
+ # can be overloaded
119
+ @patch
120
+ def call_processor(self:AICoreModule, calldata, **callargs):
121
+ return self.processor(calldata, **callargs)
122
+
123
+
124
+ # %% ../nbs/01_aicorebridge.ipynb 17
125
+ @patch
126
+ def infer(self:AICoreModule, data:dict, *_, **kwargs):
127
+ try:
128
+
129
+ msg=[
130
+ f"Startup time: {self.init_time.isoformat()}",
131
+ f"{self.processor.__name__}({self.processor_signature})",
132
+ f"init_args: {self.init_args}, init_kwargs: {self.init_kwargs}",
133
+ ]
134
+
135
+ # Pickup params, pop those that are not intended for the processor
136
+ lastSeen = kwargs.pop('lastSeen', False)
137
+ recordformat = kwargs.pop('format', "records").lower()
138
+ timezone = kwargs.get('timezone', 'UTC')
139
+ msg.append(f"lastSeen: {lastSeen}, recordformat: {recordformat}, timezone: {timezone}")
140
+
141
+ samplerPeriod = kwargs.pop('samplerPeriod', self.init_kwargs.get('samplerPeriod','h'))
142
+ samplerMethod = kwargs.pop('samplerMethod', self.init_kwargs.get('samplerMethod',None))
143
+ reversed = kwargs.pop('reversed', False)
144
+
145
+ calldata = self.get_call_data(
146
+ data,
147
+ recordformat=recordformat,
148
+ timezone=timezone)
149
+
150
+ msg.append(f"calldata shape: {calldata.shape}")
151
+
152
+ history = build_historic_args(calldata, kwargs.pop('history', {}))
153
+
154
+ callargs = self.get_callargs(kwargs, history)
155
+
156
+ for arg, val in callargs.items():
157
+ msg.append(f"{arg}: {val}")
158
+
159
+ result = timeseries_dataframe(
160
+ self.call_processor(
161
+ calldata,
162
+ **callargs),
163
+ timezone=timezone)
164
+
165
+ msg.append(f"result shape: {result.shape}")
166
+
167
+ if samplerMethod:
168
+ msg.append(f"Sampler: {samplerMethod}, period: {samplerPeriod}")
169
+ result = timeseries_dataframe_resample(result, samplerPeriod, samplerMethod)
170
+
171
+ msg.append(f"return-data shape: {result.shape}")
172
+
173
+ if reversed:
174
+ result = result[::-1]
175
+
176
+ return {
177
+ 'msg':msg,
178
+ 'data': timeseries_dataframe_to_datadict(
179
+ result if not lastSeen else result[-1:],
180
+ recordformat=recordformat,
181
+ timezone=timezone,
182
+ popNaN=True)
183
+ }
184
+ except Exception as err:
185
+ msg.append(''.join(traceback.format_exception(None, err, err.__traceback__)))
186
+ syslog.exception(f"Exception {str(err)} in infer()")
187
+ return {
188
+ 'msg': f"Unexpected {err=}, {type(err)=}",
189
+ 'data': []
190
+ }
191
+
192
+
193
+ # %% ../nbs/01_aicorebridge.ipynb 19
194
+ # Specialized types for initializing annotated parameters
195
+ # Add types by adding a tuple with the type name and a builder function
196
+ annotated_arg_builders = {
197
+ str(B[0]):B[1] for B in [
198
+ (numpy.ndarray, lambda X: numpy.array(X, dtype=X.dtype))
199
+ ]
200
+ }
201
+
202
+ # %% ../nbs/01_aicorebridge.ipynb 20
203
+ @patch
204
+ def init_annotated_param(self:AICoreModule, param_name, value):
205
+ """
206
+ Initialize argument for the processor call
207
+
208
+ param_name: name of the parameter to be initialized
209
+ value: value of the parameter read from infer data to be used for initialization
210
+
211
+ """
212
+
213
+ annotation = self.processor_signature.parameters[param_name].annotation
214
+
215
+ # try to convert value to one of the types in the annotation
216
+ for T in typing.get_args(annotation):
217
+ try:
218
+ builder = annotated_arg_builders.get(str(T), T)
219
+ return builder(value)
220
+ except TypeError as err:
221
+ continue
222
+ try:
223
+ return annotation(value)
224
+ except TypeError as err:
225
+ syslog.exception(f"Exception {str(err)} in fallback conversion to {annotation} of {type(value)}")
226
+
227
+
228
+
229
+ # %% ../nbs/01_aicorebridge.ipynb 21
230
+ @patch
231
+ def get_callargs(self:AICoreModule, kwargs, history):
232
+ "Get arguments for the processor call"
233
+
234
+ # Remove null / None values
235
+ kwargs = {k:v for k,v in kwargs.items() if v is not None}
236
+
237
+ return {
238
+ K:self.init_annotated_param(
239
+ K,
240
+ history.get(
241
+ K,
242
+ self.init_kwargs.get(
243
+ K,
244
+ kwargs.get(
245
+ K,
246
+ self.processor_signature.parameters[K].default
247
+ )
248
+ )
249
+ )
250
+ )
251
+ for K in self.call_params
252
+ }
253
+
254
+
255
+ # %% ../nbs/01_aicorebridge.ipynb 25
256
+ @patch
257
+ def get_call_data(
258
+ self:AICoreModule,
259
+ data:dict,
260
+ recordformat='records',
261
+ timezone='UTC'):
262
+
263
+ "Convert data to the processor signature"
264
+
265
+ df = set_time_index_zone(timeseries_dataframe_from_datadict(
266
+ data, ['datetimeMeasure', 'time'], recordformat), timezone)
267
+
268
+ df.sort_index(inplace=True)
269
+
270
+ if self.processor_params[self.data_param].annotation == pd.DataFrame:
271
+ return df
272
+ elif len(df.columns) > 1:
273
+ df.index = (df.index - datetime.datetime(1970,1,1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(seconds=1)
274
+ return df.to_records(index=True)
275
+ else:
276
+ df.index = (df.index - datetime.datetime(1970,1,1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(seconds=1)
277
+ return df.reset_index().to_numpy()
278
+
corebridge/core.py ADDED
@@ -0,0 +1,288 @@
1
+ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
2
+
3
+ # %% auto 0
4
+ __all__ = ['ResamplerMethods', 'ReSamplerPeriods', 'init_console_logging', 'set_time_index_zone', 'timeseries_dataframe',
5
+ 'timeseries_dataframe_from_datadict', 'pop_nan_values', 'timeseries_dataframe_to_datadict',
6
+ 'timeseries_dataframe_resample']
7
+
8
+ # %% ../nbs/00_core.ipynb 3
9
+ import typing, logging
10
+ import os
11
+ import numpy as np
12
+ import pandas as pd
13
+
14
+ # %% ../nbs/00_core.ipynb 4
15
+ def init_console_logging(name=None, level=logging.INFO):
16
+ '''Setup none-blocking stream handler for sending loggin to the console.'''
17
+
18
+ # Only if no handlers defined.
19
+ if not logging.getLogger(name).handlers:
20
+
21
+ logger = logging.getLogger()
22
+ logger.setLevel(level)
23
+
24
+ console = logging.StreamHandler()
25
+ console.setLevel(level)
26
+
27
+ # set a format which is simpler for console use
28
+ formatter = logging.Formatter("%(asctime)s %(levelname)s\t%(process)d\t%(name)s\t%(filename)s\t%(lineno)d\t%(message)s", datefmt='%Y-%m-%dT%H:%M:%S%z')
29
+
30
+ # tell the handler to use this format
31
+ console.setFormatter(formatter)
32
+
33
+ # add the handler to the root logger
34
+ logger.addHandler(console)
35
+ return logger
36
+ else:
37
+ logging.getLogger(name).info(f'There already is a logger installed for {name}.')
38
+
39
+
40
+ # %% ../nbs/00_core.ipynb 5
41
+ try:
42
+ logging.getLogger(__name__).info(f"Loading {__name__} from {__file__}")
43
+ except:
44
+ pass
45
+
46
+ # %% ../nbs/00_core.ipynb 8
47
+ def set_time_index_zone(df:pd.DataFrame, timezone):
48
+ """
49
+ Sets the time zone of the index of a pandas DataFrame.
50
+
51
+ Args:
52
+ df (pd.DataFrame): The DataFrame whose index time zone is to be set.
53
+ timezone (str): The desired time zone.
54
+
55
+ Returns:
56
+ pd.DataFrame: The DataFrame with its index time zone set to the specified time zone.
57
+
58
+ Raises:
59
+ None
60
+
61
+ Examples:
62
+ >>> df = pd.DataFrame({'A': [1, 2, 3]}, index=pd.DatetimeIndex(['2022-01-01', '2022-01-02', '2022-01-03']))
63
+ >>> set_time_index_zone(df, 'Europe/Berlin')
64
+ A
65
+ 2022-01-01 1
66
+ 2022-01-02 2
67
+ 2022-01-03 3
68
+ DatetimeIndex: 3 entries, 2022-01-01 01:00:00+01:00 to 2022-01-03 01:00:00+01:00
69
+ """
70
+
71
+ if isinstance(df.index, pd.DatetimeIndex):
72
+ df.index.name = 'time'
73
+ if not hasattr(df.index, 'tz') or not df.index.tz or not df.index.tz:
74
+ df.index = df.index.tz_localize('UTC').tz_convert(timezone)
75
+ elif str(df.index.tz) != timezone:
76
+ df.index = df.index.tz_convert(timezone)
77
+
78
+ return df
79
+
80
+
81
+ # %% ../nbs/00_core.ipynb 12
82
+ def timeseries_dataframe(
83
+ data:typing.Union[pd.DataFrame, pd.Series, dict, np.ndarray, np.recarray],
84
+ timezone='UTC',
85
+ columnnames=None):
86
+
87
+ """Convert various tabular data formats to timeseries DataFrame
88
+
89
+ Args:
90
+ data (Union[pd.DataFrame, pd.Series, dict, np.ndarray, np.recarray]): The input data to be converted.
91
+ timezone (str, optional): The timezone to set for the index of the DataFrame. Defaults to 'UTC'.
92
+ columnnames (Optional[List[str]]): The column names to use for the DataFrame. Defaults to None.
93
+
94
+ Returns:
95
+ pd.DataFrame: The converted timeseries DataFrame with the index set to the specified timezone.
96
+ """
97
+
98
+ if isinstance(data, pd.DataFrame):
99
+ df = data
100
+
101
+ elif isinstance(data, pd.Series):
102
+ df = pd.DataFrame(data)
103
+
104
+ elif isinstance(data, dict):
105
+ # assume a dict/mapping of individual arrays representing timeseries
106
+ df = pd.DataFrame({
107
+ C:pd.Series(data=A[:,1], index=pd.DatetimeIndex(A[:,0]*1e9)) if isinstance(A, np.ndarray) else A
108
+ for C,A in data.items()
109
+ })
110
+
111
+ elif data.dtype.names is not None:
112
+ # structured or recarray, we use column names from the recarray
113
+ df = pd.DataFrame(
114
+ data=data.view(dtype=np.float64).reshape(data.shape[0],len(data.dtype))[:,range(1,len(data.dtype))],
115
+ index=pd.DatetimeIndex(data.view(dtype=np.float64).reshape(data.shape[0],len(data.dtype))[:,0] * 1e9),
116
+ columns=data.dtype.names[1:]
117
+ )
118
+
119
+ else:
120
+ if data.shape[0] > 0:
121
+ # column names, either 'value' if there is only one column, or
122
+ # value_0, value_1 .... value_nn when more the one column is present
123
+ if data.shape[1]>2:
124
+ columns=[f"value_{str(i+1)}" for i in range(data.shape[1]-1)] if not columnnames else [f"{str(i)}" for i in columnnames[1:]]
125
+ else:
126
+ columns=['value']
127
+
128
+ df = pd.DataFrame(
129
+ data=data[:, 1:],
130
+ index=pd.DatetimeIndex(data[:,0]*1e9),
131
+ columns=columns
132
+ )
133
+ else:
134
+ return pd.DataFrame()
135
+
136
+ return set_time_index_zone(df, timezone)
137
+
138
+ # %% ../nbs/00_core.ipynb 15
139
+ def timeseries_dataframe_from_datadict(
140
+ data:dict,
141
+ timecolumns=None,
142
+ recordformat='records'):
143
+
144
+ """
145
+ Converts a data dict into a pandas DataFrame based on the specified record format.
146
+ Parameters:
147
+ - data: A dictionary containing the data to convert.
148
+ - timecolumns: A list of column names to be treated as time columns.
149
+ - recordformat: A string specifying the format of the data records ('records', 'table', 'split', 'index', 'tight').
150
+ Returns:
151
+ - df: A pandas DataFrame with a DatetimeIndex representing the converted data.
152
+ """
153
+
154
+ orient = recordformat.lower()
155
+ assert orient in ['records', 'table', 'split', 'index', 'tight']
156
+ assert timecolumns, 'No time columns specified'
157
+
158
+ if orient == 'records':
159
+ # data is a structured ndarray, sequence of tuples or dicts, or DataFrame
160
+ df = pd.DataFrame.from_records(data)
161
+ time_columns_in_df = [C for C in df.columns if C in timecolumns]
162
+ if not time_columns_in_df:
163
+ syslog.error(f"No column in records {df.columns} matches specification in time columns {timecolumns}, assuming first column is time")
164
+ time_column = df.columns[0]
165
+ else:
166
+ time_column = time_columns_in_df[0]
167
+
168
+ elif orient == 'table':
169
+ # data is in pandas table format
170
+ time_column = data['schema']['primaryKey'][0]
171
+ df = pd.DataFrame.from_dict(data['data']).set_index(data['schema']['primaryKey'])
172
+ df.index.name = 'time'
173
+ else:
174
+ # data is formatted according to 'orient' parameter (pandas)
175
+ df = pd.DataFrame.from_dict(data, orient=orient)
176
+ time_column = df.index.name
177
+
178
+
179
+ df.columns = list(df.columns)
180
+ df[time_column] = pd.to_datetime(df[time_column],utc=True,format='ISO8601')
181
+ df.set_index(time_column, inplace=True)
182
+ df.index = pd.DatetimeIndex(df.index).round('ms')
183
+
184
+ df.index.name = 'time'
185
+
186
+ return df
187
+
188
+
189
+ # %% ../nbs/00_core.ipynb 18
190
+ def pop_nan_values(data):
191
+ """
192
+ Recursively pop keys with nan values from dict or lists with dicts.
193
+
194
+ Args:
195
+ data (Union[list, dict]): The data to be processed.
196
+
197
+ Returns:
198
+ Union[list, dict]: The processed data with keys with nan values removed.
199
+ """
200
+
201
+ if isinstance(data, list):
202
+ return [pop_nan_values(v) for v in data if pd.notnull([v]).any()]
203
+ elif isinstance(data, dict):
204
+ return {k:pop_nan_values(v) for k, v in data.items() if pd.notnull([v]).any()}
205
+ else:
206
+ return data
207
+
208
+ # %% ../nbs/00_core.ipynb 19
209
+ def timeseries_dataframe_to_datadict(
210
+ data:typing.Union[pd.DataFrame, pd.Series, dict],
211
+ recordformat:str='records',
212
+ timezone:str='UTC',
213
+ popNaN:bool=False):
214
+
215
+ """
216
+ Convert a timeseries DataFrame or Series into a dictionary representation.
217
+
218
+ Args:
219
+ data (Union[pd.DataFrame, pd.Series, dict]): The input data to be converted. It can be a pandas DataFrame, Series, or a dictionary.
220
+ recordformat (str, optional): The format of the output records. Defaults to 'records'.
221
+ timezone (str, optional): The timezone to use for the DataFrame index. Defaults to 'UTC'.
222
+ popNaN (bool, optional): Whether to remove NaN values from the output dictionary. Defaults to False.
223
+
224
+ Returns:
225
+ Union[dict, list]: The converted dictionary representation of the input data. If `popNaN` is True, it returns a dictionary with NaN values removed. Otherwise, it returns a dictionary or a list of dictionaries depending on the `recordformat` parameter.
226
+ """
227
+
228
+ orient = recordformat.lower()
229
+
230
+ normalized_data = timeseries_dataframe(data, timezone=timezone)
231
+ if isinstance(normalized_data.index, pd.DatetimeIndex):
232
+ normalized_data.index = normalized_data.index.map(lambda x: x.isoformat())
233
+
234
+ if orient == 'records':
235
+ records = normalized_data.reset_index().to_dict(orient='records')
236
+ else:
237
+ records = normalized_data.to_dict(orient=orient)
238
+
239
+ if popNaN and normalized_data.isna().any(axis=None):
240
+ return pop_nan_values(records)
241
+
242
+ return records
243
+
244
+
245
+ # %% ../nbs/00_core.ipynb 26
246
+ #def interpolate_timeseries(sampler, period, method_args):
247
+
248
+
249
+ ResamplerMethods = dict(
250
+ count=lambda R: R.count(),
251
+ median=lambda R: R.median(),
252
+ mean=lambda R: R.mean(),
253
+ min=lambda R: R.min(),
254
+ max=lambda R: R.max(),
255
+ sum=lambda R: R.sum(),
256
+ std=lambda R: R.std(),
257
+ var=lambda R: R.var(),
258
+ nearest=lambda R: R.nearest(),
259
+ )
260
+
261
+ ReSamplerPeriods = dict(
262
+ H='h', T='min', S='sec', L='ms', U='us', N='ns'
263
+ )
264
+
265
+ def timeseries_dataframe_resample(df:pd.DataFrame, period:str, method:str):
266
+ """
267
+ Resamples a time-series DataFrame on the specified period and method.
268
+
269
+ Parameters:
270
+ df (pd.DataFrame): The input time-series DataFrame.
271
+ period (str): The resampling period.
272
+ method (str): The resampling method. Can be a string of multiple methods separated by ';'.
273
+ method_args (dict, optional): Additional arguments for the resampling method.
274
+
275
+ Returns:
276
+ pd.DataFrame: The resampled DataFrame.
277
+ """
278
+ sampler = df.resample(ReSamplerPeriods.get(period, str(period)))
279
+
280
+ dataframes = [df]
281
+ for M in str(method).split(';'):
282
+ sdf = ResamplerMethods.get(M)(sampler)
283
+ sdf.columns = [f"{C}_{M}" for C in df.columns]
284
+ dataframes.append(sdf)
285
+
286
+ return pd.concat(dataframes, axis=1, join='outer')
287
+
288
+