pygazpar 0.1.21__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pygazpar/__init__.py +10 -3
- pygazpar/__main__.py +89 -58
- pygazpar/api_client.py +228 -0
- pygazpar/client.py +98 -210
- pygazpar/datasource.py +590 -0
- pygazpar/enum.py +31 -8
- pygazpar/excelparser.py +138 -0
- pygazpar/jsonparser.py +53 -0
- pygazpar/resources/daily_data_sample.json +7802 -0
- pygazpar/resources/hourly_data_sample.json +1 -0
- pygazpar/resources/monthly_data_sample.json +146 -0
- pygazpar/resources/weekly_data_sample.json +614 -0
- pygazpar/resources/yearly_data_sample.json +18 -0
- pygazpar/version.py +3 -0
- pygazpar-0.1.21.dist-info/LICENSE.txt → pygazpar-1.3.0.dist-info/LICENSE +21 -21
- pygazpar-1.3.0.dist-info/METADATA +225 -0
- pygazpar-1.3.0.dist-info/RECORD +18 -0
- {pygazpar-0.1.21.dist-info → pygazpar-1.3.0.dist-info}/WHEEL +1 -2
- pygazpar/webdriverwrapper.py +0 -125
- pygazpar/webelementwrapper.py +0 -40
- pygazpar-0.1.21.dist-info/METADATA +0 -149
- pygazpar-0.1.21.dist-info/RECORD +0 -14
- pygazpar-0.1.21.dist-info/entry_points.txt +0 -3
- pygazpar-0.1.21.dist-info/top_level.txt +0 -2
- test/__init__.py +0 -1
- test/test_client.py +0 -50
pygazpar/datasource.py
ADDED
@@ -0,0 +1,590 @@
|
|
1
|
+
import glob
|
2
|
+
import json
|
3
|
+
import logging
|
4
|
+
import os
|
5
|
+
from abc import ABC, abstractmethod
|
6
|
+
from datetime import date, timedelta
|
7
|
+
from typing import Any, Optional, cast
|
8
|
+
|
9
|
+
import pandas as pd
|
10
|
+
|
11
|
+
from pygazpar.api_client import APIClient, ConsumptionType
|
12
|
+
from pygazpar.api_client import Frequency as APIClientFrequency
|
13
|
+
from pygazpar.enum import Frequency, PropertyName
|
14
|
+
from pygazpar.excelparser import ExcelParser
|
15
|
+
from pygazpar.jsonparser import JsonParser
|
16
|
+
|
17
|
+
Logger = logging.getLogger(__name__)
|
18
|
+
|
19
|
+
MeterReading = dict[str, Any]
|
20
|
+
|
21
|
+
MeterReadings = list[MeterReading]
|
22
|
+
|
23
|
+
MeterReadingsByFrequency = dict[str, MeterReadings]
|
24
|
+
|
25
|
+
|
26
|
+
# ------------------------------------------------------------------------------------------------------------
|
27
|
+
class IDataSource(ABC): # pylint: disable=too-few-public-methods
|
28
|
+
|
29
|
+
@abstractmethod
|
30
|
+
def login(self):
|
31
|
+
pass
|
32
|
+
|
33
|
+
@abstractmethod
|
34
|
+
def logout(self):
|
35
|
+
pass
|
36
|
+
|
37
|
+
@abstractmethod
|
38
|
+
def get_pce_identifiers(self) -> list[str]:
|
39
|
+
pass
|
40
|
+
|
41
|
+
@abstractmethod
|
42
|
+
def load(
|
43
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
44
|
+
) -> MeterReadingsByFrequency:
|
45
|
+
pass
|
46
|
+
|
47
|
+
|
48
|
+
# ------------------------------------------------------------------------------------------------------------
|
49
|
+
class WebDataSource(IDataSource): # pylint: disable=too-few-public-methods
|
50
|
+
|
51
|
+
# ------------------------------------------------------
|
52
|
+
def __init__(self, username: str, password: str):
|
53
|
+
|
54
|
+
self._api_client = APIClient(username, password)
|
55
|
+
|
56
|
+
# ------------------------------------------------------
|
57
|
+
def login(self):
|
58
|
+
|
59
|
+
if not self._api_client.is_logged_in():
|
60
|
+
self._api_client.login()
|
61
|
+
|
62
|
+
# ------------------------------------------------------
|
63
|
+
def logout(self):
|
64
|
+
|
65
|
+
if self._api_client.is_logged_in():
|
66
|
+
self._api_client.logout()
|
67
|
+
|
68
|
+
# ------------------------------------------------------
|
69
|
+
def get_pce_identifiers(self) -> list[str]:
|
70
|
+
|
71
|
+
if not self._api_client.is_logged_in():
|
72
|
+
self._api_client.login()
|
73
|
+
|
74
|
+
pce_list = self._api_client.get_pce_list()
|
75
|
+
|
76
|
+
if pce_list is None:
|
77
|
+
return []
|
78
|
+
|
79
|
+
return [pce["idObject"] for pce in pce_list]
|
80
|
+
|
81
|
+
# ------------------------------------------------------
|
82
|
+
def load(
|
83
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
84
|
+
) -> MeterReadingsByFrequency:
|
85
|
+
|
86
|
+
if not self._api_client.is_logged_in():
|
87
|
+
self._api_client.login()
|
88
|
+
|
89
|
+
res = self._loadFromSession(pceIdentifier, startDate, endDate, frequencies)
|
90
|
+
|
91
|
+
Logger.debug("The data update terminates normally")
|
92
|
+
|
93
|
+
return res
|
94
|
+
|
95
|
+
@abstractmethod
|
96
|
+
def _loadFromSession(
|
97
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
98
|
+
) -> MeterReadingsByFrequency:
|
99
|
+
pass
|
100
|
+
|
101
|
+
|
102
|
+
# ------------------------------------------------------------------------------------------------------------
|
103
|
+
class ExcelWebDataSource(WebDataSource): # pylint: disable=too-few-public-methods
|
104
|
+
|
105
|
+
DATE_FORMAT = "%Y-%m-%d"
|
106
|
+
|
107
|
+
FREQUENCY_VALUES = {
|
108
|
+
Frequency.HOURLY: "Horaire",
|
109
|
+
Frequency.DAILY: "Journalier",
|
110
|
+
Frequency.WEEKLY: "Hebdomadaire",
|
111
|
+
Frequency.MONTHLY: "Mensuel",
|
112
|
+
Frequency.YEARLY: "Journalier",
|
113
|
+
}
|
114
|
+
|
115
|
+
DATA_FILENAME = "Donnees_informatives_*.xlsx"
|
116
|
+
|
117
|
+
# ------------------------------------------------------
|
118
|
+
def __init__(self, username: str, password: str, tmpDirectory: str):
|
119
|
+
|
120
|
+
super().__init__(username, password)
|
121
|
+
|
122
|
+
self.__tmpDirectory = tmpDirectory
|
123
|
+
|
124
|
+
# ------------------------------------------------------
|
125
|
+
def _loadFromSession( # pylint: disable=too-many-branches
|
126
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
127
|
+
) -> MeterReadingsByFrequency: # pylint: disable=too-many-branches
|
128
|
+
|
129
|
+
res = {}
|
130
|
+
|
131
|
+
# XLSX is in the TMP directory
|
132
|
+
data_file_path_pattern = self.__tmpDirectory + "/" + ExcelWebDataSource.DATA_FILENAME
|
133
|
+
|
134
|
+
# We remove an eventual existing data file (from a previous run that has not deleted it).
|
135
|
+
file_list = glob.glob(data_file_path_pattern)
|
136
|
+
for filename in file_list:
|
137
|
+
if os.path.isfile(filename):
|
138
|
+
try:
|
139
|
+
os.remove(filename)
|
140
|
+
except PermissionError:
|
141
|
+
pass
|
142
|
+
|
143
|
+
if frequencies is None:
|
144
|
+
# Transform Enum in List.
|
145
|
+
frequencyList = list(Frequency)
|
146
|
+
else:
|
147
|
+
# Get distinct values.
|
148
|
+
frequencyList = list(set(frequencies))
|
149
|
+
|
150
|
+
for frequency in frequencyList:
|
151
|
+
|
152
|
+
Logger.debug(
|
153
|
+
f"Loading data of frequency {ExcelWebDataSource.FREQUENCY_VALUES[frequency]} from {startDate.strftime(ExcelWebDataSource.DATE_FORMAT)} to {endDate.strftime(ExcelWebDataSource.DATE_FORMAT)}"
|
154
|
+
)
|
155
|
+
|
156
|
+
response = self._api_client.get_pce_consumption_excelsheet(
|
157
|
+
ConsumptionType.INFORMATIVE,
|
158
|
+
startDate,
|
159
|
+
endDate,
|
160
|
+
APIClientFrequency(ExcelWebDataSource.FREQUENCY_VALUES[frequency]),
|
161
|
+
[pceIdentifier],
|
162
|
+
)
|
163
|
+
|
164
|
+
filename = response["filename"]
|
165
|
+
content = response["content"]
|
166
|
+
|
167
|
+
with open(f"{self.__tmpDirectory}/{filename}", "wb") as file:
|
168
|
+
file.write(content)
|
169
|
+
|
170
|
+
# Load the XLSX file into the data structure
|
171
|
+
file_list = glob.glob(data_file_path_pattern)
|
172
|
+
|
173
|
+
if len(file_list) == 0:
|
174
|
+
Logger.warning(f"Not any data file has been found in '{self.__tmpDirectory}' directory")
|
175
|
+
|
176
|
+
for filename in file_list:
|
177
|
+
res[frequency.value] = ExcelParser.parse(
|
178
|
+
filename, frequency if frequency != Frequency.YEARLY else Frequency.DAILY
|
179
|
+
)
|
180
|
+
try:
|
181
|
+
# openpyxl does not close the file properly.
|
182
|
+
os.remove(filename)
|
183
|
+
except PermissionError:
|
184
|
+
pass
|
185
|
+
|
186
|
+
# We compute yearly from daily data.
|
187
|
+
if frequency == Frequency.YEARLY:
|
188
|
+
res[frequency.value] = FrequencyConverter.computeYearly(res[frequency.value])
|
189
|
+
|
190
|
+
return res
|
191
|
+
|
192
|
+
|
193
|
+
# ------------------------------------------------------------------------------------------------------------
|
194
|
+
class ExcelFileDataSource(IDataSource): # pylint: disable=too-few-public-methods
|
195
|
+
|
196
|
+
def __init__(self, excelFile: str):
|
197
|
+
|
198
|
+
self.__excelFile = excelFile
|
199
|
+
|
200
|
+
# ------------------------------------------------------
|
201
|
+
def login(self):
|
202
|
+
pass
|
203
|
+
|
204
|
+
# ------------------------------------------------------
|
205
|
+
def logout(self):
|
206
|
+
pass
|
207
|
+
|
208
|
+
# ------------------------------------------------------
|
209
|
+
def get_pce_identifiers(self) -> list[str]:
|
210
|
+
|
211
|
+
return ["0123456789"]
|
212
|
+
|
213
|
+
# ------------------------------------------------------
|
214
|
+
def load(
|
215
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
216
|
+
) -> MeterReadingsByFrequency:
|
217
|
+
|
218
|
+
res = {}
|
219
|
+
|
220
|
+
if frequencies is None:
|
221
|
+
# Transform Enum in List.
|
222
|
+
frequencyList = list(Frequency)
|
223
|
+
else:
|
224
|
+
# Get unique values.
|
225
|
+
frequencyList = list(set(frequencies))
|
226
|
+
|
227
|
+
for frequency in frequencyList:
|
228
|
+
if frequency != Frequency.YEARLY:
|
229
|
+
res[frequency.value] = ExcelParser.parse(self.__excelFile, frequency)
|
230
|
+
else:
|
231
|
+
daily = ExcelParser.parse(self.__excelFile, Frequency.DAILY)
|
232
|
+
res[frequency.value] = FrequencyConverter.computeYearly(daily)
|
233
|
+
|
234
|
+
return res
|
235
|
+
|
236
|
+
|
237
|
+
# ------------------------------------------------------------------------------------------------------------
|
238
|
+
class JsonWebDataSource(WebDataSource): # pylint: disable=too-few-public-methods
|
239
|
+
|
240
|
+
INPUT_DATE_FORMAT = "%Y-%m-%d"
|
241
|
+
|
242
|
+
OUTPUT_DATE_FORMAT = "%d/%m/%Y"
|
243
|
+
|
244
|
+
# ------------------------------------------------------
|
245
|
+
def _loadFromSession(
|
246
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
247
|
+
) -> MeterReadingsByFrequency:
|
248
|
+
|
249
|
+
res = dict[str, Any]()
|
250
|
+
|
251
|
+
computeByFrequency = {
|
252
|
+
Frequency.HOURLY: FrequencyConverter.computeHourly,
|
253
|
+
Frequency.DAILY: FrequencyConverter.computeDaily,
|
254
|
+
Frequency.WEEKLY: FrequencyConverter.computeWeekly,
|
255
|
+
Frequency.MONTHLY: FrequencyConverter.computeMonthly,
|
256
|
+
Frequency.YEARLY: FrequencyConverter.computeYearly,
|
257
|
+
}
|
258
|
+
|
259
|
+
data = self._api_client.get_pce_consumption(ConsumptionType.INFORMATIVE, startDate, endDate, [pceIdentifier])
|
260
|
+
|
261
|
+
Logger.debug("Json meter data: %s", data)
|
262
|
+
|
263
|
+
# Temperatures URL: Inject parameters.
|
264
|
+
endDate = date.today() - timedelta(days=1) if endDate >= date.today() else endDate
|
265
|
+
days = max(
|
266
|
+
min((endDate - startDate).days, 730), 10
|
267
|
+
) # At least 10 days, at most 730 days, to avoid HTTP 500 error.
|
268
|
+
|
269
|
+
# Get weather data.
|
270
|
+
try:
|
271
|
+
temperatures = self._api_client.get_pce_meteo(endDate, days, pceIdentifier)
|
272
|
+
except Exception: # pylint: disable=broad-except
|
273
|
+
# Not a blocking error.
|
274
|
+
temperatures = None
|
275
|
+
|
276
|
+
Logger.debug("Json temperature data: %s", temperatures)
|
277
|
+
|
278
|
+
# Transform all the data into the target structure.
|
279
|
+
if data is None or len(data) == 0:
|
280
|
+
return res
|
281
|
+
|
282
|
+
daily = JsonParser.parse(json.dumps(data), json.dumps(temperatures), pceIdentifier)
|
283
|
+
|
284
|
+
Logger.debug("Processed daily data: %s", daily)
|
285
|
+
|
286
|
+
if frequencies is None:
|
287
|
+
# Transform Enum in List.
|
288
|
+
frequencyList = list(Frequency)
|
289
|
+
else:
|
290
|
+
# Get unique values.
|
291
|
+
frequencyList = list(set(frequencies))
|
292
|
+
|
293
|
+
for frequency in frequencyList:
|
294
|
+
res[frequency.value] = computeByFrequency[frequency](daily)
|
295
|
+
|
296
|
+
return res
|
297
|
+
|
298
|
+
|
299
|
+
# ------------------------------------------------------------------------------------------------------------
|
300
|
+
class JsonFileDataSource(IDataSource): # pylint: disable=too-few-public-methods
|
301
|
+
|
302
|
+
# ------------------------------------------------------
|
303
|
+
def __init__(self, consumptionJsonFile: str, temperatureJsonFile):
|
304
|
+
|
305
|
+
self.__consumptionJsonFile = consumptionJsonFile
|
306
|
+
self.__temperatureJsonFile = temperatureJsonFile
|
307
|
+
|
308
|
+
# ------------------------------------------------------
|
309
|
+
def login(self):
|
310
|
+
pass
|
311
|
+
|
312
|
+
# ------------------------------------------------------
|
313
|
+
def logout(self):
|
314
|
+
pass
|
315
|
+
|
316
|
+
# ------------------------------------------------------
|
317
|
+
def get_pce_identifiers(self) -> list[str]:
|
318
|
+
|
319
|
+
return ["0123456789"]
|
320
|
+
|
321
|
+
# ------------------------------------------------------
|
322
|
+
def load(
|
323
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
324
|
+
) -> MeterReadingsByFrequency:
|
325
|
+
|
326
|
+
res = {}
|
327
|
+
|
328
|
+
with open(self.__consumptionJsonFile, mode="r", encoding="utf-8") as consumptionJsonFile:
|
329
|
+
with open(self.__temperatureJsonFile, mode="r", encoding="utf-8") as temperatureJsonFile:
|
330
|
+
daily = JsonParser.parse(consumptionJsonFile.read(), temperatureJsonFile.read(), pceIdentifier)
|
331
|
+
|
332
|
+
computeByFrequency = {
|
333
|
+
Frequency.HOURLY: FrequencyConverter.computeHourly,
|
334
|
+
Frequency.DAILY: FrequencyConverter.computeDaily,
|
335
|
+
Frequency.WEEKLY: FrequencyConverter.computeWeekly,
|
336
|
+
Frequency.MONTHLY: FrequencyConverter.computeMonthly,
|
337
|
+
Frequency.YEARLY: FrequencyConverter.computeYearly,
|
338
|
+
}
|
339
|
+
|
340
|
+
if frequencies is None:
|
341
|
+
# Transform Enum in List.
|
342
|
+
frequencyList = list(Frequency)
|
343
|
+
else:
|
344
|
+
# Get unique values.
|
345
|
+
frequencyList = list(set(frequencies))
|
346
|
+
|
347
|
+
for frequency in frequencyList:
|
348
|
+
res[frequency.value] = computeByFrequency[frequency](daily)
|
349
|
+
|
350
|
+
return res
|
351
|
+
|
352
|
+
|
353
|
+
# ------------------------------------------------------------------------------------------------------------
|
354
|
+
class TestDataSource(IDataSource): # pylint: disable=too-few-public-methods
|
355
|
+
|
356
|
+
__test__ = False # Will not be discovered as a test
|
357
|
+
|
358
|
+
# ------------------------------------------------------
|
359
|
+
def __init__(self):
|
360
|
+
|
361
|
+
pass
|
362
|
+
|
363
|
+
# ------------------------------------------------------
|
364
|
+
def login(self):
|
365
|
+
pass
|
366
|
+
|
367
|
+
# ------------------------------------------------------
|
368
|
+
def logout(self):
|
369
|
+
pass
|
370
|
+
|
371
|
+
# ------------------------------------------------------
|
372
|
+
def get_pce_identifiers(self) -> list[str]:
|
373
|
+
|
374
|
+
return ["0123456789"]
|
375
|
+
|
376
|
+
# ------------------------------------------------------
|
377
|
+
def load(
|
378
|
+
self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[list[Frequency]] = None
|
379
|
+
) -> MeterReadingsByFrequency:
|
380
|
+
|
381
|
+
res = dict[str, Any]()
|
382
|
+
|
383
|
+
dataSampleFilenameByFrequency = {
|
384
|
+
Frequency.HOURLY: "hourly_data_sample.json",
|
385
|
+
Frequency.DAILY: "daily_data_sample.json",
|
386
|
+
Frequency.WEEKLY: "weekly_data_sample.json",
|
387
|
+
Frequency.MONTHLY: "monthly_data_sample.json",
|
388
|
+
Frequency.YEARLY: "yearly_data_sample.json",
|
389
|
+
}
|
390
|
+
|
391
|
+
if frequencies is None:
|
392
|
+
# Transform Enum in List.
|
393
|
+
frequencyList = list(Frequency)
|
394
|
+
else:
|
395
|
+
# Get unique values.
|
396
|
+
frequencyList = list(set(frequencies))
|
397
|
+
|
398
|
+
for frequency in frequencyList:
|
399
|
+
dataSampleFilename = (
|
400
|
+
f"{os.path.dirname(os.path.abspath(__file__))}/resources/{dataSampleFilenameByFrequency[frequency]}"
|
401
|
+
)
|
402
|
+
|
403
|
+
with open(dataSampleFilename, mode="r", encoding="utf-8") as jsonFile:
|
404
|
+
res[frequency.value] = cast(list[dict[PropertyName, Any]], json.load(jsonFile))
|
405
|
+
|
406
|
+
return res
|
407
|
+
|
408
|
+
|
409
|
+
# ------------------------------------------------------------------------------------------------------------
|
410
|
+
class FrequencyConverter:
|
411
|
+
|
412
|
+
MONTHS = [
|
413
|
+
"Janvier",
|
414
|
+
"Février",
|
415
|
+
"Mars",
|
416
|
+
"Avril",
|
417
|
+
"Mai",
|
418
|
+
"Juin",
|
419
|
+
"Juillet",
|
420
|
+
"Août",
|
421
|
+
"Septembre",
|
422
|
+
"Octobre",
|
423
|
+
"Novembre",
|
424
|
+
"Décembre",
|
425
|
+
]
|
426
|
+
|
427
|
+
# ------------------------------------------------------
|
428
|
+
@staticmethod
|
429
|
+
def computeHourly(daily: list[dict[str, Any]]) -> list[dict[str, Any]]: # pylint: disable=unused-argument
|
430
|
+
|
431
|
+
return []
|
432
|
+
|
433
|
+
# ------------------------------------------------------
|
434
|
+
@staticmethod
|
435
|
+
def computeDaily(daily: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
436
|
+
|
437
|
+
return daily
|
438
|
+
|
439
|
+
# ------------------------------------------------------
|
440
|
+
@staticmethod
|
441
|
+
def computeWeekly(daily: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
442
|
+
|
443
|
+
df = pd.DataFrame(daily)
|
444
|
+
|
445
|
+
# Trimming head and trailing spaces and convert to datetime.
|
446
|
+
df["date_time"] = pd.to_datetime(df["time_period"].str.strip(), format=JsonWebDataSource.OUTPUT_DATE_FORMAT)
|
447
|
+
|
448
|
+
# Get the first day of week.
|
449
|
+
df["first_day_of_week"] = pd.to_datetime(df["date_time"].dt.strftime("%W %Y 1"), format="%W %Y %w")
|
450
|
+
|
451
|
+
# Get the last day of week.
|
452
|
+
df["last_day_of_week"] = pd.to_datetime(df["date_time"].dt.strftime("%W %Y 0"), format="%W %Y %w")
|
453
|
+
|
454
|
+
# Reformat the time period.
|
455
|
+
df["time_period"] = (
|
456
|
+
"Du "
|
457
|
+
+ df["first_day_of_week"].dt.strftime(JsonWebDataSource.OUTPUT_DATE_FORMAT).astype(str)
|
458
|
+
+ " au "
|
459
|
+
+ df["last_day_of_week"].dt.strftime(JsonWebDataSource.OUTPUT_DATE_FORMAT).astype(str)
|
460
|
+
)
|
461
|
+
|
462
|
+
# Aggregate rows by month_year.
|
463
|
+
df = (
|
464
|
+
df[
|
465
|
+
[
|
466
|
+
"first_day_of_week",
|
467
|
+
"time_period",
|
468
|
+
"start_index_m3",
|
469
|
+
"end_index_m3",
|
470
|
+
"volume_m3",
|
471
|
+
"energy_kwh",
|
472
|
+
"timestamp",
|
473
|
+
]
|
474
|
+
]
|
475
|
+
.groupby("time_period")
|
476
|
+
.agg(
|
477
|
+
first_day_of_week=("first_day_of_week", "min"),
|
478
|
+
start_index_m3=("start_index_m3", "min"),
|
479
|
+
end_index_m3=("end_index_m3", "max"),
|
480
|
+
volume_m3=("volume_m3", "sum"),
|
481
|
+
energy_kwh=("energy_kwh", "sum"),
|
482
|
+
timestamp=("timestamp", "min"),
|
483
|
+
count=("energy_kwh", "count"),
|
484
|
+
)
|
485
|
+
.reset_index()
|
486
|
+
)
|
487
|
+
|
488
|
+
# Sort rows by month ascending.
|
489
|
+
df = df.sort_values(by=["first_day_of_week"])
|
490
|
+
|
491
|
+
# Select rows where we have a full week (7 days) except for the current week.
|
492
|
+
df = pd.concat([df[(df["count"] >= 7)], df.tail(1)[df.tail(1)["count"] < 7]])
|
493
|
+
|
494
|
+
# Select target columns.
|
495
|
+
df = df[["time_period", "start_index_m3", "end_index_m3", "volume_m3", "energy_kwh", "timestamp"]]
|
496
|
+
|
497
|
+
res = cast(list[dict[str, Any]], df.to_dict("records"))
|
498
|
+
|
499
|
+
return res
|
500
|
+
|
501
|
+
# ------------------------------------------------------
|
502
|
+
@staticmethod
|
503
|
+
def computeMonthly(daily: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
504
|
+
|
505
|
+
df = pd.DataFrame(daily)
|
506
|
+
|
507
|
+
# Trimming head and trailing spaces and convert to datetime.
|
508
|
+
df["date_time"] = pd.to_datetime(df["time_period"].str.strip(), format=JsonWebDataSource.OUTPUT_DATE_FORMAT)
|
509
|
+
|
510
|
+
# Get the corresponding month-year.
|
511
|
+
df["month_year"] = (
|
512
|
+
df["date_time"].apply(lambda x: FrequencyConverter.MONTHS[x.month - 1]).astype(str)
|
513
|
+
+ " "
|
514
|
+
+ df["date_time"].dt.strftime("%Y").astype(str)
|
515
|
+
)
|
516
|
+
|
517
|
+
# Aggregate rows by month_year.
|
518
|
+
df = (
|
519
|
+
df[["date_time", "month_year", "start_index_m3", "end_index_m3", "volume_m3", "energy_kwh", "timestamp"]]
|
520
|
+
.groupby("month_year")
|
521
|
+
.agg(
|
522
|
+
first_day_of_month=("date_time", "min"),
|
523
|
+
start_index_m3=("start_index_m3", "min"),
|
524
|
+
end_index_m3=("end_index_m3", "max"),
|
525
|
+
volume_m3=("volume_m3", "sum"),
|
526
|
+
energy_kwh=("energy_kwh", "sum"),
|
527
|
+
timestamp=("timestamp", "min"),
|
528
|
+
count=("energy_kwh", "count"),
|
529
|
+
)
|
530
|
+
.reset_index()
|
531
|
+
)
|
532
|
+
|
533
|
+
# Sort rows by month ascending.
|
534
|
+
df = df.sort_values(by=["first_day_of_month"])
|
535
|
+
|
536
|
+
# Select rows where we have a full month (more than 27 days) except for the current month.
|
537
|
+
df = pd.concat([df[(df["count"] >= 28)], df.tail(1)[df.tail(1)["count"] < 28]])
|
538
|
+
|
539
|
+
# Rename columns for their target names.
|
540
|
+
df = df.rename(columns={"month_year": "time_period"})
|
541
|
+
|
542
|
+
# Select target columns.
|
543
|
+
df = df[["time_period", "start_index_m3", "end_index_m3", "volume_m3", "energy_kwh", "timestamp"]]
|
544
|
+
|
545
|
+
res = cast(list[dict[str, Any]], df.to_dict("records"))
|
546
|
+
|
547
|
+
return res
|
548
|
+
|
549
|
+
# ------------------------------------------------------
|
550
|
+
@staticmethod
|
551
|
+
def computeYearly(daily: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
552
|
+
|
553
|
+
df = pd.DataFrame(daily)
|
554
|
+
|
555
|
+
# Trimming head and trailing spaces and convert to datetime.
|
556
|
+
df["date_time"] = pd.to_datetime(df["time_period"].str.strip(), format=JsonWebDataSource.OUTPUT_DATE_FORMAT)
|
557
|
+
|
558
|
+
# Get the corresponding year.
|
559
|
+
df["year"] = df["date_time"].dt.strftime("%Y")
|
560
|
+
|
561
|
+
# Aggregate rows by month_year.
|
562
|
+
df = (
|
563
|
+
df[["year", "start_index_m3", "end_index_m3", "volume_m3", "energy_kwh", "timestamp"]]
|
564
|
+
.groupby("year")
|
565
|
+
.agg(
|
566
|
+
start_index_m3=("start_index_m3", "min"),
|
567
|
+
end_index_m3=("end_index_m3", "max"),
|
568
|
+
volume_m3=("volume_m3", "sum"),
|
569
|
+
energy_kwh=("energy_kwh", "sum"),
|
570
|
+
timestamp=("timestamp", "min"),
|
571
|
+
count=("energy_kwh", "count"),
|
572
|
+
)
|
573
|
+
.reset_index()
|
574
|
+
)
|
575
|
+
|
576
|
+
# Sort rows by month ascending.
|
577
|
+
df = df.sort_values(by=["year"])
|
578
|
+
|
579
|
+
# Select rows where we have almost a full year (more than 360) except for the current year.
|
580
|
+
df = pd.concat([df[(df["count"] >= 360)], df.tail(1)[df.tail(1)["count"] < 360]])
|
581
|
+
|
582
|
+
# Rename columns for their target names.
|
583
|
+
df = df.rename(columns={"year": "time_period"})
|
584
|
+
|
585
|
+
# Select target columns.
|
586
|
+
df = df[["time_period", "start_index_m3", "end_index_m3", "volume_m3", "energy_kwh", "timestamp"]]
|
587
|
+
|
588
|
+
res = cast(list[dict[str, Any]], df.to_dict("records"))
|
589
|
+
|
590
|
+
return res
|
pygazpar/enum.py
CHANGED
@@ -1,12 +1,35 @@
|
|
1
1
|
from enum import Enum
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
3
|
+
|
4
|
+
# ------------------------------------------------------------------------------------------------------------
|
5
|
+
class PropertyName(Enum):
|
6
|
+
TIME_PERIOD = "time_period"
|
7
|
+
START_INDEX = "start_index_m3"
|
8
|
+
END_INDEX = "end_index_m3"
|
9
|
+
VOLUME = "volume_m3"
|
10
|
+
ENERGY = "energy_kwh"
|
11
|
+
CONVERTER_FACTOR = "converter_factor_kwh/m3"
|
12
|
+
TEMPERATURE = "temperature_degC"
|
11
13
|
TYPE = "type"
|
12
14
|
TIMESTAMP = "timestamp"
|
15
|
+
|
16
|
+
def __str__(self):
|
17
|
+
return self.value
|
18
|
+
|
19
|
+
def __repr__(self):
|
20
|
+
return self.__str__()
|
21
|
+
|
22
|
+
|
23
|
+
# ------------------------------------------------------------------------------------------------------------
|
24
|
+
class Frequency(Enum):
|
25
|
+
HOURLY = "hourly"
|
26
|
+
DAILY = "daily"
|
27
|
+
WEEKLY = "weekly"
|
28
|
+
MONTHLY = "monthly"
|
29
|
+
YEARLY = "yearly"
|
30
|
+
|
31
|
+
def __str__(self):
|
32
|
+
return self.value
|
33
|
+
|
34
|
+
def __repr__(self):
|
35
|
+
return self.__str__()
|