pygazpar 0.1.20__py3-none-any.whl → 1.3.0b1__py3-none-any.whl

pygazpar/__init__.py

@@ -1,3 +1,10 @@
-from pygazpar.enum import PropertyNameEnum
-from pygazpar.client import Client
-from pygazpar.client import LoginError
+from pygazpar.client import Client  # noqa: F401
+from pygazpar.datasource import (  # noqa: F401
+    ExcelFileDataSource,
+    ExcelWebDataSource,
+    JsonFileDataSource,
+    JsonWebDataSource,
+    TestDataSource,
+)
+from pygazpar.enum import Frequency, PropertyName  # noqa: F401
+from pygazpar.version import __version__  # noqa: F401

pygazpar/__main__.py

@@ -1,58 +1,81 @@
-import argparse
-import sys
-import json
-import traceback
-import os
-import logging
-
-from pygazpar.client import Client
-
-def main():
-    """Main function"""
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-u", "--username",
-                      required=True,
-                      help="GRDF username (email)")    
-    parser.add_argument("-p", "--password",
-                      required=True,
-                      help="GRDF password")    
-    parser.add_argument("-w", "--webdriver",
-                      required=True,
-                      help="Firefox webdriver executable (geckodriver)")    
-    parser.add_argument("-s", "--wait_time",
-                      required=False,
-                      type=int,
-                      default=30,
-                      help="Wait time in seconds (see https://selenium-python.readthedocs.io/waits.html for details)")    
-    parser.add_argument("-t", "--tmpdir",
-                      required=False,
-                      default="/tmp",
-                      help="tmp directory (default is /tmp)")
-    parser.add_argument("-l", "--lastNRows",
-                      required=False,
-                      type=int,
-                      default=0,
-                      help="Get only the last N rows (default is 0: it means all rows are retrieved)")                           
-
-    args = parser.parse_args()
-
-    # We remove the pygazpar log file.
-    geckodriverLogFile = f"{args.tmpdir}/pygazpar.log"
-    if os.path.isfile(geckodriverLogFile):
-        os.remove(geckodriverLogFile)
-
-    # Setup logging.
-    logging.basicConfig(filename=f"{args.tmpdir}/pygazpar.log", level=logging.DEBUG, format="%(asctime)s %(levelname)s [%(name)s] %(message)s")
-
-    client = Client(args.username, args.password, args.webdriver, int(args.wait_time), args.tmpdir, int(args.lastNRows))
-
-    try:
-        client.update()
-    except BaseException:
-        print('An error occured while querying PyGazpar library : %s', traceback.format_exc())
-        return 1
-
-    print(json.dumps(client.data(), indent=2))
-
-if __name__ == '__main__':
-    sys.exit(main())
+import argparse
+import json
+import logging
+import os
+import sys
+import traceback
+
+import pygazpar
+
+
+def main():
+    """Main function"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version", action="version", version=f"PyGazpar {pygazpar.__version__}")
+    parser.add_argument("-u", "--username", required=True, help="GRDF username (email)")
+    parser.add_argument("-p", "--password", required=True, help="GRDF password")
+    parser.add_argument("-c", "--pce", required=True, help="GRDF PCE identifier")
+    parser.add_argument("-t", "--tmpdir", required=False, default="/tmp", help="tmp directory (default is /tmp)")
+    parser.add_argument(
+        "-f",
+        "--frequency",
+        required=False,
+        type=lambda frequency: pygazpar.Frequency[frequency],
+        choices=list(pygazpar.Frequency),
+        default="DAILY",
+        help="Meter reading frequency (DAILY, WEEKLY, MONTHLY, YEARLY)",
+    )
+    parser.add_argument(
+        "-d",
+        "--lastNDays",
+        required=False,
+        type=int,
+        default=365,
+        help="Get only the last N days of records (default: 365 days)",
+    )
+    parser.add_argument("--datasource", required=False, default="json", help="Datasource: json | excel | test")
+
+    args = parser.parse_args()
+
+    # We create the tmp directory if not already exists.
+    if not os.path.exists(args.tmpdir):
+        os.mkdir(args.tmpdir)
+
+    # We remove the pygazpar log file.
+    pygazparLogFile = f"{args.tmpdir}/pygazpar.log"
+    if os.path.isfile(pygazparLogFile):
+        os.remove(pygazparLogFile)
+
+    # Setup logging.
+    logging.basicConfig(
+        filename=f"{pygazparLogFile}", level=logging.DEBUG, format="%(asctime)s %(levelname)s [%(name)s] %(message)s"
+    )
+
+    logging.info(f"PyGazpar {pygazpar.__version__}")
+    logging.info(f"--tmpdir {args.tmpdir}")
+    logging.info(f"--frequency {args.frequency}")
+    logging.info(f"--lastNDays {args.lastNDays}")
+    logging.info(f"--datasource {bool(args.datasource)}")
+
+    if args.datasource == "json":
+        client = pygazpar.Client(pygazpar.JsonWebDataSource(args.username, args.password))
+    elif args.datasource == "excel":
+        client = pygazpar.Client(pygazpar.ExcelWebDataSource(args.username, args.password, args.tmpdir))
+    elif args.datasource == "test":
+        client = pygazpar.Client(pygazpar.TestDataSource())
+    else:
+        raise ValueError("Invalid datasource: (json | excel | test) is expected")
+
+    try:
+        data = client.loadSince(args.pce, int(args.lastNDays), [args.frequency])
+    except BaseException:  # pylint: disable=broad-except
+        print("An error occured while querying PyGazpar library : %s", traceback.format_exc())
+        return 1
+
+    print(json.dumps(data, indent=2))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pygazpar/client.py

@@ -1,253 +1,64 @@
-import os
-import time
-import glob
-import logging
-from datetime import datetime
-from openpyxl import load_workbook
-from pygazpar.enum import PropertyNameEnum
-from .webdriverwrapper import WebDriverWrapper
-
-HOME_URL = 'https://monespace.grdf.fr'
-LOGIN_URL = HOME_URL + '/monespace/connexion'
-WELCOME_URL = HOME_URL + '/monespace/particulier/accueil'
-DATA_URL = HOME_URL + '/monespace/particulier/consommation/consommations'
-DATA_FILENAME = 'Consommations gaz_*.xlsx'
-
-DEFAULT_TMP_DIRECTORY = '/tmp'
-DEFAULT_FIREFOX_WEBDRIVER = os.getcwd() + '/geckodriver'
-DEFAULT_WAIT_TIME = 30
-
-# ------------------------------------------------------------------------------------------------------------
-class LoginError(Exception):
-    """ Client has failed to login in GrDF Web site (check username/password)"""
-    pass
-
-
-# ------------------------------------------------------------------------------------------------------------
-class Client(object):
-
-    logger = logging.getLogger(__name__)
-
-    # ------------------------------------------------------
-    def __init__(self, username: str, password: str, firefox_webdriver_executable: str = DEFAULT_FIREFOX_WEBDRIVER, wait_time: int = DEFAULT_WAIT_TIME, tmp_directory: str = DEFAULT_TMP_DIRECTORY, lastNRows: int = 0):
-        self.__username = username
-        self.__password = password        
-        self.__firefox_webdriver_executable = firefox_webdriver_executable
-        self.__wait_time = wait_time
-        self.__tmp_directory = tmp_directory
-        self.__data = []
-        self.__lastNRows = lastNRows
-
-
-    # ------------------------------------------------------
-    def data(self):
-        return self.__data
-
-
-    # ------------------------------------------------------
-    def acceptCookies(self, driver: WebDriverWrapper):
-
-        try:
-            cookies_accept_button = driver.find_element_by_xpath("//a[@id='_EPcommonPage_WAR_EPportlet_:formBandeauCnil:j_idt12']", "Cookies accept button", False)
-            cookies_accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-
-    # ------------------------------------------------------
-    def acceptPrivacyConditions(self, driver: WebDriverWrapper):
-
-        # linkText=Tout accepter
-        # css=#btn_accept_banner 
-
-        try:
-            # id=btn_accept_banner
-            accept_button = driver.find_element_by_id("btn_accept_banner", "Privacy Conditions accept button", False)
-            accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-        try:
-            # xpath=//a[contains(text(),'Tout accepter')]
-            accept_button = driver.find_element_by_xpath("//a[contains(text(),'Tout accepter')]", "Privacy Conditions accept button", False)
-            accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass            
-
-        try:
-            # xpath=//a[@id='btn_accept_banner']
-            accept_button = driver.find_element_by_xpath("//a[@id='btn_accept_banner']", "Privacy Conditions accept button", False)
-            accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-        try:
-            # xpath=//div[@id='ckieBnr_banner']/div[2]/a[2]
-            accept_button = driver.find_element_by_xpath("//div[@id='ckieBnr_banner']/div[2]/a[2]", "Privacy Conditions accept button", False)
-            accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-        try:
-            # xpath=//div[9]/div[2]/a[2]
-            accept_button = driver.find_element_by_xpath("//div[9]/div[2]/a[2]", "Privacy Conditions accept button", False)
-            accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-        
-        try:
-            # xpath=//a[contains(.,'Tout accepter')]
-            accept_button = driver.find_element_by_xpath("//a[contains(.,'Tout accepter')]", "Privacy Conditions accept button", False)
-            accept_button.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-
-    # ------------------------------------------------------
-    def closeEventualPopup(self, driver: WebDriverWrapper):
-
-        # Accept an eventual Privacy Conditions popup.
-        self.acceptPrivacyConditions(driver)
-
-        # Eventually, click Accept in the lower banner to accept cookies from the site.
-        self.acceptCookies(driver)
-
-        # Eventually, close Advertisement Popup Windows.
-        try:
-            advertisement_popup_element = driver.find_element_by_xpath("/html/body/abtasty-modal/div/div[1]", "Advertisement close button", False)
-            advertisement_popup_element.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-        # Eventually, close Survey Popup Windows : /html/body/div[12]/div[2] or //*[@id="mfbIframeClose"]
-        try:
-            survey_popup_element = driver.find_element_by_xpath("//*[@id='mfbIframeClose']", "Survey close button", False)
-            survey_popup_element.click()
-        except:
-            # Do nothing, because the Pop up may not appear.
-            pass
-
-
-    # ------------------------------------------------------
-    def update(self):
-
-        Client.logger.debug("Start updating the data...")
-
-        # XLSX is in the TMP directory
-        data_file_path_pattern = self.__tmp_directory + '/' + DATA_FILENAME
-
-        # We remove an eventual existing data file (from a previous run that has not deleted it)
-        file_list = glob.glob(data_file_path_pattern)
-        for filename in file_list:
-            if os.path.isfile(filename):
-                os.remove(filename)
-
-        # Create the WebDriver with the ability to log and take screenshot for debugging.
-        driver = WebDriverWrapper(self.__firefox_webdriver_executable, self.__wait_time, self.__tmp_directory)
-
-        try:
-          
-            ## Login URL
-            driver.get(LOGIN_URL, "Go to login page")
-
-            # Fill login form
-            email_element = driver.find_element_by_id("_EspacePerso_WAR_EPportlet_:seConnecterForm:email", "Login page: Email text field")
-            password_element = driver.find_element_by_id("_EspacePerso_WAR_EPportlet_:seConnecterForm:passwordSecretSeConnecter", "Login page: Password text field")
-            
-            email_element.send_keys(self.__username)
-            password_element.send_keys(self.__password)
-            
-            # Submit the login form.
-            submit_button_element = driver.find_element_by_id("_EspacePerso_WAR_EPportlet_:seConnecterForm:meConnecter", "Login page: 'Me connecter' button")
-            submit_button_element.click()
-            
-            # Close eventual popup Windows or Assistant appearing.
-            self.closeEventualPopup(driver)
-
-            # Once we find the 'Acceder' button from the main page, we are logged on successfully.
-            try:
-                driver.find_element_by_xpath("//div[2]/div[2]/div/a/div", "Welcome page: 'Acceder' button of 'Suivi de consommation'")
-            except:
-                # Perhaps, login has failed.
-                if driver.current_url() == WELCOME_URL:
-                    # We're good.
-                    pass
-                elif driver.current_url() == LOGIN_URL:
-                    raise LoginError("GrDF sign in has failed, please check your username/password")
-                else:
-                    raise
-
-            # Page 'votre consommation'
-            driver.get(DATA_URL, "Go to 'Consommations' page")          
-
-            # Wait for the data page to load completely.
-            time.sleep(self.__wait_time)
-
-            # Eventually, close TokyWoky assistant which may hide the Download button.
-            try:
-                tokyWoky_close_button = driver.find_element_by_xpath("//div[@id='toky_container']/div/div", "TokyWoky assistant close button")
-                tokyWoky_close_button.click()
-            except:
-                # Do nothing, because the Pop up may not appear.
-                pass    
-
-            # Select daily consumption
-            daily_consumption_element = driver.find_element_by_xpath("//table[@id='_eConsoconsoDetaille_WAR_eConsoportlet_:idFormConsoDetaille:panelTypeGranularite1']/tbody/tr/td[3]/label", "Daily consumption button")
-            daily_consumption_element.click()
-
-            # Download file
-            # xpath=//button[@id='_eConsoconsoDetaille_WAR_eConsoportlet_:idFormConsoDetaille:telechargerDonnees']/span
-            download_button_element = driver.find_element_by_xpath("//button[@onclick=\"envoieGATelechargerConsoDetaille('particulier', 'jour_kwh');\"]/span", "Download button")                                                                   
-            download_button_element.click()
-
-            # Timestamp of the data.
-            data_timestamp = datetime.now().isoformat()
-
-            # Wait a few for the download to complete
-            time.sleep(self.__wait_time)
-            
-            # Load the XLSX file into the data structure
-            file_list = glob.glob(data_file_path_pattern)            
-
-            for filename in file_list:
-
-                Client.logger.debug(f"Loading Excel data file '{filename}'...")
-                wb = load_workbook(filename = filename)
-                ws = wb['Historique par jour']
-                minRowNum = max(8, len(ws['B'])+1-self.__lastNRows) if self.__lastNRows > 0 else 8
-                maxRowNum = len(ws['B'])
-                for rownum in range(minRowNum, maxRowNum + 1):
-                    row = {}
-                    if ws.cell(column=2, row=rownum).value != None:
-                        row[PropertyNameEnum.DATE.value] = ws.cell(column=2, row=rownum).value                        
-                        row[PropertyNameEnum.START_INDEX_M3.value] = ws.cell(column=3, row=rownum).value
-                        row[PropertyNameEnum.END_INDEX_M3.value] = ws.cell(column=4, row=rownum).value
-                        row[PropertyNameEnum.VOLUME_M3.value] = ws.cell(column=5, row=rownum).value
-                        row[PropertyNameEnum.ENERGY_KWH.value] = ws.cell(column=6, row=rownum).value
-                        row[PropertyNameEnum.CONVERTER_FACTOR.value] = ws.cell(column=7, row=rownum).value
-                        row[PropertyNameEnum.LOCAL_TEMPERATURE.value] = ws.cell(column=8, row=rownum).value
-                        row[PropertyNameEnum.TYPE.value] = ws.cell(column=9, row=rownum).value
-                        row[PropertyNameEnum.TIMESTAMP.value] = data_timestamp
-                        self.__data.append(row)
-                wb.close()
-                Client.logger.debug(f"Data read successfully between row #{minRowNum} and row #{maxRowNum}")
-            
-                os.remove(filename)
-
-                Client.logger.debug("The data update terminates normally")
-
-
-        except Exception:
-            WebDriverWrapper.logger.error(f"An unexpected error occured while updating the data",  exc_info=True)
-        finally:
-            # Quit the driver
-            driver.quit()
-            
+import logging
+from datetime import date, timedelta
+from typing import List, Optional
+
+from pygazpar.datasource import IDataSource, MeterReadingsByFrequency
+from pygazpar.enum import Frequency
+
+AUTH_NONCE_URL = "https://monespace.grdf.fr/client/particulier/accueil"
+LOGIN_URL = "https://login.monespace.grdf.fr/sofit-account-api/api/v1/auth"
+LOGIN_HEADER = {"domain": "grdf.fr"}
+LOGIN_PAYLOAD = """{{
+    "email": "{0}",
+    "password": "{1}",
+    "capp": "meg",
+    "goto": "https://sofa-connexion.grdf.fr:443/openam/oauth2/externeGrdf/authorize?response_type=code&scope=openid%20profile%20email%20infotravaux%20%2Fv1%2Faccreditation%20%2Fv1%2Faccreditations%20%2Fdigiconso%2Fv1%20%2Fdigiconso%2Fv1%2Fconsommations%20new_meg&client_id=prod_espaceclient&state=0&redirect_uri=https%3A%2F%2Fmonespace.grdf.fr%2F_codexch&nonce={2}&by_pass_okta=1&capp=meg"}}"""
+DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives/telecharger?dateDebut={1}&dateFin={2}&frequence={0}&pceList%5B%5D={3}"
+DATA_FILENAME = "Donnees_informatives_*.xlsx"
+
+DEFAULT_TMP_DIRECTORY = "/tmp"
+DEFAULT_LAST_N_DAYS = 365
+
+
+Logger = logging.getLogger(__name__)
+
+
+# ------------------------------------------------------------------------------------------------------------
+class Client:
+
+    # ------------------------------------------------------
+    def __init__(self, dataSource: IDataSource):
+        self.__dataSource = dataSource
+
+    # ------------------------------------------------------
+    def loadSince(
+        self, pceIdentifier: str, lastNDays: int = DEFAULT_LAST_N_DAYS, frequencies: Optional[List[Frequency]] = None
+    ) -> MeterReadingsByFrequency:
+
+        try:
+            endDate = date.today()
+            startDate = endDate + timedelta(days=-lastNDays)
+
+            res = self.loadDateRange(pceIdentifier, startDate, endDate, frequencies)
+        except Exception:
+            Logger.error("An unexpected error occured while loading the data", exc_info=True)
+            raise
+
+        return res
+
+    # ------------------------------------------------------
+    def loadDateRange(
+        self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None
+    ) -> MeterReadingsByFrequency:
+
+        Logger.debug("Start loading the data...")
+
+        try:
+            res = self.__dataSource.load(pceIdentifier, startDate, endDate, frequencies)
+
+            Logger.debug("The data load terminates normally")
+        except Exception:
+            Logger.error("An unexpected error occured while loading the data", exc_info=True)
+            raise
+
+        return res