pygazpar 1.3.0a2__tar.gz → 1.3.0a6__tar.gz

{pygazpar-1.3.0a2/pygazpar.egg-info → pygazpar-1.3.0a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pygazpar
-Version: 1.3.0a2
+Version: 1.3.0a6
 Summary: Retrieve gas consumption from GrDF web site (French Gas Company)
 Home-page: https://github.com/ssenart/pygazpar
 Author: Stephane Senart
@@ -197,9 +197,12 @@ Description: # PyGazpar
         The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
         and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
         
-        ## [1.2.1](https://github.com/ssenart/PyGazpar/compare/1.2.0...1.2.1) - 2022-12-28
+        
+        ## [1.2.1](https://github.com/ssenart/PyGazpar/compare/1.2.0...1.2.1) - 2024-05-04
         
         ### Fixed
+        - [#64](https://github.com/ssenart/PyGazpar/issues/64): [Issue] Captcha failed issue.
+        
         - [#63](https://github.com/ssenart/PyGazpar/issues/63): [Bug] If the latest received consumption is Sunday, then the last weekly period is duplicated.
         
         ## [1.2.0](https://github.com/ssenart/PyGazpar/compare/1.1.6...1.2.0) - 2022-12-16

{pygazpar-1.3.0a2 → pygazpar-1.3.0a6}/pygazpar/datasource.py

@@ -2,7 +2,9 @@ import logging
 import glob
 import os
 import json
+import time
 import pandas as pd
+import http.cookiejar
 from abc import ABC, abstractmethod
 from typing import Any, List, Dict, cast, Optional
 from requests import Session
@@ -11,15 +13,22 @@ from pygazpar.enum import Frequency, PropertyName
 from pygazpar.excelparser import ExcelParser
 from pygazpar.jsonparser import JsonParser
 
-AUTH_NONCE_URL = "https://monespace.grdf.fr/client/particulier/accueil"
-LOGIN_URL = "https://login.monespace.grdf.fr/sofit-account-api/api/v1/auth"
-LOGIN_HEADER = {"domain": "grdf.fr"}
-LOGIN_PAYLOAD = """{{
-    "email": "{0}",
+SESSION_TOKEN_URL = "https://connexion.grdf.fr/api/v1/authn"
+SESSION_TOKEN_PAYLOAD = """{{
+    "username": "{0}",
     "password": "{1}",
-    "capp": "meg",
-    "goto": "https://sofa-connexion.grdf.fr:443/openam/oauth2/externeGrdf/authorize?response_type=code&scope=openid%20profile%20email%20infotravaux%20%2Fv1%2Faccreditation%20%2Fv1%2Faccreditations%20%2Fdigiconso%2Fv1%20%2Fdigiconso%2Fv1%2Fconsommations%20new_meg&client_id=prod_espaceclient&state=0&redirect_uri=https%3A%2F%2Fmonespace.grdf.fr%2F_codexch&nonce={2}&by_pass_okta=1&capp=meg"}}"""
-
+    "options": {{
+        "multiOptionalFactorEnroll": "false",
+        "warnBeforePasswordExpired": "false"
+    }}
+}}"""
+
+AUTH_TOKEN_URL = "https://connexion.grdf.fr/login/sessionCookieRedirect"
+AUTH_TOKEN_PARAMS = """{{
+    "checkAccountSetupComplete": "true",
+    "token": "{0}",
+    "redirectUrl": "https://monespace.grdf.fr"
+}}"""
 
 Logger = logging.getLogger(__name__)
 
@@ -50,56 +59,59 @@ class WebDataSource(IDataSource):
     # ------------------------------------------------------
     def load(self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
 
-        session = Session()
-
-        session.headers.update(LOGIN_HEADER)
-
-        self._login(session, self.__username, self.__password)
+        auth_token = self._login(self.__username, self.__password)
 
-        res = self._loadFromSession(session, pceIdentifier, startDate, endDate, frequencies)
+        res = self._loadFromSession(auth_token, pceIdentifier, startDate, endDate, frequencies)
 
         Logger.debug("The data update terminates normally")
 
         return res
 
     # ------------------------------------------------------
-    def _login(self, session: Session, username: str, password: str):
+    def _login(self, username: str, password: str) -> str:
 
-        # Get auth_nonce token.
-        session.get(AUTH_NONCE_URL)
-        if "auth_nonce" not in session.cookies:
-            raise Exception("Login error: Cannot get auth_nonce token")
-        auth_nonce = session.cookies.get("auth_nonce")
+        session = Session()
+        session.headers.update({"domain": "grdf.fr"})
+        session.headers.update({"Content-Type": "application/json"})
+        session.headers.update({"X-Requested-With": "XMLHttpRequest"})
 
-        # Build the login payload as a json string.
-        payload = LOGIN_PAYLOAD.format(username, password, auth_nonce)
+        payload = SESSION_TOKEN_PAYLOAD.format(username, password)
 
-        # Build the login payload as a python object.
-        data = json.loads(payload)
+        response = session.post(SESSION_TOKEN_URL, data=payload)
 
-        # Send the login command.
-        response = session.post(LOGIN_URL, data=data)
+        if response.status_code != 200:
+            raise Exception(f"An error occurred while logging in. Status code: {response.status_code} - {response.text}")
 
-        # Check login result.
-        loginData = response.json()
+        session_token = response.json().get("sessionToken")
 
-        response.raise_for_status()
+        Logger.debug("Session token: %s", session_token)
+
+        jar = http.cookiejar.CookieJar()
 
-        if "status" in loginData and "error" in loginData and loginData["status"] >= 400:
-            raise Exception(f"{loginData['error']} ({loginData['status']})")
+        session = Session()
+        session.headers.update({"Content-Type": "application/json"})
+        session.headers.update({"X-Requested-With": "XMLHttpRequest"})
+
+        params = json.loads(AUTH_TOKEN_PARAMS.format(session_token))
+
+        response = session.get(AUTH_TOKEN_URL, params=params, allow_redirects=True, cookies=jar)
+
+        if response.status_code != 200:
+            raise Exception(f"An error occurred while getting the auth token. Status code: {response.status_code} - {response.text}")
 
-        if "state" in loginData and loginData["state"] != "SUCCESS":
-            raise Exception(loginData["error"])
+        auth_token = session.cookies.get("auth_token", domain="monespace.grdf.fr")
+
+        return auth_token
 
     @abstractmethod
-    def _loadFromSession(self, session: Session, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
+    def _loadFromSession(self, auth_token: str, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
         pass
 
 
 # ------------------------------------------------------------------------------------------------------------
 class ExcelWebDataSource(WebDataSource):
 
-    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives/telecharger?dateDebut={0}&dateFin={1}&frequence={3}&pceList%5B%5D={2}"
+    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives/telecharger?dateDebut={0}&dateFin={1}&frequence={3}&pceList[]={2}"
 
     DATE_FORMAT = "%Y-%m-%d"
 
@@ -121,7 +133,7 @@ class ExcelWebDataSource(WebDataSource):
         self.__tmpDirectory = tmpDirectory
 
     # ------------------------------------------------------
-    def _loadFromSession(self, session: Session, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
+    def _loadFromSession(self, auth_token: str, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
 
         res = {}
 
@@ -145,11 +157,31 @@ class ExcelWebDataSource(WebDataSource):
             # Inject parameters.
             downloadUrl = ExcelWebDataSource.DATA_URL.format(startDate.strftime(ExcelWebDataSource.DATE_FORMAT), endDate.strftime(ExcelWebDataSource.DATE_FORMAT), pceIdentifier, ExcelWebDataSource.FREQUENCY_VALUES[frequency])
 
-            session.get(downloadUrl)  # First request does not return anything : strange...
-
             Logger.debug(f"Loading data of frequency {ExcelWebDataSource.FREQUENCY_VALUES[frequency]} from {startDate.strftime(ExcelWebDataSource.DATE_FORMAT)} to {endDate.strftime(ExcelWebDataSource.DATE_FORMAT)}")
 
-            self.__downloadFile(session, downloadUrl, self.__tmpDirectory)
+            # Retry mechanism.
+            retry = 10
+            while retry > 0:
+
+                # Create a session.
+                session = Session()
+                session.headers.update({"Host": "monespace.grdf.fr"})
+                session.headers.update({"Domain": "grdf.fr"})
+                session.headers.update({"X-Requested-With": "XMLHttpRequest"})
+                session.headers.update({"Accept": "application/json"})
+                session.cookies.set("auth_token", auth_token, domain="monespace.grdf.fr")
+
+                try:
+                    self.__downloadFile(session, downloadUrl, self.__tmpDirectory)
+                    break
+                except Exception as e:
+
+                    if retry == 1:
+                        raise e
+
+                    Logger.error("An error occurred while loading data. Retry in 3 seconds.")
+                    time.sleep(3)
+                    retry -= 1
 
             # Load the XLSX file into the data structure
             file_list = glob.glob(data_file_path_pattern)
@@ -159,7 +191,11 @@ class ExcelWebDataSource(WebDataSource):
 
             for filename in file_list:
                 res[frequency.value] = ExcelParser.parse(filename, frequency if frequency != Frequency.YEARLY else Frequency.DAILY)
-                os.remove(filename)
+                try:
+                    # openpyxl does not close the file properly.
+                    os.remove(filename)
+                except Exception:
+                    pass
 
             # We compute yearly from daily data.
             if frequency == Frequency.YEARLY:
@@ -172,6 +208,12 @@ class ExcelWebDataSource(WebDataSource):
 
         response = session.get(url)
 
+        if "text/html" in response.headers.get("Content-Type"):
+            raise Exception("An error occurred while loading data. Please check your credentials.")
+
+        if response.status_code != 200:
+            raise Exception(f"An error occurred while loading data. Status code: {response.status_code} - {response.text}")
+
         response.raise_for_status()
 
         filename = response.headers["Content-Disposition"].split("filename=")[1]
@@ -210,7 +252,7 @@ class ExcelFileDataSource(IDataSource):
 # ------------------------------------------------------------------------------------------------------------
 class JsonWebDataSource(WebDataSource):
 
-    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives?dateDebut={0}&dateFin={1}&pceList%5B%5D={2}"
+    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives?dateDebut={0}&dateFin={1}&pceList[]={2}"
 
     TEMPERATURES_URL = "https://monespace.grdf.fr/api/e-conso/pce/{0}/meteo?dateFinPeriode={1}&nbJours={2}"
 
@@ -222,7 +264,7 @@ class JsonWebDataSource(WebDataSource):
 
         super().__init__(username, password)
 
-    def _loadFromSession(self, session: Session, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
+    def _loadFromSession(self, auth_token: str, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
 
         res = {}
 
@@ -237,11 +279,38 @@ class JsonWebDataSource(WebDataSource):
         # Data URL: Inject parameters.
         downloadUrl = JsonWebDataSource.DATA_URL.format(startDate.strftime(JsonWebDataSource.INPUT_DATE_FORMAT), endDate.strftime(JsonWebDataSource.INPUT_DATE_FORMAT), pceIdentifier)
 
-        # First request never returns data.
-        session.get(downloadUrl)
+        # Retry mechanism.
+        retry = 10
+        while retry > 0:
+
+            # Create a session.
+            session = Session()
+            session.headers.update({"Host": "monespace.grdf.fr"})
+            session.headers.update({"Domain": "grdf.fr"})
+            session.headers.update({"X-Requested-With": "XMLHttpRequest"})
+            session.headers.update({"Accept": "application/json"})
+            session.cookies.set("auth_token", auth_token, domain="monespace.grdf.fr")
+
+            try:
+                response = session.get(downloadUrl)
+
+                if "text/html" in response.headers.get("Content-Type"):
+                    raise Exception("An error occurred while loading data. Please check your credentials.")
+
+                if response.status_code != 200:
+                    raise Exception(f"An error occurred while loading data. Status code: {response.status_code} - {response.text}")
+
+                break
+            except Exception as e:
+
+                if retry == 1:
+                    raise e
+
+                Logger.error("An error occurred while loading data. Retry in 3 seconds.")
+                time.sleep(3)
+                retry -= 1
 
-        # Get consumption data.
-        data = session.get(downloadUrl).text
+        data = response.text
 
         # Temperatures URL: Inject parameters.
         endDate = date.today() - timedelta(days=1) if endDate >= date.today() else endDate

pygazpar-1.3.0a6/pygazpar/version.py

@@ -0,0 +1 @@
+__version__ = "1.3.0a6"

{pygazpar-1.3.0a2 → pygazpar-1.3.0a6/pygazpar.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pygazpar
-Version: 1.3.0a2
+Version: 1.3.0a6
 Summary: Retrieve gas consumption from GrDF web site (French Gas Company)
 Home-page: https://github.com/ssenart/pygazpar
 Author: Stephane Senart
@@ -197,9 +197,12 @@ Description: # PyGazpar
         The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
         and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
         
-        ## [1.2.1](https://github.com/ssenart/PyGazpar/compare/1.2.0...1.2.1) - 2022-12-28
+        
+        ## [1.2.1](https://github.com/ssenart/PyGazpar/compare/1.2.0...1.2.1) - 2024-05-04
         
         ### Fixed
+        - [#64](https://github.com/ssenart/PyGazpar/issues/64): [Issue] Captcha failed issue.
+        
         - [#63](https://github.com/ssenart/PyGazpar/issues/63): [Bug] If the latest received consumption is Sunday, then the last weekly period is duplicated.
         
         ## [1.2.0](https://github.com/ssenart/PyGazpar/compare/1.1.6...1.2.0) - 2022-12-16

{pygazpar-1.3.0a2 → pygazpar-1.3.0a6}/tests/test_client.py

@@ -75,7 +75,7 @@ class TestClient:
         assert (len(data[Frequency.MONTHLY.value]) >= 12 and len(data[Frequency.MONTHLY.value]) <= 13)
 
     def test_yearly_jsonweb(self):
-        client = Client(ExcelWebDataSource(self.__username, self.__password, self.__tmp_directory))
+        client = Client(JsonWebDataSource(self.__username, self.__password))
 
         data = client.loadSince(self.__pceIdentifier, 365, [Frequency.YEARLY])
 

pygazpar-1.3.0a2/pygazpar/version.py

@@ -1 +0,0 @@
-__version__ = "1.3.0a2"