pygazpar 1.3.0a2__py39-none-any.whl → 1.3.0a9__py39-none-any.whl

pygazpar/datasource.py

@@ -2,7 +2,9 @@ import logging
 import glob
 import os
 import json
+import time
 import pandas as pd
+import http.cookiejar
 from abc import ABC, abstractmethod
 from typing import Any, List, Dict, cast, Optional
 from requests import Session
@@ -11,15 +13,22 @@ from pygazpar.enum import Frequency, PropertyName
 from pygazpar.excelparser import ExcelParser
 from pygazpar.jsonparser import JsonParser
 
-AUTH_NONCE_URL = "https://monespace.grdf.fr/client/particulier/accueil"
-LOGIN_URL = "https://login.monespace.grdf.fr/sofit-account-api/api/v1/auth"
-LOGIN_HEADER = {"domain": "grdf.fr"}
-LOGIN_PAYLOAD = """{{
-    "email": "{0}",
+SESSION_TOKEN_URL = "https://connexion.grdf.fr/api/v1/authn"
+SESSION_TOKEN_PAYLOAD = """{{
+    "username": "{0}",
     "password": "{1}",
-    "capp": "meg",
-    "goto": "https://sofa-connexion.grdf.fr:443/openam/oauth2/externeGrdf/authorize?response_type=code&scope=openid%20profile%20email%20infotravaux%20%2Fv1%2Faccreditation%20%2Fv1%2Faccreditations%20%2Fdigiconso%2Fv1%20%2Fdigiconso%2Fv1%2Fconsommations%20new_meg&client_id=prod_espaceclient&state=0&redirect_uri=https%3A%2F%2Fmonespace.grdf.fr%2F_codexch&nonce={2}&by_pass_okta=1&capp=meg"}}"""
-
+    "options": {{
+        "multiOptionalFactorEnroll": "false",
+        "warnBeforePasswordExpired": "false"
+    }}
+}}"""
+
+AUTH_TOKEN_URL = "https://connexion.grdf.fr/login/sessionCookieRedirect"
+AUTH_TOKEN_PARAMS = """{{
+    "checkAccountSetupComplete": "true",
+    "token": "{0}",
+    "redirectUrl": "https://monespace.grdf.fr"
+}}"""
 
 Logger = logging.getLogger(__name__)
 
@@ -50,56 +59,59 @@ class WebDataSource(IDataSource):
     # ------------------------------------------------------
     def load(self, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
 
-        session = Session()
-
-        session.headers.update(LOGIN_HEADER)
-
-        self._login(session, self.__username, self.__password)
+        auth_token = self._login(self.__username, self.__password)
 
-        res = self._loadFromSession(session, pceIdentifier, startDate, endDate, frequencies)
+        res = self._loadFromSession(auth_token, pceIdentifier, startDate, endDate, frequencies)
 
         Logger.debug("The data update terminates normally")
 
         return res
 
     # ------------------------------------------------------
-    def _login(self, session: Session, username: str, password: str):
+    def _login(self, username: str, password: str) -> str:
 
-        # Get auth_nonce token.
-        session.get(AUTH_NONCE_URL)
-        if "auth_nonce" not in session.cookies:
-            raise Exception("Login error: Cannot get auth_nonce token")
-        auth_nonce = session.cookies.get("auth_nonce")
+        session = Session()
+        session.headers.update({"domain": "grdf.fr"})
+        session.headers.update({"Content-Type": "application/json"})
+        session.headers.update({"X-Requested-With": "XMLHttpRequest"})
 
-        # Build the login payload as a json string.
-        payload = LOGIN_PAYLOAD.format(username, password, auth_nonce)
+        payload = SESSION_TOKEN_PAYLOAD.format(username, password)
 
-        # Build the login payload as a python object.
-        data = json.loads(payload)
+        response = session.post(SESSION_TOKEN_URL, data=payload)
 
-        # Send the login command.
-        response = session.post(LOGIN_URL, data=data)
+        if response.status_code != 200:
+            raise Exception(f"An error occurred while logging in. Status code: {response.status_code} - {response.text}")
 
-        # Check login result.
-        loginData = response.json()
+        session_token = response.json().get("sessionToken")
 
-        response.raise_for_status()
+        Logger.debug("Session token: %s", session_token)
 
-        if "status" in loginData and "error" in loginData and loginData["status"] >= 400:
-            raise Exception(f"{loginData['error']} ({loginData['status']})")
+        jar = http.cookiejar.CookieJar()
 
-        if "state" in loginData and loginData["state"] != "SUCCESS":
-            raise Exception(loginData["error"])
+        session = Session()
+        session.headers.update({"Content-Type": "application/json"})
+        session.headers.update({"X-Requested-With": "XMLHttpRequest"})
+
+        params = json.loads(AUTH_TOKEN_PARAMS.format(session_token))
+
+        response = session.get(AUTH_TOKEN_URL, params=params, allow_redirects=True, cookies=jar)
+
+        if response.status_code != 200:
+            raise Exception(f"An error occurred while getting the auth token. Status code: {response.status_code} - {response.text}")
+
+        auth_token = session.cookies.get("auth_token", domain="monespace.grdf.fr")
+
+        return auth_token
 
     @abstractmethod
-    def _loadFromSession(self, session: Session, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
+    def _loadFromSession(self, auth_token: str, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
         pass
 
 
 # ------------------------------------------------------------------------------------------------------------
 class ExcelWebDataSource(WebDataSource):
 
-    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives/telecharger?dateDebut={0}&dateFin={1}&frequence={3}&pceList%5B%5D={2}"
+    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives/telecharger?dateDebut={0}&dateFin={1}&frequence={3}&pceList[]={2}"
 
     DATE_FORMAT = "%Y-%m-%d"
 
@@ -121,7 +133,7 @@ class ExcelWebDataSource(WebDataSource):
         self.__tmpDirectory = tmpDirectory
 
     # ------------------------------------------------------
-    def _loadFromSession(self, session: Session, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
+    def _loadFromSession(self, auth_token: str, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
 
         res = {}
 
@@ -132,7 +144,10 @@ class ExcelWebDataSource(WebDataSource):
         file_list = glob.glob(data_file_path_pattern)
         for filename in file_list:
             if os.path.isfile(filename):
-                os.remove(filename)
+                try:
+                    os.remove(filename)
+                except PermissionError:
+                    pass
 
         if frequencies is None:
             # Transform Enum in List.
@@ -145,11 +160,31 @@ class ExcelWebDataSource(WebDataSource):
             # Inject parameters.
             downloadUrl = ExcelWebDataSource.DATA_URL.format(startDate.strftime(ExcelWebDataSource.DATE_FORMAT), endDate.strftime(ExcelWebDataSource.DATE_FORMAT), pceIdentifier, ExcelWebDataSource.FREQUENCY_VALUES[frequency])
 
-            session.get(downloadUrl)  # First request does not return anything : strange...
-
             Logger.debug(f"Loading data of frequency {ExcelWebDataSource.FREQUENCY_VALUES[frequency]} from {startDate.strftime(ExcelWebDataSource.DATE_FORMAT)} to {endDate.strftime(ExcelWebDataSource.DATE_FORMAT)}")
 
-            self.__downloadFile(session, downloadUrl, self.__tmpDirectory)
+            # Retry mechanism.
+            retry = 10
+            while retry > 0:
+
+                # Create a session.
+                session = Session()
+                session.headers.update({"Host": "monespace.grdf.fr"})
+                session.headers.update({"Domain": "grdf.fr"})
+                session.headers.update({"X-Requested-With": "XMLHttpRequest"})
+                session.headers.update({"Accept": "application/json"})
+                session.cookies.set("auth_token", auth_token, domain="monespace.grdf.fr")
+
+                try:
+                    self.__downloadFile(session, downloadUrl, self.__tmpDirectory)
+                    break
+                except Exception as e:
+
+                    if retry == 1:
+                        raise e
+
+                    Logger.error("An error occurred while loading data. Retry in 3 seconds.")
+                    time.sleep(3)
+                    retry -= 1
 
             # Load the XLSX file into the data structure
             file_list = glob.glob(data_file_path_pattern)
@@ -159,7 +194,11 @@ class ExcelWebDataSource(WebDataSource):
 
             for filename in file_list:
                 res[frequency.value] = ExcelParser.parse(filename, frequency if frequency != Frequency.YEARLY else Frequency.DAILY)
-                os.remove(filename)
+                try:
+                    # openpyxl does not close the file properly.
+                    os.remove(filename)
+                except PermissionError:
+                    pass
 
             # We compute yearly from daily data.
             if frequency == Frequency.YEARLY:
@@ -172,6 +211,12 @@ class ExcelWebDataSource(WebDataSource):
 
         response = session.get(url)
 
+        if "text/html" in response.headers.get("Content-Type"):
+            raise Exception("An error occurred while loading data. Please check your credentials.")
+
+        if response.status_code != 200:
+            raise Exception(f"An error occurred while loading data. Status code: {response.status_code} - {response.text}")
+
         response.raise_for_status()
 
         filename = response.headers["Content-Disposition"].split("filename=")[1]
@@ -210,7 +255,7 @@ class ExcelFileDataSource(IDataSource):
 # ------------------------------------------------------------------------------------------------------------
 class JsonWebDataSource(WebDataSource):
 
-    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives?dateDebut={0}&dateFin={1}&pceList%5B%5D={2}"
+    DATA_URL = "https://monespace.grdf.fr/api/e-conso/pce/consommation/informatives?dateDebut={0}&dateFin={1}&pceList[]={2}"
 
     TEMPERATURES_URL = "https://monespace.grdf.fr/api/e-conso/pce/{0}/meteo?dateFinPeriode={1}&nbJours={2}"
 
@@ -222,7 +267,7 @@ class JsonWebDataSource(WebDataSource):
 
         super().__init__(username, password)
 
-    def _loadFromSession(self, session: Session, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
+    def _loadFromSession(self, auth_token: str, pceIdentifier: str, startDate: date, endDate: date, frequencies: Optional[List[Frequency]] = None) -> MeterReadingsByFrequency:
 
         res = {}
 
@@ -237,11 +282,38 @@ class JsonWebDataSource(WebDataSource):
         # Data URL: Inject parameters.
         downloadUrl = JsonWebDataSource.DATA_URL.format(startDate.strftime(JsonWebDataSource.INPUT_DATE_FORMAT), endDate.strftime(JsonWebDataSource.INPUT_DATE_FORMAT), pceIdentifier)
 
-        # First request never returns data.
-        session.get(downloadUrl)
+        # Retry mechanism.
+        retry = 10
+        while retry > 0:
+
+            # Create a session.
+            session = Session()
+            session.headers.update({"Host": "monespace.grdf.fr"})
+            session.headers.update({"Domain": "grdf.fr"})
+            session.headers.update({"X-Requested-With": "XMLHttpRequest"})
+            session.headers.update({"Accept": "application/json"})
+            session.cookies.set("auth_token", auth_token, domain="monespace.grdf.fr")
+
+            try:
+                response = session.get(downloadUrl)
+
+                if "text/html" in response.headers.get("Content-Type"):
+                    raise Exception("An error occurred while loading data. Please check your credentials.")
+
+                if response.status_code != 200:
+                    raise Exception(f"An error occurred while loading data. Status code: {response.status_code} - {response.text}")
+
+                break
+            except Exception as e:
+
+                if retry == 1:
+                    raise e
+
+                Logger.error("An error occurred while loading data. Retry in 3 seconds.")
+                time.sleep(3)
+                retry -= 1
 
-        # Get consumption data.
-        data = session.get(downloadUrl).text
+        data = response.text
 
         # Temperatures URL: Inject parameters.
         endDate = date.today() - timedelta(days=1) if endDate >= date.today() else endDate

pygazpar/version.py

@@ -1 +1 @@
-__version__ = "1.3.0a2"
+__version__ = "1.3.0a9"

{pygazpar-1.3.0a2.dist-info → pygazpar-1.3.0a9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pygazpar
-Version: 1.3.0a2
+Version: 1.3.0a9
 Summary: Retrieve gas consumption from GrDF web site (French Gas Company)
 Home-page: https://github.com/ssenart/pygazpar
 Author: Stephane Senart
@@ -25,11 +25,16 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: openpyxl (>=2.6.3)
-Requires-Dist: requests (>=2.26.0)
+Requires-Dist: openpyxl >=2.6.3
+Requires-Dist: requests >=2.26.0
 Requires-Dist: pandas
 
 # PyGazpar
+
+## <span style="color:green">!!! This library is working again. CAPTCHA has been removed !!!</span>
+
+## <span style="color:red">~~!!! This library is broken since CAPTCHA is mandatory on GrDF site !!!~~</span>
+
 PyGazpar is a Python library for getting natural gas consumption from GrDF French provider.
 
 Their natural gas meter is called Gazpar. It is wireless and transmit the gas consumption once per day.
@@ -214,9 +219,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.2.1](https://github.com/ssenart/PyGazpar/compare/1.2.0...1.2.1) - 2022-12-28
+## [1.2.2](https://github.com/ssenart/PyGazpar/compare/1.2.1...1.2.2) - 2024-05-08
 
 ### Fixed
+- [#65](https://github.com/ssenart/PyGazpar/issues/65): [Bug] PermissionError happens when loading data from Excel file.
+
+## [1.2.1](https://github.com/ssenart/PyGazpar/compare/1.2.0...1.2.1) - 2024-05-04
+
+### Fixed
+- [#64](https://github.com/ssenart/PyGazpar/issues/64): [Issue] Captcha failed issue.
+
 - [#63](https://github.com/ssenart/PyGazpar/issues/63): [Bug] If the latest received consumption is Sunday, then the last weekly period is duplicated.
 
 ## [1.2.0](https://github.com/ssenart/PyGazpar/compare/1.1.6...1.2.0) - 2022-12-16

{pygazpar-1.3.0a2.dist-info → pygazpar-1.3.0a9.dist-info}/RECORD

@@ -1,11 +1,11 @@
 pygazpar/__init__.py,sha256=qshO_XZbDA2Wrt80ABDs0MoScqJytClAuIJjAnILglk,309
 pygazpar/__main__.py,sha256=Pt3PInX7QiWcs0aBKZN90NTaU8KFnrQiZ5Hsow1eR5U,3177
 pygazpar/client.py,sha256=JdVm0jZbeibwtTumcRbUSFadfXnCUClPMjL95_J6p5Y,2595
-pygazpar/datasource.py,sha256=B2mXUGpgq7MafYKZ1mrfq1z-U9fTgwLdTEaagTIihEU,18957
+pygazpar/datasource.py,sha256=nlIWxZ6SNSHf09BVBFIChSHf4dN05lC3LCxUUDRTylg,21470
 pygazpar/enum.py,sha256=3ZCk4SziXF6pxgP3MuQ1qxYfqB3X5DOV8Rtd0GHsK9w,898
 pygazpar/excelparser.py,sha256=glWlbj22pxYjHGKurOFmhzcVAoWCvfOHn7_Y6GgHUPo,5915
 pygazpar/jsonparser.py,sha256=AWdU3h7UohsOov8HpeP8GNuqcnDmM4r3I7-CI_crDvA,1804
-pygazpar/version.py,sha256=0Q-MEvpw9O5iUPwcmISVzmckFYWXa-XeZODFDjKaSvE,24
+pygazpar/version.py,sha256=M68dz_mPRENIQzrCD_i1UgDy4ON37VxutlMfPrsxFpY,24
 pygazpar/resources/daily_data_sample.json,sha256=YJovtrNUMs257magTfyxiewLmecySFypcelbGFUUeT8,199583
 pygazpar/resources/hourly_data_sample.json,sha256=N1F-Xz3GaBn2H1p7uKzhkhKCQV8QVR0t76XD6wmFtXA,3
 pygazpar/resources/monthly_data_sample.json,sha256=yrr4SqrB2MubeVU2HX_FRDZKHIhC0LXCqkO1iqnFWcg,3351
@@ -16,12 +16,12 @@ samples/excelSample.py,sha256=ltAl-bBz9-U9YI802JpcIswra-vDS7tR_KL5VNdxJ5c,765
 samples/jsonSample.py,sha256=sYAIusdEJhZdwDAMgHqoWcwDR0FA2eWhSt_2gL_mJRk,736
 samples/testSample.py,sha256=UeirdEtezHwfZDv_75oxul17YzGWn5yZuHfJYTF3Ez0,387
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/test_client.py,sha256=zpviS6C8h5PR-jP2RZNHUZnbVTAKXl1bjF-ku3l9EEg,5134
+tests/test_client.py,sha256=DVGGubMPMM56LvQW-4_pCouvG9qTCbVZVPoRTBGuCU4,5111
 tests/test_datafileparser.py,sha256=nAeUpOHtelblMpmbrrnf-2GuMjK5ai65veDoymceprE,818
-tests/test_datasource.py,sha256=2BCrnUh9ZbR_dAdvKT9498u5a-jGnC4aZafP7V9iZn8,5983
-pygazpar-1.3.0a2.dist-info/LICENSE.md,sha256=XsCJx_7_BC9tvmE0ZxS1cTNR7ekurog_ea9ybdZ-8tc,1073
-pygazpar-1.3.0a2.dist-info/METADATA,sha256=bN196PaZrGluxX8Xfko-8jmn6i_dqPeHp0fynh0qzG8,17682
-pygazpar-1.3.0a2.dist-info/WHEEL,sha256=ns_9KNZvwSNZtRgVV_clzMUG_fXjGc5Z8Tx4hxQ0gkw,93
-pygazpar-1.3.0a2.dist-info/entry_points.txt,sha256=c_FMZPYlRv1w9EqfgWhlkdJOoje7FcglI0UMm2oRLoI,53
-pygazpar-1.3.0a2.dist-info/top_level.txt,sha256=P7qn-XtanDPBLQsTvjvLV71wH8RK0DYbx8tzN_rDS70,23
-pygazpar-1.3.0a2.dist-info/RECORD,,
+tests/test_datasource.py,sha256=Fkn9BOGVKITAgrx9XipR1_ykT7rdvPyt_PAg3ftjfSU,5983
+pygazpar-1.3.0a9.dist-info/LICENSE.md,sha256=XsCJx_7_BC9tvmE0ZxS1cTNR7ekurog_ea9ybdZ-8tc,1073
+pygazpar-1.3.0a9.dist-info/METADATA,sha256=BAd0ZNGUkpdCRe0CxbqHc-9X-1wa5MwrBFLVy6AEjvg,18193
+pygazpar-1.3.0a9.dist-info/WHEEL,sha256=kG0f_S63jJ569yg8d_OdUrqQSrnKsZBVKN5Kb1RSnpA,93
+pygazpar-1.3.0a9.dist-info/entry_points.txt,sha256=c_FMZPYlRv1w9EqfgWhlkdJOoje7FcglI0UMm2oRLoI,53
+pygazpar-1.3.0a9.dist-info/top_level.txt,sha256=P7qn-XtanDPBLQsTvjvLV71wH8RK0DYbx8tzN_rDS70,23
+pygazpar-1.3.0a9.dist-info/RECORD,,

{pygazpar-1.3.0a2.dist-info → pygazpar-1.3.0a9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py39-none-any
 

tests/test_client.py

@@ -72,10 +72,10 @@ class TestClient:
 
         data = client.loadSince(self.__pceIdentifier, 365, [Frequency.MONTHLY])
 
-        assert (len(data[Frequency.MONTHLY.value]) >= 12 and len(data[Frequency.MONTHLY.value]) <= 13)
+        assert (len(data[Frequency.MONTHLY.value]) >= 11 and len(data[Frequency.MONTHLY.value]) <= 13)
 
     def test_yearly_jsonweb(self):
-        client = Client(ExcelWebDataSource(self.__username, self.__password, self.__tmp_directory))
+        client = Client(JsonWebDataSource(self.__username, self.__password))
 
         data = client.loadSince(self.__pceIdentifier, 365, [Frequency.YEARLY])
 

tests/test_datasource.py

@@ -143,7 +143,7 @@ class TestAllDataSource:
 
         assert (len(data[Frequency.WEEKLY.value]) >= 51 and len(data[Frequency.WEEKLY.value]) <= 54)
 
-        assert (len(data[Frequency.MONTHLY.value]) >= 12 and len(data[Frequency.MONTHLY.value]) <= 13)
+        assert (len(data[Frequency.MONTHLY.value]) >= 11 and len(data[Frequency.MONTHLY.value]) <= 13)
 
         assert (len(data[Frequency.YEARLY.value]) == 1)