gn2pg_client 1.9.0.dev0__tar.gz → 1.9.1__tar.gz

{gn2pg_client-1.9.0.dev0 → gn2pg_client-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: gn2pg_client
-Version: 1.9.0.dev0
+Version: 1.9.1
 Summary: Import tool from GeoNature to a PostgreSQL database through Export module API (client side)
 License: AGPL-3.0-or-later
 Keywords: GeoNature,Export,SINP,opendata,biodiversity

{gn2pg_client-1.9.0.dev0 → gn2pg_client-1.9.1}/gn2pg/api.py

@@ -22,9 +22,10 @@ from urllib.parse import urlencode
 
 import requests
 from requests.adapters import HTTPAdapter, Retry
-from requests.exceptions import RetryError
+from requests.exceptions import HTTPError, InvalidSchema, RetryError
 
 from gn2pg import _, __version__
+from gn2pg.check_conf import Gn2PgSourceConf
 
 logger = logging.getLogger(__name__)
 
@@ -33,8 +34,8 @@ class APIException(Exception):
     """An exception occurred while handling your request."""
 
 
-class HTTPError(APIException):
-    """An HTTP error occurred."""
+class ExportModuleNotFoundError(Exception):
+    """Custom exception raised when the EXPORTS module is not found."""
 
 
 class BaseAPI:
@@ -43,10 +44,10 @@ class BaseAPI:
 
     def __init__(
         self,
-        config,
-        controler,
+        config: Gn2PgSourceConf,
+        controler: str,
     ):
-        self._config = config
+        self._config: Gn2PgSourceConf = config
         max_retry = config.max_retry
         max_requests = config.max_requests
         retry_delay = config.retry_delay
@@ -65,38 +66,42 @@ class BaseAPI:
             status_forcelist=[500, 501, 502, 503, 504],
         )
         self._session.mount("https://", HTTPAdapter(max_retries=retries))
-        self._session.headers = {"Content-Type": "application/json"}
+        self._session.headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json, text/plain, */*",
+        }
         auth_payload = {
             "login": config.user_name,
             "password": config.user_password,
         }
-        login = self._session.post(
-            self._api_url + "auth/login",
-            json=auth_payload,
-        )
+
         try:
+            login = self._session.post(
+                self._api_url + "auth/login",
+                json=auth_payload,
+            )
+            login.raise_for_status()
             if login.status_code == 200:
                 logger.info(
-                    "Successfully logged in into GeoNature named %s",
+                    _("Successfully logged in into GeoNature named %s"),
                     self._config.name,
                 )
-            else:
-                logger.critical(
-                    (
-                        "Log in GeoNature named %s failed with status code %s, cause: %s",
-                        self._config.name,
-                        login.status_code,
-                        json.loads(login.content)["msg"],
-                    )
-                )
-
-        except Exception as error:
+        except HTTPError as error:
+            logger.critical(
+                _("Login into GeoNature from source %s failed with status code %s, message: %s"),
+                self._config.name,
+                error.response.status_code,
+                error.response.json(),
+            )
+            raise error
+        except InvalidSchema as error:
             logger.critical(
-                "Session failed (%s), HTTP status code is %s", error, login.status_code
+                _("There is probably an error on source URL for %s , %s"), self._config.name, error
             )
-            raise HTTPError(login.status_code) from error
+            raise error
 
         #  Find exports api path
+        self._export_api_path = None  # Initialize the variable
         try:
             modules_list = self._session.get(self._api_url + "gn_commons/modules")
             logger.info(
@@ -104,6 +109,7 @@ class BaseAPI:
                 modules_list.status_code,
                 modules_list.url,
             )
+            modules_list.raise_for_status()
             if modules_list.status_code == 200 and "login?next=" not in modules_list.url:
                 modules = json.loads(modules_list.content)
                 for item in modules:
@@ -111,6 +117,16 @@ class BaseAPI:
                         self._export_api_path = item["module_path"]
                         logger.debug(_("Export api path is %s"), self._export_api_path)
                         break
+                if self._export_api_path is None:
+                    logger.critical(
+                        _(
+                            "EXPORTS module not found in the modules list for export %s. "
+                            "User %s may not have required permissions on module."
+                        ),
+                        self._config.name,
+                        self._config.user_name,
+                    )
+                    raise ExportModuleNotFoundError("Module not found")
             else:
                 logger.critical(
                     _("Get GeoNature modules failed with status code %s, cause: %s"),
@@ -118,9 +134,11 @@ class BaseAPI:
                     json.loads(modules_list.content)["msg"],
                 )
 
-        except Exception as error:
-            logger.critical(_("Find export module failed, %s"), error)
-            raise HTTPError(login.status_code) from error
+        except HTTPError as error:
+            logger.critical(
+                _("Looking for export module failed for source %s , %s"), self._config.name, error
+            )
+            raise error
 
     @property
     def version(self) -> str:
@@ -217,9 +235,8 @@ class BaseAPI:
                     return page_list, total_filtered, status_code
         except RetryError as e:
             last_response = e.response
-            status_code = last_response.status_code
-            logger.error(_("Export seems to be unavailable : %s"), e)
-            raise RetryError from e
+            status_code = last_response.status_code if last_response is not None else None
+            raise e
 
         return None, 0, status_code
 

{gn2pg_client-1.9.0.dev0 → gn2pg_client-1.9.1}/gn2pg/app/admin_views.py

@@ -37,7 +37,6 @@ class ImportView(ReadOnlyView):
         "data_count_errors",
         "metadata_count_upserts",
         "metadata_count_errors",
-        "xfer_http_status",
         "xfer_filters",
         "comment",
     )

{gn2pg_client-1.9.0.dev0 → gn2pg_client-1.9.1}/gn2pg/app/models.py

@@ -19,7 +19,6 @@ class ImportLog(db.Model):
     )
     xfer_type = db.Column(db.String, index=True)
     xfer_status = db.Column(db.String, index=True)
-    xfer_http_status = db.Column(db.String, index=True)
     xfer_filters = db.Column(JSONB, server_default="{}")
     xfer_start_ts = db.Column(
         db.DateTime(timezone=True),

{gn2pg_client-1.9.0.dev0 → gn2pg_client-1.9.1}/gn2pg/download.py

@@ -20,10 +20,11 @@ from multiprocessing.pool import ThreadPool
 from threading import Thread
 from typing import Callable, List, Optional
 
-from requests.exceptions import RetryError
+from requests.exceptions import HTTPError, InvalidSchema, RetryError
+from urllib3.exceptions import ResponseError
 
 from gn2pg import _, __version__
-from gn2pg.api import DataAPI
+from gn2pg.api import DataAPI, ExportModuleNotFoundError
 from gn2pg.check_conf import Gn2PgSourceConf
 from gn2pg.store_postgresql import StorePostgresql
 from gn2pg.utils import XferStatus
@@ -49,6 +50,7 @@ class DownloadGn:
         self, config: Gn2PgSourceConf, api_instance: DataAPI, backend: StorePostgresql
     ) -> None:
         self._config = config
+
         self._api_instance = api_instance
         self._backend = backend
         max_retry = config.max_retry
@@ -66,6 +68,8 @@ class DownloadGn:
         self.metadata_count_errors = 0
         self.xfer_type = ""
         self.xfer_filters = {}
+        self.xfer_status = XferStatus.init
+        self.xfer_comment = None
 
         self._limits = {
             "max_retry": max_retry,
@@ -75,7 +79,7 @@ class DownloadGn:
         # Init import log
         self.import_log_id = self._backend.import_log(
             controler=self._api_instance.controler,
-            values={"xfer_status": XferStatus.init, "xfer_start_ts": datetime.now()},
+            values={"xfer_status": self.xfer_status, "xfer_start_ts": datetime.now()},
         )
 
     @property
@@ -147,28 +151,15 @@ class DownloadGn:
         # The Queue enables the report thread to get the progress from other threads
         self.queue: Queue = Queue()
         errors: List[Exception] = []
+
         # Initialize and start the report thread
         thread = Thread(target=report, args=[self.queue])
         thread.start()
+
         # Start the worker threads
         with ThreadPool(nb_threads) as thread:
             thread.map(partial(func, queue=self.queue), pages)
-        # Will stop the report thread
-        # self._backend.import_log(
-        #     controler=self._api_instance.controler,
-        #     id=self.import_log_id,
-        #     values={
-        #         "xfer_end_ts": datetime.now(),
-        #         "api_count_items": self.api_count_items,
-        #         "api_count_errors": self.api_count_errors,
-        #         "data_count_upserts": self.data_count_upserts,
-        #         "data_count_delete": self.data_count_delete,
-        #         "data_count_errors": self.data_count_errors,
-        #         "metadata_count_upserts": self.metadata_count_upserts,
-        #         "metadata_count_errors": self.metadata_count_errors,
-        #         "xfer_status": "success",
-        #     },
-        # )
+
         self.queue.put(("DONE"))
         return errors
 
@@ -244,34 +235,41 @@ class DownloadGn:
         logger.info(_("QueryStrings %s"), params)
         pages = None
         try:
-            pages, self.api_count_items, status_code = self._api_instance.page_list(
+            pages, self.api_count_items, _xfer_http_status = self._api_instance.page_list(
                 kind="data", params=params
             )
-        except RetryError:
+        except (RetryError, ResponseError) as e:
+            self.xfer_status = XferStatus.failed
+            self.xfer_comment = str(e)
+            logger.critical("%s %s %s", e, e.response, dir(e))
             logger.error(_("Could not retrieve API data from source %s"), self._config.name)
             return
 
         try:
             # input(f"FULL DOWNLOAD INPUT {self._config.name}")
             if pages:
+                self.xfer_status = XferStatus.import_data
+                self.xfer_type = "full"
+                self.xfer_filters = (json.dumps(params, default=str),)
                 self._backend.import_log(
                     controler=self._api_instance.controler,
                     values={
-                        "xfer_type": "full",
-                        "xfer_status": XferStatus.import_data,
-                        "xfer_http_status": status_code,
-                        "xfer_filters": json.dumps(params),
+                        "xfer_type": self.xfer_type,
+                        "xfer_status": self.xfer_status,
+                        "xfer_filters": self.xfer_filters,
                     },
                 )
 
                 self.launch_threads(
                     nb_threads=self._config.nb_threads, func=self.download, pages=pages
                 )
-
+                self.xfer_status = XferStatus.success
                 # Log download timestamp to download.
 
-        except RetryError as e:
+        except (RetryError, ResponseError) as e:
             self.queue.put(("EXIT"))
+            self.xfer_status = XferStatus.failed
+            self.xfer_comment = str(e)
             logger.error(
                 "A problem occured on FULL DOWNLOAD process for source %s : %s",
                 self._config.name,
@@ -279,7 +277,7 @@ class DownloadGn:
             )
             return
 
-        return
+        self.xfer_status = XferStatus.success
 
     def update(self, since: Optional[str] = None, actions: Optional[list] = None) -> None:
         """[summary]
@@ -323,16 +321,19 @@ class DownloadGn:
 
         # Process UPDATE
         try:
-            upsert_pages, self.api_count_items, status_code = self._api_instance.page_list(
+            upsert_pages, self.api_count_items, _xfer_http_status = self._api_instance.page_list(
                 kind="data", params=params
             )
+            self.xfer_type = "update"
+            self.xfer_status = XferStatus.import_data
+            self.xfer_filters = (json.dumps(params, default=str),)
+
             self._backend.import_log(
                 controler=self._api_instance.controler,
                 values={
                     "xfer_type": "update",
-                    "xfer_status": XferStatus.import_data,
-                    "xfer_http_status": status_code,
-                    "xfer_filters": json.dumps(params, default=str),
+                    "xfer_status": self.xfer_status,
+                    "xfer_filters": self.xfer_filters,
                 },
             )
             # input(f"UPDATE INPUT {self._config.name}")
@@ -342,10 +343,14 @@ class DownloadGn:
                     func=self.download,
                     pages=upsert_pages,
                 )
-        except RetryError as e:
+
+        except (RetryError, ResponseError) as e:
             self.queue.put(("EXIT"))
+            logger.critical("%s %s %s %s", dir(e), type(e), e.args, str(e))
+            self.xfer_status = XferStatus.failed
+            self.xfer_comment = str(e)
             logger.error(
-                "A problem occured on UPDATE process for source %s : %s", self._config.name, e
+                _("A problem occured on UPDATE process for source %s : %s"), self._config.name, e
             )
             return
         # Process DELETE
@@ -355,7 +360,7 @@ class DownloadGn:
             since,
         )
         try:
-            deleted_pages, _total_len, status_code = self._api_instance.page_list(
+            deleted_pages, _total_len, _xfer_http_status = self._api_instance.page_list(
                 kind="log",
                 params={
                     "meta_last_action_date": f"gte:{since}",
@@ -365,25 +370,32 @@ class DownloadGn:
                 pagination_param="page",
             )
             # input(f"DELETE INPUT {self._config.name}")
+            self.xfer_status = XferStatus.delete
             self._backend.import_log(
                 controler=self._api_instance.controler,
-                values={"xfer_status": XferStatus.delete, "xfer_http_status": status_code},
+                values={
+                    "xfer_status": self.xfer_status,
+                },
             )
             if deleted_pages:
-
                 self.launch_threads(
                     nb_threads=self._config.nb_threads,
                     func=self.delete,
                     pages=deleted_pages,
                     store=False,
                 )
-        except RetryError as e:
+
+        except (RetryError, ResponseError) as e:
             self.queue.put(("EXIT"))
+            self.xfer_status = XferStatus.failed
+            self.xfer_comment = str(e)
             logger.error(
                 "A problem occured on DELETE process for source %s : %s", self._config.name, e
             )
             return
 
+        self.xfer_status = XferStatus.success
+
     def exit(self):
         """Final log on exit"""
         self._backend.import_log(
@@ -397,7 +409,8 @@ class DownloadGn:
                 "data_count_errors": self.data_count_errors,
                 "metadata_count_upserts": self.metadata_count_upserts,
                 "metadata_count_errors": self.metadata_count_errors,
-                "xfer_status": XferStatus.success,
+                "xfer_status": self.xfer_status,
+                "comment": self.xfer_comment,
             },
         )
 
@@ -411,4 +424,7 @@ class Data(DownloadGn):
     """
 
     def __init__(self, config, backend):
-        super().__init__(config, DataAPI(config), backend)
+        try:
+            super().__init__(config, DataAPI(config), backend)
+        except (HTTPError, ExportModuleNotFoundError, InvalidSchema) as e:
+            logger.critical(e)

{gn2pg_client-1.9.0.dev0 → gn2pg_client-1.9.1}/gn2pg/helpers.py

@@ -13,6 +13,8 @@ from os.path import isfile, join
 from pathlib import Path
 from typing import Optional
 
+from requests.exceptions import InvalidSchema
+
 from gn2pg import _
 from gn2pg.download import Data
 from gn2pg.env import CONFDIR
@@ -74,19 +76,26 @@ def full_download_1source(ctrl, cfg):
 
     logger.debug(cfg)
     with StorePostgresql(cfg) as store_pg:
-        downloader = ctrl(cfg, store_pg)
-        logger.debug(
-            _("%s => Starting download using controler %s"),
-            cfg.source,
-            downloader.name,
-        )
-        downloader.store()
-        logger.info(
-            _("%s => Ending download using controler %s"),
-            cfg.source,
-            downloader.name,
-        )
-        downloader.exit()
+        try:
+
+            downloader = ctrl(cfg, store_pg)
+            logger.debug(
+                _("%s => Starting download using controler %s"),
+                cfg.source,
+                downloader.name,
+            )
+            downloader.store()
+            logger.info(
+                _("%s => Ending download using controler %s"),
+                cfg.source,
+                downloader.name,
+            )
+            downloader.exit()
+        except (AttributeError, InvalidSchema) as e:
+            logger.critical(
+                _("An error occured when trying to download data from %s: %s"), cfg.name, e
+            )
+            return
 
 
 def full_download(cfg_ctrl):
@@ -114,19 +123,25 @@ def update_1source(ctrl, cfg):
     logger.debug(_("config source name %s"), cfg.name)
     logger.debug(_("controler %s"), ctrl)
     with StorePostgresql(cfg) as store_pg:
-        downloader = ctrl(cfg, store_pg)
-        logger.debug(
-            _("%s => Starting update (%s)"),
-            cfg.source,
-            downloader.name,
-        )
-        downloader.update()
-        logger.info(
-            _("%s => Ending update (%s)"),
-            cfg.name,
-            downloader.name,
-        )
-        downloader.exit()
+        try:
+            downloader = ctrl(cfg, store_pg)
+            logger.debug(
+                _("%s => Starting update (%s)"),
+                cfg.source,
+                downloader.name,
+            )
+            downloader.update()
+            logger.info(
+                _("%s => Ending update (%s)"),
+                cfg.name,
+                downloader.name,
+            )
+            downloader.exit()
+        except AttributeError as e:
+            logger.critical(
+                _("An error occured when trying to download data from %s: %s"), cfg.name, e
+            )
+            return
 
 
 def update(cfg_ctrl):