ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/ckan_api/ckan_api_0_base.py

@@ -0,0 +1,896 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+
+"""
+from abc import ABC
+from typing import List, Dict, Callable, Union, Any, Generator, Sequence, Tuple, Collection
+from collections import OrderedDict
+import time
+import copy
+from warnings import warn
+import argparse
+import shlex
+import os
+
+import requests
+from requests.auth import AuthBase
+import pandas as pd
+
+from ckanapi_harvesters.auxiliary.error_level_message import ContextErrorLevelMessage, ErrorLevel
+from ckanapi_harvesters.auxiliary.external_code_import import unlock_external_code_execution
+from ckanapi_harvesters.auxiliary.ckan_configuration import download_external_resource_urls, \
+    unlock_external_url_resource_download, allow_no_ca, unlock_no_ca
+from ckanapi_harvesters.auxiliary.ckan_defs import environ_keyword
+from ckanapi_harvesters.auxiliary.path import path_rel_to_dir
+from ckanapi_harvesters.auxiliary.urls import urlsep, url_join
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import RequestType, max_len_debug_print, assert_or_raise
+from ckanapi_harvesters.auxiliary.proxy_config import ProxyConfig
+from ckanapi_harvesters.auxiliary.ckan_action import CkanActionResponse, CkanActionError, CkanNotFoundError
+from ckanapi_harvesters.auxiliary.ckan_errors import (MaxRequestsCountError, UnexpectedError, InvalidParameterError,
+                                                      ExternalUrlLockedError, UrlError, NoCAVerificationError, RequestError)
+from ckanapi_harvesters.auxiliary.ckan_map import CkanMap
+from ckanapi_harvesters.auxiliary.ckan_api_key import CkanApiKey
+from ckanapi_harvesters.ckan_api.ckan_api_params import CkanApiParamsBasic, CkanApiDebug
+
+CKAN_API_VERSION = 3
+
+use_ckan_owner_org_as_default:bool = True  # the owner_org field of CkanApi is destined to default the owner organization (or else it should be None)
+ckan_request_proxy_default_auth_if_ckan:bool = True  # fill authentification headers for requests with CkanApi requests proxy method if same domain is used by default
+
+## Abstract class
+class CkanApiABC(ABC):
+    pass
+
+
+
+## Main class ------------------
+class CkanApiBase(CkanApiABC):
+    """
+    CKAN Database API interface to CKAN server with helper functions using pandas DataFrames.
+    This class implements the basic parameters and request functions.
+    """
+    CKAN_URL_ENVIRON = "CKAN_URL"
+
+    def __init__(self, url:str=None, *, proxies:Union[str,dict,ProxyConfig]=None,
+                 apikey:Union[str,CkanApiKey]=None, apikey_file:str=None,
+                 owner_org:str=None, params:CkanApiParamsBasic=None,
+                 identifier=None):
+        """
+        CKAN Database API interface to CKAN server with helper functions using pandas DataFrames.
+
+        :param url: url of the CKAN server
+        :param proxies: proxies to use for requests
+        :param apikey: way to provide the API key directly (optional)
+        :param apikey_file: path to a file containing a valid API key in the first line of text (optional)
+        :param owner_org: name of the organization to limit package_search (optional)
+        :param params: other connection/behavior parameters
+        :param identifier: identifier of the ckan client
+        """
+        if identifier is None: identifier = ""
+        if apikey is None or not isinstance(apikey, CkanApiKey):
+            apikey = CkanApiKey(apikey=apikey)
+        if apikey_file is not None:
+            apikey.apikey_file = apikey_file
+        if params is None:
+            params = CkanApiParamsBasic()
+        if proxies is not None:
+            params.proxies = proxies
+        self.identifier = identifier  # variable for debugging purposes
+        self._ckan_url: str = ""
+        self.apikey: CkanApiKey = apikey
+        self.owner_org: Union[str, None] = owner_org  # name of the organization to limit package_search (optional)
+        self.params: CkanApiParamsBasic = params
+        self.ckan_session: Union[requests.Session, None] = None
+        self.extern_session: Union[requests.Session, None] = None
+        if apikey_file is not None and apikey is None:
+            self.load_apikey()
+        self.debug: CkanApiDebug = CkanApiDebug()
+        # properties
+        self.url = url  # url of the CKAN server (property)
+
+    def __del__(self):
+        self.disconnect()
+        self.apikey.__del__()
+
+    def __copy__(self):
+        return self.copy()
+
+    def copy(self, new_identifier:str=None, *, dest=None):
+        """
+        Returns a copy of the current instance.
+        Useful to use an initialized ckan object in a multithreaded context. Each thread would have its own copy.
+        It is recommended to purge the last response before doing a copy (with purge_map=False)
+        """
+        if dest is None:
+            dest = CkanApiBase()
+        dest._ckan_url = self._ckan_url
+        dest.params = self.params.copy()
+        dest.ckan_session = None
+        dest.extern_session = None
+        dest.owner_org = self.owner_org
+        dest.debug = CkanApiDebug()
+        dest.apikey = self.apikey.copy()
+        # post-copy operations
+        if new_identifier is not None:
+            dest.identifier = new_identifier
+        dest.purge()
+        # this only sets the session objects to None but lets the original instance's session open
+        dest.session = None
+        dest.extern_session = None
+        return dest
+
+    def __str__(self) -> str:
+        """
+        String representation of the instance, for debugging purposes.
+
+        :return: URL representing the CKAN server
+        """
+        return f"CKAN <{self.url}> {str(self.identifier)}"
+
+    @property
+    def url(self) -> str:
+        return self._ckan_url
+    @url.setter
+    def url(self, url:str) -> None:
+        # ensure the ckan url ends with '/' (see is_url_internal)
+        if url is None:
+            self._ckan_url = None
+        elif url.lower().strip() == environ_keyword:  # keyword
+            self.init_from_environ(init_api_key=False)
+        elif not url.endswith(urlsep):
+            self._ckan_url = url + urlsep
+        else:
+            self._ckan_url = url
+
+    def _init_session(self, *, internal:bool=False):
+        """
+        Initialize the session objects which are used to perform requests with this CKAN instance.
+        This method can be overloaded to fit your needs (proxies, certificates, cookies, headers, etc.).
+
+        :param internal:
+        :return:
+        """
+        if internal:
+            if self.ckan_session is None:
+                # the use of a session object will improve performance
+                self.ckan_session = requests.Session()
+                if self.params.proxies is not None:
+                    self.ckan_session.proxies.update(self.params.proxies)
+                self.ckan_session.auth = self.params.proxy_auth
+                self.ckan_session.verify = self.params.ckan_ca
+                self.ckan_session.headers = self.params.ckan_headers
+                # API key is applied in the headers of each request
+        else:
+            if self.extern_session is None:
+                self.extern_session = requests  # do not persist cookies between domains & requests to external resources are not meant to be numerous
+                # self.extern_session = requests.Session()
+                # if self.params.proxies is not None:
+                #     self.extern_session.proxies.update(self.params.proxies)
+                # self.extern_session.auth = self.params.proxy_auth
+                # self.extern_session.verify = self.params.extern_ca
+                # self.extern_session.headers = self.params.http_headers
+
+    def connect(self):
+        self.test_ckan_login(raise_error=True)
+
+    def disconnect(self):
+        if self.ckan_session is not None:
+            self.ckan_session.close()
+        if self.extern_session is not None and isinstance(self.extern_session, requests.Session):
+            self.extern_session.close()
+        self.ckan_session = None
+        self.extern_session = None
+
+    def full_unlock(self, unlock:bool=True,
+                    *, no_ca:bool=None, external_url_resource_download:bool=None) -> None:
+        """
+        Function to unlock full capabilities of the CKAN API
+
+        :param unlock:
+        :return:
+        """
+        if no_ca is not None:
+            unlock_no_ca(no_ca)
+        if external_url_resource_download is not None:
+            unlock_external_url_resource_download(external_url_resource_download)
+
+    def prepare_for_multithreading(self, mode_reduced:bool=True) -> None:
+        """
+        This method disables unnecessary writes to this object.
+        It is recommended to enable the reduced writes mode in a multithreaded context.
+        Do not forget to reset sessions at the beginning of each thread.
+
+        :param mode_reduced:
+        :return:
+        """
+        self.debug.store_last_response = not mode_reduced
+        self.debug.store_last_response_debug_info = not mode_reduced
+        if mode_reduced:
+            self.disconnect()
+
+    def purge(self) -> None:
+        """
+        Erase temporary data stored in this object
+
+        :param purge_map: whether to purge the map created with map_resources
+        """
+        self.debug.last_response = None
+        self.debug.ckan_request_counter = 0
+        self.debug.extern_request_counter = 0
+        self.debug.last_response_request_count = 0
+        self.debug.multi_requests_last_successful_offset = 0
+        self.debug.last_response_elapsed_time = 0.0
+
+    def set_limits(self, limit_read:Union[int,None]) -> None:
+        """
+        Set default query limits. If only one argument is provided, it applies to both limits.
+
+        :param limit_read: default limit for read requests
+        :return:
+        """
+        self.params.default_limit_read = limit_read
+        self.params.default_limit_list = limit_read
+
+    def set_verbosity(self, verbosity:bool=True, verbose_extra:bool=None) -> None:
+        """
+        Enable/disable full verbose output
+        :param verbosity: boolean. Cannot be None
+        :return:
+        """
+        self.params.verbose_multi_requests = verbosity
+        self.params.verbose_request = verbosity
+        self.params.verbose_request_error = verbosity
+        if verbose_extra is not None:
+            self.params.verbose_extra = verbose_extra
+
+    def set_proxies(self, proxies:Union[str,dict,ProxyConfig], *, default_proxies:dict=None, proxy_headers:dict=None) -> None:
+        """
+        Set up the proxy configuration
+
+        :param proxies: string or proxies dict or ProxyConfig object.
+        If a string is provided, it must be an url to a proxy or one of the following values:
+            - "environ": use the proxies specified in the environment variables "http_proxy" and "https_proxy"
+            - "noproxy": do not use any proxies
+            - "unspecified": do not specify the proxies
+            - "default": use value provided by default_proxies
+        :param default_proxies: proxies used if proxies="default"
+        :param proxy_headers: headers used to access the proxies, generally for authentication
+        :return:
+        """
+        self.params._proxy_config = ProxyConfig.from_str_or_config(proxies,
+                                                            default_proxies=default_proxies, proxy_headers=proxy_headers)
+
+    def init_from_environ(self, *, init_api_key:bool=True, error_not_found:bool=False) -> None:
+        """
+        Initialize CKAN from environment variables.
+
+        - `CKAN_URL` for the url of the CKAN server.
+
+        And optionally:
+        - `CKAN_API_KEY`: for the raw API key (it is not recommended to store API key in an environment variable)
+        - `CKAN_API_KEY_FILE`: path to a file containing a valid API key in the first line of text
+
+        :param error_not_found: raise an error if the API key file was not found
+        :return:
+        """
+        ckan_url = os.environ.get(self.CKAN_URL_ENVIRON)            # "CKAN_URL"
+        if ckan_url is not None:
+            assert not ckan_url.lower().strip() == environ_keyword  # this value would create an infinite loop
+            self.url = ckan_url
+        if init_api_key:
+            self.apikey.load_from_environ(error_not_found=error_not_found)
+
+    def _setup_cli_ckan_parser(self, parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        """
+        Define or add CLI arguments to initialize a CKAN API connection
+        parser help message:
+
+        CKAN API connection parameters initialization
+
+        options:
+          -h, --help            show this help message and exit
+          --ckan-url CKAN_URL   CKAN URL
+          --apikey APIKEY       CKAN API key
+          --apikey-file APIKEY_FILE
+                                Path to a file containing the CKAN API key (first line)
+          --policy-file POLICY_FILE
+                                Path to a file containing the CKAN data format policy (json format)
+          --owner-org OWNER_ORG
+                                CKAN Owner Organization
+          --default-limit DEFAULT_LIMIT
+                                Default number of rows per request
+          --verbose VERBOSE     Option to set verbosity
+
+        :param parser: option to provide an existing parser to add the specific fields needed to initialize a CKAN API connection
+        :return:
+        """
+        if parser is None:
+            parser = argparse.ArgumentParser(description="CKAN API connection parameters initialization")
+        parser.add_argument("--ckan-url", type=str,
+                            help="CKAN URL")
+        CkanApiKey._setup_cli_parser(parser)  # add arguments --apikey-file --apikey
+        self.params._setup_cli_ckan_parser__params(parser)
+        # parser.add_argument("--external-code", action="store_true",
+        #                     help="Enable external code execution for builder (only enable for trusted sources)")
+        return parser
+
+    def _cli_ckan_args_apply(self, args: argparse.Namespace, *, base_dir:str=None, error_not_found:bool=True,
+                             default_proxies:dict=None, proxy_headers:dict=None) -> None:
+        """
+        Apply the arguments parsed by the argument parser defined by _setup_cli_ckan_parser
+
+        :param args:
+        :param base_dir: base directory to find the CKAN API key file, if a relative path is provided
+        (recommended: leave None to use cwd)
+        :param error_not_found: option to raise an exception if the CKAN API key file is not found
+        :param default_proxies: proxies used if proxies="default"
+        :param proxy_headers: headers used to access the proxies, generally for authentication
+        :return:
+        """
+        if args.ckan_url is not None:
+            self.url = args.ckan_url
+        self.apikey._cli_args_apply(args, base_dir=base_dir, error_not_found=error_not_found)
+        self.params._cli_ckan_args_apply(args, base_dir=base_dir, error_not_found=error_not_found,
+                                         default_proxies=default_proxies, proxy_headers=proxy_headers)
+        if args.default_limit is not None:
+            self.set_limits(args.default_limit)
+        if args.verbose is not None:
+            self.set_verbosity(args.verbose)
+        # if args.external_code:
+        #     unlock_external_code_execution()
+        print(args)
+
+    def initialize_from_cli_args(self, *, args:Sequence[str]=None, base_dir:str=None,
+                                 error_not_found:bool=True, parser:argparse.ArgumentParser=None,
+                                 default_proxies:dict=None, proxy_headers:dict=None) -> None:
+        """
+        Intialize the CKAN API connection from command line arguments.
+
+        :param args: Option to provide arguments from another source.
+        :return:
+        """
+        parser = self._setup_cli_ckan_parser(parser)
+        args_parsed = parser.parse_args(args)
+        self._cli_ckan_args_apply(args_parsed, base_dir=base_dir, error_not_found=error_not_found,
+                                  default_proxies=default_proxies, proxy_headers=proxy_headers)
+
+    def input_cli_args(self, *, base_dir:str=None, error_not_found:bool=True, only_if_necessary:bool=False,
+                       default_proxies:dict=None, proxy_headers:dict=None):
+        """
+        Initialize the query for initialization parameters in the command-line format in the console window.
+
+        :return:
+        """
+        if only_if_necessary and (self.url is not None and not self.apikey.is_empty()):  # and self.proxy_object.is_defined()):
+            return
+        options_string = input("Please enter CKAN connection CLI arguments: ")
+        self.initialize_from_options_string(options_string, base_dir=base_dir, error_not_found=error_not_found,
+                                            default_proxies=default_proxies, proxy_headers=proxy_headers)
+
+    def initialize_from_options_string(self, options_string:str=None, base_dir:str=None,
+                                       error_not_found:bool=True, parser:argparse.ArgumentParser=None,
+                                       default_proxies:dict=None, proxy_headers:dict=None) -> None:
+        parser = self._setup_cli_ckan_parser(parser)
+        args = parser.parse_args(shlex.split(options_string))
+        self._cli_ckan_args_apply(args, base_dir=base_dir, error_not_found=error_not_found,
+                                  default_proxies=default_proxies, proxy_headers=proxy_headers)
+
+    def input_missing_info(self, *, base_dir:str=None, input_args:bool=False, input_args_if_necessary:bool=False,
+                           input_apikey:bool=True, error_not_found:bool=True):
+        """
+        Ask user information in the console window.
+
+        :param input_owner_org: option to ask for the owner organization.
+        :return:
+        """
+        if input_args or input_args_if_necessary:
+            self.input_cli_args(base_dir=base_dir, error_not_found=error_not_found, only_if_necessary=input_args_if_necessary)
+        if self.url is None:
+            ckan_url = input("Please enter the CKAN URL: ")
+            self.url = ckan_url
+        if self.apikey.is_empty() and input_apikey:
+            self.apikey.input()
+
+
+    ## Error management ------------------
+    def _error_print_debug_response(self, response:requests.Response, *,
+                                    url:str=None, params:dict=None, json:dict=None, error:Exception=None, headers:dict=None):
+        if self.params.verbose_request_error:
+            print(f"{self.identifier} CKAN Response error details ({str(self)})")
+            print(" ")
+            if response is None:
+                print(f"Problematic request did not obtain response ({url})")
+            else:
+                print(f"Problematic response code {response.status_code}:")
+            if response is None:
+                pass
+            elif isinstance(response.content, bytes):
+                print_str = response.content.decode()
+            else:
+                print_str = response.content
+                print(print_str[:max_len_debug_print])
+                if len(print_str) > max_len_debug_print:
+                    print("[...]")
+            if error is not None:
+                print(" ")
+                print("Exception error message:")
+                print(str(error))
+            if response is None:
+                if params is not None:
+                    print(" ")
+                    print("Request params:")
+                    print(params)
+                if json is not None:
+                    print(" ")
+                    print("Request json:")
+                    print(json)
+            else:
+                print(" ")
+                print("Request URL:")
+                print(response.request.url)
+                print(" ")
+                print("Request body:")
+                if isinstance(response.request.body, bytes):
+                    print_str = response.request.body.decode()
+                elif response.request.body is not None:
+                    print_str = response.request.body
+                else:
+                    print_str = "None"
+                print(print_str[:max_len_debug_print])
+                if len(print_str) > max_len_debug_print:
+                    print("[...]")
+                print(" ")
+                print("Response body:")
+                if response.text is not None:
+                    print_str = response.text
+                else:
+                    print_str = "None"
+                print(print_str[:max_len_debug_print])
+                if len(print_str) > max_len_debug_print:
+                    print("[...]")
+            print(" ")
+
+
+    ## Authentification ------------------
+    def load_apikey(self, apikey_file:str=None, base_dir:str=None, error_not_found:bool=True):
+        """
+        Load the CKAN API key from file.
+        The file should contain a valid API key in the first line of text.
+
+        :param apikey_file: API key file (optional if specified at the creation of the object)
+        :param base_dir: base directory, if the apikey_file is a relative path
+        :return:
+        """
+        self.apikey.load_apikey(apikey_file=apikey_file, base_dir=base_dir, error_not_found=error_not_found)
+
+    def _prepare_headers(self, headers:dict=None, include_ckan_auth:bool=False) -> dict:
+        """
+        Prepare headers for a request. If the request is destined to the CKAN server,
+        include authentication headers, if API key was provided.
+
+        :param headers: initial headers
+        :param include_ckan_auth: boolean to include CKAN authentication headers
+        :return:
+        """
+        if headers is None:
+            headers = {}
+        if self.params.user_agent is not None:
+            headers["User-Agent"] = str(self.params.user_agent)
+        headers.update(self.params._proxy_config.proxy_headers)
+        headers.update(self.params.http_headers)
+        if include_ckan_auth:
+            headers.update(self.params.ckan_headers)
+            headers.update(self.apikey.get_auth_header())
+        return headers
+
+    @staticmethod
+    def unlock_no_ca(value:bool=True):
+        """
+        This function enables you to disable the CA verification of the CKAN server.
+
+        __Warning__:
+        Only allow in a local environment!
+        """
+        unlock_no_ca(value)
+
+    @staticmethod
+    def unlock_external_url_resource_download(value:bool=True):
+        """
+        This function enables the download of resources external from the CKAN server.
+        """
+        unlock_external_url_resource_download(value)
+
+    def prepare_arguments_for_url_download_request(self, url:str, *,
+                                                   auth_if_ckan:bool=None, headers:dict=None, verify:Union[bool,str,None]=None) \
+            -> Tuple[bool, dict]:
+        """
+        Include CKAN authentication headers only if the URL points to the CKAN server.
+
+        :param url: target URL
+        :param headers: initial headers
+        :param auth_if_ckan: option to include CKAN authentication headers if the url is recognized as part of the CKAN server.
+        :return:
+        """
+        if auth_if_ckan is None:
+            auth_if_ckan = ckan_request_proxy_default_auth_if_ckan
+        verify_ca = verify
+        url_is_internal = self.is_url_internal(url)
+        if url_is_internal:
+            headers = self._prepare_headers(headers, include_ckan_auth=auth_if_ckan)
+            if verify is None:
+                verify_ca = self.params.ckan_ca
+        elif not download_external_resource_urls:
+            raise ExternalUrlLockedError(url)
+        else:
+            headers = self._prepare_headers(headers, include_ckan_auth=False)
+            if verify is None:
+                verify_ca = self.params.extern_ca
+            msg = f"Request to external url: {url}"
+            warn(msg)
+        request_kwargs = dict(headers=headers, verify=verify_ca)
+        return url_is_internal and auth_if_ckan, request_kwargs
+
+    def download_url_proxy(self, url:str, *, method:str=None, auth_if_ckan:bool=None,
+                           proxies:dict=None, headers:dict=None, auth: Union[AuthBase, Tuple[str,str]]=None, verify:Union[bool,str,None]=None) -> requests.Response:
+        """
+        Download a URL using the CKAN parameters (proxy, authentication etc.)
+
+        :param url:
+        :param proxies:
+        :param headers:
+        :return:
+        """
+        if proxies is None: proxies = self.params.proxies
+        if method is None:
+            method = "GET"
+        if auth is None:
+            auth = self.params.proxy_auth
+        url_is_internal_auth, request_kwargs = self.prepare_arguments_for_url_download_request(url, auth_if_ckan=auth_if_ckan,
+                                                                                              headers=headers, verify=verify)
+        response = None
+        self._init_session(internal=url_is_internal_auth)
+        try:
+            if self.params.dry_run:
+                response = requests.Response()
+            elif url_is_internal_auth:
+                self.debug.ckan_request_counter += 1
+                response = self.ckan_session.request(method, url, timeout=self.params.requests_timeout,
+                                                     proxies=proxies, **request_kwargs, auth=auth)
+            else:
+                self.debug.extern_request_counter += 1
+                response = self.extern_session.request(method, url, timeout=self.params.requests_timeout,
+                                                       proxies=proxies, **request_kwargs, auth=auth)
+        except Exception as e:
+            self._error_print_debug_response(response, url=url, headers=headers, error=e)
+            raise e from e
+        self.debug.last_response_request_count = 1
+        if self.params.store_last_response:
+            self.debug.last_response = response
+        return response
+
+    def download_url_proxy_test_head(self, url:str, *, raise_error:bool=False, auth_if_ckan:bool=None,
+                                     proxies:dict=None, headers:dict=None, auth: Union[AuthBase, Tuple[str,str]]=None,
+                                     verify:Union[bool,str,None]=None, context:str=None) \
+            -> Union[None,ContextErrorLevelMessage]:
+        """
+        This sends a HEAD request to the url using the CKAN connexion parameters via download_url_proxy.
+        The resource is not downloaded but the headers indicate if the url is valid.
+
+        :return: None if successful
+        """
+        if context is None:
+            context = "URL"
+        try:
+            response = self.download_url_proxy(url, method="HEAD", auth_if_ckan=auth_if_ckan, proxies=proxies, headers=headers, auth=auth, verify=verify)
+        except Exception as e:
+            if raise_error:
+                raise e from e
+            return ContextErrorLevelMessage(context, ErrorLevel.Error, f"Failed to query url {url}: {str(e)}")
+        if response.ok and response.status_code == 200:
+            return None
+        else:
+            if raise_error:
+                raise RequestError(f"Failed to query url {url}: status {response.status_code} {response.reason}")
+            return ContextErrorLevelMessage(context, ErrorLevel.Error, f"Failed to query url: {url}: status {response.status_code} {response.reason}")
+
+    ## API calls ------------------
+    def _error_empty_url(self, raise_error:bool=True) -> bool:
+        if self.url is None or self.url == "":
+            if raise_error:
+                raise UrlError("CKAN URL was not specified")
+            return True
+        return False
+
+    def _get_api_url(self, category:str=None):
+        """
+        Returns the base API url and appends the category
+
+        :param category: usually, "action"
+        :return:
+        """
+        self._error_empty_url()
+        base = url_join(self.url, "api/3")
+        if category is not None:
+            return base + urlsep + category
+        else:
+            return base
+
+    def _api_action_request(self, action:str, *, method:RequestType, params:dict=None,
+                            headers:dict=None,
+                            data:Union[dict,str,bytes]=None, json:dict=None, files:List[tuple]=None) -> CkanActionResponse:
+        """
+        Send API action request and return response.
+
+        :param action: action name
+        :param method: GET / POST
+        :param params: params to set in the url
+        :param data: information to encode in the request body (only for POST method)
+        :param json: information to encode as JSON in the request json (only for POST method)
+        :param files: files to upload in the request (only for POST method)
+        :param headers: headers for the request (authentication tokens are added by the function)
+        :return:
+        """
+        if params is None: params = {}
+        base = self._get_api_url("action")
+        url = base + urlsep + action
+        headers = self._prepare_headers(headers, include_ckan_auth=True)
+        if self.params.verbose_request:
+            if json is not None:
+                params_str = "json=" + str(json) + " / "
+            else:
+                params_str = ""
+            params_str = params_str + str(params)
+            if data is not None:
+                params_str = params_str + "data=" + str(data)[:max_len_debug_print] + " / "
+                if len(str(data)) > max_len_debug_print:
+                    params_str = params_str + "[...]"
+            params_str = params_str[:max_len_debug_print]
+            print(f"{self.identifier} API action '{action}' with arguments {params_str}")
+        start = time.time()
+        self.debug.ckan_request_counter += 1
+        response = None
+        self._init_session(internal=True)
+        try:
+            if self.params.dry_run:
+                response = requests.Response()
+            elif method == RequestType.Get:
+                assert_or_raise(data is None, UnexpectedError("data"))
+                response = self.ckan_session.get(url, params=params, headers=headers, timeout=self.params.requests_timeout,
+                                                 proxies=self.params.proxies, verify=self.params.ckan_ca, auth=self.params.proxy_auth)
+            else:
+                response = self.ckan_session.post(url, data=data, headers=headers, params=params, files=files, json=json,
+                                                  timeout=self.params.requests_timeout,
+                                                  proxies=self.params.proxies, verify=self.params.ckan_ca, auth=self.params.proxy_auth)
+        except Exception as e:
+            self._error_print_debug_response(response, url=url, params=params, headers=headers, json=json, error=e)
+            raise e from e
+        end = time.time()
+        if self.params.verbose_request and not self.params.dry_run:
+            print(f"{self.identifier} API action '{action}' done in {end-start} seconds. Received {len(response.content)} bytes")
+        if self.params.store_last_response:
+            self.debug.last_response = response
+        if self.params.store_last_response_debug_info:
+            self.debug.last_response_elapsed_time = end - start
+            self.debug.last_response_request_count = 1
+        return CkanActionResponse(response, self.params.dry_run)
+
+    def api_action_call(self, action:str, *, method:RequestType, params:dict=None,
+                            headers:dict=None,
+                            data:dict=None, json:dict=None, files:List[tuple]=None) -> CkanActionResponse:
+        # function alias of _api_action_request
+        return self._api_action_request(action=action, method=method, params=params, headers=headers, data=data, json=json, files=files)
+
+    def _url_request(self, path:str, *, method:RequestType, params:dict=None, headers:dict=None,
+                            data:dict=None, json:dict=None, files:List[tuple]=None) -> requests.Response:
+        """
+        Send request to server and return response.
+
+        :param path: relative path to server url
+        :param method: GET / POST
+        :param params: params to set in the url
+        :param data: information to encode in the request body (only for POST method)
+        :param headers: headers for the request (authentication tokens are added by the function)
+        :return:
+        """
+        if params is None: params = {}
+        self._error_empty_url()
+        url = url_join(self.url, path)
+        headers = self._prepare_headers(headers, include_ckan_auth=True)
+        if self.params.verbose_request:
+            if json is not None:
+                params_str = str(json) + " / "
+            else:
+                params_str = ""
+            params_str = params_str + str(params)
+            params_str = params_str[:min(len(params_str), max_len_debug_print)]
+            print(f"{self.identifier} URL call {url} with arguments {params_str}")
+        start = time.time()
+        self.debug.ckan_request_counter += 1
+        response = None
+        self._init_session(internal=True)
+        try:
+            if self.params.dry_run:
+                response = requests.Response()
+            elif method == RequestType.Get:
+                response = self.ckan_session.get(url, params=params, headers=headers, timeout=self.params.requests_timeout,
+                                                 proxies=self.params.proxies, verify=self.params.ckan_ca, auth=self.params.proxy_auth)
+            else:
+                response = self.ckan_session.post(url, data=data, headers=headers, params=params, timeout=self.params.requests_timeout,
+                                                  json=json, files=files,
+                                                  proxies=self.params.proxies, verify=self.params.ckan_ca, auth=self.params.proxy_auth)
+        except Exception as e:
+            self._error_print_debug_response(response, url=url, params=params, headers=headers, json=json, error=e)
+            raise e from e
+        end = time.time()
+        if self.params.verbose_request:
+            print(f"{self.identifier} URL call {url} done in {end-start} seconds. Received {len(response.content)} bytes")
+        if self.params.store_last_response:
+            self.debug.last_response = response
+        if self.params.store_last_response_debug_info:
+            self.debug.last_response_elapsed_time = end - start
+            self.debug.last_response_request_count = 1
+        return response
+
+    def api_help_show(self, action_name:str, *, print_output:bool=True) -> str:
+        """
+        API help command on a given action.
+
+        :param action_name:
+        :param print_output: Option to print the output in the command line
+        :return:
+        """
+        response = self._api_action_request("help_show", method=RequestType.Get, params={"name": action_name})
+        if response.success:
+            if print_output:
+                print(action_name + " help:")
+                print(response.result)
+            return response.result
+        elif response.status_code == 404 and response.success_json_loads and response.error_message["__type"] == "Not Found Error":
+            raise CkanNotFoundError(self, "Action", response)
+        else:
+            if print_output:
+                print(f"No documentation found for action '{action_name}'")
+            raise response.default_error(self)
+
+
+    ## Multiple queries with limited responses until full contents are obtained ------------------
+    def _request_all_results_generator(self, api_fun:Callable, *, params:dict=None,
+                                          limit:int=None, offset:int=0, search_all:bool=True,
+                                          **kwargs) -> Generator[Any, Any, None]:
+        """
+        Multiply request with a limited length until no more data is transmitted thanks to the offset parameter.
+        Lazy auxiliary function which yields a result for each request.
+
+        :param api_fun: function to call, typically a unitary request function
+        :param params: api_fun must accept params argument in order to transmit other values and enforce the offset parameter
+        :param limit: api_fun must accept limit argument in order to update the limit value
+        :param offset: api_fun must accept offset argument in order to update the offset value
+        :param search_all: if False, only the first request is operated
+        :param kwargs: additional keyword arguments to pass to api_fun
+        :return:
+        """
+        if params is None:
+            params = {}
+        if limit is None:
+            limit = self.params.default_limit_read
+        if limit is not None:
+            # params["limit"] = limit
+            assert_or_raise(limit > 0, InvalidParameterError("limit"))
+        if offset is None:
+            offset = 0
+        # params["offset"] = offset
+        if self.params.store_last_response_debug_info:
+            self.debug.multi_requests_last_successful_offset = offset
+        start = time.time()
+        requests_count = 1
+        n_received = 0
+        if self.params.verbose_multi_requests:
+            print(f"{self.identifier} Multi-requests no. {requests_count} - Requesting {limit} results from {api_fun.__name__}...")
+        result_add: Union[pd.DataFrame, CkanActionResponse, Collection] = api_fun(params=params, limit=limit, offset=offset, **kwargs)
+        if self.params.store_last_response_debug_info:
+            self.debug.multi_requests_last_successful_offset = offset
+            self.debug.last_response_request_count = requests_count
+        offset += len(result_add)
+        n_received += len(result_add)
+        yield result_add
+        current = time.time()
+        timeout = (current - start) > self.params.multi_requests_timeout
+        flag = search_all and len(result_add) > 0 and requests_count < self.params.max_requests_count and not timeout
+        while flag:
+            if self.params.multi_requests_time_between_requests > 0:
+                time.sleep(self.params.multi_requests_time_between_requests)
+            # params["offset"] = offset
+            requests_count += 1
+            if self.params.verbose_multi_requests:
+                print(f"{self.identifier} Multi-requests no. {requests_count} - Requesting {limit} results from {api_fun.__name__}...")
+            result_add = api_fun(params=params, limit=limit, offset=offset, **kwargs)
+            if self.params.store_last_response_debug_info:
+                self.debug.multi_requests_last_successful_offset = offset
+                self.debug.last_response_request_count = requests_count
+            offset += len(result_add)
+            n_received += len(result_add)
+            yield result_add
+            current = time.time()
+            timeout = (current - start) > self.params.multi_requests_timeout
+            flag = len(result_add) > 0 and requests_count < self.params.max_requests_count and not timeout
+        if timeout:
+            raise TimeoutError()
+        if requests_count >= self.params.max_requests_count:
+            raise MaxRequestsCountError()
+        current = time.time()
+        if self.params.verbose_multi_requests:
+            print(f"{self.identifier} Multi-requests {api_fun.__name__} done in {requests_count} calls and {round(current - start, 2)} seconds. {n_received} lines received.")
+        return
+
+    def _request_all_results_df(self, api_fun:Callable, *, params:dict=None, list_attrs:bool=True,
+                                limit:int=None, offset:int=0, search_all:bool=True,
+                                **kwargs) -> pd.DataFrame:
+        """
+        Multiply request with a limited length until no more data is transmitted thanks to the offset parameter.
+        DataFrame implementation returns the concatenated DataFrame from the unitary function calls.
+
+        :param api_fun: function to call, typically a unitary request function
+        :param params: api_fun must accept params argument in order to transmit other values and enforce the offset parameter
+        :param limit: api_fun must accept limit argument in order to update the limit value
+        :param offset: api_fun must accept offset argument in order to update the offset value
+        :param search_all: if False, only the first request is operated
+        :param list_attrs: option to aggregate DataFrame attrs field into lists. # False not tested
+        :param kwargs: additional keyword arguments to pass to api_fun
+        :return:
+        """
+        start = time.time()
+        iter = self._request_all_results_generator(api_fun=api_fun, params=params,
+                                                   limit=limit, offset=offset, search_all=search_all, **kwargs)
+        requests_count = 1
+        df = next(iter)
+        if list_attrs:
+            df.attrs = {key: [value] for key, value in df.attrs.items()}
+        for df_add in iter:
+            requests_count += 1
+            if len(df_add) > 0:
+                if list_attrs:
+                    attrs = df.attrs
+                df = pd.concat([df, df_add])
+                if list_attrs:
+                    df.attrs = {key: value + [df_add.attrs[key]] for key, value in attrs.items()}
+        current = time.time()
+        df.attrs["requests_count"] = requests_count
+        df.attrs["elapsed_time"] = (current - start)
+        return df
+
+    def _request_all_results_list(self, api_fun:Callable, *, params:dict=None,
+                                  limit:int=None, offset:int=0, search_all:bool=True, **kwargs) -> Union[List[CkanActionResponse], list]:
+        """
+        Multiply request with a limited length until no more data is transmitted thanks to the offset parameter.
+        List implementation returns the list of the unitary function return values.
+
+        :param api_fun: function to call, typically a unitary request function
+        :param params: api_fun must accept params argument in order to transmit other values and enforce the offset parameter
+        :param limit: api_fun must accept limit argument in order to update the limit value
+        :param offset: api_fun must accept offset argument in order to update the offset value
+        :param search_all: if False, only the first request is operated
+        :param kwargs: additional keyword arguments to pass to api_fun
+        :return:
+        """
+        return list(self._request_all_results_generator(api_fun=api_fun, params=params, limit=limit, offset=offset,
+                                                            search_all=search_all, **kwargs))
+
+    def is_url_internal(self, url:str) -> bool:
+        """
+        Tests whether a url points to the same server as the CKAN url.
+
+        :param url:
+        :return:
+        """
+        # TODO: improve the url matching test
+        return url.startswith(self.url)
+
+    def test_ckan_url_reachable(self, raise_error:bool=False) -> bool:
+        """
+        Test if the CKAN URL is reachable with a HEAD request.
+        This does not check it is really a CKAN server and does not check authentication.
+        """
+        error_message = self.download_url_proxy_test_head(self.url, raise_error=raise_error, context="CKAN URL test")
+        return error_message is None
+