ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.2__py3-none-any.whl

ckanapi_harvesters/__init__.py

@@ -1,15 +1,37 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Package with helper function for CKAN requests using pandas DataFrames.
 """
-Module for demonstrating a simple 'Hello World' function.
 
-This module imports the hello_world function from the main module
-and makes it available for public use.
+# builder_file_format_version = "0.0.1"
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:  # Python <3.8
+    from importlib_metadata import version, PackageNotFoundError
 
-Functions
----------
-hello_world()
-    Print 'Hello World!' to the console.
-"""
+try:
+    __version__ = version("ckanapi_harvesters")
+except PackageNotFoundError:
+    __version__ = None
+
+
+import os
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+from . import auxiliary
+from . import policies
+from . import harvesters
+from . import ckan_api
+from . import builder
+from . import reports
+
+# usage shortcuts
+from .auxiliary import CkanMap
+from .policies import CkanPackageDataFormatPolicy
+from .ckan_api import CkanApi, CKAN_API_VERSION
+from .builder import BUILDER_FILE_FORMAT_VERSION
+from .builder import BuilderPackage, BuilderDataStoreMultiABC, BuilderDataStoreFolder, RequestFileMapperIndexKeys
 
-from .main import hello_world
 
-__all__ = ['hello_world']

ckanapi_harvesters/auxiliary/__init__.py

@@ -0,0 +1,26 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Package with helper function for CKAN requests using pandas DataFrames.
+"""
+
+from . import ckan_defs
+from . import path
+from . import login
+from . import urls
+from . import proxy_config
+from . import external_code_import
+from . import list_records
+from . import ckan_action
+from . import ckan_errors
+from . import ckan_configuration
+from . import ckan_api_key
+from . import ckan_model
+from . import ckan_map
+from . import ckan_vocabulary_deprecated
+from . import ckan_auxiliary
+from . import deprecated
+
+from .ckan_map import CkanMap
+from .external_code_import import unlock_external_code_execution
+

ckanapi_harvesters/auxiliary/ckan_action.py

@@ -0,0 +1,93 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Action response common treatments
+"""
+from typing import Union
+import json
+
+import requests
+
+
+class CkanActionResponse:
+    """
+    Class which decodes and checks the response of a CKAN request
+    """
+    def __init__(self, response: requests.Response, dry_run: bool=False):
+        self.response: requests.Response = response  # for debug purposes
+        self.response_dict:Union[dict,None] = None
+        self.status_code:int = response.status_code
+        self.success:bool = False
+        self.success_json_loads:bool = False
+        self.result:Union[dict,None] = None
+        self.error_message: Union[None,str,dict] = None
+        self.len:Union[int,None] = None
+        self.dry_run:bool = dry_run
+
+        if response.content is None and response.request is None:
+            # dry run
+            assert(dry_run)
+            self.success = True
+            self.success_json_loads = False
+            self.status_code = 1
+            self.error_message = "Request not sent: dry run mode"
+            self.len = 0
+        else:
+            try:
+                response_dict = json.loads(response.content.decode())
+                self.response_dict = response_dict
+                self.success_json_loads = True
+                if (response.status_code == 200 and "success" in response_dict.keys() and "result" in response_dict.keys()
+                        and response_dict["success"]):
+                    self.success = True
+                    self.result = response_dict["result"]
+                else:
+                    if "error" in response_dict.keys():
+                        self.error_message = response_dict["error"]
+                    else:
+                        self.error_message = response.content.decode()
+            except Exception as json_error:
+                self.error_message = f"JSON decode error {json_error} & CKAN error {response.content.decode()}"
+
+    def __len__(self):
+        if self.len is None:
+            raise RuntimeError("queried len but does not have len")
+        return self.len
+
+    def default_error(self, ckan) -> "CkanActionError":
+        """
+        Raise specific error codes depending on response
+        """
+        if self.status_code == 404 and self.success_json_loads and self.error_message["__type"] == "Not Found Error":
+            return CkanNotFoundError(ckan, "(Generic)", self)
+        elif self.status_code == 403 and self.success_json_loads and self.error_message["__type"] == "Authorization Error":
+            return CkanAuthorizationError(ckan, self)
+        else:
+            return CkanActionError(ckan, self)
+
+## action error codes
+class CkanActionError(Exception):
+    def __init__(self, ckan, response: CkanActionResponse, display_request:bool=True):
+        super().__init__(response.error_message)
+        self.response = response
+        self.status_code = response.status_code
+        if display_request:
+            ckan._error_print_debug_response(response.response)
+
+    def __str__(self):
+        return f"Server code [{self.status_code}]: " + super().__str__()
+
+class CkanNotFoundError(CkanActionError):
+    def __init__(self, ckan, object_type:str, response: CkanActionResponse, display_request:bool=True):
+        response.error_message = f"{object_type} not found: {response.error_message}"
+        super().__init__(ckan, response, display_request=display_request)
+        self.object_type = object_type
+
+class CkanAuthorizationError(CkanActionError):
+    pass
+
+class CkanSqlCapabilityError(CkanActionError):
+    def __init__(self, ckan, response: CkanActionResponse, display_request:bool=True):
+        response.error_message = f"sql capabilities are not activated on CKAN server. See documentation for option ckan.datastore.sqlsearch.enabled"
+        super().__init__(ckan, response, display_request=display_request)
+

ckanapi_harvesters/auxiliary/ckan_api_key.py

@@ -0,0 +1,213 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Methods to load an API key
+"""
+
+import os.path
+from warnings import warn
+from typing import Dict, Union, Iterable
+import getpass
+import argparse
+
+from ckanapi_harvesters.auxiliary.ckan_errors import ApiKeyFileError
+from ckanapi_harvesters.auxiliary.path import sanitize_path, path_rel_to_dir
+from ckanapi_harvesters.auxiliary.ckan_defs import environ_keyword
+
+
+
+class ApiKey:
+    """
+    API key storage class.
+    """
+    CKAN_API_KEY_HEADER_NAME = {"Authorization", "X-CKAN-API-Key"}  # match apikey_header_name of your CKAN instance
+    CKAN_API_KEY_ENVIRON = "CKAN_API_KEY"  # not recommended to store sensitive information in environment variables
+    API_KEY_FILE_ENVIRON = "CKAN_API_KEY_FILE"
+
+    def __init__(self, *, apikey:str=None, apikey_file:str=None,
+                 api_key_header_name:Union[str, Iterable[str]]=None):
+        """
+        CKAN Database API key storage class.
+
+        :param apikey: way to provide the API key directly (optional)
+        :param apikey_file: path to a file containing a valid API key in the first line of text (optional)
+        """
+        if api_key_header_name is None:
+            api_key_header_name = "Authorization"
+        self.apikey_file: str = apikey_file  # path to a file containing a valid API key in the first line of text (optional)
+        self._apikey: str = apikey  # API key used for restricted package access
+        self.api_key_header_name = api_key_header_name
+
+    def __del__(self):
+        self.clear()
+
+    def __copy__(self):
+        return self.copy()
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = ApiKey()
+        dest.apikey_file = self.apikey_file
+        dest._apikey = self._apikey
+        return dest
+
+    def __str__(self):
+        if self._apikey is None:
+            return "None"
+        elif self._apikey == "":
+            return "<empty string>"
+        else:
+            return "*****"
+
+    @property
+    def value(self) -> Union[str,None]:
+        return self._apikey
+    @value.setter
+    def value(self, value:Union[str,None]):
+        self._apikey = value
+
+    def is_empty(self):
+        return self._apikey is None
+
+    def clear(self) -> None:
+        self._apikey = None
+
+    def load_from_environ(self, *, error_not_found:bool=False) -> bool:
+        """
+        Load CKAN API key from environment variables, by order of priority:
+
+        By default, no environment variables are used.
+        """
+        return False
+
+    def load_apikey(self, apikey_file:str=None, *, base_dir:str=None, error_not_found:bool=True) -> bool:
+        """
+        Load the API key from file.
+        The file should contain a valid API key in the first line of text.
+
+        :param apikey_file: path to the API key file. The following keywords are accepted:
+            - "environ": the API key will be looked up in the environment variable with load_from_environ
+        :param base_dir: base directory to find the API key file, if a relative path is provided
+        :param error_not_found: option to raise an exception if the API key file is not found
+        :return:
+        """
+        if apikey_file is None:
+            apikey_file = self.apikey_file
+        apikey_file = path_rel_to_dir(apikey_file, base_dir=base_dir, keyword_exceptions={environ_keyword})
+        if apikey_file is None:
+            raise ApiKeyFileError('apikey_file is required')
+        api_keyword = apikey_file.strip().lower()
+        if api_keyword == environ_keyword:
+            return self.load_from_environ(error_not_found=error_not_found)
+        if not(os.path.isfile(apikey_file)) and not error_not_found:
+            msg = f"API key file does not exist: {apikey_file}"
+            warn(msg)
+            return False
+        with open(apikey_file, 'r') as f:
+            apikey = f.readline().strip()
+            f.close()
+        self.value = apikey
+        self.apikey_file = apikey_file
+        return True
+
+    def get_auth_header(self) -> Dict[str, str]:
+        """
+        Returns the correct header with the API key for the requests needing it.
+        If no API key was loaded, returns an empty dictionary.
+        """
+        if self.value is not None:
+            apikey_encoded = self.value
+            if isinstance(self.api_key_header_name, str):
+                return {self.api_key_header_name: apikey_encoded}
+            else:
+                return {key: apikey_encoded for key in self.api_key_header_name}
+        else:
+            return {}
+
+    def input(self):
+        """
+        Prompt the user to input the API key in the console window.
+
+        :return:
+        """
+        api_key = getpass.getpass("Please enter the API key: ")
+        self._apikey = api_key
+
+    @staticmethod
+    def _setup_cli_parser(parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        if parser is None:
+            parser = argparse.ArgumentParser(description="API key initialization")
+        parser.add_argument("--apikey", type=str,
+                            help="API key")
+        parser.add_argument("--apikey-file", type=str,
+                            help="Path to a file containing the API key (first line)")
+        return parser
+
+    def _cli_args_apply(self, args: argparse.Namespace, *, base_dir: str = None, error_not_found: bool = True) -> None:
+        if args.apikey is not None:
+            self.value = args.apikey
+        if args.apikey_file is not None:
+            self.load_apikey(args.apikey_file, base_dir=base_dir, error_not_found=error_not_found)
+
+
+class CkanApiKey(ApiKey):
+    """
+    CKAN Database API key storage class.
+    """
+
+    def __init__(self, *, apikey:str=None, apikey_file:str=None):
+        """
+        CKAN Database API key storage class.
+
+        :param apikey: way to provide the API key directly (optional)
+        :param apikey_file: path to a file containing a valid API key in the first line of text (optional)
+        """
+        super().__init__(apikey=apikey, apikey_file=apikey_file, api_key_header_name=self.CKAN_API_KEY_HEADER_NAME)
+
+    def copy(self, *, dest=None) -> "CkanApiKey":
+        if dest is None:
+            dest = CkanApiKey()
+        super().copy(dest=dest)
+        return dest
+
+    def load_from_environ(self, *, error_not_found:bool=False) -> bool:
+        """
+        Load CKAN API key from environment variables, by order of priority:
+
+        - `CKAN_API_KEY`: for the raw API key (it is not recommended to store API key in an environment variable)
+        - `CKAN_API_KEY_FILE`: path to a file containing a valid API key in the first line of text
+
+        :param error_not_found: raise an error if the API key file was not found
+        :return:
+        """
+        apikey = os.environ.get(self.CKAN_API_KEY_ENVIRON)            # "CKAN_API_KEY"
+        apikey_file = sanitize_path(os.environ.get(self.API_KEY_FILE_ENVIRON))  # "CKAN_API_KEY_FILE"
+        if apikey is not None:
+            msg = f"It is not recommended to store sensitive information in environment variables such as the API key ({self.CKAN_API_KEY_ENVIRON})"
+            warn(msg)
+            self.value = apikey
+            return True
+        elif apikey_file is not None:
+            assert not apikey_file.strip().lower() == environ_keyword  # this value would create an infinite loop
+            return self.load_apikey(apikey_file, error_not_found=error_not_found)
+        else:
+            msg = f"No API key was found in the environment variable {self.CKAN_API_KEY_ENVIRON}"
+            warn(msg)
+            return False
+
+    def input(self):
+        """
+        Prompt the user to input the API key in the console window.
+
+        :return:
+        """
+        api_key = getpass.getpass("Please enter the CKAN API key: ")
+        self._apikey = api_key
+
+    @staticmethod
+    def _setup_cli_parser(parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        if parser is None:
+            parser = argparse.ArgumentParser(description="CKAN API key initialization")
+        ApiKey._setup_cli_parser(parser=parser)
+        return parser
+

ckanapi_harvesters/auxiliary/ckan_auxiliary.py

@@ -0,0 +1,293 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Data model to represent a CKAN database architecture
+"""
+from typing import Iterable, Union, Set, Tuple, final
+from enum import IntEnum
+import json
+import numbers
+import os
+import io
+import shlex
+import argparse
+import re
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.auxiliary.path import path_rel_to_dir, make_path_relative
+
+
+ckan_package_name_re = "^[0-9a-z-_]*$"
+datastore_id_col = "_id"
+
+
+class CkanIdFieldTreatment(IntEnum):
+    Keep = 0
+    SetIndex = 1
+    Remove = 2
+
+re_geometry = r"geometry\((\w+),\s*(\d+)\)"
+def parse_geometry_native_type(geometry_type:str) -> Tuple[str,int]:
+    match = re.search(re_geometry, geometry_type)
+    geometry_type = match.group(1)
+    geo_epsg = int(match.group(2))
+    return geometry_type, geo_epsg
+
+class CkanFieldInternalAttrs:
+    """
+    Custom information for internal use
+    """
+    def __init__(self):
+        self.geometry_as_source: Union[bool, None] = None
+        self.geometry_type: Union[str, None] = None
+        self.epsg_target:Union[int,None] = None
+        self.epsg_source:Union[int,None] = None
+
+    def __copy__(self):
+        return self.copy()
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    def copy(self) -> "CkanFieldInternalAttrs":
+        dest = CkanFieldInternalAttrs()
+        # from: native type (geometries)
+        dest.geometry_type = self.geometry_type
+        dest.epsg_target = self.epsg_target
+        # user options
+        dest.epsg_source = self.epsg_source
+        return dest
+
+    def merge(self, new_values: "CkanFieldInternalAttrs") -> "CkanFieldInternalAttrs":
+        dest = self.copy()
+        if new_values.geometry_type is not None:
+            dest.geometry_type = new_values.geometry_type
+        if new_values.epsg_source is not None:
+            dest.epsg_source = new_values.epsg_source
+        if new_values.epsg_target is not None:
+            dest.epsg_target = new_values.epsg_target
+        return dest
+
+    @staticmethod
+    def _setup_cli_ckan_parser(parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        if parser is None:
+            parser = argparse.ArgumentParser(description="CKAN internal field parameters")
+        parser.add_argument("--epsg-src", type=int,
+                            help="Source EPSG (geographic coordinate system) for the column, used by data_cleaner")
+        return parser
+
+    def _cli_ckan_args_apply(self, args: argparse.Namespace) -> None:
+        if args.epsg_src:
+            self.epsg_source = args.epsg_src
+
+    def init_from_options_string(self, options_string:str) -> None:
+        if options_string is None:
+            return
+        parser = self._setup_cli_ckan_parser()
+        args = parser.parse_args(shlex.split(options_string))
+        self._cli_ckan_args_apply(args)
+
+    def init_from_native_type(self, native_type:str) -> None:
+        if native_type is None:
+            return
+        if native_type.lower().strip().startswith("geometry("):
+            geometry_type, geo_epsg = parse_geometry_native_type(native_type)
+            self.geometry_type = geometry_type
+            self.epsg_target = geo_epsg
+
+    def update_from_ckan(self, ckan):
+        if self.epsg_source is not None:
+            self.epsg_target = ckan.params.ckan_default_target_epsg
+
+
+## Requests ------------------
+json_headers = {"Content-Type": "application/json", 'Accept': 'text/plain'}
+max_len_debug_print = 5000
+
+
+def json_encode_params(params:dict) -> Tuple[str, dict]:
+    """
+    For upload requests, with a records field, it is necessary to specify the params in the data argument
+    instead of the json argument of requests.
+    In the case there are NaN values, these are not supported by the requests encoder.
+
+    ___Requirement___: add headers=json_headers !!!
+
+    :param params:
+    :return:
+    """
+    data_payload = json.dumps(params, separators=(',', ':'))
+    return data_payload, json_headers
+
+class RequestType(IntEnum):
+    Get = 1
+    Post = 2
+
+def requests_multipart_data(json_dict:dict, files:dict) -> dict:
+    """
+    Generate the multipart data for a request containing json and a file.
+    Used to fill the files argument of requests.post
+    json_headers must not be used
+
+    :param json_dict:
+    :param files:
+    :return:
+    """
+    json_payload = json.dumps(json_dict)
+    multipart_data = {"json": (None, json_payload, "application/json")}
+    assert_or_raise(isinstance(files, dict) and not "json" in files.keys(), ValueError("files"))
+    multipart_data.update(files)
+    return multipart_data
+
+df_upload_to_csv_kwargs = dict()
+df_download_to_csv_kwargs = dict()
+
+def upload_prepare_requests_files_arg(*, files:dict=None, file_path:str=None, df:pd.DataFrame=None,
+                                      payload:Union[bytes, io.BufferedIOBase]=None, payload_name:str=None) -> dict:
+    """
+    Create files argument for requests.post, by order of priority:
+
+    :param files: files pass through argument to the requests.post function. Use to send other data formats.
+    :param payload: bytes to upload as a file
+    :param payload_name: name of the payload to use (associated with the payload argument) - this determines the format recognized in CKAN viewers.
+    :param file_path: path of the file to transmit (binary and text files are supported here)
+    :param df: pandas DataFrame to replace resource
+
+    :return:
+    """
+    if files is not None:
+        assert (file_path is None and df is None and payload is None)
+    elif payload is not None:
+        assert (file_path is None and df is None)
+        if payload_name is not None:
+            payload_file_name = payload_name
+            files = {"upload": (payload_file_name, payload)}
+        else:
+            files = {"upload": payload}
+    elif file_path is not None:
+        # tested with text files only, use files pass-through argument for other formats
+        assert (df is None)
+        file_name = os.path.basename(file_path)
+        payload_file_name = file_name
+        # files = {file_name: (os.path.basename(file_path), open(file_path, "r"), "text/plain")}
+        files = {"upload": (payload_file_name, open(file_path, "r"))}
+    elif df is not None:
+        payload_file_name = "file.csv"
+        files = {"upload": (payload_file_name, df.to_csv(index=False, **df_upload_to_csv_kwargs), "text/plain")}
+    else:
+        files = None
+    return files
+
+
+## Path for specific objects ------------------
+def ca_file_rel_to_dir(ca_file:Union[str,None], base_dir:str=None) -> Tuple[Union[bool,str,None], Union[str,None]]:
+    if ca_file is not None:
+        bool_keyword = ca_file.strip().lower()
+        if bool_keyword == "true":
+            return True, None
+        elif bool_keyword == "false":
+            return False, None
+        else:
+            return path_rel_to_dir(ca_file, base_dir), ca_file
+    else:
+        return None, None
+
+def ca_arg_to_str(ca_cert:Union[bool,str,None], base_dir:str=None, source_string:str=None) -> Union[str,None]:
+    if ca_cert is not None and isinstance(ca_cert, bool) and not ca_cert:
+        return "False"
+    elif ca_cert is not None and isinstance(ca_cert, str):
+        return make_path_relative(ca_cert, base_dir, source_string=source_string)
+    else:
+        return None
+
+def ssl_arguments_decompose(ca_cert:Union[bool,str,None], *, default_ssl:bool=True) -> Tuple[bool, Union[str,None]]:
+    """
+    Decompose requirements argument verify into boolean and path to a certificate file.
+
+    :param ca_cert:
+    :param default_ssl: option to indicate if SSL should be enabled if ca_cert is None
+    :return: Tuple ssl, ssl_certfile
+    """
+    if ca_cert is None:
+        return default_ssl, None
+    elif isinstance(ca_cert, bool):
+        return ca_cert, None
+    elif isinstance(ca_cert, str):
+        return True, ca_cert
+
+## Auxiliary functions ------------------
+def assert_or_raise(condition: bool, e: Exception) -> None:
+    if not condition:
+        raise e
+
+def find_duplicates(list_str:Iterable) -> list:
+    seen = set()
+    uniq = []
+    duplicates = []
+    for x in list_str:
+        if x not in seen:
+            seen.add(x)
+            uniq.append(x)
+        else:
+            duplicates.append(x)
+    return duplicates
+
+def dict_recursive_update(d:dict,u:dict) -> dict:
+    for k,v in u.items():
+        if isinstance(v, dict):
+            d[k] = dict_recursive_update(d.get(k, {}),v)
+        else:
+            d[k] = v
+    return d
+
+def _bool_from_string(string:str, default_value:Union[bool,None]=False) -> Union[bool,None]:
+    if isinstance(string, bool):
+        return string
+    else:
+        keyword = string.lower().strip()
+        if keyword == "true":
+            return True
+        elif keyword == "false":
+            return False
+        else:
+            return default_value
+
+def _string_from_element(element: pd.Series, empty_value=None) -> str:
+    if isinstance(element, pd.Series):
+        value = element.values[0]
+    else:
+        value = element
+    if ((value is None)
+            or (isinstance(value, numbers.Number) and np.isnan(value))
+            or (isinstance(value, str) and len(value) == 0)):
+        return empty_value
+    else:
+        return value
+
+def bytes_to_megabytes(size_bytes:int) -> float:
+    return round(size_bytes / 1024 / 1024, 2)
+
+## json
+def _jsons_repl_func(match):
+        return " ".join(match.group().split())
+def to_jsons_indent_lists_single_line(obj, *args, reduced_size:bool=False, **kwargs) -> str:
+    """
+    Modified json representation of an object.
+    Lists with strings / integers are displayed on one line.
+
+    :param obj: object to encode
+    :param args: args to pass to json.dumps()
+    :param reduced_size: option to not indent the json output (not human-readable)
+    :param kwargs: kwargs to pass to json.dumps()
+    :return:
+    """
+    if reduced_size:
+        return json.dumps(obj, *args, **kwargs)
+    else:
+        output = json.dumps(obj, *args, indent=4, **kwargs)
+        output = re.sub(r"(?<=\[)[^\[\]\{\}]+(?=\])", _jsons_repl_func, output)
+        # output = re.sub(r"(?<=\{)[^\[\]\{\}]+(?=\})", _jsons_repl_func, output)
+        return output
+