ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/auxiliary/ckan_vocabulary_deprecated.py

@@ -0,0 +1,104 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+CKAN tag vocabulary information
+"""
+from typing import List, Dict, Union
+import copy
+
+from ckanapi_harvesters.auxiliary.ckan_model import CkanTagInfo
+from ckanapi_harvesters.auxiliary.ckan_map import CkanMapABC
+from ckanapi_harvesters.auxiliary.ckan_errors import NotMappedObjectNameError
+
+class CkanTagVocabularyInfo:
+    def __init__(self, d: dict):
+        self.vocabulary_name: str = d["name"]
+        self.tags: Dict[str, CkanTagInfo] = {tag_dict["name"]: CkanTagInfo.from_dict(tag_dict) for tag_dict in d["tags"]}
+        self.id: str = d["id"]
+        self.details:dict = d
+
+    def __str__(self):
+        return f"Vocabulary '{self.vocabulary_name}' ({self.id})"
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.vocabulary_name,
+                  "tags": [tag_info.to_dict() for tag_info in self.tags.values()]})
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanTagVocabularyInfo":
+        return CkanTagVocabularyInfo(d)
+
+
+class CkanVocabularyMap(CkanMapABC):
+    def __init__(self):
+        self.vocabularies: Dict[str, CkanTagVocabularyInfo] = {}  # id -> info
+        self.vocabulary_id_index: Dict[str, str] = {}  # name -> id
+        self.vocabularies_listed: bool = False
+        self._mapping_query_vocabulary_list: bool = True
+
+    def purge(self):
+        self.vocabularies = None
+        self.vocabulary_id_index = None
+        self.vocabularies_listed = False
+
+    def copy(self) -> "CkanVocabularyMap":
+        return copy.deepcopy(self)
+
+    def to_dict(self) -> dict:
+        return {"vocabularies":[vocabulary_info.to_dict() for vocabulary_info in self.vocabularies.values()],
+                }
+
+    def update_from_dict(self, data:dict) -> None:
+        for vocabulary_dict in data["packages"]:
+            self._update_vocabulary_info(CkanTagVocabularyInfo.from_dict(vocabulary_dict))
+
+    @staticmethod
+    def from_dict(d: dict) -> "CkanVocabularyMap":
+        map = CkanVocabularyMap()
+        map.update_from_dict(d)
+        return map
+
+    ## Vocabulary functions
+    def get_vocabulary_id(self, vocabulary_name:str, *, error_not_mapped:bool=True, search_title:bool=True) -> Union[str,None]:
+        """
+        Retrieve the vocabulary id for a given vocabulary name based on the vocabulary map.
+
+        :param vocabulary_name: vocabulary name or id.
+        :return:
+        """
+        if vocabulary_name is None:
+            raise ValueError("vocabulary_name cannot be None")
+        if vocabulary_name in self.vocabularies.keys():
+            # recognized vocabulary_id
+            vocabulary_id = vocabulary_name
+        elif vocabulary_name in self.vocabulary_id_index.keys():
+            vocabulary_id = self.vocabulary_id_index[vocabulary_name]
+        elif error_not_mapped:
+            raise NotMappedObjectNameError(f"Vocabulary {vocabulary_name} is not mapped or does not exist.")
+        else:
+            vocabulary_id = None
+        return vocabulary_id
+
+    def _update_vocabulary_info(self, vocabulary_info:Union[CkanTagVocabularyInfo, List[CkanTagVocabularyInfo]],
+                                vocabularies_listed:bool=False) -> None:
+        """
+        Internal function to update the information of a vocabulary.
+        """
+        if not(isinstance(vocabulary_info, list)):
+            vocabulary_info = [vocabulary_info]
+        self.vocabularies.update({vocab_info.id: vocab_info for vocab_info in vocabulary_info})
+        self.vocabulary_id_index.update({vocab_info.vocabulary_name: vocab_info.id for vocab_info in vocabulary_info})
+        if vocabularies_listed:
+            self.vocabularies_listed = True
+
+    def _record_vocabulary_delete(self, vocabulary_id: str) -> None:
+        # only pass in delete state
+        vocabulary_info = self.vocabularies[vocabulary_id]
+        self.vocabulary_id_index.pop(vocabulary_info.vocabulary_name)
+        self.vocabularies.pop(vocabulary_id)
+
+

ckanapi_harvesters/auxiliary/deprecated.py

@@ -0,0 +1,82 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Dead code from auxiliary functions
+"""
+
+from enum import IntEnum
+
+import numpy as np
+
+
+class CkanBasicDataFieldType(IntEnum):
+    Default = 0  # no override
+    Text = 1
+    Numeric = 2
+    TimeStamp = 3
+
+    def __str__(self):
+        if self == CkanBasicDataFieldType.Default:
+            return ""
+        else:
+            return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "text":
+            return CkanBasicDataFieldType.Text
+        elif s == "numeric":
+            return CkanBasicDataFieldType.Numeric
+        elif s == "timestamp":
+            return CkanBasicDataFieldType.TimeStamp
+        elif s == "" or np.isnan(s):
+            return CkanBasicDataFieldType.Default
+        else:
+            raise ValueError(s)
+
+class CkanCollaboratorCapacity(IntEnum):
+    """
+    Collaboration capacities of users associated to a package/dataset
+    """
+    Excluded = 0
+    Member = 1
+    Editor = 2
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "excluded":
+            return CkanCollaboratorCapacity.Excluded
+        elif s == "member":
+            return CkanCollaboratorCapacity.Member
+        elif s == "editor":
+            return CkanCollaboratorCapacity.Editor
+        else:
+            raise ValueError(s)
+
+class CkanGroupCapacity(IntEnum):
+    """
+    Capacities of users in a group
+    """
+    Excluded = 0
+    Member = 1
+    Admin = 3
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "excluded":
+            return CkanGroupCapacity.Excluded
+        elif s == "member":
+            return CkanGroupCapacity.Member
+        elif s == "admin":
+            return CkanGroupCapacity.Admin
+        else:
+            raise ValueError(s)

ckanapi_harvesters/auxiliary/error_level_message.py

@@ -0,0 +1,51 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Functions to define messages with an error level
+"""
+from enum import IntEnum
+from collections import OrderedDict
+from warnings import warn
+
+
+class ErrorLevel(IntEnum):
+    Information = 0
+    Warning = 1
+    Error = 2
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "information":
+            return ErrorLevel.Information
+        elif s == "warning":
+            return ErrorLevel.Warning
+        elif s == "error":
+            return ErrorLevel.Error
+        else:
+            raise ValueError(s)
+
+
+class ErrorLevelMessage(Exception):
+    def __init__(self, error_level:ErrorLevel, message: str):
+        super().__init__(message)
+        self.error_level: ErrorLevel = error_level
+        self.message: str = message
+
+    def to_dict(self) -> dict:
+        return OrderedDict([
+            ("level", str(self.error_level)),
+            ("message", self.message),
+        ])
+
+
+class ContextErrorLevelMessage(ErrorLevelMessage):
+    def __init__(self, context:str, error_level:ErrorLevel, specific_message: str):
+        message = f"In {context} / {error_level.name}: {specific_message}"
+        super().__init__(error_level, message)
+        self.context: str = context
+        self.specific_message: str = specific_message
+

ckanapi_harvesters/auxiliary/external_code_import.py

@@ -0,0 +1,98 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+This implements functionality to dynamically call functions specified by the user.
+This functionality is disabled by default. You must call unlock_external_code_execution to enable external code execution.
+__Warning__:
+only run code if you trust the source!
+"""
+from typing import Callable
+import os
+import re
+from warnings import warn
+import importlib
+import importlib.util
+
+from ckanapi_harvesters.auxiliary.path import path_rel_to_dir
+
+
+
+def unlock_external_code_execution(value:bool=True) -> None:
+    """
+    This function enables external code execution for the PythonUserCode class.
+
+    __Warning__:
+    only run code if you trust the source!
+
+    :return:
+    """
+    PythonUserCode.enable_external_code = value
+    if value:
+        msg = "External code is enabled. Only run code if you trust the source!"
+        warn(msg)
+
+
+var_name_subs_re = '\W|^(?=\d)'
+
+def clean_var_name(variable_name: str) -> str:
+    return re.sub(var_name_subs_re,'_', variable_name)
+
+
+class ExternalUserCodeDisabledException(Exception):
+    def __init__(self, function_name:str, source_file:str) -> None:
+        super().__init__(f"{function_name} in {source_file} cannot be executed because the external code execution is locked. Use unlock_external_code_execution to unlock. Warning: Only run code if you trust the source!")
+
+
+class PythonUserCode:
+    """
+    This class imports an arbitrary Python file as a module and makes it available to the rest of the code.
+    This functionality is disabled by default. You must call unlock_external_code_execution to enable external code execution.
+
+    __Warning__:
+    only run code if you trust the source!
+    """
+    enable_external_code = False  # remain False to ensure no custom code is executed from the builder specification
+
+    def __init__(self, python_file:str, base_dir:str=None):
+        self.python_file: str = ""
+        if python_file is None:
+            # only use None argument to initialize copy
+            self.python_file = ""
+        else:
+            self.python_file = path_rel_to_dir(python_file, base_dir=base_dir)
+        self.module = None
+        self.imported_module_name: str = ""
+        if PythonUserCode.enable_external_code and python_file is not None:
+            module_base = clean_var_name(os.path.splitext(os.path.split(self.python_file)[1])[0])
+            self.imported_module_name = "ckan_builder_aux_funcs__" + module_base
+            spec = importlib.util.spec_from_file_location(self.imported_module_name, self.python_file)
+            self.module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(self.module)
+        elif python_file is not None:  # and not enable_external_code
+            # External python script execution is locked
+            raise ExternalUserCodeDisabledException("code", os.path.split(self.python_file)[1])
+
+    def __copy__(self):
+        return self.copy()
+
+    def copy(self) -> "PythonUserCode":
+        # do not execute module twice and do not make copies either
+        dest = PythonUserCode(python_file=None, base_dir=None)
+        dest.module = self.module
+        dest.imported_module_name = self.imported_module_name
+        dest.python_file = self.python_file
+        return dest
+
+    def function_pointer(self, function_name:str) -> Callable:
+        """
+        Obtain function pointer for a given name in the loaded Python module.
+
+        :param function_name:
+        :return:
+        """
+        if not PythonUserCode.enable_external_code:
+            raise ExternalUserCodeDisabledException(function_name, os.path.split(self.python_file)[1])
+        fun = getattr(self.module, function_name)
+        assert(isinstance(fun, Callable))
+        return fun
+

ckanapi_harvesters/auxiliary/list_records.py

@@ -0,0 +1,60 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Give partial DataFrame behavior to a list of dictionaries
+"""
+import pandas as pd
+import numpy as np
+from typing import Union, List
+
+class _ListRecords_index:
+    def __init__(self, parent) -> None:
+        self.parent = parent
+
+    def __getitem__(self, slice):
+        return self.parent[slice]
+
+    # def __setitem__(self, slice, value):
+    #     self.parent[slice] = value
+
+class ListRecords(list):  # List[dict]
+    """
+    Give partial DataFrame behavior to a list of dictionaries
+    """
+    def __init__(self,*args,**kwargs):
+        super().__init__(*args,**kwargs)
+        self.columns: Union[List[str],None] = None
+        if len(self) > 0:
+            self.columns = list(self[0].keys())
+        else:
+            self.columns = []
+
+    @property
+    def iloc(self):
+        return _ListRecords_index(self)
+
+def records_to_df(records: Union[List[dict], ListRecords], df_args:dict=None, *,
+                  missing_value="", none_value="None") -> pd.DataFrame:
+    """
+    Keep source values (lesser type inference) and replace cells with missing keys with a fixed value.
+    None values are also preserved using the none_value.
+
+    :param records: input data
+    :param df_args: arguments to pass to DataFrame constructor
+    :param missing_value: value to set if a column is not specified on a row.
+    :param none_value: value to set if a value is None in the input data.
+    :return:
+    """
+    if df_args is None:
+        df_args = {}
+    df = pd.DataFrame(records, dtype=object, **df_args)
+    fieldnames = df.columns
+    nrows = len(df)
+    # df.fillna(np.nan, inplace=True, downcast="object")
+    for (i, row), record in zip(df.iterrows(), records):
+        for field in fieldnames:
+            if field not in record.keys():
+                df.loc[i, field] = missing_value
+            elif record[field] is None:
+                df.loc[i, field] = none_value
+    return df

ckanapi_harvesters/auxiliary/login.py

@@ -0,0 +1,163 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Methods to load authentication credentials (user, password)
+"""
+from typing import Union, Tuple, Dict
+import getpass
+from warnings import warn
+import os
+import argparse
+import shlex
+
+from ckanapi_harvesters.auxiliary.path import path_rel_to_dir
+from ckanapi_harvesters.auxiliary.ckan_defs import environ_keyword
+from ckanapi_harvesters.auxiliary.ckan_errors import ApiKeyFileError
+
+
+class Login:
+    def __init__(self, username:str=None, password:str=None, login_file:str=None):
+        self._username:Union[str,None] = username
+        self._password:Union[str,None] = password
+        self.login_file: str = login_file  # path to a file containing a valid API key in the first line of text (optional)
+
+    def __del__(self):
+        self.clear()
+
+    def __copy__(self):
+        return self.copy()
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = Login()
+        dest.login_file = self.login_file
+        dest._username = self._username
+        dest._password = self._password
+        return dest
+
+    def clear(self) -> None:
+        self._username = None
+        self._password = None
+
+    def is_empty(self):
+        return self._username is None or self._password is None
+
+    def __str__(self):
+        if self.is_empty():
+            return "None"
+        elif self._username == "" and self._password == "":
+            return "<empty string>"
+        elif self._username == "":
+            return "<empty user>"
+        elif self._password == "":
+            return "<empty password>"
+        else:
+            return "*****"
+
+    @property
+    def username(self) -> Union[str,None]:
+        return self._username
+    @username.setter
+    def username(self, value:Union[str,None]):
+        self._username = value
+    @property
+    def password(self) -> Union[str,None]:
+        return self._password
+    @password.setter
+    def password(self, value:Union[str,None]):
+        self._password = value
+
+    def load_from_file(self, login_file:str=None, *, base_dir:str=None, error_not_found:bool=True) -> bool:
+        """
+        Load the credentials from file.
+        The file should contain username in first line and password in second line.
+
+        :param login_file: path to the API key file. The following keywords are accepted:
+            - "environ": the API key will be looked up in the environment variable with load_from_environ
+        :param base_dir: base directory to find the API key file, if a relative path is provided
+        :param error_not_found: option to raise an exception if the API key file is not found
+        :return:
+        """
+        if login_file is None:
+            login_file = self.login_file
+        login_file = path_rel_to_dir(login_file, base_dir=base_dir, keyword_exceptions={environ_keyword})
+        if login_file is None:
+            raise ApiKeyFileError('login_file is required')
+        api_keyword = login_file.strip().lower()
+        if api_keyword == environ_keyword:
+            return False  # self.load_from_environ(error_not_found=error_not_found)
+        if not(os.path.isfile(login_file)) and not error_not_found:
+            msg = f"Login file does not exist: {login_file}"
+            warn(msg)
+            return False
+        with open(login_file, 'r') as f:
+            self.username = f.readline().strip()
+            self.password = f.readline().rstrip('\n')
+            f.close()
+        self.login_file = login_file
+        return True
+
+    def input(self):
+        """
+        Prompt the user to input the login credentials in the console window.
+
+        :return:
+        """
+        value = input("Please enter user name: ")
+        self.username = value
+        value = getpass.getpass("Please enter password: ")
+        self.password = value
+
+    @staticmethod
+    def _setup_cli_parser(parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        if parser is None:
+            parser = argparse.ArgumentParser(description="Login credentials initialization")
+        parser.add_argument("--login-file", type=str,
+                            help="Path to a text file containing login credentials for authentification (user, password)")
+        return parser
+
+    def _cli_args_apply(self, args: argparse.Namespace, *, base_dir: str = None, error_not_found: bool = True) -> None:
+        if args.login_file is not None:
+            self.load_from_file(args.login_file, base_dir=base_dir, error_not_found=error_not_found)
+
+    def to_tuple(self) -> Tuple[str,str]:
+        return self.username, self.password
+
+    @staticmethod
+    def from_tuple(values: Tuple[str,str]) -> "Login":
+        login = Login(*values)
+        return login
+
+    def to_dict(self) -> Dict[str,str]:
+        return {"username": self.username, "password": self.password}
+
+    @staticmethod
+    def from_dict(values: Dict[str,str]) -> "Login":
+        login = Login(**values)
+        return login
+
+
+class SSHLogin(Login):
+    @staticmethod
+    def _setup_cli_parser(parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        if parser is None:
+            parser = argparse.ArgumentParser(description="SSH login credentials initialization")
+        parser.add_argument("--ssh-login-file", type=str,
+                            help="Path to a text file containing SSH login credentials for authentification (user, password)")
+        return parser
+
+    def _cli_args_apply(self, args: argparse.Namespace, *, base_dir: str = None, error_not_found: bool = True) -> None:
+        if args.login_file is not None:
+            self.load_from_file(args.ssh_login_file, base_dir=base_dir, error_not_found=error_not_found)
+
+    def input(self):
+        """
+        Prompt the user to input the login credentials in the console window.
+
+        :return:
+        """
+        value = input("Please enter SSH user name: ")
+        self.username = value
+        value = getpass.getpass("Please enter SSH password: ")
+        self.password = value
+