ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/builder/mapper_datastore_multi.py

@@ -0,0 +1,262 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Code to define the bondage between a file and a database query
+in the context of a large DataStore defined by the concatenation of multiple files.
+"""
+from warnings import warn
+from abc import ABC, abstractmethod
+from typing import Dict, List, Iterable, Callable, Any, Tuple, Generator, Set, Union
+
+import numpy as np
+import pandas as pd
+
+from ckanapi_harvesters.builder.builder_resource_datastore import DataSchemeConversion
+from ckanapi_harvesters.auxiliary.ckan_model import UpsertChoice
+from ckanapi_harvesters.auxiliary.ckan_defs import ckan_tags_sep
+from ckanapi_harvesters.ckan_api import CkanApi
+
+
+class RequestMapperABC(DataSchemeConversion, ABC):
+    """
+    Class to define how to reconstruct a file from the full dataset
+    This class overloads some data scheme conversion class functions
+    This abstract class can be derived to specify custom data treatments
+    """
+    def __init__(self,
+                 *, df_upload_fun:Callable[[pd.DataFrame], Any] = None,
+                 df_download_fun:Callable[[pd.DataFrame], Any] = None):
+        super().__init__(df_upload_fun=df_upload_fun, df_download_fun=df_download_fun)
+        self.upsert_only_missing_rows:bool = False
+
+    ## upsert request preparation  ----------------
+    def get_file_query_of_df(self, df_upload:pd.DataFrame) -> Union[dict,None]:
+        """
+        Return the dict of {field: value} combinations representing the arguments of the query to reconstruct a file
+
+        :param df_upload: the DataFrame representing the file
+        :return:
+        """
+        return None
+
+    def last_inserted_row_request(self, ckan:CkanApi, resource_id:str, file_query:dict) -> Union[pd.DataFrame,None]:
+        """
+        Request in CKAN the last inserted row(s) corresponding to a given file_query
+
+        :param ckan:
+        :param resource_id:
+        :param file_query: a dict of {field: value} combinations representing the arguments of the query to reconstruct a file
+        :return: The last row(s) in the database or None (if no specific method was defined)
+        """
+        return None
+
+    def last_inserted_index_request(self, ckan:CkanApi, resource_id:str, file_query:dict, df_upload:pd.DataFrame) -> Tuple[int, bool, int, Union[pd.DataFrame,None]]:
+        """
+        Knowing the data which needs to be uploaded, this function compares the last known row(s) to the dataframe
+        and returns the index to restart the upload process.
+
+        :param ckan:
+        :param resource_id:
+        :param file_query: a dict of {field: value} combinations representing the arguments of the query to reconstruct a file
+        :param df_upload: the known data corresponding to the file_query to be sent
+        :return: a tuple (i_restart, upload_needed, row_count, df_last_row):
+         - i_restart: the last known index in the dataframe
+         - upload_needed: a boolean indicating if an update is necessary
+         - row_count: the number of rows corresponding to the file_query
+         - df_last_row: the last found row in the dataframe
+        """
+        return 0, True, -1, None
+
+    ## download preparation  ----------------
+    @abstractmethod
+    def download_file_query_list(self, ckan: CkanApi, resource_id: str) -> List[dict]:
+        """
+        Function to list the {key: value} combinations present in the CKAN datastore to reconstruct the file database before downloading.
+
+        :param ckan:
+        :param resource_id:
+        :return: a list of query arguments defining each file
+        """
+        raise NotImplementedError()
+
+    def download_file_query_generator(self, ckan: CkanApi, resource_id: str) -> Generator[dict, Any, None]:
+        """
+        Generator for download_file_query_list which can be customized
+
+        :param ckan:
+        :param resource_id:
+        :return:
+        """
+        for file_query in self.download_file_query_list(ckan=ckan, resource_id=resource_id):
+            yield file_query
+
+    def download_file_query(self, ckan: CkanApi, resource_id: str, file_query:dict) -> pd.DataFrame:
+        return ckan.datastore_search(resource_id=resource_id, **file_query, search_all=True)
+
+
+class RequestFileMapperABC(RequestMapperABC, ABC):
+    """
+    Class to define how to reconstruct a file from the full dataset
+    This abstract class is oriented to treating files in the file system
+    """
+    def __init__(self,
+                 *, df_upload_fun:Callable[[pd.DataFrame], Any] = None,
+                 df_download_fun:Callable[[pd.DataFrame], Any] = None):
+        super().__init__(df_upload_fun=df_upload_fun, df_download_fun=df_download_fun)
+        self.file_name_prefix:str = "table_"
+        self.file_name_suffix:str = ".csv"
+        self.file_name_function:Union[Callable[[dict], str], None] = None
+
+    def get_file_name_of_query(self, file_query:dict) -> str:
+        if self.file_name_function is None:
+            file_filters_str = '_'.join([str(key)+'_'+str(value) for key,value in file_query.items()])
+        else:
+            file_filters_str = self.file_name_function(file_query)
+        return f"{self.file_name_prefix}{file_filters_str}{self.file_name_suffix}"
+
+
+class RequestFileMapperUser(RequestFileMapperABC):
+    """
+    Use this basic implementation if the file query list is provided by the user or if the builder is only used to upload files.
+    """
+    def __init__(self, file_query_list: Iterable[Tuple[str, dict]],
+                 *, df_upload_fun:Callable[[pd.DataFrame], Any] = None,
+                 df_download_fun:Callable[[pd.DataFrame], Any] = None):
+        super().__init__(df_upload_fun=df_upload_fun, df_download_fun=df_download_fun)
+        # file_query_list must be stored in the BuilderDataStoreMultiAbc instance
+
+    def download_file_query_list(self, ckan: CkanApi, resource_id: str) -> List[dict]:
+        raise RuntimeError("File query list is provided by user")
+
+
+class RequestFileMapperLimit(RequestFileMapperABC):
+    """
+    In this implementation, a file is defined by a certain amount of rows
+    """
+    default_limit = 10000
+
+    def __init__(self, limit:int=None,
+                 *, df_upload_fun:Callable[[pd.DataFrame], Any] = None,
+                 df_download_fun:Callable[[pd.DataFrame], Any] = None):
+        super().__init__(df_upload_fun=df_upload_fun, df_download_fun=df_download_fun)
+        if limit is None:
+            limit = RequestFileMapperLimit.default_limit
+        self.limit:int = limit
+
+    ## download preparation  ----------------
+    def get_file_name_of_query(self, file_query:dict) -> str:
+        if self.file_name_function is None:
+            # file_filters_str = str(file_query["offset"] // self.limit)
+            file_filters_str = f'{file_query["offset"]}_{file_query["offset"]+self.limit-1}'
+        else:
+            file_filters_str = self.file_name_function(file_query)
+        return f"{self.file_name_prefix}{file_filters_str}{self.file_name_suffix}"
+
+    def download_file_query_list(self, ckan: CkanApi, resource_id: str) -> List[dict]:
+        # get number of rows and return a list of [offset,limit] combinations
+        row_count = ckan.datastore_search_row_count(resource_id)
+        return [{"offset": self.limit*counter, "limit": self.limit} for counter in range(row_count // self.limit + 1)]
+
+    def download_file_query(self, ckan: CkanApi, resource_id: str, file_query:dict) -> pd.DataFrame:
+        return ckan.datastore_search(resource_id=resource_id, offset=file_query["offset"], limit=file_query["limit"], search_all=True)
+
+
+class RequestFileMapperIndexKeys(RequestFileMapperABC):
+    """
+    In this implementation, a file is defined by a combination of file_keys values
+    It is optionally ordered by an index_keys which enables to restart a transfer when interrupted
+    By default, the index_keys is the last field of the primary key
+    and the file_keys are the fields preceding the index_keys in the primary key
+    """
+    last_rows_limit = 1
+    def __init__(self, group_by_keys:List[str], sort_by_keys:List[str] = None,
+                 *, df_upload_fun:Callable[[pd.DataFrame], Any] = None,
+                 df_download_fun:Callable[[pd.DataFrame], Any] = None):
+        super().__init__(df_upload_fun=df_upload_fun, df_download_fun=df_download_fun)
+        self.group_by_keys: List[str] = group_by_keys        # fields to filter to obtain one file
+        self.sort_by_keys: Union[List[str],None] = None      # field to order the document
+        if sort_by_keys is not None:
+            self.sort_by_keys = sort_by_keys
+
+    def get_necessary_fields(self) -> Set[str]:
+        fields = set(self.group_by_keys)
+        if self.sort_by_keys is not None:
+            fields = fields.union(set(self.sort_by_keys))
+        return fields
+
+    def df_upload_alter(self, df_local: pd.DataFrame, file_name:str=None, mapper_kwargs:dict=None, **kwargs) -> pd.DataFrame:
+        # overload of df_upload_alter calling self.df_upload_fun
+        # order dataframes before sending to database in order to be able to restart transfer from last transmitted index
+        df_database = super().df_upload_alter(df_local, file_name=file_name, mapper_kwargs=mapper_kwargs, **kwargs)
+        if self.sort_by_keys is not None:
+            if self.df_upload_fun is None:
+                df_database = df_database.copy()
+            df_database.sort_values(self.sort_by_keys, inplace=True)
+        return df_database
+
+    ## upsert request preparation  ----------------
+    def get_file_query_of_df(self, df_upload:pd.DataFrame) -> Union[dict,None]:
+        df_file_query = df_upload[self.group_by_keys].drop_duplicates(subset=self.group_by_keys)
+        if len(df_file_query) == 1:
+            return {"filters": df_file_query.to_dict(orient="records")[0]}
+        else:
+            return None
+
+    def last_inserted_row_request(self, ckan:CkanApi, resource_id:str, file_query:dict) -> Union[pd.DataFrame,None]:
+        if self.sort_by_keys is None or not self.upsert_only_missing_rows:
+            return None
+        else:
+            df = ckan.datastore_search(resource_id, filters=file_query["filters"], sort=ckan_tags_sep.join(self.sort_by_keys) + " desc",
+                                       limit=self.last_rows_limit, search_all=False)  #, fields=self.file_keys + self.index_keys)
+            return df
+
+    def last_inserted_index_request(self, ckan:CkanApi, resource_id:str, file_query:dict, df_upload:pd.DataFrame) -> Tuple[int, bool, int, pd.DataFrame]:
+        # df_upload is in the database format (df_upload_fun has been applied)
+        # df_last_row has just been downloaded but no field typing has been applied
+        df_last_row = self.last_inserted_row_request(ckan=ckan, resource_id=resource_id, file_query=file_query)
+        if df_last_row is None or df_last_row.empty:
+            return 0, True, df_last_row.attrs["total"] if df_last_row is not None else 0, df_last_row
+        else:
+            for key in self.sort_by_keys:
+                if key in df_upload.columns:
+                    # apply field typing from df_upload in order to perform line-by-line comparison
+                    df_last_row[key] = df_last_row[key].astype(df_upload[key].dtype)
+            match_table = np.column_stack([df_upload[key] == df_last_row[key].iloc[0] for key in self.sort_by_keys])
+            match_array = np.logical_and.reduce(match_table, 1)
+            i_restart = np.argwhere(match_array) + 1
+            if len(i_restart) == 1 and len(i_restart[0]) == 1:
+                i_restart_py = int(i_restart[0][0])
+                return i_restart_py, i_restart_py < len(df_upload), df_last_row.attrs["total"], df_last_row
+            else:
+                msg = "Multiple results obtained when querying the last inserted index"
+                warn(msg)
+                return 0, True, df_last_row.attrs["total"], df_last_row
+
+    ## download preparation  ----------------
+    def get_file_name_of_query(self, file_query:dict) -> str:
+        if self.file_name_function is None:
+            file_filters_str = '_'.join([str(key)+'_'+str(value) for key,value in file_query['filters'].items()])
+        else:
+            file_filters_str = self.file_name_function(file_query['filters'])
+        return f"{self.file_name_prefix}{file_filters_str}{self.file_name_suffix}"
+
+    def download_file_query_list(self, ckan: CkanApi, resource_id: str) -> List[dict]:
+        # function to list the files which are defined by unique file_keys combinations in the database
+        df_list = ckan.datastore_search(resource_id, fields=self.group_by_keys, search_all=True, distinct=True)
+        # df_list = ckan.datastore_search(resource_id, filters={key: 0 for key in self.order_keys}, fields=self.file_keys, search_all=True)
+        filters_list = df_list.to_dict(orient="records")
+        return [{"filters": file_filter} for file_filter in filters_list]
+
+#    def download_file_query(self, ckan: CkanApi, resource_id: str, file_query:dict) -> pd.DataFrame:
+#        return ckan.datastore_search(resource_id=resource_id, filters=file_query["filters"], search_all=True)
+
+
+def default_file_mapper_from_primary_key(primary_key:List[str]=None, file_query_list: Iterable[Tuple[str,dict]]=None) -> RequestFileMapperABC:
+    if primary_key is None or len(primary_key) <= 1:
+        if file_query_list is not None:
+            return RequestFileMapperUser(file_query_list)
+        else:
+            return RequestFileMapperLimit()
+    else:
+        return RequestFileMapperIndexKeys(group_by_keys=primary_key[:-1], sort_by_keys=[primary_key[-1]])
+

ckanapi_harvesters/builder/specific/__init__.py

@@ -0,0 +1,11 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Default package builders
+"""
+
+from . import configuration_builder
+
+# alias
+from .configuration_builder import ConfigurationBuilder
+

ckanapi_harvesters/builder/specific/configuration_builder.py

@@ -0,0 +1,66 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+CKAN configuration builder
+"""
+from typing import Union, List, Dict
+
+from ckanapi_harvesters.auxiliary import ckan_configuration
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.policies.data_format_policy import CkanPackageDataFormatPolicy
+from ckanapi_harvesters.policies.data_format_policy_errors import DataPolicyError
+from ckanapi_harvesters.builder.builder_resource import BuilderResourceUnmanaged
+from ckanapi_harvesters.builder.specific_builder_abc import SpecificBuilderABC
+
+
+class ConfigurationBuilder(SpecificBuilderABC):
+    def __init__(self, ckan:CkanApi, organization_name:str):
+        super().__init__(ckan, package_name=ckan_configuration.configuration_package_name, organization_name=organization_name,
+                         title="Configuration for scripts",
+                         description="Configuration for use with Python scripts",
+                         private=True,
+                         )
+        self.resource_builders[ckan_configuration.policy_resource] = \
+            BuilderResourceUnmanaged(name=ckan_configuration.policy_resource, format="JSON",
+                                     description="CKAN Data format policy for use with Python scripts")
+
+    def patch_policy(self, ckan:CkanApi, policy: CkanPackageDataFormatPolicy,
+                     *, reduced_size:bool=None, update_ckan:bool=True):
+        package_info = self.patch_request_package(ckan)
+        package_id = package_info.id
+        if policy is not None:
+            payload = policy.to_jsons(reduced_size=reduced_size).encode()
+            policy_builder: BuilderResourceUnmanaged = self.resource_builders[ckan_configuration.policy_resource]
+            policy_builder.patch_request(ckan, package_id, payload=payload, reupload=True)
+        else:
+            # delete data format policy
+            self.resource_builders[ckan_configuration.policy_resource].delete_request(ckan, package_id)
+        if update_ckan:
+            ckan.policy = policy
+
+    def load_default_policy(self, ckan:CkanApi) -> CkanPackageDataFormatPolicy:
+        return ckan.load_default_policy(force=True)
+
+    def policy_check(self, ckan: CkanApi,
+                    package_list: Union[str, List[str]] = None, *, owner_org:str=None,
+                    policy:CkanPackageDataFormatPolicy=None, buffer:Dict[str, List[DataPolicyError]]=None,
+                    raise_error:bool=False, verbose:bool=None) -> bool:
+        """
+        Check package list against currently loaded data format policy loaded in CKAN (or the one provided by argument).
+        If not provided, the package list is the full list of packages, restrained to an organization (requires an API request).
+        :param ckan:
+        :param package_list:
+        :param owner_org:
+        :param policy:
+        :param buffer:
+        :param raise_error:
+        :param verbose:
+        :return:
+        """
+        # recommended to run load_default_policy before
+        package_list = ckan.complete_package_list(package_list, owner_org=owner_org)
+        ckan.map_resources(package_list, owner_org=owner_org)
+        return ckan.policy_check(package_list, policy=policy, buffer=buffer, verbose=verbose, raise_error=raise_error)
+
+
+

ckanapi_harvesters/builder/specific_builder_abc.py

@@ -0,0 +1,23 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Abstract class to implement specific builders from code
+"""
+from abc import ABC
+from typing import List
+
+from ckanapi_harvesters.auxiliary.ckan_model import CkanState
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.builder.builder_package import BuilderPackage
+
+class SpecificBuilderABC(BuilderPackage, ABC):
+    def __init__(self, ckan:CkanApi, package_name:str, organization_name:str, *,
+                 title: str = None, description: str = None, private: bool = None, state: CkanState = None,
+                 version: str = None,
+                 url: str = None, tags: List[str] = None,
+                 license_name:str=None):
+        super().__init__(package_name=package_name, title=title,
+                         description=description, private=private, state=state, version=version, url=url,
+                         tags=tags, organization_name=organization_name, license_name=license_name)
+        self.ckan_builder.from_ckan(ckan)
+

ckanapi_harvesters/ckan_api/__init__.py

@@ -0,0 +1,20 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Package with helper functions for CKAN requests using pandas DataFrames.
+"""
+
+from . import ckan_api_params
+from . import ckan_api_0_base
+from . import ckan_api_1_map
+from . import ckan_api_2_readonly
+from . import ckan_api_3_policy
+from . import ckan_api_4_readwrite
+from . import ckan_api_5_manage
+from . import ckan_api
+# from . import deprecated
+
+# usage shortcuts
+from ckanapi_harvesters.ckan_api.ckan_api import CkanApi, CkanApiParams, CkanApiABC, CKAN_API_VERSION, CkanApiMap
+
+

ckanapi_harvesters/ckan_api/ckan_api.py

@@ -0,0 +1,11 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Alias to most complete CkanApi implementation
+"""
+
+from ckanapi_harvesters.ckan_api.ckan_api_0_base import CkanApiABC, CKAN_API_VERSION
+from ckanapi_harvesters.ckan_api.ckan_api_1_map import CkanApiMap
+from ckanapi_harvesters.ckan_api.ckan_api_5_manage import CkanApiManage as CkanApi  # alias
+from ckanapi_harvesters.ckan_api.ckan_api_5_manage import CkanApiExtendedParams as CkanApiParams  # alias
+