ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/builder/builder_resource_init.py

@@ -0,0 +1,126 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Code to initialize a resource builder from a row
+"""
+from typing import Union
+
+import pandas as pd
+
+from ckanapi_harvesters.ckan_api import  CkanApiMap
+from ckanapi_harvesters.auxiliary.ckan_model import CkanResourceInfo, CkanDataStoreInfo
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import assert_or_raise
+from ckanapi_harvesters.auxiliary.ckan_defs import ckan_tags_sep
+from ckanapi_harvesters.auxiliary.ckan_errors import (UnexpectedError)
+from ckanapi_harvesters.builder.builder_errors import MissingDataStoreInfoError
+from ckanapi_harvesters.builder.builder_resource import (BuilderResourceABC, BuilderFileBinary, BuilderUrl,
+                                                         BuilderResourceUnmanaged)
+from ckanapi_harvesters.builder.builder_resource_multi_file import BuilderMultiFile
+from ckanapi_harvesters.builder.builder_resource_datastore import (BuilderDataStoreABC, BuilderDataStoreFile,
+                                                                   BuilderResourceIgnored)
+from ckanapi_harvesters.builder.builder_resource_multi_datastore import BuilderMultiDataStore
+from ckanapi_harvesters.builder.builder_resource_datastore_url import BuilderDataStoreUrl
+from ckanapi_harvesters.builder.builder_resource_datastore_multi_harvester import BuilderDataStoreHarvester
+from ckanapi_harvesters.builder.builder_resource_datastore_unmanaged import BuilderDataStoreUnmanaged
+from ckanapi_harvesters.builder.builder_resource_datastore_multi_abc import BuilderDataStoreMultiABC
+from ckanapi_harvesters.builder.builder_resource_datastore_multi_folder import BuilderDataStoreFolder
+from ckanapi_harvesters.builder.builder_field import BuilderField
+
+
+import_as_folder_row_count_threshold: Union[int,None] = None
+
+
+def init_resource_from_df(row: pd.Series, base_dir:str=None) -> BuilderResourceABC:
+    """
+    Function mapping keywords to a resource builder type.
+
+    :param row:
+    :return:
+    """
+    mode = row["mode"].lower().strip()
+    if mode == "file":
+        resource_builder = BuilderFileBinary()
+    elif mode == "url":
+        resource_builder = BuilderUrl()
+    elif mode == "datastore from file":
+        resource_builder = BuilderDataStoreFile()
+    elif mode == "datastore from folder":
+        resource_builder = BuilderDataStoreFolder()
+    elif mode == "datastore from url":
+        resource_builder = BuilderDataStoreUrl()
+    elif mode == "datastore from harvester":
+        resource_builder = BuilderDataStoreHarvester()
+    elif mode == "unmanaged":
+        resource_builder = BuilderResourceUnmanaged()
+    elif mode == "unmanaged datastore":
+        resource_builder = BuilderDataStoreUnmanaged()
+    elif mode == "multifile":
+        resource_builder = BuilderMultiFile()
+    elif mode == "multidatastore":
+        resource_builder = BuilderMultiDataStore()
+    elif mode == "ignored":
+        resource_builder = BuilderResourceIgnored()
+    else:
+        raise ValueError(f"{mode} is not a valid mode")
+    resource_builder._load_from_df_row(row=row, base_dir=base_dir)
+    return resource_builder
+
+
+def init_resource_from_ckan(ckan: CkanApiMap, resource_info: CkanResourceInfo) -> BuilderResourceABC:
+    """
+    Function initiating a resource builder based on information provided by the CKAN API.
+
+    :return:
+    """
+    # assert_or_raise(ckan.map._mapping_query_datastore_info, MissingDataStoreInfoError())
+    assert_or_raise(resource_info.datastore_queried(), MissingDataStoreInfoError())
+    d = {
+        "name": resource_info.name,
+        "format": resource_info.format,
+        "description": resource_info.description,
+        "state": resource_info.state.name if resource_info.state is not None else "",
+        "file/url": resource_info.name,
+        "primary key": "",
+        "indexes": "",
+        "known id": resource_info.id,
+        "known url": resource_info.download_url,
+    }
+    if (isinstance(resource_info.datastore_info, CkanDataStoreInfo)
+            and resource_info.datastore_info.row_count is not None
+            and len(resource_info.datastore_info.fields_id_list) > 0):
+        # DataStore
+        d["indexes"] = ckan_tags_sep.join(resource_info.datastore_info.index_fields)
+        d["aliases"] = ckan_tags_sep.join(resource_info.datastore_info.aliases)
+        if len(resource_info.download_url) > 0 and not ckan.is_url_internal(resource_info.download_url):
+            d["file/url"] = resource_info.download_url
+            row = pd.Series(d)
+            resource = BuilderDataStoreUrl()
+            resource._load_from_df_row(row=row)
+        elif resource_info.format.lower() == "csv":
+            row = pd.Series(d)
+            resource = BuilderDataStoreUnmanaged()
+            resource._load_from_df_row(row=row)
+            if import_as_folder_row_count_threshold is not None and resource_info.datastore_info.row_count > import_as_folder_row_count_threshold:
+                resource = BuilderDataStoreFolder.from_file_datastore(resource)
+        else:
+            raise UnexpectedError(f"Format of data store {resource_info.name} ({resource_info.format}) is not recognized")
+        # load fields information
+        resource.field_builders = {}
+        for field_id in resource_info.datastore_info.fields_id_list:
+            field_info = resource_info.datastore_info.fields_dict[field_id]
+            resource.field_builders[field_id] = BuilderField._from_ckan_field(field_info)
+    elif len(resource_info.download_url) > 0 and not ckan.is_url_internal(resource_info.download_url):
+        # external resource
+        d["file/url"] = resource_info.download_url
+        row = pd.Series(d)
+        resource = BuilderUrl()
+        resource._load_from_df_row(row=row)
+        assert_or_raise(not resource_info.datastore_active and not isinstance(resource_info.datastore_info, CkanResourceInfo), UnexpectedError())
+    else:
+        # file
+        row = pd.Series(d)
+        resource = BuilderResourceUnmanaged()
+        resource._load_from_df_row(row=row)
+    resource.package_name = resource_info.package_id
+    return resource
+

ckanapi_harvesters/builder/builder_resource_multi_abc.py

@@ -0,0 +1,361 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Code to upload metadata to the CKAN server to create/update an existing package
+The metadata is defined by the user in an Excel worksheet
+This file implements the basic resources. See builder_datastore for specific functions to initiate datastores.
+"""
+from concurrent.futures import ThreadPoolExecutor
+import threading
+from threading import current_thread
+from typing import Any, Generator, Union, Callable, Set, List, Dict, Tuple
+from abc import ABC, abstractmethod
+import io
+import os
+import glob
+import fnmatch
+from warnings import warn
+import copy
+
+import pandas as pd
+import requests
+
+from ckanapi_harvesters.auxiliary.error_level_message import ContextErrorLevelMessage, ErrorLevel
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import _string_from_element
+from ckanapi_harvesters.auxiliary.ckan_defs import ckan_tags_sep
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.auxiliary.ckan_model import UpsertChoice, CkanResourceInfo
+from ckanapi_harvesters.builder.builder_aux import positive_end_index
+from ckanapi_harvesters.builder.builder_errors import ResourceFileNotExistMessage
+from ckanapi_harvesters.builder.builder_field import BuilderField
+from ckanapi_harvesters.builder.builder_resource import BuilderResourceABC
+
+multi_file_exclude_other_files:bool = True
+
+
+def default_progress_callback(index:int, total:int, info:Any, *, context:str=None, **kwargs) -> None:
+    if context is None:
+        context = ""
+    if index == total:
+        # info is None
+        print(f"{context} Finished {index}/{total} (100%)")
+    elif info is None:
+        print(f"{context} Request {index}/{total} ({index/total*100.0:.2f}%)")
+    else:
+        if isinstance(info, str):
+            info_str = info
+        elif isinstance(info, pd.DataFrame):
+            if "source" in info.attrs.keys():
+                info_str = str(info.attrs["source"])
+            else:
+                info_str = "<DataFrame>"
+        elif isinstance(info, list):
+            info_str = "<records>"
+        else:
+            info_str = str(info)
+        print(f"{context} Request {index}/{total} ({index/total*100.0:.2f}%): " + info_str)
+
+
+class BuilderMultiABC(ABC):
+    def __init__(self):
+        self.progress_callback: Union[Callable[[int, int, Any], None], None] = default_progress_callback
+        self.progress_callback_kwargs: dict = {}
+        self.stop_event = threading.Event()
+        self.thread_ckan: Dict[str, CkanApi] = {}
+        self.enable_multi_threaded_upload:bool = True
+        self.enable_multi_threaded_download:bool = True
+        # from Resource (for code validation)
+        self.name:str = ""
+        self.enable_download:bool = True
+
+    def copy(self, *, dest=None):
+        dest.progress_callback = self.progress_callback
+        dest.progress_callback_kwargs = copy.deepcopy(self.progress_callback_kwargs)
+        dest.enable_multi_threaded_upload = self.enable_multi_threaded_upload
+        dest.enable_multi_threaded_download = self.enable_multi_threaded_download
+        # do not copy stop_event
+        return dest
+
+    def _call_progress_callback(self, index:int, total:int, *, info:Any=None, context:str=None) -> None:
+        if self.progress_callback is not None:
+            self.progress_callback(index, total, info=info, context=context, **self.progress_callback_kwargs)
+
+    def _prepare_for_multithreading(self, ckan: CkanApi):
+        self.stop_event.clear()
+        self.thread_ckan.clear()
+
+    def _init_thread(self, ckan: CkanApi):
+        thread_name = current_thread().name
+        ckan_thread = ckan.copy(new_identifier=thread_name)
+        ckan_thread.prepare_for_multithreading(True)  # prepare CKAN object for multi-threading
+        self.thread_ckan[thread_name] = ckan_thread
+
+    def _terminate_thread(self):
+        for ckan in self.thread_ckan.values():
+            ckan.disconnect()
+        self.thread_ckan.clear()
+
+
+    ## upload -----------------------------------------------------------------
+    @abstractmethod
+    def init_local_files_list(self, resources_base_dir:str, cancel_if_present:bool=True, **kwargs) -> List[str]:
+        """
+        Behavior to list parts of an upload.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_local_file_len(self) -> int:
+        """
+        Get the number of parts of the upload.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_local_file_generator(self, resources_base_dir:str, **kwargs) -> Generator[Any, None, None]:
+        """
+        Returns an iterator over the parts of the upload.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def upload_request_final(self, ckan:CkanApi, *, force:bool=False) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _unit_upload_apply(self, *, ckan:CkanApi, file:Any,
+                           index:int, start_index:int, end_index:int, total:int, **kwargs) -> Any:
+        """
+        Unitary function deciding whether to perform upload and making the steps for the upload.
+        """
+        raise NotImplementedError()
+
+    def upload_request_full(self, ckan:CkanApi, resources_base_dir:str, *,
+                            threads:int=1, external_stop_event=None,
+                            start_index:int=0, end_index:int=None, **kwargs) -> None:
+        """
+        Perform all the upload requests.
+
+        :param ckan:
+        :param resources_base_dir:
+        :param threads:
+        :param external_stop_event:
+        :param only_missing:
+        :param start_index:
+        :param end_index:
+        :return:
+        """
+        if threads < 0:
+            # cancel large uploads in this case
+            return None
+        elif (threads is None or threads > 1) and self.enable_multi_threaded_upload:
+            return self.upload_request_full_multi_threaded(ckan=ckan, resources_base_dir=resources_base_dir,
+                                                           threads=threads, external_stop_event=external_stop_event,
+                                                           start_index=start_index, end_index=end_index, **kwargs)
+        else:
+            self.init_local_files_list(resources_base_dir=resources_base_dir, cancel_if_present=True, **kwargs)
+            if ckan.params.verbose_extra:
+                print(f"Launching single-threaded upload of multi-file resource {self.name}")
+            total = self.get_local_file_len()
+            end_index = positive_end_index(end_index, total)
+            for index, file_path in enumerate(self.get_local_file_generator(resources_base_dir=resources_base_dir, **kwargs)):
+                if external_stop_event is not None and external_stop_event.is_set():
+                    print(f"{ckan.identifier} Interrupted")
+                    return
+                self._unit_upload_apply(ckan=ckan, file=file_path, index=index,
+                                        start_index=start_index, end_index=end_index, total=total, **kwargs)
+            self._call_progress_callback(total, total, context=f"{ckan.identifier} single-thread upload")
+            # at last, apply final actions:
+            self.upload_request_final(ckan)
+
+    def upload_request_graceful(self, ckan:CkanApi, file_path: str, *, index:int,
+                                external_stop_event=None,
+                                start_index:int=0, end_index:int=None, **kwargs) -> None:
+        """
+        Calls upload_file with checks specific to multi-threading.
+
+        :return:
+        """
+        # ckan.session_reset()
+        # ckan.identifier = current_thread().name
+        ckan = self.thread_ckan[current_thread().name]
+        total = self.get_local_file_len()
+        end_index = positive_end_index(end_index, total)
+        if self.stop_event.is_set():
+            return
+        if external_stop_event is not None and external_stop_event.is_set():
+            print(f"{ckan.identifier} Interrupted")
+            return
+        try:
+            self._unit_upload_apply(ckan=ckan, file=file_path, index=index,
+                                    start_index=start_index, end_index=end_index, total=total, **kwargs)
+        except Exception as e:
+            self.stop_event.set()  # Ensure all threads stop
+            if ckan.params.verbose_extra:
+                print(f"Stopping all threads because an exception occurred in thread: {e}")
+            raise e from e
+
+    def upload_request_full_multi_threaded(self, ckan:CkanApi, resources_base_dir:str,
+                            threads:int=1, external_stop_event=None,
+                            start_index:int=0, end_index:int=None, **kwargs):
+        """
+        Multi-threaded implementation of upload_request_full, using ThreadPoolExecutor.
+        """
+        self.init_local_files_list(resources_base_dir=resources_base_dir, cancel_if_present=True, **kwargs)
+        self._prepare_for_multithreading(ckan)
+        try:
+            with ThreadPoolExecutor(max_workers=threads, initializer=self._init_thread, initargs=(ckan,)) as executor:
+                if ckan.params.verbose_extra:
+                    print(f"Launching multi-threaded upload of multi-file resource {self.name}")
+                futures = [executor.submit(self.upload_request_graceful, ckan=ckan, file_path=file_path, index=index,
+                                           start_index=start_index, end_index=end_index, external_stop_event=external_stop_event,
+                                           **kwargs)
+                           for index, file_path in enumerate(self.get_local_file_generator(resources_base_dir=resources_base_dir, **kwargs))]
+                for future in futures:
+                    future.result()  # This will propagate the exception
+            total = self.get_local_file_len()
+            self._call_progress_callback(total, total, context=f"{ckan.identifier} multi-thread upload")
+        except Exception as e:
+            self.stop_event.set()  # Ensure all threads stop
+            if ckan.params.verbose_extra:
+                print(f"Stopping all threads because an exception occurred: {e}")
+            raise e from e
+        finally:
+            # self.stop_event.set()  # Ensure all threads stop
+            if ckan.params.verbose_extra:
+                print("End of multi-threaded upload...")
+        # at last, apply final actions:
+        self._terminate_thread()
+        self.upload_request_final(ckan)
+
+    ## download -------------------------------------------------------------
+    @abstractmethod
+    def init_download_file_query_list(self, ckan: CkanApi, out_dir: str, cancel_if_present: bool = True, **kwargs) -> List[Any]:
+        """
+        Determine the list of queries to download to reconstruct the uploaded parts.
+        By default, the unique combinations of the first columns of the primary key are used.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_file_query_generator(self) -> Generator[Any, Any, None]:
+        """
+        Returns an iterator on all the file_queries.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_file_query_len(self) -> int:
+        """
+        Returns the total number of file_queries.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def download_file_query_item(self, ckan: CkanApi, out_dir: str, file_query_item: Any) -> Any:
+        """
+        Download the file_query item with the its arguments
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def download_request_generator(self, ckan: CkanApi, out_dir: str) -> Generator[Any, Any, None]:
+        """
+        Generator to apply treatments after each request (single-threaded).
+
+        :param ckan:
+        :param out_dir:
+        :return:
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _unit_download_apply(self, ckan:CkanApi, file_query_item:Any, out_dir:str,
+                           index:int, start_index:int, end_index:int, total:int, **kwargs) -> Any:
+        """
+        Unitary function deciding whether to perform download and making the steps for the request.
+        """
+        raise NotImplementedError()
+
+    def download_request_full(self, ckan: CkanApi, out_dir: str, threads:int=1, external_stop_event=None,
+                              start_index:int=0, end_index:int=None, force:bool=False, **kwargs) -> None:
+
+        if (not self.enable_download) and (not force):
+            msg = f"Did not download resource {self.name} because download was disabled."
+            warn(msg)
+            return None
+        if threads < 0:
+            # do not download large datasets in this case
+            return None
+        elif (threads is None or threads > 1) and self.enable_multi_threaded_download:
+            return self.download_request_full_multi_threaded(ckan=ckan, out_dir=out_dir,
+                                                             threads=threads, external_stop_event=external_stop_event,
+                                                             start_index=start_index, end_index=end_index, **kwargs)
+        else:
+            self.init_download_file_query_list(ckan=ckan, out_dir=out_dir, cancel_if_present=True, **kwargs)
+            if ckan.params.verbose_extra:
+                print(f"Launching single-threaded download of multi-file resource {self.name}")
+            total = self.get_file_query_len()
+            end_index = positive_end_index(end_index, total)
+            for index, file_query_item in enumerate(self.get_file_query_generator()):
+                if external_stop_event is not None and external_stop_event.is_set():
+                    print(f"{ckan.identifier} Interrupted")
+                    return
+                self._unit_download_apply(ckan=ckan, file_query_item=file_query_item, out_dir=out_dir,
+                                          index=index, start_index=start_index, end_index=end_index, total=total, **kwargs)
+            self._call_progress_callback(total, total, context=f"{ckan.identifier} single-thread download")
+
+    def download_file_query_item_graceful(self, ckan: CkanApi, out_dir: str, file_query_item: Any, index:int,
+                                          external_stop_event=None, start_index:int=0, end_index:int=None, **kwargs) -> None:
+        """
+        Implementation of download_file_query_item with checks for a multi-threaded download.
+        """
+        # ckan.session_reset()
+        # ckan.identifier = current_thread().name
+        ckan = self.thread_ckan[current_thread().name]
+        total = self.get_file_query_len()
+        end_index = positive_end_index(end_index, total)
+        if self.stop_event.is_set():
+            return
+        if external_stop_event is not None and external_stop_event.is_set():
+            print(f"{ckan.identifier} Interrupted")
+            return
+        try:
+            self._unit_download_apply(ckan=ckan, file_query_item=file_query_item, out_dir=out_dir,
+                                      index=index, start_index=start_index, end_index=end_index, total=total, **kwargs)
+        except Exception as e:
+            self.stop_event.set()  # Ensure all threads stop
+            if ckan.params.verbose_extra:
+                print(f"Stopping all threads because an exception occurred in thread: {e}")
+            raise e from e
+
+    def download_request_full_multi_threaded(self, ckan: CkanApi, out_dir: str,
+                                             threads: int = None, external_stop_event=None,
+                                             start_index:int=0, end_index:int=-1, **kwargs) -> None:
+        """
+        Multi-threaded implementation of download_request_full using ThreadPoolExecutor.
+        """
+        self.init_download_file_query_list(ckan=ckan, out_dir=out_dir, cancel_if_present=True, **kwargs)
+        self._prepare_for_multithreading(ckan)
+        try:
+            with ThreadPoolExecutor(max_workers=threads, initializer=self._init_thread, initargs=(ckan,)) as executor:
+                if ckan.params.verbose_extra:
+                    print(f"Launching multi-threaded download of multi-file resource {self.name}")
+                futures = [executor.submit(self.download_file_query_item_graceful, ckan=ckan, out_dir=out_dir, file_query_item=file_query_item,
+                                           index=index, external_stop_event=external_stop_event, start_index=start_index, end_index=end_index, **kwargs)
+                           for index, file_query_item in enumerate(self.get_file_query_generator())]
+                for future in futures:
+                    future.result()  # This will propagate the exception
+            total = self.get_file_query_len()
+            self._call_progress_callback(total, total, context=f"multi-thread download")
+        except Exception as e:
+            self.stop_event.set()  # Ensure all threads stop
+            if ckan.params.verbose_extra:
+                print(f"Stopping all threads because an exception occurred: {e}")
+            raise e from e
+        finally:
+            # self.stop_event.set()  # Ensure all threads stop
+            if ckan.params.verbose_extra:
+                print("End of multi-threaded download...")
+        # at last, apply final actions:
+        self._terminate_thread()

ckanapi_harvesters/builder/builder_resource_multi_datastore.py

@@ -0,0 +1,146 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Code to upload metadata to the CKAN server to create/update an existing package
+The metadata is defined by the user in an Excel worksheet
+This file implements the basic resources. See builder_datastore for specific functions to initiate datastores.
+"""
+from typing import Any, Generator, Union, Set, List, Dict, Tuple
+import os
+import requests
+import copy
+
+import pandas as pd
+
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import _string_from_element, find_duplicates
+from ckanapi_harvesters.auxiliary.ckan_defs import ckan_tags_sep
+from ckanapi_harvesters.auxiliary.ckan_errors import DuplicateNameError
+from ckanapi_harvesters.auxiliary.path import resolve_rel_path, glob_rm_glob
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.harvesters.data_cleaner.data_cleaner_abc import CkanDataCleanerABC
+from ckanapi_harvesters.auxiliary.ckan_model import CkanResourceInfo
+from ckanapi_harvesters.builder.builder_field import BuilderField
+from ckanapi_harvesters.builder.builder_resource_datastore import BuilderDataStoreFile
+from ckanapi_harvesters.builder.builder_resource_multi_file import BuilderMultiFile
+
+
+class BuilderMultiDataStore(BuilderMultiFile):
+    def __init__(self, *, name:str=None, format:str=None, description:str=None,
+                 resource_id:str=None, download_url:str=None):
+        super().__init__(name=name, format=format, description=description, resource_id=resource_id, download_url=download_url)
+        self.field_builders: Union[Dict[str, BuilderField],None] = None
+        self.primary_key: Union[List[str],None] = None
+        self.indexes: Union[List[str],None] = None
+        self.aux_upload_fun_name:str = ""
+        self.aux_download_fun_name:str = ""
+        self.data_cleaner_upload:Union[CkanDataCleanerABC,None] = None
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = BuilderMultiDataStore()
+        super().copy(dest=dest)
+        dest.field_builders = copy.deepcopy(self.field_builders)
+        dest.primary_key = copy.deepcopy(self.primary_key)
+        dest.indexes = copy.deepcopy(self.indexes)
+        dest.aux_upload_fun_name = self.aux_upload_fun_name
+        dest.aux_download_fun_name = self.aux_download_fun_name
+        return dest
+
+    def _load_from_df_row(self, row: pd.Series, base_dir:str=None):
+        super()._load_from_df_row(row=row)
+        primary_keys_string: str = _string_from_element(row["primary key"])
+        indexes_string: str = _string_from_element(row["indexes"])
+        if primary_keys_string is not None:
+            if primary_keys_string.lower() == "none":
+                self.primary_key = []
+            else:
+                self.primary_key = [field.strip() for field in primary_keys_string.split(ckan_tags_sep)]
+        if indexes_string is not None:
+            if indexes_string.lower() == "none":
+                self.indexes = []
+            else:
+                self.indexes = [field.strip() for field in indexes_string.split(ckan_tags_sep)]
+        if "upload function" in row.keys():
+            self.aux_upload_fun_name: str = _string_from_element(row["upload function"], empty_value="")
+        if "download function" in row.keys():
+            self.aux_download_fun_name: str = _string_from_element(row["download function"], empty_value="")
+
+    def _load_fields_df(self, fields_df: pd.DataFrame):
+        fields_df.columns = fields_df.columns.map(str.lower)
+        fields_df.columns = fields_df.columns.map(str.strip)
+        self.field_builders = {}
+        for index, row in fields_df.iterrows():
+            field_builder = BuilderField()
+            field_builder._load_from_df_row(row=row)
+            self.field_builders[field_builder.name] = field_builder
+
+    def _check_field_duplicates(self):
+        duplicates = find_duplicates([field_builder.name for field_builder in self.field_builders.values()])
+        if len(duplicates) > 0:
+            raise DuplicateNameError("Field", duplicates)
+
+    def _get_fields_dict(self) -> Dict[str, dict]:
+        self._check_field_duplicates()
+        if self.field_builders is not None:
+            fields_dict = {field_builder.name: field_builder._to_dict() for field_builder in self.field_builders.values()}
+        else:
+            fields_dict = None
+        return fields_dict
+
+    def _get_fields_df(self) -> pd.DataFrame:
+        fields_dict_list = [value for value in self._get_fields_dict().values()]
+        fields_df = pd.DataFrame.from_records(fields_dict_list)
+        return fields_df
+
+    @staticmethod
+    def resource_mode_str() -> str:
+        return "MultiDataStore"
+
+    def _to_dict(self, include_id:bool=True) -> dict:
+        d = super()._to_dict(include_id=include_id)
+        d["Primary key"] = ckan_tags_sep.join(self.primary_key) if self.primary_key else ""
+        d["Indexes"] = ckan_tags_sep.join(self.indexes) if self.indexes is not None else ""
+        return d
+
+    def _data_store_builder_of_file(self, file_path:str) -> Tuple[BuilderDataStoreFile, str]:
+        file_dir, file_name = os.path.split(file_path)
+        ds_builder = BuilderDataStoreFile(name=file_name, description=self.description, download_url=self.download_url,
+                                          format=self.format, file_name=file_name)
+        ds_builder.field_builders = self.field_builders
+        ds_builder.primary_key = self.primary_key
+        ds_builder.indexes = self.indexes
+        ds_builder.package_name = self.package_name
+        ds_builder.aux_upload_fun_name = self.aux_upload_fun_name
+        ds_builder.aux_download_fun_name = self.aux_download_fun_name
+        ds_builder.aliases = None
+        ds_builder.data_cleaner_upload = self.data_cleaner_upload
+        return ds_builder, file_dir
+
+
+    ## Upload ----------------
+    def upload_file(self, ckan:CkanApi, package_id:str, file_path:str, *,
+                    reupload:bool=False, cancel_if_present:bool=True) -> CkanResourceInfo:
+        ds_builder, file_dir = self._data_store_builder_of_file(file_path=file_path)
+        return ds_builder.patch_request(ckan=ckan, package_id=package_id, reupload=reupload,
+                                        resources_base_dir=file_dir)
+
+
+    ## Download --------------
+    def download_file_query_item_df(self, ckan: CkanApi, out_dir: str, file_query_item: str, full_download:bool=True) -> Tuple[str, pd.DataFrame]:
+        resource_name = file_query_item
+        ds_builder, _ = self._data_store_builder_of_file(file_path=resource_name)
+        file_dir = resolve_rel_path(out_dir, glob_rm_glob(self.dir_name), field=f"File/URL of resource {self.name}")
+        df = ds_builder.download_request(ckan, out_dir=file_dir, full_download=full_download)
+        return ds_builder.downloaded_destination, df
+
+    def download_file_query_item(self, ckan: CkanApi, out_dir: str, file_query_item: str, full_download:bool=True) -> Tuple[Union[str,None], Union[requests.Response,None]]:
+        downloaded_destination, df = self.download_file_query_item_df(ckan=ckan, out_dir=out_dir, file_query_item=file_query_item,full_download=full_download)
+        return downloaded_destination, None
+
+    def download_request_generator_df(self, ckan: CkanApi, out_dir: str,
+                                   excluded_resource_names:Set[str]=None) -> Generator[Tuple[Union[str,None], Union[pd.DataFrame,None]], Any, None]:
+        self.init_download_file_query_list(ckan=ckan, out_dir=out_dir, cancel_if_present=True,
+                                           excluded_resource_names=excluded_resource_names)
+        for file_query_item in self.get_file_query_generator():
+            yield self.download_file_query_item_df(ckan=ckan, out_dir=out_dir, file_query_item=file_query_item)
+