ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/harvesters/pymongo_data_cleaner.py

@@ -0,0 +1,173 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Harvest from a mongo database using pymongo
+"""
+from typing import Union, List, Any, Dict, Set
+from types import SimpleNamespace
+from collections import OrderedDict
+from warnings import warn
+import copy
+
+import pandas as pd
+
+
+try:
+    import bson
+except ImportError:
+    bson = SimpleNamespace(ObjectId=None, DBRef=None)
+
+
+from ckanapi_harvesters.auxiliary.ckan_model import CkanField
+from ckanapi_harvesters.auxiliary.list_records import ListRecords, records_to_df
+from ckanapi_harvesters.harvesters.data_cleaner.data_cleaner_errors import CleanerRequirementError
+from ckanapi_harvesters.harvesters.data_cleaner.data_cleaner_upload import CkanDataCleanerUpload, _pd_series_type_detect
+
+
+mongodb_keep_id_column_trace:bool = True
+mongodb_keep_class_column_trace:bool = True
+mongodb_id_column:str = "_id"
+mongodb_id_new_column:str = "ObjectId"
+mongodb_id_datatype_numeric:bool = False  # option to store ids as a numeric datatype - there is no numeric datatype which corresponds
+mongodb_id_alt_type_numeric:str = "int12"  # the ObjectIds are 12 byte integers (int96) - there is no such data type in Postgre (oid?)
+mongodb_dbref_extract_new_id_column_max_level:int = 1  # option to create a new column if a DBRef is found in a json field
+mongodb_dbref_alt_type:str = "json"  # used if not resumed in one column
+
+
+def pymongo_default_df_conversion(documents: List[dict], **kwargs) -> Union[pd.DataFrame, ListRecords]:
+    df = ListRecords(documents)
+    # if df.columns is not None:
+    #     for i, field_name in enumerate(df.columns):
+    #         if field_name == mongodb_id_column:
+    #             df.columns[i] = mongodb_id_new_column
+    # df = records_to_df(documents)
+    return df
+
+
+class BrokenMongoRefError(Exception):
+    pass
+
+
+class MongoDataCleanerUpload(CkanDataCleanerUpload):
+    """
+    Data cleaner operations specific to MongoDB objects.
+    """
+    def __init__(self):
+        super().__init__()
+        # change default parameters
+        self.param_field_subs[mongodb_id_column] = mongodb_id_new_column
+        self.param_field_primary_key = [mongodb_id_new_column]
+        self.param_apply_field_changes = True
+        # specific options
+        self.param_mongodb_dbref_as_one_column:bool = True  # option to extract only the ObjectId referenced by the DBRefs
+        self.collection_refs:Dict[str,Set[str]] = {}
+        self.database_refs:Dict[str,Set[str]] = {}
+        self.broken_collection_refs:List[str] = []
+        self.broken_database_refs:List[str] = []
+        if bson.DBRef is None:
+            raise CleanerRequirementError("bson", "DBRef, ObjectId")
+
+    def clear_outputs_new_dataframe(self):
+        super().clear_outputs_new_dataframe()
+        self.broken_collection_refs:List[str] = []
+        self.broken_database_refs:List[str] = []
+
+    def clear_all_outputs(self):
+        super().clear_all_outputs()
+        self.collection_refs:Dict[str,Set[str]] = {}
+        self.database_refs:Dict[str,Set[str]] = {}
+
+    def copy(self, dest=None) -> "MongoDataCleanerUpload":
+        if dest is None:
+            dest = MongoDataCleanerUpload()
+        super().copy(dest=dest)
+        dest.param_mongodb_dbref_as_one_column = self.param_mongodb_dbref_as_one_column
+        dest.collection_refs = copy.deepcopy(self.collection_refs)
+        dest.database_refs = copy.deepcopy(self.database_refs)
+        return dest
+
+    def _detect_standard_field_bypass(self, field_name: str, values: Union[Any, pd.Series]) -> Union[CkanField,None]:
+        if _pd_series_type_detect(values, bson.DBRef):
+            if self.param_mongodb_dbref_as_one_column:
+                return CkanField(field_name, mongodb_id_alt_type_numeric if mongodb_id_datatype_numeric else "text")
+            else:
+                return CkanField(field_name, mongodb_dbref_alt_type)
+        elif _pd_series_type_detect(values, bson.ObjectId):
+            return CkanField(field_name, mongodb_id_alt_type_numeric if mongodb_id_datatype_numeric else "text")
+        return None
+
+    def _replace_non_standard_subvalue(self, subvalue:Any, field:CkanField, path:str, level:int,
+                                       *, field_data_type:str) -> Any:
+        field_name = field.name if field is not None else None
+        if isinstance(subvalue, bson.ObjectId):
+            if mongodb_id_datatype_numeric:
+                new_subvalue = int(str(subvalue), 16)
+                # new_subvalue = str(subvalue)
+            else:
+                new_subvalue = str(subvalue)
+            if level == 0:
+                self.field_suggested_index.add(field_name)
+            return new_subvalue
+        elif isinstance(subvalue, bson.DBRef):
+            id_field = path.replace(".", "_")
+            if level == 0 and self.param_mongodb_dbref_as_one_column:
+                id_path = None
+            elif self.param_mongodb_dbref_as_one_column:  # and level > 0
+                id_path = path
+            elif level == 0 and not self.param_mongodb_dbref_as_one_column:
+                id_path = path + "." + mongodb_id_new_column
+                id_field = id_field + "_id"
+            else:
+                id_path = path + "." + mongodb_id_new_column
+            if (level <= mongodb_dbref_extract_new_id_column_max_level and
+                    id_path is not None and id_path not in self.field_subs_path.keys()):
+                self._add_field_from_path(id_path,
+                                          data_type=mongodb_id_alt_type_numeric if mongodb_id_datatype_numeric else "text",
+                                          new_field_name=id_field,
+                                          notes=f"Column extracted from {id_path}")
+            if mongodb_id_datatype_numeric:
+                id_value = int(str(subvalue.id), 16)
+                # id_value = str(subvalue.id)
+            else:
+                id_value = str(subvalue.id)
+            if self.param_mongodb_dbref_as_one_column:
+                if path in self.collection_refs.keys():
+                    self.collection_refs[path].add(str(subvalue.collection))
+                    self.database_refs[path].add(str(subvalue.database))
+                else:
+                    self.collection_refs[path] = {str(subvalue.collection)}
+                    self.database_refs[path] = {str(subvalue.database)}
+                new_subvalue = id_value
+            else:
+                new_subvalue = {mongodb_id_new_column: id_value,
+                                "collection": subvalue.collection,
+                                "database": subvalue.database,
+                                }
+                if id_path in self.field_subs_path.keys():
+                    self._new_columns_in_row[id_path] = new_subvalue[mongodb_id_new_column]
+            return new_subvalue
+        elif level == 0:
+            return super()._replace_non_standard_value(subvalue, field, field_data_type=field_data_type)
+        else:
+            return super()._replace_non_standard_subvalue(subvalue, field, path, level, field_data_type=field_data_type)
+
+    def _replace_non_standard_value(self, value:Any, field:CkanField,
+                                       *, field_data_type:str) -> Any:
+        field_name = field.name if field is not None else None
+        return self._replace_non_standard_subvalue(value, field, path=field_name,
+                                                   level=0, field_data_type=field_data_type)
+
+    def _extra_checks(self, records: Union[List[dict], pd.DataFrame], fields:Union[OrderedDict[str, CkanField], None]) -> None:
+        self.broken_collection_refs = [path for path, refs in self.collection_refs.items() if len(refs) > 1]
+        self.broken_database_refs = [path for path, refs in self.database_refs.items() if len(refs) > 1]
+        if len(self.broken_collection_refs) > 0 or len(self.broken_database_refs) > 0:
+            broken_refs = set(self.broken_collection_refs).union(set(self.broken_database_refs))
+            msg = f"DBRefs do not point to an unique collection: {', '.join(broken_refs)}"
+            if self.param_raise_error or self.param_mongodb_dbref_as_one_column:
+                raise BrokenMongoRefError(msg)
+            else:
+                warn(msg)
+
+def pymongo_default_data_cleaner() -> MongoDataCleanerUpload:
+    return MongoDataCleanerUpload()
+

ckanapi_harvesters/harvesters/pymongo_harvester.py

@@ -0,0 +1,355 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Harvest from a mongo database using pymongo
+"""
+from typing import Union, List, Any, Dict
+from types import SimpleNamespace
+from collections import OrderedDict
+import json
+import argparse
+
+
+try:
+    import pymongo
+    import pymongo.client_session
+    import pymongo.database
+except ImportError:
+    pymongo = SimpleNamespace(MongoClient=None, client_session=SimpleNamespace(ClientSession=None),
+                              database=SimpleNamespace(Database=None), collection=SimpleNamespace(Collection=None))
+
+
+from ckanapi_harvesters.harvesters.harvester_errors import (HarvesterRequirementError, HarvesterArgumentRequiredError, ResourceNotFoundError)
+from ckanapi_harvesters.harvesters.harvester_abc import TableHarvesterABC, DatasetHarvesterABC, DatabaseHarvesterABC
+from ckanapi_harvesters.harvesters.harvester_model import TableMetadata, DatasetMetadata
+from ckanapi_harvesters.harvesters.harvester_params import DatasetParams, DatabaseParams
+from ckanapi_harvesters.harvesters.pymongo_data_cleaner import pymongo_default_data_cleaner, pymongo_default_df_conversion
+from ckanapi_harvesters.harvesters.pymongo_data_cleaner import mongodb_keep_id_column_trace, mongodb_id_new_column
+from ckanapi_harvesters.harvesters.pymongo_params import TableParamsMongoCollection
+from ckanapi_harvesters.auxiliary.urls import url_join
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import ssl_arguments_decompose, assert_or_raise
+from ckanapi_harvesters.auxiliary.ckan_errors import UrlError
+from ckanapi_harvesters.harvesters.data_cleaner.data_cleaner_abc import CkanDataCleanerABC
+from ckanapi_harvesters.auxiliary.error_level_message import ContextErrorLevelMessage, ErrorLevel
+
+
+mongodb_excluded_collections = {"system.profile"}
+
+
+class DatabaseHarvesterMongoServer(DatabaseHarvesterABC):
+    """
+    This class manages the connection to a MongoDB server.
+    It can list datasets (MongoDB databases) but this call could lead to an error.
+    """
+    def __init__(self, params:DatabaseParams=None):
+        super().__init__(params)
+        if pymongo.MongoClient is None:
+            raise HarvesterRequirementError("pymongo", "pymongo")
+        self.params.harvest_method = "Pymongo"
+        self.mongo_client: Union[pymongo.MongoClient,None] = None
+        self.mongo_session: Union[pymongo.client_session.ClientSession,None] = None
+        if self.params.auth_url is None and self.params.port is None and self.params.host is None:
+            raise HarvesterArgumentRequiredError("auth-url", "pymongo", "This argument defines the url used to authenticate.")
+
+    @staticmethod
+    def init_from_options_string(options_string:str, base_dir:str=None) -> "DatabaseHarvesterMongoServer":
+        params = DatabaseParams()
+        params.parse_options_string(options_string, base_dir=base_dir)
+        return DatabaseHarvesterMongoServer(params)
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = DatabaseHarvesterMongoServer()
+        return super().copy(dest=dest)
+
+    def connect(self, *, cancel_if_connected:bool=True) -> Any:
+        if cancel_if_connected and self.mongo_client is not None:
+            return self.mongo_client
+        else:
+            if self.mongo_client is not None:
+                self.mongo_session.end_session()
+                self.mongo_client.close()
+                self.mongo_session = None
+                self.mongo_client = None
+            ssl, ssl_certfile = ssl_arguments_decompose(self.params.verify_ca)
+            auth_url = self.params.auth_url
+            if auth_url is None:
+                if self.params.url is not None:
+                    auth_url = self.params.url
+                elif self.params.host is not None:
+                    auth_url = f"mongodb://{self.params.host}"
+                    if self.params.port is not None:
+                        auth_url += f":{self.params.port}"
+                else:
+                    raise UrlError("No Mongo URL provided")
+                if self.params.auth_url_suffix is not None:
+                    auth_url = url_join(auth_url, self.params.auth_url_suffix)
+                self.params.auth_url = auth_url
+            self.mongo_client = pymongo.MongoClient(auth_url, username=self.params.login.username, password=self.params.login.password,
+                                                    ssl=ssl, tlscafile=ssl_certfile,
+                                                    timeoutMS=self.params.timeout*1000.0 if self.params.timeout is not None else None)
+            self.mongo_session = self.mongo_client.start_session()
+            if self.params.host is None and self.params.port is None:
+                # complete with host and port parsed by MongoClient
+                mongo_address = self.mongo_client.address
+                if mongo_address is not None:
+                    self.params.host, self.params.port = mongo_address
+            return self.mongo_client
+
+    def is_connected(self) -> bool:
+        return self.mongo_client is not None
+
+    def disconnect(self) -> None:
+        if self.mongo_client is not None:
+            if self.mongo_session is not None:
+                self.mongo_session.end_session()
+            self.mongo_client.close()
+            self.mongo_client = None
+            self.mongo_session = None
+
+    def check_connection(self, *, new_connection:bool=False, raise_error:bool=False) -> Union[None, ContextErrorLevelMessage]:
+        try:
+            self.connect(cancel_if_connected=not new_connection)
+            # the following line requires specific admin rights (unexpected)
+            # remote_collections = self.mongo_client.list_database_names(self.mongo_session)  # this tests the database connection
+        except Exception as e:
+            if raise_error:
+                raise e from e
+            else:
+                return ContextErrorLevelMessage("Mongo Harvester", ErrorLevel.Error, f"Failed to connect to {self.params.auth_url}: {e}")
+
+    def get_dataset_harvester(self, dataset_name: str) -> "DatasetHarvesterMongoDatabase":
+        params_dataset = self.params.copy(dest=DatasetParams())
+        params_dataset.dataset = dataset_name
+        dataset_harvester = DatasetHarvesterMongoDatabase(params_dataset)
+        self.copy(dest=dataset_harvester)
+        dataset_harvester.params = params_dataset
+        dataset_harvester._finalize_connection()
+        return dataset_harvester
+
+    def list_datasets(self, return_metadata: bool = True) -> Union[List[str], OrderedDict[str, DatasetMetadata]]:
+        # this would raise an unauthorized error !
+        self.connect()
+        dataset_list = self.mongo_client.list_database_names(self.mongo_session)
+        if return_metadata:
+            return OrderedDict([(name, self.get_dataset_harvester(name).query_dataset_metadata()) for name in dataset_list])
+        else:
+            return dataset_list
+
+
+class DatasetHarvesterMongoDatabase(DatabaseHarvesterMongoServer, DatasetHarvesterABC):
+    """
+    A CKAN dataset corresponds to a MongoDB database (set of collections).
+    """
+    def __init__(self, params:DatasetParams=None):
+        super().__init__(params)
+        self.mongo_database: Union[pymongo.database.Database,None] = None
+        self.dataset_metadata: Union[DatasetMetadata, None] = None  # DatasetHarvesterABC
+        # use database attribute if dataset is not specified (ambiguity on name - database attribute is not used above)
+        if self.params.dataset is None:
+            self.params.dataset = self.params.database
+        if self.params.dataset is None:
+            raise HarvesterArgumentRequiredError("dataset", "pymongo", "This argument defines the mongo database used")
+
+    @staticmethod
+    def init_from_options_string(options_string:str, base_dir:str=None) -> "DatasetHarvesterMongoDatabase":
+        params = DatasetParams()
+        params.parse_options_string(options_string, base_dir=base_dir)
+        return DatasetHarvesterMongoDatabase(params)
+
+    def _finalize_connection(self):
+        if super().is_connected() and self.mongo_database is None:
+            # remote_datasets = self.mongo_client.list_database_names(self.mongo_session)
+            # assert_or_raise(self.params.dataset in remote_datasets, ResourceNotFoundError("Database", self.params.dataset, self.params.auth_url))
+            self.mongo_database = self.mongo_client[self.params.dataset]
+
+    def connect(self, *, cancel_if_connected:bool=True) -> Any:
+        if not (cancel_if_connected and self.is_connected()):
+            super().connect(cancel_if_connected=cancel_if_connected)
+            self._finalize_connection()
+        return self.mongo_client
+
+    def is_connected(self) -> bool:
+        return super().is_connected()
+
+    def disconnect(self) -> None:
+        if super().is_connected():
+            self.mongo_database = None
+            super().disconnect()
+
+    def check_connection(self, *, new_connection: bool = False, raise_error: bool = False) -> Union[None, ContextErrorLevelMessage]:
+        try:
+            super().check_connection(new_connection=new_connection, raise_error=raise_error)
+            remote_collections = self.mongo_database.list_collection_names(self.mongo_session)  # this tests the database connection
+        except Exception as e:
+            if raise_error:
+                raise e from e
+            else:
+                return ContextErrorLevelMessage("Mongo Harvester", ErrorLevel.Error,
+                                                f"Failed to connect to {self.params.auth_url}: {e}")
+        if self.mongo_database is None:
+            return ContextErrorLevelMessage("Mongo Harvester", ErrorLevel.Error, f"Failed to connect to {self.params.auth_url}: <no error message>")
+        else:
+            return None
+
+    def query_dataset_metadata(self, cancel_if_present:bool=True) -> DatasetMetadata:
+        self.connect()
+        if cancel_if_present and self.dataset_metadata is not None:
+            return self.dataset_metadata
+        else:
+            self.dataset_metadata = DatasetMetadata()
+            self.dataset_metadata.name = self.mongo_database.name
+            self.dataset_metadata.tables = self.list_tables(return_metadata=True)
+            return self.dataset_metadata
+
+    def clean_dataset_metadata(self) -> DatasetMetadata:
+        return self.query_dataset_metadata().copy()
+
+    def get_table_harvester(self, table_name:str) -> "TableHarvesterMongoCollection":
+        params_table = self.params.copy(dest=TableParamsMongoCollection())
+        if self.params.options_string is not None:
+            # reparse options_string for table-specific arguments
+            params_table.parse_options_string(self.params.options_string, base_dir=self.params.base_dir)
+        params_table.table = table_name
+        table_harvester = TableHarvesterMongoCollection(params_table)
+        self.copy(dest=table_harvester)
+        table_harvester.params = params_table
+        table_harvester._finalize_connection()
+        return table_harvester
+
+    def list_tables(self, return_metadata:bool=True) -> Union[List[str], OrderedDict[str, TableMetadata]]:
+        self.connect()
+        remote_collections = [collection_name for collection_name in self.mongo_database.list_collection_names(session=self.mongo_session) if collection_name not in mongodb_excluded_collections]
+        if return_metadata:
+            return OrderedDict([(name, self.get_table_harvester(name).query_table_metadata()) for name in remote_collections])
+        else:
+            return remote_collections
+
+
+class TableHarvesterMongoCollection(DatasetHarvesterMongoDatabase, TableHarvesterABC):
+    """
+    A table (CKAN DataStore) corresponds to a MongoDB collection.
+    """
+    _default_upload_fun = pymongo_default_df_conversion
+    _default_primary_key = [mongodb_id_new_column]
+
+    def __init__(self, params:TableParamsMongoCollection=None):
+        super().__init__(params)
+        self.params: TableParamsMongoCollection = params
+        self.mongo_collection: Union[pymongo.collection.Collection,None] = None
+        self.table_metadata: Union[TableMetadata, None] = None  # TableHarvesterABC
+        if self.params.file_url_attr is not None:
+            # File/URL attribute has priority over CLI
+            self.params.table = self.params.file_url_attr
+        if self.params.table is None:
+            raise HarvesterArgumentRequiredError("table", "pymongo", "This argument defines the mongo collection used")
+
+    @staticmethod
+    def init_from_options_string(options_string:str, *, base_dir:str=None, file_url_attr:str=None) -> "TableHarvesterMongoCollection":
+        params = TableParamsMongoCollection()
+        params.parse_options_string(options_string, file_url_attr=file_url_attr, base_dir=base_dir)
+        return TableHarvesterMongoCollection(params)
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = TableHarvesterMongoCollection()
+        super().copy(dest=dest)
+        return dest
+
+    def disconnect(self) -> None:
+        if super().is_connected():
+            self.mongo_collection = None
+            super().disconnect()
+
+    def _finalize_connection(self):
+        super()._finalize_connection()
+        if super().is_connected() and self.mongo_collection is None:
+            mongo_database = self.mongo_database
+            remote_collections = mongo_database.list_collection_names()
+            collection_name = self.params.table
+            assert_or_raise(collection_name in remote_collections, ResourceNotFoundError("Collection", self.params.dataset + '.' + collection_name, self.params.auth_url))
+            collection = mongo_database[collection_name]
+            self.mongo_collection = collection
+
+    def connect(self, *, cancel_if_connected:bool=True) -> Any:
+        if not (cancel_if_connected and self.is_connected()):
+            super().connect()
+            self._finalize_connection()
+        return self.mongo_client
+
+    def check_connection(self, *, new_connection:bool=False, raise_error:bool=False) -> Union[None, ContextErrorLevelMessage]:
+        super().check_connection(new_connection=new_connection, raise_error=raise_error)
+        if self.mongo_collection is None:
+            return ContextErrorLevelMessage("Mongo Harvester", ErrorLevel.Error, f"Failed to connect to {self.params.auth_url}: <no error message>")
+        else:
+            return None
+
+    def query_table_metadata(self, cancel_if_present:bool=True) -> TableMetadata:
+        self.connect()
+        if cancel_if_present and self.table_metadata is not None:
+            return self.table_metadata
+        else:
+            # TODO: query at least primary key and indexes
+            self.table_metadata = TableMetadata()
+            index_dict = self.mongo_collection.index_information(session=self.mongo_session)
+            self.table_metadata.name = self.mongo_collection.name
+            self.table_metadata.indexes = sum([[key[0] for key in index["key"]] for index in index_dict.values()], [])
+            return self.table_metadata
+
+    def clean_table_metadata(self) -> TableMetadata:
+        clean_metadata = self.query_table_metadata().copy()
+        if clean_metadata.indexes is not None:
+            i_rm = []
+            for i, name in enumerate(clean_metadata.indexes):
+                if name == "_id":
+                    if mongodb_keep_id_column_trace:
+                        clean_metadata.indexes[i] = mongodb_id_new_column
+                    else:
+                        i_rm.append(i)
+                elif "." in name:
+                    i_rm.append(i)
+            for i in reversed(i_rm):
+                clean_metadata.indexes.pop(i)
+        clean_metadata.indexes = None  # finally, do not specify indexes at all
+        return clean_metadata
+
+    def get_default_primary_key(self) -> List[str]:
+        table_metadata = self.query_table_metadata()
+        if table_metadata.primary_key is not None:
+            return table_metadata.primary_key
+        else:
+            return TableHarvesterMongoCollection._default_primary_key
+
+    def get_default_data_cleaner(self) -> Union[CkanDataCleanerABC, None]:
+        data_cleaner = pymongo_default_data_cleaner()
+        data_cleaner.param_mongodb_dbref_as_one_column = not self.params.dbref_expand
+        return data_cleaner
+
+    def list_queries(self, *, new_connection:bool=False) -> List[Dict[str,Any]]:
+        self.connect(cancel_if_connected=not new_connection)
+        assert(self.mongo_collection is not None)
+        query = json.loads(self.params.query_string) if self.params.query_string is not None else {}
+        if self.params.verbose_harvester:
+            print(f"Counting documents of table {self.params.table}")
+        # num_rows = self.mongo_collection.count_documents(query, session=self.mongo_session)
+        num_rows = self.mongo_collection.estimated_document_count()
+        num_queries = num_rows // self.params.limit + 1
+        if self.params.single_request:
+            return [OrderedDict([("$match", query), ("$skip", i * self.params.limit), ("$limit", self.params.limit)]) for i in range(1)]
+        else:
+            queries_exact = [OrderedDict([("$match", query), ("$skip", i * self.params.limit), ("$limit", self.params.limit)]) for i in range(num_queries)]
+            query_extra = OrderedDict([("$match", query), ("$skip", num_queries * self.params.limit)])
+            return queries_exact + [query_extra]
+
+    def query_data(self, query:Dict[str,Any]) -> List[dict]:
+        assert(self.mongo_collection is not None)
+        if isinstance(query, str):
+            query = json.loads(query)
+        if self.params.verbose_harvester:
+            print(f"Pymongo request {query} on table {self.params.table}")
+        cursor = self.mongo_collection.find(query["$match"], session=self.mongo_session).skip(query["$skip"])
+        if "$limit" in query.keys():
+            cursor = cursor.limit(query["$limit"])
+        documents = list(cursor)
+        return documents
+

ckanapi_harvesters/harvesters/pymongo_params.py

@@ -0,0 +1,54 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Harvest from a mongo database using pymongo
+"""
+import argparse
+
+from ckanapi_harvesters.harvesters.harvester_params import TableParams
+
+
+class TableParamsMongoCollection(TableParams):
+    """
+    A table (CKAN DataStore) corresponds to a MongoDB collection.
+    This subclass of TableParams implements an alias attribute for table name called collection.
+    """
+    def __init__(self, source: "TableParamsMongoCollection" =None):
+        super().__init__(source)
+        self.dbref_expand:bool = False
+        if source is not None:
+            source.copy(dest=self)
+
+    # alias property for the table name setting: collection in MongoDB
+    @property
+    def collection(self) -> str:
+        return self.table
+    @collection.setter
+    def collection(self, value: str):
+        self.table = value
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = TableParamsMongoCollection()
+        super().copy(dest=dest)
+        dest.dbref_expand = self.dbref_expand
+        return dest
+
+    @staticmethod
+    def setup_cli_harvester_parser(parser: argparse.ArgumentParser = None) -> argparse.ArgumentParser:
+        parser = TableParams.setup_cli_harvester_parser(parser=parser)
+        parser.add_argument("--collection", type=str,
+                            help="MongoDB collection name")  # normally specified in the File/URL attribute of builder
+        # parser.add_argument("--table", help=argparse.SUPPRESS)  # do not display in help ==> conflict
+        parser.add_argument("--dbref-expand",
+                            help="Option to expand DBRefs",
+                            action="store_true", default=False)  # applies to data cleaner
+        return parser
+
+    def initialize_from_cli_args(self, args: argparse.Namespace, base_dir: str = None, error_not_found: bool = True,
+                                 default_proxies: dict = None, proxy_headers: dict = None) -> None:
+        super().initialize_from_cli_args(args, base_dir=base_dir, error_not_found=error_not_found,
+                                         default_proxies=default_proxies, proxy_headers=proxy_headers)
+        self.dbref_expand = args.dbref_expand
+        if args.collection is not None:
+            self.collection = args.collection

ckanapi_harvesters/policies/__init__.py

@@ -0,0 +1,20 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Package to enforce CKAN data policies.
+"""
+
+POLICY_FILE_FORMAT_VERSION = "0.0.0"  # version of the data format policy file format
+
+from . import data_format_policy_defs
+from . import data_format_policy_errors
+from . import data_format_policy_abc
+from . import data_format_policy_lists
+from . import data_format_policy_tag_groups
+from . import data_format_policy_custom_fields
+from . import data_format_policy
+
+# usage shortcuts
+from ckanapi_harvesters.policies.data_format_policy import CkanPackageDataFormatPolicy
+
+