ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.2__py3-none-any.whl

ckanapi_harvesters/harvesters/file_formats/shp_format.py

@@ -0,0 +1,129 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Shapefile format support
+"""
+from typing import Union, Dict
+from types import SimpleNamespace
+import io
+from warnings import warn
+from enum import IntEnum
+
+import pandas as pd
+try:
+    import geopandas as gpd
+except ImportError:
+    gpd = SimpleNamespace(GeoDataFrame=None)
+try:
+    import pyproj
+except ImportError:
+    pyproj = None
+
+from ckanapi_harvesters.auxiliary.list_records import ListRecords
+from ckanapi_harvesters.auxiliary.ckan_model import CkanField
+from ckanapi_harvesters.auxiliary.ckan_errors import FileFormatRequirementError, UnknownTargetCRSError
+from ckanapi_harvesters.auxiliary.ckan_configuration import default_ckan_target_epsg
+from ckanapi_harvesters.harvesters.file_formats.file_format_abc import FileFormatABC
+
+
+shp_upload_read_file_kwargs = dict(encoding='utf-8')
+
+class DownloadedShapeFileConversion(IntEnum):
+    CsvWkb = 0
+    ShapefileProjection = 2
+    ShapefileAsIs = 3
+
+class ShapeFileFormat(FileFormatABC):
+    def __init__(self, read_file_kwargs=None) -> None:
+        if gpd.GeoDataFrame is None:
+            raise FileFormatRequirementError("geopandas", "SHP")
+        if pyproj is None:
+            raise FileFormatRequirementError("pyproj", "SHP")
+        if read_file_kwargs is None: read_file_kwargs = shp_upload_read_file_kwargs
+        self.read_file_kwargs:dict = read_file_kwargs
+        self.require_field_crs:bool = True
+        self.download_conversion = DownloadedShapeFileConversion.ShapefileProjection
+
+    # loading a file before upload ----------------
+    def read_file(self, file_path: Union[str,io.StringIO], fields: Union[Dict[str, CkanField],None]) -> Union[pd.DataFrame, ListRecords]:
+        # target EPSG = EPSG used in CKAN, source EPSG read from SHP file
+        gdf = gpd.read_file(file_path, **self.read_file_kwargs)
+        geo_columns = list(gdf.select_dtypes('geometry'))
+        for field_name in geo_columns:
+            gdf.set_geometry(field_name, inplace=True)  # select the current column for geometry computations
+            crs_source = gdf.crs
+            if field_name in fields.keys():
+                field = fields[field_name]
+                field_data_type = field.data_type
+                epsg_target = field.internal_attrs.epsg_target
+                epsg_source_from_params = field.internal_attrs.epsg_source
+            else:
+                # default field data type, with a generic geometry type and the default EPSG
+                epsg_target = default_ckan_target_epsg
+                field_data_type = f"geometry(geometry,{epsg_target})"
+                fields[field_name] = CkanField(field_name, field_data_type)  # TODO: update field data type in caller? user can change data type afterwards?
+                epsg_source_from_params = None
+                msg = f"PostGIS geometric destination type was not specified and will not be transmitted to CKAN. Assuming default {field_data_type}."
+                warn(msg)
+            if field_data_type == "geometry" or field_data_type.startswith("geometry("):  # and field.internal_attrs.geometry_as_source:
+                if epsg_target is not None:
+                    crs_target = pyproj.CRS.from_epsg(epsg_target)
+                    if not crs_source == crs_target:
+                        gdf.to_crs(crs_target, inplace=True)
+                elif self.require_field_crs:
+                    raise UnknownTargetCRSError(crs_source, file_path)
+                if epsg_source_from_params is not None:
+                    crs_source_from_params = pyproj.CRS.from_epsg(epsg_source_from_params)
+                    if not crs_source_from_params == crs_source:
+                        msg = f"EPSG in SHP file ({crs_source}) does not match given source EPSG ({crs_source_from_params}). The downloaded result will differ from original format."
+                        warn(msg)
+            else:
+                raise NotImplementedError(f"Field {field_data_type} is not implemented or not compatible with geometric representations.")
+        df = gdf.to_wkb(hex=True)  # converts all geometric fields to WKB and returns a standard DataFrame object
+        return df
+
+    def read_buffer(self, buffer: io.StringIO, fields: Union[Dict[str, CkanField],None]) -> Union[pd.DataFrame, ListRecords]:
+        return self.read_file(buffer, fields=fields)
+
+    # saving a file after download -------------
+    def downloaded_df_to_gdf(self, df: pd.DataFrame, *, fields: Union[Dict[str, CkanField],None], context:str=None) -> gpd.GeoDataFrame:
+        # NB: target EPSG = CRS in database (required), source = option to recover original CRS
+        gdf = gpd.GeoDataFrame(df)
+        if self.download_conversion == DownloadedShapeFileConversion.CsvWkb:
+            # do not look at CRS information and leave in WKB format
+            return gdf
+        for field_name in df.columns:
+            if field_name in fields.keys():
+                field = fields[field_name]
+                field_data_type = field.data_type
+                if field_data_type == "geometry" or field_data_type.startswith("geometry("):
+                    crs_target = pyproj.CRS.from_epsg(field.internal_attrs.epsg_target)
+                    if crs_target is None and self.require_field_crs:
+                        raise UnknownTargetCRSError(field.internal_attrs.epsg_source, context)
+                    gdf[field_name] = gpd.geoseries.from_wkb(df[field_name], crs=crs_target)
+                    if (self.download_conversion == DownloadedShapeFileConversion.ShapefileProjection
+                            and field.internal_attrs.epsg_source is not None and crs_target is not None
+                            and not field.internal_attrs.epsg_target == field.internal_attrs.epsg_source):
+                        crs_source = pyproj.CRS.from_epsg(field.internal_attrs.epsg_source)
+                        if not crs_source == crs_target:
+                            gdf.to_crs(crs_source, inplace=True)
+        return gdf
+
+    def write_file(self, df: pd.DataFrame, file_path: str, fields: Union[Dict[str, CkanField],None]) -> None:
+        # this writes the shp file and auxiliary shx, dbf, cpg, prj files
+        gdf = self.downloaded_df_to_gdf(df, fields=fields, context=file_path)
+        gdf.to_file(file_path, driver="ESRI Shapefile")
+
+    def write_in_memory(self, df: pd.DataFrame, fields: Union[Dict[str, CkanField],None]) -> bytes:
+        # how could this work because there are multiple files?
+        gdf = self.downloaded_df_to_gdf(df, fields=fields)
+        buffer = io.StringIO()
+        gdf.to_file(buffer, driver="ESRI Shapefile")
+        return buffer.getvalue().encode("utf8")
+
+    def copy(self):
+        dest = ShapeFileFormat(self.read_file_kwargs)
+        dest.download_conversion = self.download_conversion
+        dest.require_field_crs = self.require_field_crs
+        return dest
+

ckanapi_harvesters/harvesters/harvester_abc.py

@@ -0,0 +1,190 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Harvester base class
+"""
+from typing import Union, List, Any, Callable
+from collections import OrderedDict
+from abc import ABC, abstractmethod
+
+import pandas as pd
+
+from ckanapi_harvesters.auxiliary.error_level_message import ContextErrorLevelMessage
+from ckanapi_harvesters.harvesters.harvester_model import DatasetMetadata, TableMetadata
+from ckanapi_harvesters.harvesters.harvester_params import DatabaseParams, DatasetParams, TableParams
+from ckanapi_harvesters.harvesters.data_cleaner.data_cleaner_abc import CkanDataCleanerABC
+
+
+class HarvesterConnectABC(ABC):
+    def __del__(self):
+        self.disconnect()
+
+    @abstractmethod
+    def connect(self, *, cancel_if_connected:bool=True) -> Any:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _finalize_connection(self):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def is_connected(self) -> bool:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def check_connection(self, *, new_connection:bool=False, raise_error:bool=False) -> Union[None, ContextErrorLevelMessage]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def disconnect(self) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def update_from_ckan(self, ckan):
+        raise NotImplementedError()
+
+
+class DatabaseHarvesterABC(HarvesterConnectABC, ABC):
+    def __init__(self, params:DatabaseParams=None):
+        if params is None:
+            params = DatabaseParams()
+        self.params: DatabaseParams = params
+
+    @abstractmethod
+    def copy(self, *, dest=None):
+        dest.params = self.params.copy()
+        return dest
+
+    @staticmethod
+    @abstractmethod
+    def init_from_options_string(options_string:str, *, base_dir:str=None) -> "DatabaseHarvesterABC":
+        raise NotImplementedError()
+
+    def _finalize_connection(self):
+        pass
+
+    ## query methods interface ---------------
+    @abstractmethod
+    def get_dataset_harvester(self, dataset_name:str) -> "DatasetHarvesterABC":
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_datasets(self, return_metadata:bool=True) -> Union[List[str], OrderedDict[str, DatasetMetadata]]:
+        raise NotImplementedError()
+
+    def update_from_ckan(self, ckan):
+        self.params._update_from_ckan(ckan)
+
+
+class DatasetHarvesterABC(DatabaseHarvesterABC, ABC):
+    def __init__(self, params:DatasetParams=None):
+        if params is None:
+            params = DatasetParams()
+        super().__init__(params)
+        self.params: DatasetParams = params
+        self.dataset_metadata: Union[DatasetMetadata, None] = None
+
+    def __del__(self):
+        self.disconnect()
+
+    @abstractmethod
+    def _finalize_connection(self):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def copy(self, *, dest=None):
+        super().copy(dest=dest)
+        dest.dataset_metadata = self.dataset_metadata.copy() if self.dataset_metadata is not None else None
+        return dest
+
+    @staticmethod
+    @abstractmethod
+    def init_from_options_string(options_string:str, *, base_dir:str=None) -> "DatasetHarvesterABC":
+        raise NotImplementedError()
+
+    ## metadata interface ---------------
+    @abstractmethod
+    def query_dataset_metadata(self, cancel_if_present:bool=True) -> DatasetMetadata:
+        self.connect()
+        if cancel_if_present and self.dataset_metadata is not None:
+            return self.dataset_metadata
+        else:
+            self.dataset_metadata = DatasetMetadata()
+            # user needs to complete here
+            self.dataset_metadata.tables = self.list_tables(return_metadata=True)
+            return self.dataset_metadata
+
+    def clean_dataset_metadata(self) -> DatasetMetadata:
+        return self.query_dataset_metadata().copy()
+
+    ## query methods interface ---------------
+    @abstractmethod
+    def get_table_harvester(self, table_name:str) -> "TableHarvesterABC":
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list_tables(self, return_metadata:bool=True) -> Union[List[str], OrderedDict[str, TableMetadata]]:
+        raise NotImplementedError()
+
+
+class TableHarvesterABC(DatasetHarvesterABC, ABC):
+    _default_upload_fun: Union[Callable[[Any], pd.DataFrame], None] = None
+    _default_primary_key: Union[List[str], None] = None
+
+    def __init__(self, params:TableParams=None):
+        if params is None:
+            params = TableParams()
+        super().__init__(params)
+        self.params: TableParams = params
+        self.table_metadata: Union[TableMetadata, None] = None
+
+    def __del__(self):
+        self.disconnect()
+
+    @abstractmethod
+    def copy(self, *, dest=None):
+        super().copy(dest=dest)
+        dest.table_metadata = self.table_metadata.copy() if self.table_metadata is not None else None
+        return dest
+
+    @staticmethod
+    @abstractmethod
+    def init_from_options_string(options_string:str, *, base_dir:str=None, file_url_attr:str=None) -> "TableHarvesterABC":
+        raise NotImplementedError()
+
+    ## metadata interface ---------------
+    @abstractmethod
+    def query_table_metadata(self, cancel_if_present:bool=True) -> TableMetadata:
+        self.connect()
+        if cancel_if_present and self.table_metadata is not None:
+            return self.table_metadata
+        else:
+            self.table_metadata = TableMetadata()
+            # user needs to complete here
+            return self.table_metadata
+
+    def clean_table_metadata(self) -> TableMetadata:
+        return self.query_table_metadata().copy()
+
+    @classmethod
+    def get_default_df_upload_fun(cls) -> Union[Callable[[Any], pd.DataFrame], None]:
+        return cls._default_upload_fun
+
+    def get_default_data_cleaner(self) -> Union[CkanDataCleanerABC, None]:
+        return None
+
+    @abstractmethod
+    def get_default_primary_key(self) -> List[str]:
+        return []
+
+    ## query methods interface ---------------
+    @abstractmethod
+    def list_queries(self, *, new_connection:bool=False) -> List[Any]:
+        self.connect(cancel_if_connected=not new_connection)
+        raise NotImplementedError()
+
+    @abstractmethod
+    def query_data(self, query:Any) -> Union[List[dict], pd.DataFrame]:
+        raise NotImplementedError()
+
+

ckanapi_harvesters/harvesters/harvester_errors.py

@@ -0,0 +1,31 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Errors specific to harvesting data
+"""
+
+from ckanapi_harvesters.auxiliary.ckan_errors import RequirementError
+from ckanapi_harvesters.harvesters.data_cleaner.data_cleaner_errors import CleanError, CleanerRequirementError  # alias
+
+
+class HarvestMethodRequiredError(Exception):
+    def __init__(self):
+        super().__init__("The harvesting method argument --harvester is required.")
+
+class HarvesterArgumentError(Exception):
+    pass
+
+class HarvesterArgumentRequiredError(HarvesterArgumentError):
+    def __init__(self, argument:str, harvest_method:str, help:str=None):
+        if help is None: help = ""
+        super().__init__(f"The argument {argument} is required for harvest method {harvest_method}. " + help)
+
+class HarvesterRequirementError(RequirementError):
+    def __init__(self, requirement:str, harvest_method:str):
+        super().__init__(f"The package {requirement} is required for this harvester ({harvest_method}).")
+
+class ResourceNotFoundError(Exception):
+    def __init__(self, resource_type:str, table_name:str, host:str):
+        super().__init__(f"{resource_type} {table_name} was not found on host ({host}).")
+
+

ckanapi_harvesters/harvesters/harvester_init.py

@@ -0,0 +1,30 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Harvester initialization from the options_string arguments
+"""
+from ckanapi_harvesters.harvesters.harvester_abc import TableHarvesterABC, DatasetHarvesterABC
+from ckanapi_harvesters.harvesters.harvester_params import TableParams
+from ckanapi_harvesters.harvesters.harvester_params import DatasetParams
+from ckanapi_harvesters.harvesters.postgre_harvester import TableHarvesterPostgre, DatasetHarvesterPostgre
+from ckanapi_harvesters.harvesters.pymongo_harvester import TableHarvesterMongoCollection, DatasetHarvesterMongoDatabase
+
+
+def init_table_harvester_from_options_string(options_string:str, *, file_url_attr:str, base_dir:str=None) -> TableHarvesterABC:
+    harvest_method = TableParams.parse_harvest_method(options_string)
+    if harvest_method == "pymongo":
+        return TableHarvesterMongoCollection.init_from_options_string(options_string, file_url_attr=file_url_attr, base_dir=base_dir)
+    elif harvest_method == "postgre":
+        return TableHarvesterPostgre.init_from_options_string(options_string, file_url_attr=file_url_attr, base_dir=base_dir)
+    else:
+        raise NotImplementedError(f"harvester method {harvest_method} not implemented")
+
+
+def init_dataset_harvester_from_options_string(options_string:str, *, base_dir:str=None) -> DatasetHarvesterABC:
+    harvest_method = DatasetParams.parse_harvest_method(options_string)
+    if harvest_method == "pymongo":
+        return DatasetHarvesterMongoDatabase.init_from_options_string(options_string, base_dir=base_dir)
+    elif harvest_method == "postgre":
+        return DatasetHarvesterPostgre.init_from_options_string(options_string, base_dir=base_dir)
+    else:
+        raise NotImplementedError(f"harvester method {harvest_method} not implemented")

ckanapi_harvesters/harvesters/harvester_model.py

@@ -0,0 +1,49 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Harvester base class
+"""
+from typing import Union, List
+from collections import OrderedDict
+import copy
+
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import CkanFieldInternalAttrs
+
+
+class FieldMetadata:
+    def __init__(self):
+        self.name:str = ""
+        self.description:Union[str,None] = None
+        self.label:Union[str,None] = None
+        self.data_type:Union[str,None] = None
+        self.is_index:Union[bool,None] = None
+        self.uniquekey:Union[bool,None] = None
+        self.notnull:Union[bool,None] = None
+        self.internal_attrs: CkanFieldInternalAttrs = CkanFieldInternalAttrs()
+        self.harvester_attrs: dict = {}
+
+    def copy(self):
+        return copy.deepcopy(self)
+
+
+class TableMetadata:
+    def __init__(self):
+        self.name: str = ""
+        self.primary_key: Union[List[str],None] = None
+        self.indexes: Union[List[str],None] = None
+        self.unique_keys: Union[List[str],None] = None
+        self.description: Union[str,None] = None
+        self.fields: Union[OrderedDict[str,FieldMetadata],None] = None
+
+    def copy(self):
+        return copy.deepcopy(self)
+
+
+class DatasetMetadata:
+    def __init__(self):
+        self.name:str = ""
+        self.description:Union[str,None] = None
+        self.tables:Union[OrderedDict[str,TableMetadata],None] = None
+
+    def copy(self):
+        return copy.deepcopy(self)