ckanapi-harvesters 0.0.4__tar.gz → 0.0.6__tar.gz

{ckanapi_harvesters-0.0.4/src/ckanapi_harvesters.egg-info → ckanapi_harvesters-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ckanapi_harvesters
-Version: 0.0.4
+Version: 0.0.6
 Summary: Package helping to upload local resources to a CKAN server using the CKAN API. Metadata and the list of resources can be defined in an Excel spreadsheet. The package includes requests to download resources and checks metadata against formatting rules.
 Author-email: ifpen-gp <63413841+ifpen-gp@users.noreply.github.com>
 Maintainer-email: ifpen-gp <63413841+ifpen-gp@users.noreply.github.com>

{ckanapi_harvesters-0.0.4 → ckanapi_harvesters-0.0.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ckanapi_harvesters"
-version = '0.0.4'
+version = '0.0.6'
 authors = [
 	{ name = "ifpen-gp", email = "63413841+ifpen-gp@users.noreply.github.com" },
 ]

{ckanapi_harvesters-0.0.4 → ckanapi_harvesters-0.0.6}/src/ckanapi_harvesters/builder/example/builder_example_test_sql.py

@@ -37,12 +37,33 @@ def run(ckan:CkanApi = None):
         ckan.api_help_show("datastore_search_sql")
     except CkanNotFoundError:
         print("No datastore_search_sql help")
+
     users_table_id = mdl.get_or_query_resource_id(ckan, "users.csv")
+    traces_table_id = mdl.get_or_query_resource_id(ckan, "traces.csv")
+
+    # datastore_search simple API
+    cursor = ckan.datastore_search_cursor(users_table_id, limit=1)
+    document = next(cursor)
+    user_id = document["user_id"]
+    cursor = ckan.datastore_search_cursor(traces_table_id, filters={"user_id": int(user_id)}, limit=10)
+    for document in cursor:
+        print(document)
+
     try:
         ckan.datastore_search_sql(f'SELECT * FROM "{users_table_id}"')
     except CkanSqlCapabilityError:
         print("datastore_search_sql is not accessible")
 
+    query = f"""
+    SELECT t.*, u.* FROM "{traces_table_id}" t
+    JOIN "{users_table_id}" u ON t.user_id = u.user_id
+    WHERE t.user_id = {user_id}
+    LIMIT 10
+    """
+    cursor = ckan.datastore_search_sql_cursor(query, return_df=False)
+    for document in cursor:
+        print(document)
+
     print("Tests done.")
 
 

{ckanapi_harvesters-0.0.4 → ckanapi_harvesters-0.0.6}/src/ckanapi_harvesters/ckan_api/ckan_api_2_readonly.py

@@ -280,7 +280,7 @@ class CkanApiReadOnly(CkanApiMap):
     ### Search method ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     def _api_datastore_search_raw(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                   distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None,
-                                  params:dict=None, compute_len:int=False) -> CkanActionResponse:
+                                  params:dict=None, compute_len:bool=False) -> CkanActionResponse:
         """
         API call to datastore_search. Performs queries on the DataStore.
 
@@ -338,7 +338,8 @@ class CkanApiReadOnly(CkanApiMap):
             raise response.default_error(self)
 
     def _api_datastore_search_df(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
-                                 distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None, params:dict=None) -> pd.DataFrame:
+                                 distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None,
+                                 params:dict=None, compute_len:bool=True) -> pd.DataFrame:
         """
         Convert output of _api_datastore_search_raw to pandas DataFrame.
         """
@@ -377,7 +378,7 @@ class CkanApiReadOnly(CkanApiMap):
     def _api_datastore_search_all(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                   distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None,
                                   search_all:bool=True, params:dict=None, return_df:bool=True, compute_len:bool=False) \
-            -> Union[pd.DataFrame, Tuple[ListRecords, OrderedDict], Any]:
+            -> Union[pd.DataFrame, ListRecords, Any]:
         """
         Successive calls to _api_datastore_search_df until an empty list is received.
 
@@ -409,7 +410,7 @@ class CkanApiReadOnly(CkanApiMap):
                                             search_all=search_all, resource_id=resource_id, filters=filters, q=q, fields=fields, distinct=distinct, sort=sort, format=format, compute_len=compute_len)
             # aggregate results, depending on the format
             if self.params.dry_run:
-                return [], {}
+                return []
             if format is not None:
                 format = format.lower()
             if len(responses) > 0:
@@ -420,12 +421,12 @@ class CkanApiReadOnly(CkanApiMap):
                 fields_type_dict = None
                 df_args = {}
             if format is None or format == "objects":
-                return ListRecords(sum([response.result["records"] for response in responses], [])), fields_type_dict
+                return ListRecords(sum([response.result["records"] for response in responses], []))
             else:
                 if format == "lists":
-                    return sum([response.result["records"] for response in responses], []), fields_type_dict
+                    return sum([response.result["records"] for response in responses], [])
                 else:
-                    return "\n".join([response.result["records"] for response in responses]), fields_type_dict
+                    return "\n".join([response.result["records"] for response in responses])
 
     def _api_datastore_search_all_generator(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                             distinct:bool=None, sort:str=None, limit:int=None, offset:int=0,
@@ -486,6 +487,10 @@ class CkanApiReadOnly(CkanApiMap):
             params["limit"] = limit
         response = self._api_action_request(f"datastore_search_sql", method=RequestType.Post, params=params)
         if response.success:
+            if response.dry_run:
+                return response
+            else:
+                response.len = len(response.result["records"])
             return response
         elif response.status_code == 400 and response.success_json_loads and response.response.text == '"Bad request - Action name not known: datastore_search_sql"':
             raise CkanSqlCapabilityError(self, response)
@@ -513,7 +518,7 @@ class CkanApiReadOnly(CkanApiMap):
 
     def _api_datastore_search_sql_all(self, sql:str, *, params:dict=None,
                                       search_all:bool=True, limit:int=None, offset:int=0, return_df:bool=True) \
-            -> Union[pd.DataFrame, Tuple[ListRecords, dict]]:
+            -> Union[pd.DataFrame, ListRecords]:
         """
         Successive calls to _api_datastore_search_sql until an empty list is received.
 
@@ -543,7 +548,7 @@ class CkanApiReadOnly(CkanApiMap):
                 fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
             else:
                 fields_type_dict = None
-            return ListRecords(sum([response.result["records"] for response in responses], [])), fields_type_dict
+            return ListRecords(sum([response.result["records"] for response in responses], []))
 
     def _api_datastore_search_sql_all_generator(self, sql:str, *, params:dict=None,
                                                 search_all:bool=True, limit:int=None, offset:int=0, return_df:bool=True) \
@@ -659,8 +664,8 @@ class CkanApiReadOnly(CkanApiMap):
 
     def datastore_search_cursor(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                    distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, params:dict=None,
-                                   search_all:bool=False, search_method:bool=True, format:str=None, return_df:bool=True) \
-            -> Generator[Union[pd.Series, Tuple[dict,dict], Tuple[list,dict], Tuple[str,dict]], Any, None]:
+                                   search_all:bool=False, search_method:bool=True, format:str=None, return_df:bool=False) \
+            -> Generator[Union[pd.Series, dict, list, str], Any, None]:
         """
         Cursor on rows
         """
@@ -682,12 +687,12 @@ class CkanApiReadOnly(CkanApiMap):
                 for response in generator:
                     fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                     for element in response.result["records"]:
-                        yield element, fields_type_dict
+                        yield element
             else:
                 for response in generator:
                     fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                     for element in response.result["records"]:
-                        yield element, fields_type_dict
+                        yield element
         else:
             raise TypeError("dumping datastore without parsing with a DataFrame does not return an iterable object")
 
@@ -752,8 +757,8 @@ class CkanApiReadOnly(CkanApiMap):
         return self._api_datastore_search_sql_all_generator(sql, params=params, limit=limit, offset=offset, search_all=search_all, return_df=return_df)
 
     def datastore_search_sql_cursor(self, sql:str, *, params:dict=None, search_all:bool=False,
-                                    limit:int=None, offset:int=0, return_df:bool=True) \
-            -> Generator[Union[pd.Series,Tuple[dict,dict]], Any, None]:
+                                    limit:int=None, offset:int=0, return_df:bool=False) \
+            -> Generator[Union[pd.Series,dict], Any, None]:
         generator = self.datastore_search_sql_generator(sql, params=params, search_all=search_all,
                                                         limit=limit, offset=offset, return_df=return_df)
         if return_df:
@@ -769,7 +774,7 @@ class CkanApiReadOnly(CkanApiMap):
             for response in generator:
                 fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                 for element in response.result["records"]:
-                    yield element, fields_type_dict
+                    yield element
 
     def datastore_search_sql_find_one(self, sql:str, *, params:dict=None,
                                       offset:int=0, return_df:bool=True) -> Union[pd.DataFrame, Tuple[ListRecords, dict]]:
@@ -912,7 +917,7 @@ class CkanApiReadOnly(CkanApiMap):
     ## Mapping of resource aliases from table
     def list_datastore_aliases(self) -> List[CkanAliasInfo]:
         alias_resource_id = "_table_metadata"  # resource name of table containing CKAN aliases
-        alias_list_dict, _ = self.datastore_search(alias_resource_id, search_all=True, return_df=False, format="objects", search_method=True)
+        alias_list_dict = self.datastore_search(alias_resource_id, search_all=True, return_df=False, format="objects", search_method=True)
         alias_list = [CkanAliasInfo(alias_dict) for alias_dict in alias_list_dict]
         for alias_info in alias_list:
             if alias_info.alias_of is not None:

{ckanapi_harvesters-0.0.4 → ckanapi_harvesters-0.0.6/src/ckanapi_harvesters.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ckanapi_harvesters
-Version: 0.0.4
+Version: 0.0.6
 Summary: Package helping to upload local resources to a CKAN server using the CKAN API. Metadata and the list of resources can be defined in an Excel spreadsheet. The package includes requests to download resources and checks metadata against formatting rules.
 Author-email: ifpen-gp <63413841+ifpen-gp@users.noreply.github.com>
 Maintainer-email: ifpen-gp <63413841+ifpen-gp@users.noreply.github.com>