hdx-python-api 6.7.2__py3-none-any.whl → 6.7.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hdx/api/_version.py CHANGED
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '6.7.2'
22
- __version_tuple__ = version_tuple = (6, 7, 2)
21
+ __version__ = version = '6.7.4'
22
+ __version_tuple__ = version_tuple = (6, 7, 4)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -0,0 +1,86 @@
1
+ """HDX site-wide statistics"""
2
+
3
+ from hdx.api.configuration import Configuration
4
+ from hdx.data.hdxobject import HDXError
5
+
6
+
7
+ def _call_stats_action(
8
+ action: str, configuration: Configuration | None, data: dict | None = None
9
+ ) -> dict:
10
+ """Call an HDX statistics action
11
+
12
+ Args:
13
+ action: CKAN action name
14
+ configuration: HDX configuration. Defaults to global configuration.
15
+ data: Parameters to pass to the action. Defaults to None.
16
+
17
+ Returns:
18
+ Statistics dictionary
19
+ """
20
+ if configuration is None:
21
+ configuration = Configuration.read()
22
+ try:
23
+ return configuration.call_remoteckan(action, data or {})
24
+ except Exception as e:
25
+ raise HDXError(f"Failed when trying to call {action}!") from e
26
+
27
+
28
+ def get_general_statistics(configuration: Configuration | None = None) -> dict:
29
+ """Get general statistics about datasets, organizations and users on HDX
30
+
31
+ Args:
32
+ configuration: HDX configuration. Defaults to global configuration.
33
+
34
+ Returns:
35
+ Dictionary with keys datasets, organizations and users
36
+ """
37
+ return _call_stats_action("hdx_general_statistics", configuration)
38
+
39
+
40
+ def push_general_stats(configuration: Configuration | None = None) -> dict:
41
+ """Compute general statistics on HDX and push them to analytics. Requires
42
+ permission to manage basic scheduled tasks.
43
+
44
+ Args:
45
+ configuration: HDX configuration. Defaults to global configuration.
46
+
47
+ Returns:
48
+ Statistics that were pushed
49
+ """
50
+ return _call_stats_action("hdx_push_general_stats", configuration)
51
+
52
+
53
+ def get_user_statistics(
54
+ start_date: str | None = None,
55
+ end_date: str | None = None,
56
+ configuration: Configuration | None = None,
57
+ ) -> dict:
58
+ """Get the number of activities of each active user on HDX, optionally
59
+ between start and end dates. Requires sysadmin.
60
+
61
+ Args:
62
+ start_date: Count activities on or after this date eg. 2026-01-01. Defaults to None.
63
+ end_date: Count activities on or before this date. Defaults to None.
64
+ configuration: HDX configuration. Defaults to global configuration.
65
+
66
+ Returns:
67
+ Dictionary from user id to dictionary with keys id, name and no_of_activities
68
+ """
69
+ data = {}
70
+ if start_date:
71
+ data["start_date"] = start_date
72
+ if end_date:
73
+ data["end_date"] = end_date
74
+ return _call_stats_action("hdx_user_statistics", configuration, data)
75
+
76
+
77
+ def get_organization_statistics(configuration: Configuration | None = None) -> dict:
78
+ """Get active and inactive organizations on HDX. Requires sysadmin.
79
+
80
+ Args:
81
+ configuration: HDX configuration. Defaults to global configuration.
82
+
83
+ Returns:
84
+ Dictionary with keys active_organizations and inactive_organizations
85
+ """
86
+ return _call_stats_action("hdx_organization_statistics", configuration)
hdx/data/activity.py CHANGED
@@ -129,7 +129,9 @@ class Activity(HDXObject):
129
129
  result = activity._write_to_hdx(action, data)
130
130
  return [cls(x, configuration=configuration) for x in result]
131
131
  data = dict(data)
132
- limit = data.setdefault("limit", 100)
132
+ # HDX silently caps limit at ckan.activity_list_limit_max
133
+ limit = min(data.get("limit", 100), 100)
134
+ data["limit"] = limit
133
135
  offset = data.get("offset", 0)
134
136
  activities = []
135
137
  while True:
hdx/data/dataset.py CHANGED
@@ -147,6 +147,14 @@ class Dataset(HDXObject):
147
147
  "list": "package_list",
148
148
  "autocomplete": "package_autocomplete",
149
149
  "create_default_views": "package_create_default_resource_views",
150
+ "fs_check_info": "fs_check_info_show",
151
+ "fs_check_reset": "hdx_fs_check_package_reset",
152
+ "shape_info": "shape_info_show",
153
+ "recommend_tags": "hdx_recommend_tags",
154
+ "allowed_for_datastore": "hdx_is_package_allowed_for_datastore",
155
+ "push_to_datastore": "hdx_push_resource_to_datastore",
156
+ "qa_completed": "hdx_mark_qa_completed",
157
+ "qa_revise_resources": "hdx_qa_package_revise_resource",
150
158
  }
151
159
 
152
160
  def __setitem__(self, key: Any, value: Any) -> None:
@@ -1447,13 +1455,7 @@ class Dataset(HDXObject):
1447
1455
  Returns:
1448
1456
  List of Activity objects
1449
1457
  """
1450
- return Activity.get_activities(
1451
- "package",
1452
- self.data.get("id") or self.data.get("name"),
1453
- self.configuration,
1454
- all_pages,
1455
- **kwargs,
1456
- )
1458
+ return self._get_activities("package", all_pages, **kwargs)
1457
1459
 
1458
1460
  @classmethod
1459
1461
  def autocomplete(
@@ -2333,6 +2335,155 @@ class Dataset(HDXObject):
2333
2335
  return None
2334
2336
  return f"{self.configuration.get_hdx_site_url()}/api/3/action/package_show?id={name_or_id}"
2335
2337
 
2338
+ def get_fs_check_info(self) -> list[dict]:
2339
+ """Get file structure check information for the dataset's resources
2340
+ from HDX. Each list item is a dictionary from resource name to a list
2341
+ of check events, each of which can have keys such as state, timestamp,
2342
+ sheet_changes and hxl_proxy_response.
2343
+
2344
+ Returns:
2345
+ List of dictionaries from resource name to file structure checks
2346
+ """
2347
+ success, result = self._read_from_hdx(
2348
+ "dataset",
2349
+ self.get_name_or_id(prefer_name=False),
2350
+ "id",
2351
+ self.actions()["fs_check_info"],
2352
+ )
2353
+ if success:
2354
+ return result
2355
+ return []
2356
+
2357
+ def _set_from_package(self, package: dict) -> None:
2358
+ """Set the dataset's metadata and resources from a package metadata
2359
+ dictionary returned by HDX
2360
+
2361
+ Args:
2362
+ package: Package metadata dictionary
2363
+
2364
+ Returns:
2365
+ None
2366
+ """
2367
+ self.data = package
2368
+ self.init_resources()
2369
+ self.separate_resources()
2370
+
2371
+ def reset_fs_check(self, key: str = "fs_check_info") -> None:
2372
+ """Clear a file structure check field of all the dataset's resources
2373
+ in HDX. Requires sysadmin or permission to manage QA.
2374
+
2375
+ Args:
2376
+ key: Resource field to clear. Defaults to fs_check_info.
2377
+
2378
+ Returns:
2379
+ None
2380
+ """
2381
+ result = self._write_to_hdx(
2382
+ "fs_check_reset", {"package_id": self.data["id"], "key": key}, "package_id"
2383
+ )
2384
+ self._set_from_package(result["package"])
2385
+
2386
+ def set_qa_completed(self, qa_completed: bool) -> None:
2387
+ """Mark QA of the dataset in HDX as completed or not. Requires
2388
+ sysadmin or permission to manage QA.
2389
+
2390
+ Args:
2391
+ qa_completed: Whether QA is completed
2392
+
2393
+ Returns:
2394
+ None
2395
+ """
2396
+ result = self._write_to_hdx(
2397
+ "qa_completed", {"id": self.data["id"], "qa_completed": qa_completed}, "id"
2398
+ )
2399
+ self._set_from_package(result["package"])
2400
+
2401
+ def qa_revise_resources(self, key: str, value: Any) -> None:
2402
+ """Set a QA field of all the dataset's resources in HDX. Requires
2403
+ sysadmin or permission to manage QA.
2404
+
2405
+ Args:
2406
+ key: Resource field eg. pii
2407
+ value: Value to set
2408
+
2409
+ Returns:
2410
+ None
2411
+ """
2412
+ result = self._write_to_hdx(
2413
+ "qa_revise_resources",
2414
+ {"id": self.data["id"], "key": key, "value": value},
2415
+ "id",
2416
+ )
2417
+ self._set_from_package(result["package"])
2418
+
2419
+ def get_shape_info(self) -> list[dict]:
2420
+ """Get geopreview shape information for the dataset's resources from
2421
+ HDX
2422
+
2423
+ Returns:
2424
+ List of dictionaries from resource name to shape information
2425
+ """
2426
+ success, result = self._read_from_hdx(
2427
+ "dataset",
2428
+ self.get_name_or_id(prefer_name=False),
2429
+ "id",
2430
+ self.actions()["shape_info"],
2431
+ )
2432
+ if success:
2433
+ return result
2434
+ return []
2435
+
2436
+ def get_recommended_tags(self) -> list[str]:
2437
+ """Get tags recommended by HDX for the dataset based on its title and
2438
+ organization. Tags are ordered from most to least recommended.
2439
+
2440
+ Returns:
2441
+ List of recommended tags
2442
+ """
2443
+ data = {"title": self.data.get("title")}
2444
+ organization = self.data.get("owner_org")
2445
+ if organization:
2446
+ data["organization"] = organization
2447
+ try:
2448
+ result = self.configuration.call_remoteckan(
2449
+ self.actions()["recommend_tags"], data
2450
+ )
2451
+ except Exception as e:
2452
+ raise HDXError("Failed when trying to get recommended tags!") from e
2453
+ return [tag["name"] for tag in result]
2454
+
2455
+ def is_allowed_for_datastore(self) -> bool:
2456
+ """Return if the dataset or its organization is in HDX's allowlist for
2457
+ automatic loading of resources into the datastore
2458
+
2459
+ Returns:
2460
+ Whether dataset is allowed for datastore
2461
+ """
2462
+ success, result = self._read_from_hdx(
2463
+ "dataset",
2464
+ self.data["id"],
2465
+ "package_id",
2466
+ self.actions()["allowed_for_datastore"],
2467
+ )
2468
+ if success:
2469
+ return result
2470
+ logger.debug(result)
2471
+ return False
2472
+
2473
+ def push_to_datastore(self) -> bool:
2474
+ """Submit all the dataset's CSV resources to DataPusher+ using HDX's
2475
+ wrapper action. Requires sysadmin.
2476
+
2477
+ Returns:
2478
+ True if submitted, False if not (eg. no CSV resources)
2479
+ """
2480
+ result = self._write_to_hdx(
2481
+ "push_to_datastore", {"dataset_id": self.data["id"]}, "dataset_id"
2482
+ )
2483
+ if not result["success"]:
2484
+ logger.debug(result["message"])
2485
+ return result["success"]
2486
+
2336
2487
  def generate_resource(
2337
2488
  self,
2338
2489
  folder: Path | str,
hdx/data/hdxobject.py CHANGED
@@ -9,7 +9,7 @@ from collections import UserDict
9
9
  from collections.abc import Sequence
10
10
  from os.path import isfile
11
11
  from pathlib import Path
12
- from typing import Any, Optional, Union
12
+ from typing import TYPE_CHECKING, Any, Optional, Union
13
13
 
14
14
  from ckanapi.errors import NotFound
15
15
  from hdx.utilities.dictandlist import merge_two_dictionaries
@@ -20,6 +20,9 @@ from hdx.utilities.loader import (
20
20
 
21
21
  from hdx.api.configuration import Configuration
22
22
 
23
+ if TYPE_CHECKING:
24
+ from hdx.data.activity import Activity
25
+
23
26
  logger = logging.getLogger(__name__)
24
27
 
25
28
 
@@ -376,6 +379,30 @@ class HDXObject(UserDict, ABC):
376
379
  **kwargs,
377
380
  )
378
381
 
382
+ def _get_activities(
383
+ self, object_type: str, all_pages: bool = False, **kwargs: Any
384
+ ) -> list["Activity"]:
385
+ """Helper method to get the activity stream of this object
386
+
387
+ Args:
388
+ object_type: One of package, organization, group or user
389
+ all_pages: Whether to get all pages of results. Defaults to False.
390
+ **kwargs: Parameters to pass to Activity.get_activities
391
+
392
+ Returns:
393
+ List of Activity objects
394
+ """
395
+ # Imported here as hdx.data.activity imports this module
396
+ from hdx.data.activity import Activity
397
+
398
+ return Activity.get_activities(
399
+ object_type,
400
+ self.data.get("id") or self.data.get("name"),
401
+ self.configuration,
402
+ all_pages,
403
+ **kwargs,
404
+ )
405
+
379
406
  def _write_to_hdx(
380
407
  self,
381
408
  action: str,
hdx/data/location.py CHANGED
@@ -43,6 +43,8 @@ class Location(HDXObject):
43
43
  "delete": "group_delete",
44
44
  "list": "group_list",
45
45
  "autocomplete": "group_autocomplete",
46
+ "data_grid": "hdx_datagrid_show",
47
+ "invalidate_data_grid": "invalidate_data_completeness_for_location",
46
48
  }
47
49
 
48
50
  @classmethod
@@ -218,6 +220,47 @@ class Location(HDXObject):
218
220
  if len(group["name"]) == 3 and group.get("data_completeness") == "active"
219
221
  )
220
222
 
223
+ @staticmethod
224
+ def get_data_grid(
225
+ iso3: str, configuration: Configuration | None = None
226
+ ) -> dict | None:
227
+ """Get the Data Grid of a country. The dictionary has keys including
228
+ title, date, stats and categories. Each category has subcategories,
229
+ each with a list of datasets. Complementary datasets have
230
+ is_complementary set to True. Requires sysadmin.
231
+
232
+ Args:
233
+ iso3: Country ISO 3-letter code
234
+ configuration: HDX configuration. Defaults to global configuration.
235
+
236
+ Returns:
237
+ Data Grid dictionary or None if not found
238
+ """
239
+ location = Location(configuration=configuration)
240
+ success, result = location._read_from_hdx(
241
+ "location", iso3.lower(), "id", location.actions()["data_grid"]
242
+ )
243
+ if success:
244
+ return result
245
+ logger.debug(result)
246
+ return None
247
+
248
+ @staticmethod
249
+ def invalidate_data_grid(
250
+ iso3: str, configuration: Configuration | None = None
251
+ ) -> None:
252
+ """Clear HDX's cached Data Grid of a country. Requires sysadmin.
253
+
254
+ Args:
255
+ iso3: Country ISO 3-letter code
256
+ configuration: HDX configuration. Defaults to global configuration.
257
+
258
+ Returns:
259
+ None
260
+ """
261
+ location = Location(configuration=configuration)
262
+ location._write_to_hdx("invalidate_data_grid", {"name": iso3.lower()}, "name")
263
+
221
264
  def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
222
265
  """Get the activity stream of this location. One page of results (whose size
223
266
  is given by limit) is returned unless all_pages is True.
@@ -232,13 +275,7 @@ class Location(HDXObject):
232
275
  Returns:
233
276
  List of Activity objects
234
277
  """
235
- return Activity.get_activities(
236
- "group",
237
- self.data.get("id") or self.data.get("name"),
238
- self.configuration,
239
- all_pages,
240
- **kwargs,
241
- )
278
+ return self._get_activities("group", all_pages, **kwargs)
242
279
 
243
280
  @classmethod
244
281
  def autocomplete(
hdx/data/organization.py CHANGED
@@ -326,13 +326,7 @@ class Organization(HDXObject):
326
326
  Returns:
327
327
  List of Activity objects
328
328
  """
329
- return Activity.get_activities(
330
- "organization",
331
- self.data.get("id") or self.data.get("name"),
332
- self.configuration,
333
- all_pages,
334
- **kwargs,
335
- )
329
+ return self._get_activities("organization", all_pages, **kwargs)
336
330
 
337
331
  @classmethod
338
332
  def autocomplete(
hdx/data/resource.py CHANGED
@@ -72,6 +72,15 @@ class Resource(HDXObject):
72
72
  "datastore_upsert": "datastore_upsert",
73
73
  "datastore_search": "datastore_search",
74
74
  "datastore_search_sql": "datastore_search_sql",
75
+ "datapusher_submit": "datapusher_submit",
76
+ "datapusher_status": "datapusher_status",
77
+ "push_to_datastore": "hdx_push_resource_to_datastore",
78
+ "p_coded": "hdx_p_coded_resource_update",
79
+ "in_hapi": "hdx_mark_resource_in_hapi",
80
+ "in_quarantine": "hdx_mark_resource_in_quarantine",
81
+ "qa_patch": "hdx_qa_resource_patch",
82
+ "fs_check_revise": "hdx_fs_check_resource_revise",
83
+ "fs_check_reset": "hdx_fs_check_resource_reset",
75
84
  }
76
85
 
77
86
  def update_from_yaml(
@@ -319,6 +328,69 @@ class Resource(HDXObject):
319
328
  data_dictionary, separators=(",", ":")
320
329
  )
321
330
 
331
+ def update_hdx_data_dictionary(
332
+ self,
333
+ data_dictionary: Sequence[dict],
334
+ replace: bool = False,
335
+ update_datastore: bool = True,
336
+ ) -> None:
337
+ """Update the resource's data dictionary (column definitions) in HDX
338
+ without changing last_modified, so no dataset update notifications are
339
+ sent. Column definitions are as for set_hdx_data_dictionary. Unless
340
+ replace is True, they are merged by field into the existing data
341
+ dictionary in the resource metadata (so call read_from_hdx first).
342
+ The resource's hdx_data_dictionary field is patched, then if
343
+ update_datastore is True and the resource has a datastore, the
344
+ datastore field labels and descriptions are updated using
345
+ datastore_create without reloading the data.
346
+
347
+ Args:
348
+ data_dictionary: Sequence of column definition dictionaries
349
+ replace: Replace rather than merge into existing data dictionary. Defaults to False.
350
+ update_datastore: Update datastore field labels and descriptions. Defaults to True.
351
+
352
+ Returns:
353
+ None
354
+ """
355
+ if replace:
356
+ merged = [dict(column) for column in data_dictionary]
357
+ else:
358
+ new_columns = {column["field"]: column for column in data_dictionary}
359
+ merged = []
360
+ for column in self.get_hdx_data_dictionary() or []:
361
+ new_column = new_columns.pop(column.get("field"), None)
362
+ merged.append({**column, **new_column} if new_column else column)
363
+ merged.extend(new_columns.values())
364
+ self.set_hdx_data_dictionary(merged)
365
+ self._write_to_hdx(
366
+ "patch",
367
+ {
368
+ "id": self.data["id"],
369
+ "hdx_data_dictionary": self.data["hdx_data_dictionary"],
370
+ },
371
+ "id",
372
+ )
373
+ if not update_datastore or not self.has_datastore():
374
+ return
375
+ columns = {column["field"]: column for column in merged}
376
+ fields = []
377
+ for field in self.get_datastore_fields():
378
+ info = dict(field.get("info") or {})
379
+ column = columns.pop(field["id"], None)
380
+ if column:
381
+ info["label"] = column["label"]
382
+ info["notes"] = column["description"]
383
+ fields.append({"id": field["id"], "type": field["type"], "info": info})
384
+ if columns:
385
+ logger.warning(
386
+ f"Data dictionary fields {', '.join(columns)} not in datastore for resource {self.data['id']}!"
387
+ )
388
+ self._write_to_hdx(
389
+ "datastore_create",
390
+ {"resource_id": self.data["id"], "force": True, "fields": fields},
391
+ "resource_id",
392
+ )
393
+
322
394
  def clean_format(self) -> str:
323
395
  """Clean the resource's format, setting it to None if it is invalid and
324
396
  cannot be mapped
@@ -683,6 +755,30 @@ class Resource(HDXObject):
683
755
  logger.debug(result)
684
756
  return resources
685
757
 
758
+ def _download_file(
759
+ self,
760
+ url: str,
761
+ folder: Path | str | None,
762
+ filename: str,
763
+ retriever: Retrieve | None,
764
+ ) -> Path:
765
+ """Helper method to download a file from HDX
766
+
767
+ Args:
768
+ url: URL to download
769
+ folder: Folder to download file to. None means temporary folder.
770
+ filename: Filename to use
771
+ retriever: Retrieve object to use. None means do not use one.
772
+
773
+ Returns:
774
+ Path to downloaded file
775
+ """
776
+ session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
777
+ with Download(session=session) as downloader:
778
+ if retriever:
779
+ downloader = retriever.clone(downloader)
780
+ return downloader.download_file(url, folder=folder, filename=filename)
781
+
686
782
  def download(
687
783
  self, folder: Path | str | None = None, retriever: Retrieve | None = None
688
784
  ) -> tuple[str, Path]:
@@ -706,12 +802,7 @@ class Resource(HDXObject):
706
802
  file_format = f".{self.get_format()}"
707
803
  if not filename.endswith(file_format):
708
804
  filename = f"{filename}{file_format}"
709
- session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
710
- with Download(session=session) as downloader:
711
- if retriever:
712
- downloader = retriever.clone(downloader)
713
- path = downloader.download_file(url, folder=folder, filename=filename)
714
- return url, path
805
+ return url, self._download_file(url, folder, filename, retriever)
715
806
 
716
807
  def get_datastore_dump_url(self, file_format: str = "csv") -> str:
717
808
  """Get the URL from which the resource datastore can be downloaded in
@@ -755,19 +846,15 @@ class Resource(HDXObject):
755
846
  Returns:
756
847
  (URL downloaded, Path to downloaded file)
757
848
  """
849
+ file_format = file_format.lower()
758
850
  url = self.get_datastore_dump_url(file_format)
759
851
  logger.debug(f"Downloading {url}")
760
852
  if filename is None:
761
853
  filename = self.data["name"]
762
- extension = f".{file_format.lower()}"
854
+ extension = f".{file_format}"
763
855
  if not filename.endswith(extension):
764
856
  filename = f"{filename}{extension}"
765
- session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
766
- with Download(session=session) as downloader:
767
- if retriever:
768
- downloader = retriever.clone(downloader)
769
- path = downloader.download_file(url, folder=folder, filename=filename)
770
- return url, path
857
+ return url, self._download_file(url, folder, filename, retriever)
771
858
 
772
859
  @staticmethod
773
860
  def get_all_resource_ids_in_datastore(
@@ -930,11 +1017,18 @@ class Resource(HDXObject):
930
1017
  sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None.
931
1018
  distinct: Whether to return only distinct rows. Defaults to False.
932
1019
  page_size: Number of rows to request per call. Defaults to 10000.
933
- **kwargs: Other parameters to pass to datastore_search eg. plain, language.
1020
+ **kwargs: Other parameters to pass to datastore_search eg. plain, language. records_format csv and tsv are not supported.
934
1021
 
935
1022
  Returns:
936
1023
  Iterator of record dictionaries
937
1024
  """
1025
+ if page_size < 1:
1026
+ raise HDXError("page_size must be at least 1!")
1027
+ records_format = kwargs.get("records_format")
1028
+ if records_format in ("csv", "tsv"):
1029
+ raise HDXError(
1030
+ f"records_format {records_format} is not supported! Use search_datastore instead."
1031
+ )
938
1032
  offset = 0
939
1033
  while True:
940
1034
  result = self.search_datastore(
@@ -950,9 +1044,11 @@ class Resource(HDXObject):
950
1044
  )
951
1045
  records = result["records"]
952
1046
  yield from records
953
- if len(records) < page_size:
1047
+ # HDX caps limit at its maximum rows per call and returns the limit applied
1048
+ limit = result["limit"]
1049
+ if len(records) < limit:
954
1050
  return
955
- offset += page_size
1051
+ offset += limit
956
1052
 
957
1053
  def get_datastore_fields(self) -> list[dict]:
958
1054
  """Get the field definitions of the resource datastore excluding the
@@ -971,7 +1067,8 @@ class Resource(HDXObject):
971
1067
  ) -> list[dict]:
972
1068
  """Run a read-only SQL query against the HDX datastore using CKAN
973
1069
  datastore_search_sql. Tables are named by resource id, which must be
974
- double quoted eg. SELECT * FROM "RESOURCE_ID".
1070
+ double quoted eg. SELECT * FROM "RESOURCE_ID". Logs a warning if HDX
1071
+ truncates the results.
975
1072
 
976
1073
  Args:
977
1074
  sql: SQL SELECT statement
@@ -982,7 +1079,12 @@ class Resource(HDXObject):
982
1079
  """
983
1080
  resource = Resource(configuration=configuration)
984
1081
  result = resource._write_to_hdx("datastore_search_sql", {"sql": sql})
985
- return result["records"]
1082
+ records = result["records"]
1083
+ if result.get("records_truncated"):
1084
+ logger.warning(
1085
+ f"SQL query results truncated by HDX to {len(records)} rows!"
1086
+ )
1087
+ return records
986
1088
 
987
1089
  def delete_datastore(self) -> None:
988
1090
  """Delete a resource from the HDX datastore
@@ -1000,6 +1102,55 @@ class Resource(HDXObject):
1000
1102
  if not success:
1001
1103
  logger.debug(result)
1002
1104
 
1105
+ def submit_to_datapusher(self, ignore_hash: bool = False) -> bool:
1106
+ """Submit the resource to DataPusher+ to load its file into the
1107
+ datastore. This does not change last_modified. Unlike the automatic
1108
+ submission on file upload, HDX's datastore allowlist and format checks
1109
+ are not applied.
1110
+
1111
+ Args:
1112
+ ignore_hash: Reload even if the file hash is unchanged. Defaults to False.
1113
+
1114
+ Returns:
1115
+ True if submitted, False if not (eg. a job is already pending)
1116
+ """
1117
+ return self._write_to_hdx(
1118
+ "datapusher_submit",
1119
+ {"resource_id": self.data["id"], "ignore_hash": ignore_hash},
1120
+ "resource_id",
1121
+ )
1122
+
1123
+ def push_to_datastore(self) -> bool:
1124
+ """Submit the resource to DataPusher+ using HDX's wrapper action.
1125
+ Unlike submit_to_datapusher, this requires sysadmin.
1126
+
1127
+ Returns:
1128
+ True if submitted, False if not
1129
+ """
1130
+ result = self._write_to_hdx(
1131
+ "push_to_datastore", {"resource_id": self.data["id"]}, "resource_id"
1132
+ )
1133
+ if not result["success"]:
1134
+ logger.debug(result["message"])
1135
+ return result["success"]
1136
+
1137
+ def get_datapusher_status(self) -> dict | None:
1138
+ """Get the status of the resource's latest DataPusher+ job
1139
+
1140
+ Returns:
1141
+ Status dictionary with keys including status and error or None if no job
1142
+ """
1143
+ success, result = self._read_from_hdx(
1144
+ "datapusher",
1145
+ self.data["id"],
1146
+ "resource_id",
1147
+ self.actions()["datapusher_status"],
1148
+ )
1149
+ if success:
1150
+ return result
1151
+ logger.debug(result)
1152
+ return None
1153
+
1003
1154
  def get_resource_views(self) -> list[ResourceView]:
1004
1155
  """Get any resource views in the resource
1005
1156
 
@@ -1159,6 +1310,174 @@ class Resource(HDXObject):
1159
1310
  else:
1160
1311
  logger.debug(result)
1161
1312
 
1313
+ def get_fs_check_info(self) -> list[dict]:
1314
+ """Get file structure check information for the resource from its
1315
+ fs_check_info field. Each check event can have keys such as state,
1316
+ timestamp, sheet_changes and hxl_proxy_response.
1317
+
1318
+ Returns:
1319
+ List of file structure check events
1320
+ """
1321
+ fs_check_info = self.data.get("fs_check_info")
1322
+ if not fs_check_info:
1323
+ return []
1324
+ if isinstance(fs_check_info, str):
1325
+ return json.loads(fs_check_info)
1326
+ return fs_check_info
1327
+
1328
+ def _set_from_package(self, package: dict) -> None:
1329
+ """Set the resource's metadata from the matching resource in a package
1330
+ metadata dictionary returned by HDX
1331
+
1332
+ Args:
1333
+ package: Package metadata dictionary
1334
+
1335
+ Returns:
1336
+ None
1337
+ """
1338
+ for resource in package.get("resources", []):
1339
+ if resource["id"] == self.data["id"]:
1340
+ self.data = resource
1341
+ return
1342
+
1343
+ def revise_fs_check(self, key: str, value: Any) -> None:
1344
+ """Set a file structure check field of the resource in HDX. If key is
1345
+ fs_check_info and value is a dictionary, it is appended to the
1346
+ existing check events (the last 20 are kept) rather than replacing
1347
+ them. Requires sysadmin or permission to manage QA.
1348
+
1349
+ Args:
1350
+ key: Resource field eg. fs_check_info
1351
+ value: Value to set
1352
+
1353
+ Returns:
1354
+ None
1355
+ """
1356
+ result = self._write_to_hdx(
1357
+ "fs_check_revise",
1358
+ {
1359
+ "package_id": self.data["package_id"],
1360
+ "id": self.data["id"],
1361
+ "key": key,
1362
+ "value": value,
1363
+ },
1364
+ "id",
1365
+ )
1366
+ self._set_from_package(result["package"])
1367
+
1368
+ def reset_fs_check(self, key: str = "fs_check_info") -> None:
1369
+ """Clear a file structure check field of the resource in HDX. Requires
1370
+ sysadmin or permission to manage QA.
1371
+
1372
+ Args:
1373
+ key: Resource field to clear. Defaults to fs_check_info.
1374
+
1375
+ Returns:
1376
+ None
1377
+ """
1378
+ result = self._write_to_hdx(
1379
+ "fs_check_reset",
1380
+ {"package_id": self.data["package_id"], "id": self.data["id"], "key": key},
1381
+ "id",
1382
+ )
1383
+ self._set_from_package(result["package"])
1384
+
1385
+ def is_p_coded(self) -> bool | None:
1386
+ """Return if the resource is marked as containing p-codes
1387
+
1388
+ Returns:
1389
+ Whether resource is p-coded or None if not set
1390
+ """
1391
+ p_coded = self.data.get("p_coded")
1392
+ if p_coded is None:
1393
+ return None
1394
+ if isinstance(p_coded, str):
1395
+ return p_coded.lower() == "true"
1396
+ return p_coded
1397
+
1398
+ def set_p_coded(self, p_coded: bool) -> None:
1399
+ """Mark the resource in HDX as containing p-codes or not. Requires
1400
+ sysadmin or permission to manage p-codes.
1401
+
1402
+ Args:
1403
+ p_coded: Whether resource is p-coded
1404
+
1405
+ Returns:
1406
+ None
1407
+ """
1408
+ self.data = self._write_to_hdx(
1409
+ "p_coded", {"id": self.data["id"], "p_coded": p_coded}, "id"
1410
+ )
1411
+
1412
+ def is_in_hapi(self) -> bool:
1413
+ """Return if the resource is marked as being in HAPI
1414
+
1415
+ Returns:
1416
+ Whether resource is in HAPI
1417
+ """
1418
+ return self.data.get("in_hapi") == "yes"
1419
+
1420
+ def set_in_hapi(self, in_hapi: bool) -> None:
1421
+ """Mark the resource in HDX as being in HAPI or not. Requires sysadmin
1422
+ or permission to manage the in HAPI flag.
1423
+
1424
+ Args:
1425
+ in_hapi: Whether resource is in HAPI
1426
+
1427
+ Returns:
1428
+ None
1429
+ """
1430
+ self.data = self._write_to_hdx(
1431
+ "in_hapi",
1432
+ {"id": self.data["id"], "in_hapi": "yes" if in_hapi else "no-data"},
1433
+ "id",
1434
+ )
1435
+
1436
+ def is_in_quarantine(self) -> bool:
1437
+ """Return if the resource is in quarantine
1438
+
1439
+ Returns:
1440
+ Whether resource is in quarantine
1441
+ """
1442
+ in_quarantine = self.data.get("in_quarantine", False)
1443
+ if isinstance(in_quarantine, str):
1444
+ return in_quarantine.lower() == "true"
1445
+ return in_quarantine
1446
+
1447
+ def set_in_quarantine(self, in_quarantine: bool) -> None:
1448
+ """Put the resource in HDX in quarantine or take it out. Quarantining
1449
+ also removes the resource's shape_info and geopreview. Requires
1450
+ sysadmin or permission to manage QA.
1451
+
1452
+ Args:
1453
+ in_quarantine: Whether resource is in quarantine
1454
+
1455
+ Returns:
1456
+ None
1457
+ """
1458
+ self.data = self._write_to_hdx(
1459
+ "in_quarantine",
1460
+ {
1461
+ "id": self.data["id"],
1462
+ "in_quarantine": "true" if in_quarantine else "false",
1463
+ },
1464
+ "id",
1465
+ )
1466
+
1467
+ def qa_patch(self, **kwargs: Any) -> None:
1468
+ """Patch QA fields of the resource in HDX eg. pii, sdc_report_flag,
1469
+ broken_link. Requires sysadmin or permission to manage QA.
1470
+
1471
+ Args:
1472
+ **kwargs: Resource fields to patch
1473
+
1474
+ Returns:
1475
+ None
1476
+ """
1477
+ self.data = self._write_to_hdx(
1478
+ "qa_patch", {"id": self.data["id"], **kwargs}, "id"
1479
+ )
1480
+
1162
1481
  def is_marked_data_updated(self) -> bool:
1163
1482
  """Return if the resource's data is marked to be updated
1164
1483
 
hdx/data/user.py CHANGED
@@ -45,6 +45,7 @@ class User(HDXObject):
45
45
  "list": "user_list",
46
46
  "listorgs": "organization_list_for_user",
47
47
  "token_list": "api_token_list",
48
+ "token_info": "hdx_token_info",
48
49
  "autocomplete": "user_autocomplete",
49
50
  }
50
51
 
@@ -334,7 +335,7 @@ class User(HDXObject):
334
335
  Returns:
335
336
  List of organization dicts in HDX that logged in user is a member of
336
337
  """
337
- user = User(configuration=configuration)
338
+ user = cls(configuration=configuration)
338
339
  try:
339
340
  return user.configuration.call_remoteckan(
340
341
  cls.actions()["listorgs"], {"permission": permission}
@@ -470,13 +471,25 @@ class User(HDXObject):
470
471
  Returns:
471
472
  List of Activity objects
472
473
  """
473
- return Activity.get_activities(
474
- "user",
475
- self.data.get("id") or self.data.get("name"),
476
- self.configuration,
477
- all_pages,
478
- **kwargs,
479
- )
474
+ return self._get_activities("user", all_pages, **kwargs)
475
+
476
+ @classmethod
477
+ def get_current_token_info(cls, configuration: Configuration | None = None) -> dict:
478
+ """Get information about the API token in the configuration: the md5
479
+ hash of the token owner's email (email_hash) and the token name
480
+ (token_name).
481
+
482
+ Args:
483
+ configuration: HDX configuration. Defaults to global configuration.
484
+
485
+ Returns:
486
+ Dictionary with keys email_hash and token_name
487
+ """
488
+ user = cls(configuration=configuration)
489
+ try:
490
+ return user.configuration.call_remoteckan(cls.actions()["token_info"], {})
491
+ except Exception as e:
492
+ raise HDXError("Failed when trying to get API token info!") from e
480
493
 
481
494
  @classmethod
482
495
  def autocomplete(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: hdx-python-api
3
- Version: 6.7.2
3
+ Version: 6.7.4
4
4
  Summary: HDX Python API for interacting with the Humanitarian Data Exchange
5
5
  Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
6
6
  Author: Michael Rans
@@ -1,5 +1,5 @@
1
1
  hdx/api/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
2
- hdx/api/_version.py,sha256=CgL0y60DiHh1gIc1CUZuiYkj-TMX2gdAy0attQGD48w,520
2
+ hdx/api/_version.py,sha256=Dbk4PNxS0tMC6IYkWs454Z4ZwWohuENXxYAfScTC2M8,520
3
3
  hdx/api/configuration.py,sha256=9WvuVI-3th2Dv70_t_4VoyrlJwB_9qUSN5_zUKbp4sQ,23805
4
4
  hdx/api/hdx_base_configuration.yaml,sha256=jj2t5b7NnI499rHVwOjj3Q5BAjE8i7NDGVKnWZ5P5-4,2222
5
5
  hdx/api/locations.py,sha256=bFl_cKz_AEQzkE_xWOkrRlUQ0GT2LMPSk-G_LsbV-Ns,4099
@@ -9,24 +9,25 @@ hdx/api/utilities/date_helper.py,sha256=sXxjRtXz-2rpBVZBbJAk80gfjpYKl_AtHplWr-rn
9
9
  hdx/api/utilities/filestore_helper.py,sha256=0FcUpGHpYlAzlNDpDxdXbSqV6VHFfGCLakCmK37hieU,6917
10
10
  hdx/api/utilities/hdx_error_handler.py,sha256=x1rb2qBO0C6CJjqToSFj_XB1F3pf3nHpA8otxXfTFzA,8711
11
11
  hdx/api/utilities/hdx_state.py,sha256=KzrK0A-H5R6cRPqwF4d23_S_zfIa1ow4K5u46FesGmY,2442
12
+ hdx/api/utilities/hdx_stats.py,sha256=JlEcF9LN-OjcVeh6C0te-nAlcO7W-JIpD5JJWTiWxs0,2847
12
13
  hdx/api/utilities/url_utils.py,sha256=Pb4r-BAqIHYmeoB_i7QjutDp6nOAUh_OHvCtiNLda3Y,4534
13
14
  hdx/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
- hdx/data/activity.py,sha256=vBnkaL7QVaa05Y1dzZ484dzqgaijb8dTCe60cERlAJQ,9101
15
- hdx/data/dataset.py,sha256=cPccjApN5td3DDW9r2PrmLM30ThzKNwf3hwlb60OxzU,100697
16
- hdx/data/hdxobject.py,sha256=_jlwYjb_CcjaXsTXObknMfq-OiNOcpJfLJ9b1vKV1Zo,27494
17
- hdx/data/location.py,sha256=HonWohDZOHSvlwoiv4vc5-9BzZ2iIG8FIos1C1Q814w,9738
18
- hdx/data/organization.py,sha256=JVxyrbobBDcAREKJzSq5Rc7uUfkH-tzLj1jwQDyMnBs,13280
19
- hdx/data/resource.py,sha256=iVARfA3_8gGsGjXgNxomm7umAadvxzFsGGqxTpBB3WQ,45196
15
+ hdx/data/activity.py,sha256=YNa7hAgsMP96EIUX7UMInxAXnJ8TynE1KpecRrv8Vso,9200
16
+ hdx/data/dataset.py,sha256=-GQO08PK8u19O0qRWttg3AuPskPCwjffcE2mPq-7dQE,105903
17
+ hdx/data/hdxobject.py,sha256=EG5KkVpi-yF6m6HVF79EqDneq1W7mo5M-bPJHG79T0I,28383
18
+ hdx/data/location.py,sha256=UJPGKc8euAYFJsUFkdQtViLUwqNWYOWwaLELApAMHk4,11168
19
+ hdx/data/organization.py,sha256=IRen2F2NsXLC8r-PVTS9eQp7LTn_44OcDUa1S7SCKiQ,13140
20
+ hdx/data/resource.py,sha256=QDQJp7XoCDn9DmyWPgA2YCn8Z4HGwezEkFyae_YlFKI,56507
20
21
  hdx/data/resource_matcher.py,sha256=Yzk9pWa4zk1p09m6zcZMj0XZf5Y16Pa3KToOdUt7rP8,4247
21
22
  hdx/data/resource_view.py,sha256=GcYkh0iDAe6_m2ErY_A6i3Z7ZFX949eGt1TLnLZv_8s,7346
22
23
  hdx/data/showcase.py,sha256=vnRZ45xd9Ct65eFQkWZtMIJPFifZfKOrH4obO5D5Pvc,14062
23
- hdx/data/user.py,sha256=rHyFPzGR_u_LmXvgMQ6uqKkYFIW3uG89-M6nnG3pkkU,16697
24
+ hdx/data/user.py,sha256=NXBuvUu-pNVSEelL59wdNQgeWmDgeWke_6JxViww3Cc,17320
24
25
  hdx/data/vocabulary.py,sha256=sDDVh9ivCbZlPKXIdOoj0ltTdRufL0ypfGZUirIb2IA,20776
25
26
  hdx/facades/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
27
  hdx/facades/infer_arguments.py,sha256=DvyDuozfOPmoaV7HjhQySojGIrMWW9nos31auQBVVl8,4032
27
28
  hdx/facades/keyword_arguments.py,sha256=aVapHCfqBO6H7xxBgTdK9ISdgrBFsNypPDWihfQ3edU,1125
28
29
  hdx/facades/simple.py,sha256=QWsBkaHG_UM6X3Zu2u3LWVOQzeR24q-EUClrfsL_mkM,1064
29
- hdx_python_api-6.7.2.dist-info/METADATA,sha256=lv8ylKpAePSbZ6KMPwYlnHeNlOnfI4FrRA_30YKUilM,4946
30
- hdx_python_api-6.7.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
31
- hdx_python_api-6.7.2.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
32
- hdx_python_api-6.7.2.dist-info/RECORD,,
30
+ hdx_python_api-6.7.4.dist-info/METADATA,sha256=k7b1Nasb2yiwkyq-LG1-aBgVTy-D6aaAwjOaooTOS2Q,4946
31
+ hdx_python_api-6.7.4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
32
+ hdx_python_api-6.7.4.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
33
+ hdx_python_api-6.7.4.dist-info/RECORD,,