hdx-python-api 6.7.0__py3-none-any.whl → 6.7.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hdx/api/_version.py +2 -2
- hdx/data/activity.py +245 -0
- hdx/data/dataset.py +27 -0
- hdx/data/location.py +23 -0
- hdx/data/organization.py +23 -0
- hdx/data/resource.py +180 -1
- hdx/data/user.py +25 -0
- {hdx_python_api-6.7.0.dist-info → hdx_python_api-6.7.2.dist-info}/METADATA +3 -3
- {hdx_python_api-6.7.0.dist-info → hdx_python_api-6.7.2.dist-info}/RECORD +11 -10
- {hdx_python_api-6.7.0.dist-info → hdx_python_api-6.7.2.dist-info}/WHEEL +1 -1
- {hdx_python_api-6.7.0.dist-info → hdx_python_api-6.7.2.dist-info}/licenses/LICENSE +0 -0
hdx/api/_version.py
CHANGED
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '6.7.
|
|
22
|
-
__version_tuple__ = version_tuple = (6, 7,
|
|
21
|
+
__version__ = version = '6.7.2'
|
|
22
|
+
__version_tuple__ = version_tuple = (6, 7, 2)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
hdx/data/activity.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Activity class containing all logic for reading activity streams."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
from hdx.api.configuration import Configuration
|
|
8
|
+
from hdx.data.hdxobject import HDXError, HDXObject
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Activity(HDXObject):
|
|
14
|
+
"""Activity class containing all logic for reading activity streams. Activities
|
|
15
|
+
are read only.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
initial_data: Initial activity metadata dictionary. Defaults to None.
|
|
19
|
+
configuration: HDX configuration. Defaults to global configuration.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
initial_data: dict | None = None,
|
|
25
|
+
configuration: Configuration | None = None,
|
|
26
|
+
) -> None:
|
|
27
|
+
if not initial_data:
|
|
28
|
+
initial_data = {}
|
|
29
|
+
super().__init__(initial_data, configuration=configuration)
|
|
30
|
+
|
|
31
|
+
@staticmethod
|
|
32
|
+
def actions() -> dict[str, str]:
|
|
33
|
+
"""Dictionary of actions that can be performed on object
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
Dictionary of actions that can be performed on object
|
|
37
|
+
"""
|
|
38
|
+
return {
|
|
39
|
+
"show": "activity_show",
|
|
40
|
+
"data_show": "activity_data_show",
|
|
41
|
+
"diff": "activity_diff",
|
|
42
|
+
"package_list": "package_activity_list",
|
|
43
|
+
"organization_list": "organization_activity_list",
|
|
44
|
+
"group_list": "group_activity_list",
|
|
45
|
+
"user_list": "user_activity_list",
|
|
46
|
+
"recently_changed_packages": "recently_changed_packages_activity_list",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def read_from_hdx(
|
|
51
|
+
cls, identifier: str, configuration: Configuration | None = None
|
|
52
|
+
) -> Optional["Activity"]:
|
|
53
|
+
"""Reads the activity given by identifier from HDX and returns Activity object.
|
|
54
|
+
HDX requires permission to edit the object the activity relates to.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
identifier: Identifier of activity
|
|
58
|
+
configuration: HDX configuration. Defaults to global configuration.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
Activity object if successful read, None if not
|
|
62
|
+
"""
|
|
63
|
+
return cls._read_from_hdx_class("activity", identifier, configuration)
|
|
64
|
+
|
|
65
|
+
def get_object_type(self) -> str:
|
|
66
|
+
"""Get the CKAN object type the activity relates to eg. package,
|
|
67
|
+
organization, group or user. This is the last word of the activity type
|
|
68
|
+
(eg. "changed package") with "dataset" (from "follow dataset") mapped to
|
|
69
|
+
"package".
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
Object type
|
|
73
|
+
"""
|
|
74
|
+
object_type = self.data["activity_type"].split(" ")[-1]
|
|
75
|
+
if object_type == "dataset":
|
|
76
|
+
return "package"
|
|
77
|
+
return object_type
|
|
78
|
+
|
|
79
|
+
def get_data(self) -> dict:
|
|
80
|
+
"""Get the data of the activity which includes a snapshot of the object
|
|
81
|
+
at the time of the activity. HDX requires permission to edit the object
|
|
82
|
+
the activity relates to.
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
Activity data
|
|
86
|
+
"""
|
|
87
|
+
return self._write_to_hdx("data_show", {"id": self.data["id"]}, "id")
|
|
88
|
+
|
|
89
|
+
def get_diff(self, diff_type: str = "unified") -> dict:
|
|
90
|
+
"""Get a diff of the object compared to its previous version. HDX requires
|
|
91
|
+
permission to edit the object the activity relates to.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
diff_type: One of unified, context or html. Defaults to unified.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
Activity diff
|
|
98
|
+
"""
|
|
99
|
+
data = {
|
|
100
|
+
"id": self.data["id"],
|
|
101
|
+
"object_type": self.get_object_type(),
|
|
102
|
+
"diff_type": diff_type,
|
|
103
|
+
}
|
|
104
|
+
return self._write_to_hdx("diff", data, "id")
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
def _get_list(
|
|
108
|
+
cls,
|
|
109
|
+
action: str,
|
|
110
|
+
data: dict,
|
|
111
|
+
configuration: Configuration | None = None,
|
|
112
|
+
all_pages: bool = False,
|
|
113
|
+
max_activities: int | None = None,
|
|
114
|
+
) -> list["Activity"]:
|
|
115
|
+
"""Helper method to get a list of activities from HDX
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
action: Action to perform
|
|
119
|
+
data: Data to pass to CKAN including any offset and limit
|
|
120
|
+
configuration: HDX configuration. Defaults to global configuration.
|
|
121
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
122
|
+
max_activities: Maximum number of activities if all_pages is True. Defaults to None (no maximum).
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
List of Activity objects
|
|
126
|
+
"""
|
|
127
|
+
activity = cls(configuration=configuration)
|
|
128
|
+
if not all_pages:
|
|
129
|
+
result = activity._write_to_hdx(action, data)
|
|
130
|
+
return [cls(x, configuration=configuration) for x in result]
|
|
131
|
+
data = dict(data)
|
|
132
|
+
limit = data.setdefault("limit", 100)
|
|
133
|
+
offset = data.get("offset", 0)
|
|
134
|
+
activities = []
|
|
135
|
+
while True:
|
|
136
|
+
data["offset"] = offset
|
|
137
|
+
result = activity._write_to_hdx(action, data)
|
|
138
|
+
activities.extend(cls(x, configuration=configuration) for x in result)
|
|
139
|
+
if max_activities is not None and len(activities) >= max_activities:
|
|
140
|
+
return activities[:max_activities]
|
|
141
|
+
if len(result) < limit:
|
|
142
|
+
return activities
|
|
143
|
+
offset += limit
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def get_activities(
|
|
147
|
+
cls,
|
|
148
|
+
object_type: str,
|
|
149
|
+
identifier: str,
|
|
150
|
+
configuration: Configuration | None = None,
|
|
151
|
+
all_pages: bool = False,
|
|
152
|
+
**kwargs: Any,
|
|
153
|
+
) -> list["Activity"]:
|
|
154
|
+
"""Get the activity stream of a CKAN object. One page of results (whose
|
|
155
|
+
size is given by limit) is returned unless all_pages is True.
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
object_type: One of package, organization, group or user
|
|
159
|
+
identifier: Id or name of object
|
|
160
|
+
configuration: HDX configuration. Defaults to global configuration.
|
|
161
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
162
|
+
**kwargs: See below
|
|
163
|
+
offset (int): Where to start getting activity items from. Defaults to 0.
|
|
164
|
+
limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
|
|
165
|
+
after (int | str): Only activities after this timestamp (package and user only)
|
|
166
|
+
before (int | str): Only activities before this timestamp (package and user only)
|
|
167
|
+
include_hidden_activity (bool): Include hidden activity, sysadmins only (package, organization and group only)
|
|
168
|
+
activity_types (list): Activity types to include (package only)
|
|
169
|
+
exclude_activity_types (list): Activity types to exclude (package only)
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
List of Activity objects
|
|
173
|
+
"""
|
|
174
|
+
if not identifier:
|
|
175
|
+
raise HDXError(f"No id or name for {object_type} to get activities!")
|
|
176
|
+
data = {"id": identifier}
|
|
177
|
+
data.update(kwargs)
|
|
178
|
+
return cls._get_list(f"{object_type}_list", data, configuration, all_pages)
|
|
179
|
+
|
|
180
|
+
@classmethod
|
|
181
|
+
def get_recently_changed_packages(
|
|
182
|
+
cls,
|
|
183
|
+
configuration: Configuration | None = None,
|
|
184
|
+
all_pages: bool = False,
|
|
185
|
+
max_activities: int | None = 1000,
|
|
186
|
+
**kwargs: Any,
|
|
187
|
+
) -> list["Activity"]:
|
|
188
|
+
"""Get the activity stream of all recently added or changed datasets.
|
|
189
|
+
One page of results (whose size is given by limit) is returned unless
|
|
190
|
+
all_pages is True. all_pages is capped by max_activities because the
|
|
191
|
+
stream covers the whole history of HDX.
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
configuration: HDX configuration. Defaults to global configuration.
|
|
195
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
196
|
+
max_activities: Maximum number of activities if all_pages is True. Defaults to 1000. None means no maximum (very slow).
|
|
197
|
+
**kwargs: See below
|
|
198
|
+
offset (int): Where to start getting activity items from. Defaults to 0.
|
|
199
|
+
limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
|
|
200
|
+
|
|
201
|
+
Returns:
|
|
202
|
+
List of Activity objects
|
|
203
|
+
"""
|
|
204
|
+
return cls._get_list(
|
|
205
|
+
"recently_changed_packages",
|
|
206
|
+
kwargs,
|
|
207
|
+
configuration,
|
|
208
|
+
all_pages,
|
|
209
|
+
max_activities,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
def check_required_fields(self, ignore_fields: Sequence[str] = ()) -> None:
|
|
213
|
+
"""Activities are read only so this raises an exception
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
ignore_fields: Fields to ignore. Default is ().
|
|
217
|
+
|
|
218
|
+
Returns:
|
|
219
|
+
None
|
|
220
|
+
"""
|
|
221
|
+
raise HDXError("Activities are read only!")
|
|
222
|
+
|
|
223
|
+
def update_in_hdx(self, **kwargs: Any) -> None:
|
|
224
|
+
"""Activities are read only so this raises an exception
|
|
225
|
+
|
|
226
|
+
Returns:
|
|
227
|
+
None
|
|
228
|
+
"""
|
|
229
|
+
raise HDXError("Activities are read only!")
|
|
230
|
+
|
|
231
|
+
def create_in_hdx(self, **kwargs: Any) -> None:
|
|
232
|
+
"""Activities are read only so this raises an exception
|
|
233
|
+
|
|
234
|
+
Returns:
|
|
235
|
+
None
|
|
236
|
+
"""
|
|
237
|
+
raise HDXError("Activities are read only!")
|
|
238
|
+
|
|
239
|
+
def delete_from_hdx(self) -> None:
|
|
240
|
+
"""Activities are read only so this raises an exception
|
|
241
|
+
|
|
242
|
+
Returns:
|
|
243
|
+
None
|
|
244
|
+
"""
|
|
245
|
+
raise HDXError("Activities are read only!")
|
hdx/data/dataset.py
CHANGED
|
@@ -40,6 +40,7 @@ from hdx.api.configuration import Configuration
|
|
|
40
40
|
from hdx.api.locations import Locations
|
|
41
41
|
from hdx.api.utilities.date_helper import DateHelper
|
|
42
42
|
from hdx.api.utilities.filestore_helper import FilestoreHelper
|
|
43
|
+
from hdx.data.activity import Activity
|
|
43
44
|
from hdx.data.hdxobject import HDXError, HDXObject
|
|
44
45
|
from hdx.data.resource_matcher import ResourceMatcher
|
|
45
46
|
|
|
@@ -1428,6 +1429,32 @@ class Dataset(HDXObject):
|
|
|
1428
1429
|
resources.append(resource)
|
|
1429
1430
|
return resources
|
|
1430
1431
|
|
|
1432
|
+
def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
|
|
1433
|
+
"""Get the activity stream of this dataset. One page of results (whose size
|
|
1434
|
+
is given by limit) is returned unless all_pages is True.
|
|
1435
|
+
|
|
1436
|
+
Args:
|
|
1437
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
1438
|
+
**kwargs: See below
|
|
1439
|
+
offset (int): Where to start getting activity items from. Defaults to 0.
|
|
1440
|
+
limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
|
|
1441
|
+
after (int | str): Only activities after this timestamp
|
|
1442
|
+
before (int | str): Only activities before this timestamp
|
|
1443
|
+
include_hidden_activity (bool): Include hidden activity (sysadmins only)
|
|
1444
|
+
activity_types (list): Activity types to include
|
|
1445
|
+
exclude_activity_types (list): Activity types to exclude
|
|
1446
|
+
|
|
1447
|
+
Returns:
|
|
1448
|
+
List of Activity objects
|
|
1449
|
+
"""
|
|
1450
|
+
return Activity.get_activities(
|
|
1451
|
+
"package",
|
|
1452
|
+
self.data.get("id") or self.data.get("name"),
|
|
1453
|
+
self.configuration,
|
|
1454
|
+
all_pages,
|
|
1455
|
+
**kwargs,
|
|
1456
|
+
)
|
|
1457
|
+
|
|
1431
1458
|
@classmethod
|
|
1432
1459
|
def autocomplete(
|
|
1433
1460
|
cls,
|
hdx/data/location.py
CHANGED
|
@@ -5,6 +5,7 @@ from collections.abc import Sequence
|
|
|
5
5
|
from typing import Any, Optional
|
|
6
6
|
|
|
7
7
|
from hdx.api.configuration import Configuration
|
|
8
|
+
from hdx.data.activity import Activity
|
|
8
9
|
from hdx.data.hdxobject import HDXObject
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger(__name__)
|
|
@@ -217,6 +218,28 @@ class Location(HDXObject):
|
|
|
217
218
|
if len(group["name"]) == 3 and group.get("data_completeness") == "active"
|
|
218
219
|
)
|
|
219
220
|
|
|
221
|
+
def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
|
|
222
|
+
"""Get the activity stream of this location. One page of results (whose size
|
|
223
|
+
is given by limit) is returned unless all_pages is True.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
227
|
+
**kwargs: See below
|
|
228
|
+
offset (int): Where to start getting activity items from. Defaults to 0.
|
|
229
|
+
limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
|
|
230
|
+
include_hidden_activity (bool): Include hidden activity (sysadmins only)
|
|
231
|
+
|
|
232
|
+
Returns:
|
|
233
|
+
List of Activity objects
|
|
234
|
+
"""
|
|
235
|
+
return Activity.get_activities(
|
|
236
|
+
"group",
|
|
237
|
+
self.data.get("id") or self.data.get("name"),
|
|
238
|
+
self.configuration,
|
|
239
|
+
all_pages,
|
|
240
|
+
**kwargs,
|
|
241
|
+
)
|
|
242
|
+
|
|
220
243
|
@classmethod
|
|
221
244
|
def autocomplete(
|
|
222
245
|
cls,
|
hdx/data/organization.py
CHANGED
|
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
|
|
|
8
8
|
import hdx.data.dataset
|
|
9
9
|
import hdx.data.user as user_module
|
|
10
10
|
from hdx.api.configuration import Configuration
|
|
11
|
+
from hdx.data.activity import Activity
|
|
11
12
|
from hdx.data.hdxobject import HDXError, HDXObject
|
|
12
13
|
|
|
13
14
|
if TYPE_CHECKING:
|
|
@@ -311,6 +312,28 @@ class Organization(HDXObject):
|
|
|
311
312
|
|
|
312
313
|
return final_orgs
|
|
313
314
|
|
|
315
|
+
def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
|
|
316
|
+
"""Get the activity stream of this organization. One page of results (whose size
|
|
317
|
+
is given by limit) is returned unless all_pages is True.
|
|
318
|
+
|
|
319
|
+
Args:
|
|
320
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
321
|
+
**kwargs: See below
|
|
322
|
+
offset (int): Where to start getting activity items from. Defaults to 0.
|
|
323
|
+
limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
|
|
324
|
+
include_hidden_activity (bool): Include hidden activity (sysadmins only)
|
|
325
|
+
|
|
326
|
+
Returns:
|
|
327
|
+
List of Activity objects
|
|
328
|
+
"""
|
|
329
|
+
return Activity.get_activities(
|
|
330
|
+
"organization",
|
|
331
|
+
self.data.get("id") or self.data.get("name"),
|
|
332
|
+
self.configuration,
|
|
333
|
+
all_pages,
|
|
334
|
+
**kwargs,
|
|
335
|
+
)
|
|
336
|
+
|
|
314
337
|
@classmethod
|
|
315
338
|
def autocomplete(
|
|
316
339
|
cls,
|
hdx/data/resource.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
-
from collections.abc import Sequence
|
|
5
|
+
from collections.abc import Iterator, Sequence
|
|
6
6
|
from datetime import datetime
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import Any, Optional
|
|
@@ -34,6 +34,7 @@ class Resource(HDXObject):
|
|
|
34
34
|
"""
|
|
35
35
|
|
|
36
36
|
_formats_dict = None
|
|
37
|
+
_valid_datastore_dump_formats = frozenset({"csv", "tsv", "json", "xml"})
|
|
37
38
|
_valid_data_dictionary_types = frozenset(
|
|
38
39
|
{"text", "numeric", "date", "timestamp without time zone"}
|
|
39
40
|
)
|
|
@@ -70,6 +71,7 @@ class Resource(HDXObject):
|
|
|
70
71
|
"datastore_insert": "datastore_insert",
|
|
71
72
|
"datastore_upsert": "datastore_upsert",
|
|
72
73
|
"datastore_search": "datastore_search",
|
|
74
|
+
"datastore_search_sql": "datastore_search_sql",
|
|
73
75
|
}
|
|
74
76
|
|
|
75
77
|
def update_from_yaml(
|
|
@@ -711,6 +713,62 @@ class Resource(HDXObject):
|
|
|
711
713
|
path = downloader.download_file(url, folder=folder, filename=filename)
|
|
712
714
|
return url, path
|
|
713
715
|
|
|
716
|
+
def get_datastore_dump_url(self, file_format: str = "csv") -> str:
|
|
717
|
+
"""Get the URL from which the resource datastore can be downloaded in
|
|
718
|
+
full.
|
|
719
|
+
|
|
720
|
+
Args:
|
|
721
|
+
file_format: Dump format (csv, tsv, json or xml). Defaults to csv.
|
|
722
|
+
|
|
723
|
+
Returns:
|
|
724
|
+
Datastore dump URL
|
|
725
|
+
"""
|
|
726
|
+
file_format = file_format.lower()
|
|
727
|
+
if file_format not in self._valid_datastore_dump_formats:
|
|
728
|
+
valid_formats = ", ".join(sorted(self._valid_datastore_dump_formats))
|
|
729
|
+
raise HDXError(
|
|
730
|
+
f"Invalid datastore dump format '{file_format}'! Must be one of: {valid_formats}"
|
|
731
|
+
)
|
|
732
|
+
url = (
|
|
733
|
+
f"{self.configuration.get_hdx_site_url()}/datastore/dump/{self.data['id']}"
|
|
734
|
+
)
|
|
735
|
+
if file_format != "csv":
|
|
736
|
+
url = f"{url}?format={file_format}"
|
|
737
|
+
return url
|
|
738
|
+
|
|
739
|
+
def download_datastore(
|
|
740
|
+
self,
|
|
741
|
+
folder: Path | str | None = None,
|
|
742
|
+
filename: str | None = None,
|
|
743
|
+
file_format: str = "csv",
|
|
744
|
+
retriever: Retrieve | None = None,
|
|
745
|
+
) -> tuple[str, Path]:
|
|
746
|
+
"""Download the resource datastore in full to provided folder or
|
|
747
|
+
temporary folder if no folder supplied
|
|
748
|
+
|
|
749
|
+
Args:
|
|
750
|
+
folder: Folder to download datastore to. Defaults to None.
|
|
751
|
+
filename: Filename to use. Defaults to None (resource name with file_format extension).
|
|
752
|
+
file_format: Dump format (csv, tsv, json or xml). Defaults to csv.
|
|
753
|
+
retriever: Retrieve object to use. Defaults to None.
|
|
754
|
+
|
|
755
|
+
Returns:
|
|
756
|
+
(URL downloaded, Path to downloaded file)
|
|
757
|
+
"""
|
|
758
|
+
url = self.get_datastore_dump_url(file_format)
|
|
759
|
+
logger.debug(f"Downloading {url}")
|
|
760
|
+
if filename is None:
|
|
761
|
+
filename = self.data["name"]
|
|
762
|
+
extension = f".{file_format.lower()}"
|
|
763
|
+
if not filename.endswith(extension):
|
|
764
|
+
filename = f"{filename}{extension}"
|
|
765
|
+
session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
|
|
766
|
+
with Download(session=session) as downloader:
|
|
767
|
+
if retriever:
|
|
768
|
+
downloader = retriever.clone(downloader)
|
|
769
|
+
path = downloader.download_file(url, folder=folder, filename=filename)
|
|
770
|
+
return url, path
|
|
771
|
+
|
|
714
772
|
@staticmethod
|
|
715
773
|
def get_all_resource_ids_in_datastore(
|
|
716
774
|
configuration: Configuration | None = None,
|
|
@@ -805,6 +863,127 @@ class Resource(HDXObject):
|
|
|
805
863
|
}
|
|
806
864
|
self._write_to_hdx("datastore_upsert", data, "resource_id")
|
|
807
865
|
|
|
866
|
+
def search_datastore(
|
|
867
|
+
self,
|
|
868
|
+
filters: dict | None = None,
|
|
869
|
+
q: str | dict | None = None,
|
|
870
|
+
fields: Sequence[str] | None = None,
|
|
871
|
+
sort: str | None = None,
|
|
872
|
+
distinct: bool = False,
|
|
873
|
+
limit: int = 100,
|
|
874
|
+
offset: int = 0,
|
|
875
|
+
include_total: bool = True,
|
|
876
|
+
**kwargs: Any,
|
|
877
|
+
) -> dict:
|
|
878
|
+
"""Search the resource datastore using CKAN datastore_search.
|
|
879
|
+
|
|
880
|
+
Args:
|
|
881
|
+
filters: Dictionary of field name to value(s) to match. Defaults to None.
|
|
882
|
+
q: Full text query string or dictionary of field name to query. Defaults to None.
|
|
883
|
+
fields: Fields to return. Defaults to None (all fields).
|
|
884
|
+
sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None.
|
|
885
|
+
distinct: Whether to return only distinct rows. Defaults to False.
|
|
886
|
+
limit: Maximum number of rows to return. Defaults to 100.
|
|
887
|
+
offset: Number of rows to skip. Defaults to 0.
|
|
888
|
+
include_total: Whether to include total number of matching rows. Defaults to True.
|
|
889
|
+
**kwargs: Other parameters to pass to datastore_search eg. plain, language.
|
|
890
|
+
|
|
891
|
+
Returns:
|
|
892
|
+
datastore_search result dictionary with keys including records, fields and total
|
|
893
|
+
"""
|
|
894
|
+
data: dict = {
|
|
895
|
+
"resource_id": self.data["id"],
|
|
896
|
+
"limit": limit,
|
|
897
|
+
"offset": offset,
|
|
898
|
+
"include_total": include_total,
|
|
899
|
+
}
|
|
900
|
+
if filters is not None:
|
|
901
|
+
data["filters"] = filters
|
|
902
|
+
if q is not None:
|
|
903
|
+
data["q"] = q
|
|
904
|
+
if fields is not None:
|
|
905
|
+
data["fields"] = ",".join(fields)
|
|
906
|
+
if sort is not None:
|
|
907
|
+
data["sort"] = sort
|
|
908
|
+
if distinct:
|
|
909
|
+
data["distinct"] = True
|
|
910
|
+
data.update(kwargs)
|
|
911
|
+
return self._write_to_hdx("datastore_search", data, "resource_id")
|
|
912
|
+
|
|
913
|
+
def get_datastore_records(
|
|
914
|
+
self,
|
|
915
|
+
filters: dict | None = None,
|
|
916
|
+
q: str | dict | None = None,
|
|
917
|
+
fields: Sequence[str] | None = None,
|
|
918
|
+
sort: str | None = None,
|
|
919
|
+
distinct: bool = False,
|
|
920
|
+
page_size: int = 10000,
|
|
921
|
+
**kwargs: Any,
|
|
922
|
+
) -> Iterator[dict]:
|
|
923
|
+
"""Iterate over all matching records in the resource datastore, paging
|
|
924
|
+
through datastore_search results.
|
|
925
|
+
|
|
926
|
+
Args:
|
|
927
|
+
filters: Dictionary of field name to value(s) to match. Defaults to None.
|
|
928
|
+
q: Full text query string or dictionary of field name to query. Defaults to None.
|
|
929
|
+
fields: Fields to return. Defaults to None (all fields).
|
|
930
|
+
sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None.
|
|
931
|
+
distinct: Whether to return only distinct rows. Defaults to False.
|
|
932
|
+
page_size: Number of rows to request per call. Defaults to 10000.
|
|
933
|
+
**kwargs: Other parameters to pass to datastore_search eg. plain, language.
|
|
934
|
+
|
|
935
|
+
Returns:
|
|
936
|
+
Iterator of record dictionaries
|
|
937
|
+
"""
|
|
938
|
+
offset = 0
|
|
939
|
+
while True:
|
|
940
|
+
result = self.search_datastore(
|
|
941
|
+
filters=filters,
|
|
942
|
+
q=q,
|
|
943
|
+
fields=fields,
|
|
944
|
+
sort=sort,
|
|
945
|
+
distinct=distinct,
|
|
946
|
+
limit=page_size,
|
|
947
|
+
offset=offset,
|
|
948
|
+
include_total=False,
|
|
949
|
+
**kwargs,
|
|
950
|
+
)
|
|
951
|
+
records = result["records"]
|
|
952
|
+
yield from records
|
|
953
|
+
if len(records) < page_size:
|
|
954
|
+
return
|
|
955
|
+
offset += page_size
|
|
956
|
+
|
|
957
|
+
def get_datastore_fields(self) -> list[dict]:
|
|
958
|
+
"""Get the field definitions of the resource datastore excluding the
|
|
959
|
+
internal _id field.
|
|
960
|
+
|
|
961
|
+
Returns:
|
|
962
|
+
List of field dictionaries each with id and type keys
|
|
963
|
+
"""
|
|
964
|
+
result = self.search_datastore(limit=0, include_total=False)
|
|
965
|
+
return [field for field in result["fields"] if field["id"] != "_id"]
|
|
966
|
+
|
|
967
|
+
@staticmethod
|
|
968
|
+
def search_datastore_sql(
|
|
969
|
+
sql: str,
|
|
970
|
+
configuration: Configuration | None = None,
|
|
971
|
+
) -> list[dict]:
|
|
972
|
+
"""Run a read-only SQL query against the HDX datastore using CKAN
|
|
973
|
+
datastore_search_sql. Tables are named by resource id, which must be
|
|
974
|
+
double quoted eg. SELECT * FROM "RESOURCE_ID".
|
|
975
|
+
|
|
976
|
+
Args:
|
|
977
|
+
sql: SQL SELECT statement
|
|
978
|
+
configuration: HDX configuration. Defaults to global configuration.
|
|
979
|
+
|
|
980
|
+
Returns:
|
|
981
|
+
List of record dictionaries
|
|
982
|
+
"""
|
|
983
|
+
resource = Resource(configuration=configuration)
|
|
984
|
+
result = resource._write_to_hdx("datastore_search_sql", {"sql": sql})
|
|
985
|
+
return result["records"]
|
|
986
|
+
|
|
808
987
|
def delete_datastore(self) -> None:
|
|
809
988
|
"""Delete a resource from the HDX datastore
|
|
810
989
|
|
hdx/data/user.py
CHANGED
|
@@ -7,6 +7,7 @@ from typing import Any, Optional
|
|
|
7
7
|
|
|
8
8
|
import hdx.data.organization
|
|
9
9
|
from hdx.api.configuration import Configuration
|
|
10
|
+
from hdx.data.activity import Activity
|
|
10
11
|
from hdx.data.hdxobject import HDXError, HDXObject
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
@@ -453,6 +454,30 @@ class User(HDXObject):
|
|
|
453
454
|
return result
|
|
454
455
|
return []
|
|
455
456
|
|
|
457
|
+
def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]:
|
|
458
|
+
"""Get the activity stream of this user. One page of results (whose size
|
|
459
|
+
is given by limit) is returned unless all_pages is True. HDX requires
|
|
460
|
+
authorisation to view the user's profile.
|
|
461
|
+
|
|
462
|
+
Args:
|
|
463
|
+
all_pages: Whether to get all pages of results. Defaults to False.
|
|
464
|
+
**kwargs: See below
|
|
465
|
+
offset (int): Where to start getting activity items from. Defaults to 0.
|
|
466
|
+
limit (int): Maximum number of activities per page. Defaults to 31 (HDX maximum 100) or 100 if all_pages is True.
|
|
467
|
+
after (int | str): Only activities after this timestamp
|
|
468
|
+
before (int | str): Only activities before this timestamp
|
|
469
|
+
|
|
470
|
+
Returns:
|
|
471
|
+
List of Activity objects
|
|
472
|
+
"""
|
|
473
|
+
return Activity.get_activities(
|
|
474
|
+
"user",
|
|
475
|
+
self.data.get("id") or self.data.get("name"),
|
|
476
|
+
self.configuration,
|
|
477
|
+
all_pages,
|
|
478
|
+
**kwargs,
|
|
479
|
+
)
|
|
480
|
+
|
|
456
481
|
@classmethod
|
|
457
482
|
def autocomplete(
|
|
458
483
|
cls,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: hdx-python-api
|
|
3
|
-
Version: 6.7.
|
|
3
|
+
Version: 6.7.2
|
|
4
4
|
Summary: HDX Python API for interacting with the Humanitarian Data Exchange
|
|
5
5
|
Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-api
|
|
6
6
|
Author: Michael Rans
|
|
@@ -28,8 +28,8 @@ Requires-Python: >=3.10
|
|
|
28
28
|
Requires-Dist: ckanapi>=4.11
|
|
29
29
|
Requires-Dist: defopt>=7.0.0
|
|
30
30
|
Requires-Dist: email-validator
|
|
31
|
-
Requires-Dist: hdx-python-country>=4.
|
|
32
|
-
Requires-Dist: hdx-python-utilities>=4.1
|
|
31
|
+
Requires-Dist: hdx-python-country>=4.2.3
|
|
32
|
+
Requires-Dist: hdx-python-utilities>=4.2.1
|
|
33
33
|
Requires-Dist: makefun
|
|
34
34
|
Requires-Dist: requests
|
|
35
35
|
Provides-Extra: docs
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
hdx/api/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
|
|
2
|
-
hdx/api/_version.py,sha256=
|
|
2
|
+
hdx/api/_version.py,sha256=CgL0y60DiHh1gIc1CUZuiYkj-TMX2gdAy0attQGD48w,520
|
|
3
3
|
hdx/api/configuration.py,sha256=9WvuVI-3th2Dv70_t_4VoyrlJwB_9qUSN5_zUKbp4sQ,23805
|
|
4
4
|
hdx/api/hdx_base_configuration.yaml,sha256=jj2t5b7NnI499rHVwOjj3Q5BAjE8i7NDGVKnWZ5P5-4,2222
|
|
5
5
|
hdx/api/locations.py,sha256=bFl_cKz_AEQzkE_xWOkrRlUQ0GT2LMPSk-G_LsbV-Ns,4099
|
|
@@ -11,21 +11,22 @@ hdx/api/utilities/hdx_error_handler.py,sha256=x1rb2qBO0C6CJjqToSFj_XB1F3pf3nHpA8
|
|
|
11
11
|
hdx/api/utilities/hdx_state.py,sha256=KzrK0A-H5R6cRPqwF4d23_S_zfIa1ow4K5u46FesGmY,2442
|
|
12
12
|
hdx/api/utilities/url_utils.py,sha256=Pb4r-BAqIHYmeoB_i7QjutDp6nOAUh_OHvCtiNLda3Y,4534
|
|
13
13
|
hdx/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
hdx/data/
|
|
14
|
+
hdx/data/activity.py,sha256=vBnkaL7QVaa05Y1dzZ484dzqgaijb8dTCe60cERlAJQ,9101
|
|
15
|
+
hdx/data/dataset.py,sha256=cPccjApN5td3DDW9r2PrmLM30ThzKNwf3hwlb60OxzU,100697
|
|
15
16
|
hdx/data/hdxobject.py,sha256=_jlwYjb_CcjaXsTXObknMfq-OiNOcpJfLJ9b1vKV1Zo,27494
|
|
16
|
-
hdx/data/location.py,sha256=
|
|
17
|
-
hdx/data/organization.py,sha256=
|
|
18
|
-
hdx/data/resource.py,sha256=
|
|
17
|
+
hdx/data/location.py,sha256=HonWohDZOHSvlwoiv4vc5-9BzZ2iIG8FIos1C1Q814w,9738
|
|
18
|
+
hdx/data/organization.py,sha256=JVxyrbobBDcAREKJzSq5Rc7uUfkH-tzLj1jwQDyMnBs,13280
|
|
19
|
+
hdx/data/resource.py,sha256=iVARfA3_8gGsGjXgNxomm7umAadvxzFsGGqxTpBB3WQ,45196
|
|
19
20
|
hdx/data/resource_matcher.py,sha256=Yzk9pWa4zk1p09m6zcZMj0XZf5Y16Pa3KToOdUt7rP8,4247
|
|
20
21
|
hdx/data/resource_view.py,sha256=GcYkh0iDAe6_m2ErY_A6i3Z7ZFX949eGt1TLnLZv_8s,7346
|
|
21
22
|
hdx/data/showcase.py,sha256=vnRZ45xd9Ct65eFQkWZtMIJPFifZfKOrH4obO5D5Pvc,14062
|
|
22
|
-
hdx/data/user.py,sha256=
|
|
23
|
+
hdx/data/user.py,sha256=rHyFPzGR_u_LmXvgMQ6uqKkYFIW3uG89-M6nnG3pkkU,16697
|
|
23
24
|
hdx/data/vocabulary.py,sha256=sDDVh9ivCbZlPKXIdOoj0ltTdRufL0ypfGZUirIb2IA,20776
|
|
24
25
|
hdx/facades/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
26
|
hdx/facades/infer_arguments.py,sha256=DvyDuozfOPmoaV7HjhQySojGIrMWW9nos31auQBVVl8,4032
|
|
26
27
|
hdx/facades/keyword_arguments.py,sha256=aVapHCfqBO6H7xxBgTdK9ISdgrBFsNypPDWihfQ3edU,1125
|
|
27
28
|
hdx/facades/simple.py,sha256=QWsBkaHG_UM6X3Zu2u3LWVOQzeR24q-EUClrfsL_mkM,1064
|
|
28
|
-
hdx_python_api-6.7.
|
|
29
|
-
hdx_python_api-6.7.
|
|
30
|
-
hdx_python_api-6.7.
|
|
31
|
-
hdx_python_api-6.7.
|
|
29
|
+
hdx_python_api-6.7.2.dist-info/METADATA,sha256=lv8ylKpAePSbZ6KMPwYlnHeNlOnfI4FrRA_30YKUilM,4946
|
|
30
|
+
hdx_python_api-6.7.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
31
|
+
hdx_python_api-6.7.2.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
|
|
32
|
+
hdx_python_api-6.7.2.dist-info/RECORD,,
|
|
File without changes
|