oda-data 2.0.2__tar.gz → 2.0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {oda_data-2.0.2 → oda_data-2.0.3}/PKG-INFO +9 -11
  2. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/__init__.py +8 -1
  3. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/api/sources.py +179 -54
  4. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/schema.py +140 -0
  5. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/config.py +1 -1
  6. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac1/common.py +0 -2
  7. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac1/one.py +1 -1
  8. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/compatibility.py +1 -1
  9. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/names/create_mapping.py +1 -1
  10. oda_data-2.0.3/pyproject.toml +35 -0
  11. oda_data-2.0.2/pyproject.toml +0 -31
  12. {oda_data-2.0.2 → oda_data-2.0.3}/LICENSE +0 -0
  13. {oda_data-2.0.2 → oda_data-2.0.3}/README.md +0 -0
  14. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/.raw_data/README.md +0 -0
  15. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/api/__init__.py +0 -0
  16. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/api/constants.py +0 -0
  17. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/api/oecd.py +0 -0
  18. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/api/representations.py +0 -0
  19. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/__init__.py +0 -0
  20. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/channels.py +0 -0
  21. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/common.py +0 -0
  22. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/crs_channel_mapping.csv +0 -0
  23. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/dtypes.py +0 -0
  24. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/clean_data/validation.py +0 -0
  25. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/__init__.py +0 -0
  26. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/common.py +0 -0
  27. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/crs/__init__.py +0 -0
  28. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/crs/common.py +0 -0
  29. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/crs/crs_functions.py +0 -0
  30. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/crs/crs_indicators.json +0 -0
  31. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/crs/oecd.py +0 -0
  32. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/crs/one.py +0 -0
  33. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac1/__init__.py +0 -0
  34. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac1/dac1_functions.py +0 -0
  35. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac1/dac1_indicators.json +0 -0
  36. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac1/oecd.py +0 -0
  37. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac2a/__init__.py +0 -0
  38. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac2a/common.py +0 -0
  39. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac2a/dac2a_functions.py +0 -0
  40. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac2a/dac2a_indicators.json +0 -0
  41. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac2a/oecd.py +0 -0
  42. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/dac2a/one.py +0 -0
  43. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/indicator.py +0 -0
  44. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/research/__init__.py +0 -0
  45. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/research/eu.py +0 -0
  46. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/research/policy_markers.py +0 -0
  47. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/indicators/research/sector_imputations.py +0 -0
  48. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/logger.py +0 -0
  49. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/crs_flow_types.json +0 -0
  50. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/crs_modalities.json +0 -0
  51. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/crs_policy_markers.json +0 -0
  52. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/crs_purpose.json +0 -0
  53. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/crs_type_of_finance.json +0 -0
  54. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/dac1_aid_flow_types.json +0 -0
  55. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/dac2a_aid_type.json +0 -0
  56. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/flow_types.json +0 -0
  57. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/indicators.json +0 -0
  58. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/key_columns.json +0 -0
  59. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/provider_groupings.json +0 -0
  60. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/recipient_groupings.json +0 -0
  61. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/settings/v1_indicators_mapping.json +0 -0
  62. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/__init__.py +0 -0
  63. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/cache.py +0 -0
  64. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/gni.py +0 -0
  65. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/groupings.py +0 -0
  66. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/names/__init__.py +0 -0
  67. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/names/add.py +0 -0
  68. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/names/crs_names.json +0 -0
  69. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/names/dac1_names.json +0 -0
  70. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/names/dac2a_names.json +0 -0
  71. {oda_data-2.0.2 → oda_data-2.0.3}/oda_data/tools/sector_lists.py +0 -0
@@ -1,28 +1,26 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: oda_data
3
- Version: 2.0.2
3
+ Version: 2.0.3
4
4
  Summary: A python package to work with Official Development Assistance data from the OECD DAC.
5
+ License: MIT
5
6
  Author: Jorge Rivera
6
7
  Author-email: jorge.rivera@one.org
7
- Requires-Python: >=3.10, <4.0
8
+ Requires-Python: >=3.10,<4.0
9
+ Classifier: License :: OSI Approved :: MIT License
8
10
  Classifier: Programming Language :: Python :: 3
9
11
  Classifier: Programming Language :: Python :: 3.10
10
12
  Classifier: Programming Language :: Python :: 3.11
11
13
  Classifier: Programming Language :: Python :: 3.12
12
14
  Classifier: Programming Language :: Python :: 3.13
13
- Provides-Extra: dev
14
- Requires-Dist: black (>=24) ; extra == "dev"
15
- Requires-Dist: cachetools (>=5.5.1,<6.0.0)
16
- Requires-Dist: oda-reader (>=1.1.3)
15
+ Requires-Dist: cachetools (>=5.5.1,<7.0.0)
16
+ Requires-Dist: oda-reader (>=1.2,<2.0)
17
17
  Requires-Dist: pandas (>=2)
18
- Requires-Dist: pyarrow (>=14,<20)
18
+ Requires-Dist: pyarrow (>=14,<22)
19
19
  Requires-Dist: pydeflate (>=2.0.1)
20
- Requires-Dist: pytest (>=8) ; extra == "dev"
21
- Requires-Dist: pytest-cov (>=4.0.0) ; extra == "dev"
22
- Requires-Dist: pytest-mock (>=3.10.0) ; extra == "dev"
23
20
  Requires-Dist: thefuzz (>=0.22.1)
24
- Project-URL: Bug_Tracker, https://github.com/ONEcampaign/oda_data_package/issues
21
+ Project-URL: Documentation, https://github.com/ONEcampaign/oda_data_package
25
22
  Project-URL: Homepage, https://github.com/ONEcampaign/oda_data_package
23
+ Project-URL: Repository, https://github.com/ONEcampaign/oda_data_package
26
24
  Description-Content-Type: text/markdown
27
25
 
28
26
  [![pypi](https://img.shields.io/pypi/v/oda_data.svg)](https://pypi.org/project/oda_data/)
@@ -5,7 +5,13 @@ __version__ = version("oda_data")
5
5
  from oda_data import tools
6
6
  from oda_reader import set_cache_dir, disable_cache, enable_cache, clear_cache
7
7
  from oda_data.api.oecd import OECDClient
8
- from oda_data.api.sources import DAC1Data, DAC2AData, MultiSystemData, CRSData
8
+ from oda_data.api.sources import (
9
+ DAC1Data,
10
+ DAC2AData,
11
+ MultiSystemData,
12
+ CRSData,
13
+ AidDataData,
14
+ )
9
15
  from oda_data.indicators.research.policy_markers import bilateral_policy_marker
10
16
  from oda_data.tools.compatibility import ODAData
11
17
  from oda_data.tools.gni import add_gni_share_column
@@ -33,6 +39,7 @@ __all__ = [
33
39
  "DAC2AData",
34
40
  "MultiSystemData",
35
41
  "CRSData",
42
+ "AidDataData",
36
43
  "bilateral_policy_marker",
37
44
  "provider_groupings",
38
45
  "recipient_groupings",
@@ -1,4 +1,5 @@
1
1
  from abc import abstractmethod
2
+ from pathlib import Path
2
3
  from typing import Optional
3
4
 
4
5
  import pandas as pd
@@ -10,6 +11,7 @@ from oda_reader import (
10
11
  download_dac2a,
11
12
  bulk_download_multisystem,
12
13
  download_multisystem,
14
+ download_aiddata,
13
15
  )
14
16
  from oda_reader._cache import memory
15
17
 
@@ -18,7 +20,7 @@ from oda_data.clean_data.common import (
18
20
  clean_raw_df,
19
21
  convert_dot_stat_to_data_explorer_codes,
20
22
  )
21
- from oda_data.clean_data.schema import ODASchema
23
+ from oda_data.clean_data.schema import ODASchema, AidDataSchema
22
24
  from oda_data.clean_data.validation import (
23
25
  validate_years_providers_recipients,
24
26
  check_integers,
@@ -41,8 +43,8 @@ def _filters_to_query(filters: list[tuple[str, str, list]]) -> str:
41
43
  return query
42
44
 
43
45
 
44
- class DACSource:
45
- """Base class for accessing DAC datasets.
46
+ class Source:
47
+ """Base class for accessing any dataset.
46
48
 
47
49
  This class handles validation of parameters and manages the retrieval
48
50
  of data in the form of a pandas DataFrame.
@@ -57,6 +59,45 @@ class DACSource:
57
59
  self.de_indicators = None
58
60
  self.filters = None
59
61
  self.de_sectors = None
62
+ self.schema = None
63
+
64
+ def _add_filter(self, column: str, predicate: str, value: str | int | list) -> None:
65
+ """Adds a filter to the dataset, ensuring no duplicate columns.
66
+
67
+ If a filter for the same column already exists, it is replaced with a warning.
68
+
69
+ Args:
70
+ column (str): The name of the column to filter.
71
+ predicate (str): The condition to apply (e.g., "in").
72
+ value (str | int | list): The value(s) to filter by.
73
+ """
74
+ existing_filter = next((f for f in self.filters if f[0] == column), None)
75
+
76
+ if existing_filter:
77
+ logger.warning(
78
+ f"A filter for {column} already exists. Replacing with new filter."
79
+ )
80
+
81
+ # Remove existing filter (if any) and add the new one
82
+ self.filters = [(c, p, v) for c, p, v in self.filters if c != column]
83
+ self.filters.append((column, predicate, value))
84
+
85
+
86
+ class DACSource(Source):
87
+ """Base class for accessing DAC datasets.
88
+
89
+ This class handles validation of parameters and manages the retrieval
90
+ of data in the form of a pandas DataFrame.
91
+ """
92
+
93
+ memory_cache = TTLCache(maxsize=20, ttl=6000)
94
+ disk_cache = OnDiskCache(config.ODAPaths.raw_data, ttl_seconds=86400)
95
+
96
+ def __init__(self):
97
+ super().__init__()
98
+
99
+ self.schema = ODASchema
100
+ self._param_hash = None
60
101
 
61
102
  def _init_filters(
62
103
  self,
@@ -80,53 +121,32 @@ class DACSource:
80
121
  if self.years is not None:
81
122
  self.start = min(self.years)
82
123
  self.end = max(self.years)
83
- self.add_filter(column=ODASchema.YEAR, predicate="in", value=self.years)
124
+ self._add_filter(column=self.schema.YEAR, predicate="in", value=self.years)
84
125
  else:
85
126
  self.start, self.end = None, None
86
127
 
87
128
  if self.providers is not None:
88
- self.add_filter(
89
- column=ODASchema.PROVIDER_CODE, predicate="in", value=self.providers
129
+ self._add_filter(
130
+ column=self.schema.PROVIDER_CODE, predicate="in", value=self.providers
90
131
  )
91
132
  self.de_providers = convert_dot_stat_to_data_explorer_codes(
92
133
  codes=self.providers
93
134
  )
94
135
 
95
136
  if self.recipients is not None:
96
- self.add_filter(
97
- column=ODASchema.RECIPIENT_CODE, predicate="in", value=self.recipients
137
+ self._add_filter(
138
+ column=self.schema.RECIPIENT_CODE, predicate="in", value=self.recipients
98
139
  )
99
140
  self.de_recipients = convert_dot_stat_to_data_explorer_codes(
100
141
  codes=self.recipients
101
142
  )
102
143
 
103
144
  if self.sectors is not None:
104
- self.add_filter(
105
- column=ODASchema.SECTOR_CODE, predicate="in", value=self.sectors
145
+ self._add_filter(
146
+ column=self.schema.SECTOR_CODE, predicate="in", value=self.sectors
106
147
  )
107
148
  self.de_sectors = check_strings(sectors)
108
149
 
109
- def add_filter(self, column: str, predicate: str, value: str | int | list) -> None:
110
- """Adds a filter to the dataset, ensuring no duplicate columns.
111
-
112
- If a filter for the same column already exists, it is replaced with a warning.
113
-
114
- Args:
115
- column (str): The name of the column to filter.
116
- predicate (str): The condition to apply (e.g., "in").
117
- value (str | int | list): The value(s) to filter by.
118
- """
119
- existing_filter = next((f for f in self.filters if f[0] == column), None)
120
-
121
- if existing_filter:
122
- logger.warning(
123
- f"A filter for {column} already exists. Replacing with new filter."
124
- )
125
-
126
- # Remove existing filter (if any) and add the new one
127
- self.filters = [(c, p, v) for c, p, v in self.filters if c != column]
128
- self.filters.append((column, predicate, value))
129
-
130
150
  def _get_filtered_download_filters(self):
131
151
  """Generates a dictionary of filters for downloading the dataset.
132
152
 
@@ -194,9 +214,16 @@ class DACSource:
194
214
 
195
215
  return cached_df
196
216
 
217
+ def _bulk_filename(self) -> Path:
218
+ return self.disk_cache.get_file_path(
219
+ dataset_name=self.__class__.__name__, param_hash=self._param_hash
220
+ )
221
+
197
222
  def _cache_dataset(self, param_hash: str, df: pd.DataFrame) -> None:
198
223
  """Caches the dataset in memory and on disk."""
199
- self.memory_cache[param_hash] = df.copy(deep=True)
224
+ size = df.memory_usage(deep=True).sum() / (1024 * 1024) # in MB
225
+ if size < 50:
226
+ self.memory_cache[param_hash] = df.copy(deep=True)
200
227
  self.disk_cache.save(self.__class__.__name__, param_hash, df)
201
228
 
202
229
  def _apply_query_and_cache(
@@ -245,6 +272,8 @@ class DACSource:
245
272
  else generate_param_hash(filters=filters)
246
273
  )
247
274
 
275
+ self._param_hash = param_hash
276
+
248
277
  # If caching is not active, clear the cache
249
278
  if not memory().store_backend:
250
279
  self.disk_cache.cleanup(hash_str=param_hash, force=True)
@@ -257,10 +286,17 @@ class DACSource:
257
286
 
258
287
  # If not cached, download the data
259
288
  if df is None:
260
- df = self.download(bulk=using_bulk_download).pipe(clean_raw_df)
261
- df = self._apply_query_and_cache(
262
- df=df, query=query, param_hash=param_hash, bulk=using_bulk_download
263
- )
289
+ df = self.download(bulk=using_bulk_download)
290
+ if df is None:
291
+ df = self._get_cached_dataset(
292
+ param_hash=param_hash, filters=filters, columns=columns
293
+ )
294
+ else:
295
+ df = self._apply_query_and_cache(
296
+ df=df, query=query, param_hash=param_hash, bulk=using_bulk_download
297
+ )
298
+
299
+ df = df.pipe(clean_raw_df)
264
300
 
265
301
  if columns:
266
302
  df = df.filter(columns)
@@ -295,8 +331,8 @@ class DAC1Data(DACSource):
295
331
  self.indicators = check_integers(indicators) if indicators else None
296
332
 
297
333
  if self.indicators is not None:
298
- self.add_filter(
299
- column=ODASchema.AIDTYPE_CODE, predicate="in", value=self.indicators
334
+ self._add_filter(
335
+ column=self.schema.AIDTYPE_CODE, predicate="in", value=self.indicators
300
336
  )
301
337
  self.de_indicators = check_strings(indicators)
302
338
 
@@ -322,6 +358,8 @@ class DAC1Data(DACSource):
322
358
 
323
359
 
324
360
  class DAC2AData(DACSource):
361
+ """Class to handle the DAC2A data."""
362
+
325
363
  def __init__(
326
364
  self,
327
365
  years: Optional[list[int] | range | int] = None,
@@ -343,8 +381,8 @@ class DAC2AData(DACSource):
343
381
  self.indicators = check_integers(indicators) if indicators else None
344
382
 
345
383
  if self.indicators is not None:
346
- self.add_filter(
347
- column=ODASchema.AIDTYPE_CODE, predicate="in", value=self.indicators
384
+ self._add_filter(
385
+ column=self.schema.AIDTYPE_CODE, predicate="in", value=self.indicators
348
386
  )
349
387
  self.de_indicators = check_strings(indicators)
350
388
 
@@ -389,6 +427,7 @@ class CRSData(DACSource):
389
427
  """
390
428
  super().__init__()
391
429
  self._init_filters(years=years, providers=providers, recipients=recipients)
430
+ self._param_hash = None
392
431
 
393
432
  def download(self, bulk: bool = True):
394
433
  """Downloads CRS data, either filtered or full dataset.
@@ -403,19 +442,16 @@ class CRSData(DACSource):
403
442
  end_year=self.end,
404
443
  filters=self._get_filtered_download_filters(),
405
444
  )
445
+ logger.info(f"CRS data downloaded successfully.")
446
+ return clean_raw_df(df)
447
+
406
448
  else:
407
449
  logger.info(
408
450
  "The full, detailed CRS is only available as a large file (>1GB). "
409
451
  "The package will now download the data, but it may take a while."
410
452
  )
411
- df = bulk_download_crs()
412
-
413
- # Clean the DataFrame
414
- df = clean_raw_df(df)
415
-
416
- logger.info(f"CRS data downloaded successfully.")
417
-
418
- return df
453
+ bulk_download_crs(save_to_path=self._bulk_filename())
454
+ return None
419
455
 
420
456
 
421
457
  class MultiSystemData(DACSource):
@@ -447,8 +483,8 @@ class MultiSystemData(DACSource):
447
483
  self.indicators = check_strings(indicators) if indicators else None
448
484
 
449
485
  if self.indicators is not None:
450
- self.add_filter(
451
- column=ODASchema.AID_TO_THRU, predicate="in", value=self.indicators
486
+ self._add_filter(
487
+ column=self.schema.AID_TO_THRU, predicate="in", value=self.indicators
452
488
  )
453
489
  self.de_indicators = indicators
454
490
 
@@ -465,16 +501,105 @@ class MultiSystemData(DACSource):
465
501
  end_year=self.end,
466
502
  filters=self._get_filtered_download_filters(),
467
503
  )
504
+ logger.info(f"Multisystem data downloaded successfully.")
505
+ return clean_raw_df(df)
506
+
468
507
  else:
469
508
  logger.info(
470
509
  "The full, detailed Multisystem is a large file (>1GB). "
471
510
  "The package will now download the data, but it may take a while."
472
511
  )
473
- df = bulk_download_multisystem()
512
+ bulk_download_multisystem(save_to_path=self._bulk_filename())
513
+ return None
514
+
515
+
516
+ class AidDataSource(Source):
517
+ """Base class for accessing AidData datasets.
518
+
519
+ This class handles validation of parameters and manages the retrieval
520
+ of data in the form of a pandas DataFrame.
521
+ """
522
+
523
+ memory_cache = TTLCache(maxsize=20, ttl=6000)
524
+ disk_cache = OnDiskCache(config.ODAPaths.raw_data, ttl_seconds=86400)
525
+
526
+ def __init__(self):
527
+ super().__init__()
528
+
529
+ self.schema = AidDataSchema
530
+
531
+ def _init_filters(
532
+ self,
533
+ years: Optional[list[int] | range | int] = None,
534
+ recipients: Optional[list[str] | str] = None,
535
+ sectors: Optional[list[int] | int] = None,
536
+ ):
537
+ self.years = check_integers(years)
538
+ self.recipients = check_strings(recipients)
539
+ self.sectors = check_integers(sectors)
540
+
541
+ self.filters = []
542
+
543
+ if self.years is not None:
544
+ self.start = min(self.years)
545
+ self.end = max(self.years)
546
+ self._add_filter(
547
+ column=self.schema.COMMITMENT_YEAR, predicate="in", value=self.years
548
+ )
549
+ else:
550
+ self.start, self.end = None, None
551
+
552
+ if self.recipients is not None:
553
+ self._add_filter(
554
+ column=self.schema.RECIPIENT_NAME, predicate="in", value=self.recipients
555
+ )
556
+
557
+ if self.sectors is not None:
558
+ self._add_filter(
559
+ column=self.schema.SECTOR_CODE, predicate="in", value=self.sectors
560
+ )
561
+
562
+
563
+ class AidDataData(AidDataSource):
564
+ """Class to handle the AidData data."""
565
+
566
+ memory_cache = TTLCache(maxsize=20, ttl=6000)
567
+ disk_cache = OnDiskCache(config.ODAPaths.raw_data, ttl_seconds=86400)
568
+
569
+ def __init__(
570
+ self,
571
+ years: Optional[list[int] | range | int] = None,
572
+ recipients: Optional[list[str] | str] = None,
573
+ sectors: Optional[list[int] | int] = None,
574
+ ):
575
+ """Initializes the AidData data handler.
576
+
577
+ Args:
578
+ years (Optional[list[int] | range | int]): Years to filter based on commitment year.
579
+ recipients (Optional[list[str] | str]): Recipient names to filter.
580
+ sectors (Optional[list[int] | int]): Sector codes to filter.
581
+ """
582
+ super().__init__()
583
+
584
+ self._init_filters(years=years, recipients=recipients, sectors=sectors)
585
+
586
+ def download(self, additional_filters: Optional[list[tuple]] = None):
587
+ """Downloads the full AidData dataset and applies filters.
588
+
589
+ Args:
590
+ additional_filters (Optional[list[tuple]], optional): Additional filters to apply. Defaults to None.
591
+ """
592
+
593
+ df = download_aiddata()
594
+
595
+ logger.info(f"Filtering the data")
596
+
597
+ self.filters = (
598
+ self.filters + additional_filters if additional_filters else self.filters
599
+ )
474
600
 
475
- # Clean the DataFrame
476
- df = clean_raw_df(df)
601
+ query = _filters_to_query(self.filters)
477
602
 
478
- logger.info(f"Multisystem data downloaded successfully.")
603
+ df = df.query(query)
479
604
 
480
605
  return df
@@ -277,3 +277,143 @@ CRS_MAPPING: dict[str, str] = {
277
277
  "de_regioncode": ODASchema.RECIPIENT_REGION_CODE_DE,
278
278
  "incomegroup_name": ODASchema.RECIPIENT_INCOME,
279
279
  }
280
+
281
+
282
+ @dataclass
283
+ class AidDataSchema:
284
+ AIDDATA_ID: str = "aiddata_id"
285
+ RECOMMENDED_FOR_AGGREGATES: str = "recommended_for_aggregates"
286
+ AIDDATA_PARENT_ID: str = "aiddata_parent_id"
287
+ UMBRELLA: str = "umbrella"
288
+ PROVIDER_NAME: str = "donor_name"
289
+ RECIPIENT_NAME: str = "recipient_name"
290
+ RECIPIENT_ISO3: str = "recipient_iso3"
291
+ REGION_NAME: str = "region_name"
292
+ COMMITMENT_YEAR: str = "commitment_year"
293
+ IMPLEMENTATION_START_YEAR: str = "start_year"
294
+ COMPLETION_YEAR: str = "completion_year"
295
+ PROJECT_TITLE: str = "project_title"
296
+ LONG_DESCRIPTION: str = "long_description"
297
+ STAFF_COMMENTS: str = "staff_comments"
298
+ PROJECT_STATUS: str = "project_status"
299
+ PROJECT_INTENT: str = "project_intent"
300
+ FLOW_TYPE: str = "flow_type"
301
+ FLOW_TYPE_SIMPLIFIED: str = "flow_type_simplified"
302
+ ODA_CONCESSIONALITY_THRESHOLD: str = "oda_concessionality_threshold"
303
+ FLOW_CLASS: str = "flow_class"
304
+ SECTOR_CODE: str = "sector_code"
305
+ SECTOR_NAME: str = "sector_name"
306
+ INFRASTRUCTURE: str = "infrastructure"
307
+ COVID: str = "covid"
308
+ DONOR_AGENCIES_NAME: str = "donor_agencies_name"
309
+ DONOR_AGENCIES_TYPE: str = "donor_agencies_type"
310
+ CO_FINANCED: str = "co_financed"
311
+ CO_FINANCING_AGENCIES_NAME: str = "co_financing_agencies_name"
312
+ CO_FINANCING_AGENCIES_TYPE: str = "co_financing_agencies_type"
313
+ DIRECT_RECIPIENT_AGENCIES_NAME: str = "direct_recipient_agencies_name"
314
+ DIRECT_RECIPIENT_AGENCIES_TYPE: str = "direct_recipient_agencies_type"
315
+ INDIRECT_RECIPIENT_AGENCIES_NAME: str = "indirect_recipient_agencies_name"
316
+ INDIRECT_RECIPIENT_AGENCIES_TYPE: str = "indirect_recipient_agencies_type"
317
+ ON_LENDING: str = "on_lending"
318
+ IMPLEMENTING_AGENCIES_NAME: str = "implementing_agencies_name"
319
+ IMPLEMENTING_AGENCIES_TYPE: str = "implementing_agencies_type"
320
+ GUARANTEE_PROVIDED: str = "guarantee_provided"
321
+ GUARANTOR_NAME: str = "guarantor_name"
322
+ GUARANTOR_TYPE: str = "guarantor_type"
323
+ INSURANCE_PROVIDED: str = "insurance_provided"
324
+ INSURANCE_PROVIDER_NAME: str = "insurance_provider_name"
325
+ INSURANCE_PROVIDER_TYPE: str = "insurance_provider_type"
326
+ COLLATERALIZED: str = "collateralized"
327
+ COLLATERAL_PROVIDER_NAME: str = "collateral_provider_name"
328
+ COLLATERAL_PROVIDER_TYPE: str = "collateral_provider_type"
329
+ COLLATERAL_AGENT_NAME: str = "collateral_agent_name"
330
+ COLLATERAL_AGENT_TYPE: str = "collateral_agent_type"
331
+ COLLATERAL: str = "collateral"
332
+ VALUE_ORIGINAL_CURRENCY: str = "commitment_original_currency"
333
+ ORIGINAL_CURRENCY: str = "original_currency"
334
+ VALUE_ESTIMATED: str = "amount_estimated"
335
+ VALUE_CONSTANT_USD_2021: str = "usd_commitment_constant"
336
+ VALUE_CURRENT_USD: str = "usd_commitment"
337
+ ADJUSTED_VALUE_ORIGINAL_CURRENCY: str = "adjustment_original_currency"
338
+ ADJUSTED_VALUE_CONSTANT_USD_2021: str = "usd_adjustment_constant"
339
+ ADJUSTED_VALUE_CURRENT_2021: str = "usd_adjustment"
340
+ FINANCIAL_DISTRESS: str = "financial_distress"
341
+ COMMITMENT_DATE: str = "commitment_date"
342
+ COMMITMENT_DATE_ESTIMATED: str = "commitment_date_estimated"
343
+ PLANNED_IMPLEMENTATION_DATE: str = "expected_start_date"
344
+ ACTUAL_IMPLEMENTATION_DATE: str = "actual_start_date"
345
+ ACTUAL_IMPLEMENTATION_DATE_ESTIMATED: str = "actual_start_date_estimated"
346
+ DEVIATION_PLANNED_ACTUAL_IMPLEMENTATION_DATE: str = (
347
+ "deviation_expected_actual_start_date"
348
+ )
349
+ PLANNED_COMPLETION_DATE: str = "expected_completion_date"
350
+ ACTUAL_COMPLETION_DATE: str = "actual_completion_date"
351
+ ACTUAL_COMPLETION_DATE_ESTIMATED: str = "actual_completion_date_estimated"
352
+ DEVIATION_PLANNED_ACTUAL_COMPLETION_DATE: str = (
353
+ "deviation_expected_actual_completion_date"
354
+ )
355
+ MATURITY: str = "maturity"
356
+ INTEREST_RATE: str = "interest_rate"
357
+ GRACE_PERIOD: str = "grace_period"
358
+ MANAGEMENT_FEE: str = "management_fee"
359
+ COMMITMENT_FEE: str = "commitment_fee"
360
+ INSURANCE_FEE_PERCENT: str = "insurance_fee_percent"
361
+ INSURANCE_FEE_CURRENT_USD: str = "insurance_fee_usd"
362
+ DEFAULT_INTEREST_RATE: str = "default_interest_rate"
363
+ FIRST_LOAN_REPAYMENT_DATE: str = "repaydate1"
364
+ LAST_LOAN_REPAYMENT_DATE: str = "repaydate2"
365
+ GRANT_ELEMENT_FLOW: str = "grant_element_flow"
366
+ GRANT_ELEMENT_GRANT_EQUIVALENT: str = "grant_element_grant_equiv"
367
+ GRANT_ELEMENT_IMF: str = "grant_element_imf"
368
+ NUMBER_OF_LENDERS: str = "number_of_lenders"
369
+ EXPORT_BUYERS_CREDIT: str = "export_buyers_credit"
370
+ SUPPLIERS_CREDIT_EXPORT_SELLERS_CREDIT: str = (
371
+ "suppliers_credit_export_sellers_credit"
372
+ )
373
+ INTEREST_FREE_LOAN: str = "interest_free_loan"
374
+ REFINANCING: str = "refinancing"
375
+ INVESTMENT_PROJECT_LOAN: str = "investment_project_loan"
376
+ M_A: str = "ma"
377
+ WORKING_CAPITAL: str = "working_capital"
378
+ EPCF: str = "epcf"
379
+ LEASE: str = "lease"
380
+ FXSL_BOP: str = "fxsl_bop"
381
+ CC_IRS: str = "cc_irs"
382
+ RCF: str = "rcf"
383
+ GCL: str = "gcl"
384
+ PBC: str = "pbc"
385
+ PXF_COMMODITY_PREPAYMENT: str = "pxf_commodity_prepayment"
386
+ INTER_BANK_LOAN: str = "inter_bank_loan"
387
+ OVERSEAS_PROJECT_CONTRACTING_LOAN: str = "overseas_project_contracting_loan"
388
+ DPA: str = "dpa"
389
+ PROJECT_FINANCE: str = "project_finance"
390
+ INVOLVING_MULTILATERAL: str = "involving_multilateral"
391
+ NON_CHINESE_FINANCIER: str = "non_chinese_financier"
392
+ SHORT_TERM: str = "short_term"
393
+ RESCUE: str = "rescue"
394
+ PROJECT_JV_SPV_HOST_GOVERNMENT_OWNERSHIP: str = (
395
+ "project_jv_spv_host_government_ownership"
396
+ )
397
+ PROJECT_JV_SPV_CHINESE_GOVERNMENT_OWNERSHIP: str = (
398
+ "project_jv_spv_chinese_government_ownership"
399
+ )
400
+ LEVEL_OF_PUBLIC_LIABILITY: str = "level_of_public_liability"
401
+ TOTAL_SOURCE_COUNT: str = "total_source_count"
402
+ OFFICIAL_SOURCE_COUNT: str = "official_source_count"
403
+ SOURCE_URLS: str = "source_urls"
404
+ SOURCE_TITLES: str = "source_titles"
405
+ SOURCE_PUBLISHERS: str = "source_publishers"
406
+ SOURCE_RESOURCE_TYPES: str = "source_resource_types"
407
+ CONTACT_NAME: str = "contact_name"
408
+ CONTACT_POSITION: str = "contact_position"
409
+ ODA_ELIGIBLE_RECIPIENT: str = "oda_eligible_recipient"
410
+ INCOME_GROUP_NAME: str = "income_group_name"
411
+ LOCATION_NARRATIVE: str = "location_narrative"
412
+ GEOGRAPHIC_PRECISION_LEVEL: str = "geographic_precision_level"
413
+ ADM1_LEVEL_AVAILABLE: str = "adm1_level_available"
414
+ ADM2_LEVEL_AVAILABLE: str = "adm2_level_available"
415
+ GEOSPATIAL_FEATURE_AVAILABLE: str = "geospatial_feature_available"
416
+ SOURCE_QUALITY: str = "source_quality"
417
+ DATA_COMPLETENESS: str = "data_completeness"
418
+ IMPLEMENTATION_DETAIL: str = "implementation_detail"
419
+ LOAN_DETAIL: str = "loan_detail"
@@ -9,7 +9,7 @@ class ODAPaths:
9
9
  raw_data = scripts / ".raw_data"
10
10
  pydeflate = raw_data / ".pydeflate"
11
11
  indicators = scripts / "indicators"
12
- names = scripts/ "tools"/ "names"
12
+ names = scripts / "tools" / "names"
13
13
  cleaning = scripts / "clean_data"
14
14
  settings = scripts / "settings"
15
15
  sectors = indicators / "sectors"
@@ -36,5 +36,3 @@ def dac1_aid_name_mapping() -> dict[int, str]:
36
36
  file_path = ODAPaths.settings / "dac1_aid_flow_types.json"
37
37
  mapping = load_json_file(file_path)
38
38
  return {int(k): v["aidtype_name"] for k, v in mapping.items()}
39
-
40
-
@@ -76,7 +76,7 @@ def core_oda() -> Indicator:
76
76
  code="ONE.10.1010C",
77
77
  name="Total Core ODA (ONE Definition)",
78
78
  description="Official Development Assistance (ODA) excluding in-donor spending but"
79
- "including administrative costs.",
79
+ "including administrative costs.",
80
80
  sources=["DAC1"],
81
81
  filters={
82
82
  "DAC1": {
@@ -49,7 +49,7 @@ class ODAData:
49
49
  currency=self.currency,
50
50
  base_year=self.base_year,
51
51
  use_bulk_download=self.use_bulk_download
52
- | self._mapping[indicator]["bulk"],
52
+ | self._mapping[indicator]["bulk"],
53
53
  ).get_indicators(indicators=self._mapping[indicator]["indicator"])
54
54
 
55
55
  self.indicators[indicator] = indicator_data
@@ -172,4 +172,4 @@ def get_merged_names_mapping():
172
172
  # Remove keys with empty dictionaries
173
173
  merged_mapping = {k: v for k, v in merged_mapping.items() if v}
174
174
 
175
- return merged_mapping
175
+ return merged_mapping
@@ -0,0 +1,35 @@
1
+ [tool.poetry]
2
+ name = "oda_data"
3
+ version = "2.0.3"
4
+ description = "A python package to work with Official Development Assistance data from the OECD DAC."
5
+ readme = "README.md"
6
+ authors = [
7
+ "Jorge Rivera <jorge.rivera@one.org>",
8
+ "Miguel Haro Ruiz <miguel.haroruiz@one.org>"
9
+ ]
10
+ license = "MIT"
11
+ homepage = "https://github.com/ONEcampaign/oda_data_package"
12
+ repository = "https://github.com/ONEcampaign/oda_data_package"
13
+ documentation = "https://github.com/ONEcampaign/oda_data_package"
14
+
15
+ [tool.poetry.dependencies]
16
+ python = ">=3.10,<4.0"
17
+ pandas = ">=2"
18
+ thefuzz = ">=0.22.1"
19
+ pyarrow = ">=14,<22"
20
+ pydeflate = ">=2.0.1"
21
+ cachetools = ">=5.5.1,<7.0.0"
22
+ oda-reader = "^1.2"
23
+
24
+ [tool.poetry.group.dev.dependencies]
25
+ black = ">=25"
26
+ pytest = ">=8"
27
+
28
+
29
+ [tool.poetry-dynamic-versioning]
30
+ enable = true
31
+ style = "pep440"
32
+
33
+ [build-system]
34
+ requires = ["poetry-core>=2.0.0,<3.0.0"]
35
+ build-backend = "poetry.core.masonry.api"
@@ -1,31 +0,0 @@
1
- [project]
2
- name = "oda_data"
3
- version = "2.0.2"
4
- description = "A python package to work with Official Development Assistance data from the OECD DAC."
5
- readme = "README.md"
6
- authors = [{ name = "Jorge Rivera", email = "jorge.rivera@one.org" }, { name = "Miguel Haro Ruiz", email = "miguel.haroruiz@one.org" }]
7
- requires-python = ">=3.10, <4.0"
8
- dependencies = ["pandas >=2",
9
- "thefuzz >=0.22.1",
10
- "pyarrow >=14, <20",
11
- "oda-reader >=1.1.3",
12
- "pydeflate >=2.0.1",
13
- "cachetools (>=5.5.1,<6.0.0)"]
14
-
15
- [project.optional-dependencies]
16
- dev = ["black >=24",
17
- "pytest >=8",
18
- "pytest-cov >=4.0.0",
19
- "pytest-mock >=3.10.0"]
20
-
21
- [tool.poetry-dynamic-versioning]
22
- enable = true
23
- style = "pep440"
24
-
25
- [project.urls]
26
- Homepage = "https://github.com/ONEcampaign/oda_data_package"
27
- Bug_Tracker = "https://github.com/ONEcampaign/oda_data_package/issues"
28
-
29
- [build-system]
30
- requires = ["poetry-core>=2.0.0,<3.0.0"]
31
- build-backend = "poetry.core.masonry.api"
File without changes
File without changes
File without changes
File without changes
File without changes