hdx-python-scraper 2.2.0__py3-none-any.whl → 2.2.1__py3-none-any.whl

hdx/scraper/_version.py

@@ -1,6 +1,16 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-from __future__ import annotations
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
 
-__version__ = version = '2.2.0'  # type: str
-__version_tuple__ = version_tuple = (2, 2, 0)  # type: tuple[int | str, ...]
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '2.2.1'
+__version_tuple__ = version_tuple = (2, 2, 1)

hdx/scraper/configurable/scraper.py

@@ -122,6 +122,7 @@ class ConfigurableScraper(BaseScraper):
                     "input": datasetinfo.get("input", []),
                     "transform": datasetinfo.get("transform", {}),
                     "population_key": datasetinfo.get("population_key"),
+                    "list": datasetinfo.get("list", []),
                     "process": datasetinfo.get("process", []),
                     "input_keep": datasetinfo.get("input_keep", []),
                     "input_append": datasetinfo.get("input_append", []),
@@ -292,6 +293,7 @@ class ConfigurableScraper(BaseScraper):
                 filter = subset["filter"]
                 input_ignore_vals = subset.get("input_ignore_vals", [])
                 input_transforms = subset.get("transform", {})
+                list_cols = subset.get("list")
                 sum_cols = subset.get("sum")
                 process_cols = subset.get("process")
                 input_append = subset.get("input_append", [])
@@ -304,6 +306,8 @@ class ConfigurableScraper(BaseScraper):
                         val = eval(input_transform.replace(valcol, "val"))
                     if sum_cols or process_cols:
                         dict_of_lists_add(valuedict, adm, val)
+                    elif list_cols and valcol in list_cols:
+                        dict_of_lists_add(valuedict, adm, val)
                     else:
                         curval = valuedict.get(adm)
                         if valcol in input_append:
@@ -326,6 +330,7 @@ class ConfigurableScraper(BaseScraper):
                 population_str = "self.population_lookup[adm]"
             else:
                 population_str = "self.population_lookup[population_key]"
+            subset.get("list")
             process_cols = subset.get("process")
             input_keep = subset.get("input_keep", [])
             sum_cols = subset.get("sum")
@@ -440,7 +445,7 @@ class ConfigurableScraper(BaseScraper):
                             valcols[i], f"newvaldicts[{i}][adm]"
                         )
                     formula = formula.replace("#pzbgvjh", population_str)
-                    for adm in valdicts[0].keys():
+                    for adm in valdicts[0]:
                         try:
                             val = eval(formula)
                         except (ValueError, TypeError, KeyError):

hdx/scraper/runner.py

@@ -1147,7 +1147,10 @@ class Runner:
     def get_hapi_metadata(
         self, names: Optional[ListTuple[str]] = None
     ) -> Dict:
-        """Get HAPI metadata for all datasets
+        """Get HAPI metadata for all datasets. A dictionary is returned that
+        maps from dataset ids to a dictionary. The dictionary has keys for
+        dataset metadata and a key resources under which is a dictionary that
+        maps from resource ids to resource metadata.
 
         Args:
             names (Optional[ListTuple[str]]): Names of scrapers
@@ -1184,12 +1187,15 @@ class Runner:
         metadata) for scrapers limiting to those in names if given and limiting
         further to those that have been set in the constructor if previously
         given. By default, only scrapers marked as having run are returned
-        unless has_run is set to False. A dictionary is returned where key is
-        HDX dataset id and value is a dictionary that has HAPI dataset metadata
-        as well as a results key. The value associated with the results key is
-        a dictionary where each key is an admin level. Each admin level key has
-        a value dictionary with headers, values and HAPI resource metadata.
-        Headers is a tuple of (column headers, hxl hashtags). Values is a list.
+        unless has_run is set to False.
+
+        A dictionary is returned where key is HDX dataset id and value is a
+        dictionary that has HAPI dataset metadata as well as a results key.
+        The value associated with the results key is a dictionary where each
+        key is an admin level. Each admin level key has a value dictionary with
+        headers, values and HAPI resource metadata. Headers is a tuple of
+        (column headers, hxl hashtags). Values is a list. HAPI resource
+        metadata is a dictionary.
 
         Args:
             names (Optional[ListTuple[str]]): Names of scrapers. Defaults to None (all scrapers).

hdx/scraper/utilities/reader.py

@@ -340,9 +340,16 @@ class Read(Retrieve):
     def read_hdx_metadata(
         self, datasetinfo: Dict, do_resource_check: bool = True
     ) -> Optional[Resource]:
-        """Read metadata from HDX dataset and add to input dictionary. If url is not
-        supplied, will look through resources for one that matches specified format and
-        use its url unless do_resource_check is False.
+        """Read metadata from HDX dataset and add to input dictionary. If url
+        is not supplied, will look through resources for one that matches
+        specified format and use its url unless do_resource_check is False.
+        The dataset key of the parameter datasetinfo will usually point to a
+        string (single dataset) but where sources vary across HXL tags can be
+        a dictionary that maps from HXL tags to datasets with the key
+        default_dataset setting a default for HXL tags. For a single dataset,
+        the keys hapi_dataset_metadata and hapi_resource_metadata will be
+        populated with more detailed dataset and resource information required
+        by HAPI.
 
         Args:
             datasetinfo (Dict): Dictionary of information about dataset

{hdx_python_scraper-2.2.0.dist-info → hdx_python_scraper-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hdx-python-scraper
-Version: 2.2.0
+Version: 2.2.1
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,12 +26,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.1.2
+Requires-Dist: hdx-python-api>=6.1.3
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: pandas
-Requires-Dist: pandas>=2.0.3; extra == 'pandas'
+Requires-Dist: pandas>=2.1.1; extra == 'pandas'
 Provides-Extra: test
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'

{hdx_python_scraper-2.2.0.dist-info → hdx_python_scraper-2.2.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 hdx/scraper/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
-hdx/scraper/_version.py,sha256=XTy2WNrumA7Z8HDQ_NiV6KQTscT0Qx-QCHUYar0vV7w,240
+hdx/scraper/_version.py,sha256=R_Wr7clGXr8a07n6uqFj88MyYFGydFRXYBI10R9k_uw,411
 hdx/scraper/base_scraper.py,sha256=IaUDqnrSxB0kbEQynX-81NEyv9DLxypWKwEDAEr9GWg,14628
-hdx/scraper/runner.py,sha256=xqn2awE70l08DgoPIpRAX5ifKF8S272fmp6Bq9HbKh0,50923
+hdx/scraper/runner.py,sha256=-7L-L9WGZdTGl5mWNAPgvpTreU9bvbdxklruGCRzjRs,51217
 hdx/scraper/configurable/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/scraper/configurable/aggregator.py,sha256=xC7bOF-wrQ17LlvdjSZUnUGuZHlNMH5jlmLSgyz5pe0,14976
 hdx/scraper/configurable/resource_downloader.py,sha256=vK8zNFy7T_Rj1h8Tj676-3B2oYYXFNKsrM9dxz7RZC8,1537
 hdx/scraper/configurable/rowparser.py,sha256=h7a0W2xvVJSAu94nS5CAXvZSZXdwZ-isFHHNaIce0gM,14635
-hdx/scraper/configurable/scraper.py,sha256=kVQpVMHEYizSq94PeSP119gi24a1XR_mzL4mtlAvK2M,20217
+hdx/scraper/configurable/scraper.py,sha256=TyB7ipTzhVpOC3in0ZBIMwbcTAOR0Ul-W6Np85NnogI,20468
 hdx/scraper/configurable/timeseries.py,sha256=uhnENo7Wsy0-YVjglm7OQkXI72-te61DkepkihbQrP8,2982
 hdx/scraper/outputs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/scraper/outputs/base.py,sha256=UBVFPANdd7wawifbKkPQWKwVC-Tr7Jg5ax1eLTmWX3M,2566
@@ -15,11 +15,11 @@ hdx/scraper/outputs/googlesheets.py,sha256=gPjzikxP4wmMBGL5LW50MXUcDq5nwCRMW74G1
 hdx/scraper/outputs/json.py,sha256=uw9_yAVpHVPWQ8LtMUZKTH88okyrHQs_SVjT6HJOxZ4,9498
 hdx/scraper/utilities/__init__.py,sha256=iBjD7bc8wEzQhwkcx2mOZwYmu28VHjl5px66quqWJ8E,2491
 hdx/scraper/utilities/fallbacks.py,sha256=08tvqVFuFV_gsvS7jqEiJUr7gqNILKCakDa8xMuIMpI,6186
-hdx/scraper/utilities/reader.py,sha256=BrCAxBibIPYp2gnkVacOTiPfeMKLEVULAqXzpaweCUY,17858
+hdx/scraper/utilities/reader.py,sha256=awm24AUWlweJmJVE1h0iid7xb6njvF7Taf0afbGXIG4,18331
 hdx/scraper/utilities/region_lookup.py,sha256=VSfIoBGmhS0lNgwe4kKIhHqP7k0DlJYI2JDdABAAmoM,3917
 hdx/scraper/utilities/sources.py,sha256=h27PjBADqIhqDwmhzMXt1OjwJWZc2iVnIBwJuAJKHwo,11204
 hdx/scraper/utilities/writer.py,sha256=x-3xnOjvZEMUR2Op42eiBbaSmtNM6MY86adnL_Cob9s,16726
-hdx_python_scraper-2.2.0.dist-info/METADATA,sha256=9xsBUu6DpgDmJZah70hsfj1Y7l9f-GgBtU2THR4UFv4,3289
-hdx_python_scraper-2.2.0.dist-info/WHEEL,sha256=9QBuHhg6FNW7lppboF2vKVbCGTVzsFykgRQjjlajrhA,87
-hdx_python_scraper-2.2.0.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
-hdx_python_scraper-2.2.0.dist-info/RECORD,,
+hdx_python_scraper-2.2.1.dist-info/METADATA,sha256=hnYCmTG7ZlGqfc4QKCHjBKSesZ2q7ooTbdtyAuuhkqs,3289
+hdx_python_scraper-2.2.1.dist-info/WHEEL,sha256=9QBuHhg6FNW7lppboF2vKVbCGTVzsFykgRQjjlajrhA,87
+hdx_python_scraper-2.2.1.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_scraper-2.2.1.dist-info/RECORD,,