hdx-python-scraper 2.3.0__py3-none-any.whl → 2.3.2__py3-none-any.whl

hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.0'
-__version_tuple__ = version_tuple = (2, 3, 0)
+__version__ = version = '2.3.2'
+__version_tuple__ = version_tuple = (2, 3, 2)

hdx/scraper/base_scraper.py

@@ -16,6 +16,7 @@ class BaseScraper(ABC):
         datasetinfo (Dict): Information about dataset
         headers (Dict[str, Tuple]): Headers to be oytput at each level_name
         source_configuration (Dict): Configuration for sources. Defaults to empty dict (use defaults).
+        reader (str): Reader to use. Defaults to "" (datasetinfo reader falling back on name).
     """
 
     population_lookup = {}
@@ -26,15 +27,20 @@ class BaseScraper(ABC):
         datasetinfo: Dict,
         headers: Dict[str, Tuple],
         source_configuration: Dict = {},
+        reader: str = "",
     ) -> None:
-        self.setup(name, headers, source_configuration)
+        self.name = name
+        if reader:
+            self.reader = reader
+        else:
+            self.reader = datasetinfo.get("reader", name)
+        self.setup(headers, source_configuration)
         self.datasetinfo = deepcopy(datasetinfo)
         self.errors_on_exit = None
         self.can_fallback = True
 
     def setup(
         self,
-        name: str,
         headers: Dict[str, Tuple],
         source_configuration: Dict = {},
     ) -> None:
@@ -42,14 +48,12 @@ class BaseScraper(ABC):
         {"national": (("School Closure",), ("#impact+type",)), ...},
 
         Args:
-            name (str): Name of scraper
             headers (Dict[str, Tuple]): Headers to be output at each level_name
             source_configuration (Dict): Configuration for sources. Defaults to empty dict (use defaults).
 
         Returns:
              None
         """
-        self.name = name
         self.headers = headers
         self.initialise_values_sources(source_configuration)
         self.has_run = False
@@ -92,7 +96,7 @@ class BaseScraper(ABC):
              None
         """
         if not name:
-            name = self.name
+            name = self.reader
         reader = Read.get_reader(name)
         return reader
 
@@ -361,10 +365,16 @@ class BaseScraper(ABC):
             return None
         if "is_hxl" in hapi_resource_metadata:
             return hapi_resource_metadata
-        reader = self.get_reader(self.name)
+        reader = self.get_reader()
         filename = self.datasetinfo.get("filename")
+        file_prefix = self.datasetinfo.get("file_prefix", self.name)
+        if filename:
+            kwargs = {"filename": filename}
+        else:
+            kwargs = {"file_prefix": file_prefix}
         hxl_info = reader.hxl_info_hapi_resource_metadata(
-            hapi_resource_metadata, filename=filename, file_prefix=self.name
+            hapi_resource_metadata,
+            **kwargs,
         )
         is_hxl = False
         if hxl_info:

hdx/scraper/configurable/scraper.py

@@ -70,6 +70,8 @@ class ConfigurableScraper(BaseScraper):
         errors_on_exit: Optional[ErrorsOnExit] = None,
         **kwargs: Any,
     ):
+        self.name = name
+        self.reader = datasetinfo.get("reader", name)
         self.level = level
         datelevel = datasetinfo.get("date_level")
         if datelevel is None:
@@ -98,11 +100,11 @@ class ConfigurableScraper(BaseScraper):
             use_hxl = self.datasetinfo.get("use_hxl", False)
             if use_hxl:
                 try:
-                    file_headers, iterator = self.get_iterator(name)
+                    file_headers, iterator = self.get_iterator()
                     self.use_hxl(headers, file_headers, iterator)
                 except DownloadError:
                     self.can_fallback = False
-        self.setup(name, headers, source_configuration)
+        self.setup(headers, source_configuration)
 
     @staticmethod
     def get_subsets_from_datasetinfo(datasetinfo: Dict) -> List[Dict]:
@@ -136,20 +138,18 @@ class ConfigurableScraper(BaseScraper):
             ]
         return subsets
 
-    def get_iterator(self, name: str) -> Tuple[List[str], Iterator[Dict]]:
-        """Get the iterator from the preconfigured reader for the given scraper name
-
-        Args:
-            name (str): Name of scraper
+    def get_iterator(self) -> Tuple[List[str], Iterator[Dict]]:
+        """Get the iterator from the preconfigured reader for this scraper
 
         Returns:
             Tuple[List[str],Iterator[Dict]]: Tuple (headers, iterator where each row is a dictionary)
         """
-        return self.get_reader(name).read(
-            self.datasetinfo,
-            file_prefix=name,
-            **self.variables,
-        )
+        if (
+            "filename" not in self.datasetinfo
+            and "file_prefix" not in self.datasetinfo
+        ):
+            self.datasetinfo["file_prefix"] = self.name
+        return self.get_reader().read(self.datasetinfo, **self.variables)
 
     def add_sources(self) -> None:
         """Add source for each HXL hashtag
@@ -466,7 +466,7 @@ class ConfigurableScraper(BaseScraper):
         Returns:
             None
         """
-        file_headers, iterator = self.get_iterator(self.name)
+        file_headers, iterator = self.get_iterator()
         header_to_hxltag = self.use_hxl(None, file_headers, iterator)
         if "source_url" not in self.datasetinfo:
             self.datasetinfo["source_url"] = self.datasetinfo["url"]

hdx/scraper/configurable/timeseries.py

@@ -50,7 +50,7 @@ class TimeSeries(BaseScraper):
             "output_hxl"
         ]
         rows = [headers, hxltags]
-        file_headers, iterator = self.get_reader(self.name).read(
+        file_headers, iterator = self.get_reader().read(
             self.datasetinfo, file_prefix=self.name
         )
         for inrow in iterator:

hdx/scraper/runner.py

@@ -10,7 +10,7 @@ from .configurable.resource_downloader import ResourceDownloader
 from .configurable.scraper import ConfigurableScraper
 from .configurable.timeseries import TimeSeries
 from .outputs.base import BaseOutput
-from .utilities import get_startend_dates_from_reference_period
+from .utilities import get_startend_dates_from_time_period
 from .utilities.fallbacks import Fallbacks
 from .utilities.reader import Read
 from .utilities.sources import Sources
@@ -29,7 +29,7 @@ class Runner:
         countryiso3s (ListTuple[str]): List of ISO3 country codes to process
         today (datetime): Value to use for today. Defaults to now_utc().
         errors_on_exit (ErrorsOnExit): ErrorsOnExit object that logs errors on exit
-        scrapers_to_run (Optional[ListTuple[str]]): Scrapers to run. Defaults to None.
+        scrapers_to_run (Optional[ListTuple[str]]): Scrapers to run. Defaults to None (all scrapers).
     """
 
     def __init__(
@@ -1061,7 +1061,7 @@ class Runner:
                 if dataset_name:
                     dataset = reader.read_dataset(dataset_name)
                     if date is None:
-                        date = get_startend_dates_from_reference_period(
+                        date = get_startend_dates_from_time_period(
                             dataset, today=self.today
                         )
                     if source_name is None:
@@ -1209,32 +1209,52 @@ class Runner:
         """
         if not names:
             names = self.scrapers.keys()
-        results = {}
+        hapi_results = {}
 
         def add_results(scraper_level, scrap, levels_used):
-            nonlocal results
+            nonlocal hapi_results
 
             if scraper_level in levels_used:
                 return
             headers = scrap.headers.get(scraper_level)
             if headers is None:
                 return
+            headings = headers[0]
+            hxltags = headers[1]
             values = scrap.get_values(scraper_level)
             hapi_dataset_metadata = scrap.get_hapi_dataset_metadata()
+            if not hapi_dataset_metadata:
+                return
             hapi_resource_metadata = scrap.get_hapi_resource_metadata()
+            if not hapi_resource_metadata:
+                return
             dataset_id = hapi_dataset_metadata["hdx_id"]
-            hapi_metadata = results.get(
+            hapi_metadata = hapi_results.get(
                 dataset_id, copy(hapi_dataset_metadata)
             )
-            level_results = hapi_metadata.get("results", {})
-            level_results[scraper_level] = {
-                "headers": headers,
-                "values": values,
-                "hapi_resource_metadata": hapi_resource_metadata,
-            }
-            hapi_metadata["results"] = level_results
+            results = hapi_metadata.get("results", {})
+            level_results = results.get(scraper_level)
+            if level_results is None:
+                level_results = {
+                    "headers": ([], []),
+                    "values": [],
+                    "hapi_resource_metadata": hapi_resource_metadata,
+                }
+                results[scraper_level] = level_results
+            lev_headings = level_results["headers"][0]
+            lev_hxltags = level_results["headers"][1]
+            lev_values = level_results["values"]
+            for i, hxltag in enumerate(hxltags):
+                if hxltag in lev_hxltags:
+                    index = lev_hxltags.index(hxltag)
+                    lev_values[index].update(values[i])
+                else:
+                    lev_headings.append(headings[i])
+                    lev_hxltags.append(hxltag)
+                    lev_values.append(values[i])
+            hapi_metadata["results"] = results
             levels_used.add(scraper_level)
-            results[dataset_id] = hapi_metadata
+            hapi_results[dataset_id] = hapi_metadata
 
         for name in names:
             if self.scrapers_to_run and not any(
@@ -1247,4 +1267,4 @@ class Runner:
             lvls_used = set()
             for scrap_level in scraper.headers:
                 add_results(scrap_level, scraper, lvls_used)
-        return results
+        return hapi_results

hdx/scraper/utilities/__init__.py

@@ -57,22 +57,22 @@ def get_rowval(row: Dict, valcol: str) -> Any:
         return result
 
 
-def get_startend_dates_from_reference_period(
+def get_startend_dates_from_time_period(
     dataset: Dataset, today: Optional[datetime] = None
 ) -> Optional[Dict]:
-    """Return the reference period in form required for source_date
+    """Return the time period in form required for source_date
 
     Args:
         dataset (Dataset): Dataset object
         today (Optional[datetime]): Date to use for today. Defaults to None (datetime.utcnow)
 
     Returns:
-        Optional[Dict]: Reference period in form required for source_date
+        Optional[Dict]: Time period in form required for source_date
     """
     if today is None:
-        date_info = dataset.get_reference_period()
+        date_info = dataset.get_time_period()
     else:
-        date_info = dataset.get_reference_period(today=today)
+        date_info = dataset.get_time_period(today=today)
     startdate = date_info.get("startdate")
     enddate = date_info.get("enddate")
     if enddate is None:

hdx/scraper/utilities/reader.py

@@ -8,7 +8,7 @@ import hxl
 from hxl.input import InputOptions, munge_url
 from slugify import slugify
 
-from . import get_startend_dates_from_reference_period, match_template
+from . import get_startend_dates_from_time_period, match_template
 from .sources import Sources
 from hdx.data.dataset import Dataset
 from hdx.data.resource import Resource
@@ -224,6 +224,13 @@ class Read(Retrieve):
             filename = datasetinfo.get("filename")
             if filename:
                 kwargs["filename"] = filename
+        if filename:
+            # remove file_prefix if filename provided
+            kwargs.pop("file_prefix", None)
+        elif "file_prefix" not in kwargs:
+            file_prefix = datasetinfo.get("file_prefix")
+            if file_prefix:
+                kwargs["file_prefix"] = file_prefix
         return self.get_tabular_rows(
             url,
             dict_form=True,
@@ -311,7 +318,7 @@ class Read(Retrieve):
         """
         return self.construct_filename_and_download(
             resource["name"],
-            resource.get_file_type(),
+            resource.get_format(),
             resource["url"],
             **kwargs,
         )
@@ -331,7 +338,7 @@ class Read(Retrieve):
             "title": dataset["title"],
             "hdx_provider_stub": dataset["organization"]["name"],
             "hdx_provider_name": dataset["organization"]["title"],
-            "reference_period": dataset.get_reference_period(today=self.today),
+            "reference_period": dataset.get_time_period(today=self.today),
         }
 
     @staticmethod
@@ -477,7 +484,7 @@ class Read(Retrieve):
             if "source_date" not in datasetinfo:
                 datasetinfo[
                     "source_date"
-                ] = get_startend_dates_from_reference_period(
+                ] = get_startend_dates_from_time_period(
                     dataset, today=self.today
                 )
             if "source" not in datasetinfo:
@@ -510,7 +517,7 @@ class Read(Retrieve):
                     key = "default_date"
                 else:
                     key = hxltag
-                source_date[key] = get_startend_dates_from_reference_period(
+                source_date[key] = get_startend_dates_from_time_period(
                     dataset, today=self.today
                 )
             if source is not None:
@@ -555,10 +562,14 @@ class Read(Retrieve):
             datasetinfo["filename"] = filename
         filename = datasetinfo.get("filename")
         if resource and not filename:
-            # prefix is added later
             filename = self.construct_filename(
-                resource["name"], resource.get_file_type()
+                resource["name"], resource.get_format()
             )
+            file_prefix = kwargs.get("file_prefix")
+            if not file_prefix:
+                file_prefix = datasetinfo.get("file_prefix")
+            if file_prefix:
+                filename = f"{file_prefix}_{filename}"
             datasetinfo["filename"] = filename
         return self.read_tabular(datasetinfo, **kwargs)
 

{hdx_python_scraper-2.3.0.dist-info → hdx_python_scraper-2.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hdx-python-scraper
-Version: 2.3.0
+Version: 2.3.2
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,7 +26,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.1.4
+Requires-Dist: hdx-python-api>=6.2.0
 Requires-Dist: hdx-python-country>=3.6.3
 Requires-Dist: regex
 Provides-Extra: dev

{hdx_python_scraper-2.3.0.dist-info → hdx_python_scraper-2.3.2.dist-info}/RECORD

@@ -1,25 +1,25 @@
 hdx/scraper/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
-hdx/scraper/_version.py,sha256=ChsIHG8bRc-eXUbXOgv4Fm4DstSKLq9FpsTAsaMeR08,411
-hdx/scraper/base_scraper.py,sha256=OZoC8X3woecKbMxTtjx_aRr027SeJCS2gbtyB20n31o,15079
-hdx/scraper/runner.py,sha256=fojFcfEh3mZXe1dY3Jpis22dr9Zc6VY-0XTMiabuXFE,51366
+hdx/scraper/_version.py,sha256=aKqtdIqWETcZnGj_9koZ-EQK7itBfKLMIKY7ucdTIMI,411
+hdx/scraper/base_scraper.py,sha256=oo9oMqCUpK8_hPwcTz2PAKabzoyU0BQu5dgWgsFa55Y,15431
+hdx/scraper/runner.py,sha256=KIEVLSJwEw9fzQxqsN92c50yDG3CRYAVDO7A6Zv_KJY,52262
 hdx/scraper/configurable/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/scraper/configurable/aggregator.py,sha256=xC7bOF-wrQ17LlvdjSZUnUGuZHlNMH5jlmLSgyz5pe0,14976
 hdx/scraper/configurable/resource_downloader.py,sha256=lCIQpNZtcCTRc3z0FFM2_JxRtoua9GEq2XiKRZ9fqZk,1549
 hdx/scraper/configurable/rowparser.py,sha256=h7a0W2xvVJSAu94nS5CAXvZSZXdwZ-isFHHNaIce0gM,14635
-hdx/scraper/configurable/scraper.py,sha256=kBkS-bm4zIQ9jbzFcwVoAnyji_9PTV_KKrNJVLTuYa4,20498
-hdx/scraper/configurable/timeseries.py,sha256=lWoQJApml-onTN4l9YnTAYnhj5uuTc-Luk05DIT7O9k,3036
+hdx/scraper/configurable/scraper.py,sha256=4f4kNbG0HCIfPe1ft93T247s841rk1fP4cIpkFQ6NWU,20594
+hdx/scraper/configurable/timeseries.py,sha256=oAby_sGL6NmRoKnDG_fMB952W9zvzujPIsXkbqcXv-o,3027
 hdx/scraper/outputs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/scraper/outputs/base.py,sha256=UBVFPANdd7wawifbKkPQWKwVC-Tr7Jg5ax1eLTmWX3M,2566
 hdx/scraper/outputs/excelfile.py,sha256=bKBj1aYUJCIXhvpmGXAJ0FLoKwjnj-2E0LlR64RcFdY,2197
 hdx/scraper/outputs/googlesheets.py,sha256=gPjzikxP4wmMBGL5LW50MXUcDq5nwCRMW74G1Ep39QY,3087
 hdx/scraper/outputs/json.py,sha256=uw9_yAVpHVPWQ8LtMUZKTH88okyrHQs_SVjT6HJOxZ4,9498
-hdx/scraper/utilities/__init__.py,sha256=iBjD7bc8wEzQhwkcx2mOZwYmu28VHjl5px66quqWJ8E,2491
+hdx/scraper/utilities/__init__.py,sha256=1IaNOMhAxjGRDKUHSM_ENFcPRn0vw499K9iTX4LvCS0,2466
 hdx/scraper/utilities/fallbacks.py,sha256=08tvqVFuFV_gsvS7jqEiJUr7gqNILKCakDa8xMuIMpI,6186
-hdx/scraper/utilities/reader.py,sha256=9cXrk8_NrE4kHIm3wrM3KHgKX6bho_eCyibMDBairiU,21499
+hdx/scraper/utilities/reader.py,sha256=HaR0da1my59P1T4sYe15GwX5cf5m4UbMo1r2uR9yvP8,21963
 hdx/scraper/utilities/region_lookup.py,sha256=VSfIoBGmhS0lNgwe4kKIhHqP7k0DlJYI2JDdABAAmoM,3917
 hdx/scraper/utilities/sources.py,sha256=h27PjBADqIhqDwmhzMXt1OjwJWZc2iVnIBwJuAJKHwo,11204
 hdx/scraper/utilities/writer.py,sha256=x-3xnOjvZEMUR2Op42eiBbaSmtNM6MY86adnL_Cob9s,16726
-hdx_python_scraper-2.3.0.dist-info/METADATA,sha256=E5b13txhk44RjnOSKJu_SkaypNFXxe5YDLUBCWKA7Pk,3318
-hdx_python_scraper-2.3.0.dist-info/WHEEL,sha256=0wCxn4rnLsvRWBK-NC7mK2QMIQ_aZSl7Qvk-8IWl_pY,87
-hdx_python_scraper-2.3.0.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
-hdx_python_scraper-2.3.0.dist-info/RECORD,,
+hdx_python_scraper-2.3.2.dist-info/METADATA,sha256=cDyJQpQAf7U486xjcUYFbyoVm2fPLZPu8mLMG7GWVMU,3318
+hdx_python_scraper-2.3.2.dist-info/WHEEL,sha256=mRYSEL3Ih6g5a_CVMIcwiF__0Ae4_gLYh01YFNwiq1k,87
+hdx_python_scraper-2.3.2.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_scraper-2.3.2.dist-info/RECORD,,

{hdx_python_scraper-2.3.0.dist-info → hdx_python_scraper-2.3.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.19.1
+Generator: hatchling 1.21.0
 Root-Is-Purelib: true
 Tag: py3-none-any