hdx-python-scraper 2.3.8__py3-none-any.whl → 2.4.0__py3-none-any.whl

hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.8'
-__version_tuple__ = version_tuple = (2, 3, 8)
+__version__ = version = '2.4.0'
+__version_tuple__ = version_tuple = (2, 4, 0)

hdx/scraper/configurable/rowparser.py

@@ -185,20 +185,14 @@ class RowParser:
         Returns:
             Iterator[Dict]: Input data with prefilter applied if specified and sorted if specified or deemed necessary
         """
-        rows = []
-        for row in iterator:
-            if self.header_to_hxltag:
-                newrow = {}
-                for header in row:
-                    newrow[self.header_to_hxltag[header]] = row[header]
-                row = newrow
-            if self.stop_row:
-                if all(
-                    row[key] == value for key, value in self.stop_row.items()
-                ):
-                    break
-            for newrow in self.flatten(row):
-                rows.append(newrow)
+        if self.header_to_hxltag:
+            iterator = self.header_to_hxltag_rows(iterator)
+        if self.stop_row:
+            iterator = self.stop_rows(iterator)
+        if self.flatteninfo:
+            iterator = self.flatten_rows(iterator)
+        if self.prefilter:
+            iterator = (row for row in iterator if eval(self.prefilter))
         if not self.sort:
             if self.datecol:
                 for subset in self.subsets:
@@ -212,15 +206,59 @@ class RowParser:
                         )
                         self.sort = {"keys": [self.datecol], "reverse": True}
                         break
-        if self.prefilter:
-            rows = [row for row in rows if eval(self.prefilter)]
         if self.sort:
             keys = self.sort["keys"]
             reverse = self.sort.get("reverse", False)
-            rows = sorted(rows, key=itemgetter(*keys), reverse=reverse)
-        return rows
+            iterator = sorted(iterator, key=itemgetter(*keys), reverse=reverse)
+        return iterator
+
+    def header_to_hxltag_rows(
+        self, iterator: Iterator[Dict]
+    ) -> Generator[Dict, None, None]:
+        """Convert headers to HXL tags in keys
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Rows where keys are HXL tags
+        """
+        for row in iterator:
+            newrow = {}
+            for header in row:
+                newrow[self.header_to_hxltag[header]] = row[header]
+            yield newrow
+
+    def stop_rows(
+        self, iterator: Iterator[Dict]
+    ) -> Generator[Dict, None, None]:
+        """Stop processing rows after condition met
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Rows up to stop condition
+        """
+        for row in iterator:
+            if all(row[key] == value for key, value in self.stop_row.items()):
+                break
+            yield row
+
+    def flatten_rows(self, iterator: Iterator[Dict]) -> Iterator[Dict]:
+        """Flatten rows
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Flattened rows
+        """
+        for row in iterator:
+            for newrow in self.flatten_row(row):
+                yield newrow
 
-    def flatten(self, row: Dict) -> Generator[Dict, None, None]:
+    def flatten_row(self, row: Dict) -> Generator[Dict, None, None]:
         """Flatten a wide spreadsheet format into a long one
 
         Args:
@@ -229,9 +267,6 @@ class RowParser:
         Returns:
             Generator[Dict]: Flattened row(s)
         """
-        if not self.flatteninfo:
-            yield row
-            return
         counters = [-1 for _ in self.flatteninfo]
         while True:
             newrow = copy.deepcopy(row)

hdx/scraper/utilities/reader.py

@@ -206,15 +206,18 @@ class Read(Retrieve):
         if headers is None:
             headers = 1
             datasetinfo["headers"] = 1
-        kwargs["headers"] = headers
-        if isinstance(headers, list):
-            kwargs["fill_merged_cells"] = True
         format = datasetinfo["format"]
         kwargs["format"] = format
-        if not sheet and format in ("xls", "xlsx"):
-            sheet = 1
+        if format in ("xls", "xlsx"):
+            if not sheet:
+                sheet = 1
+            if isinstance(headers, list):
+                kwargs["fill_merged_cells"] = True
+            elif "fill_merged_cells" not in kwargs:
+                kwargs["fill_merged_cells"] = False
         if sheet:
             kwargs["sheet"] = sheet
+        kwargs["headers"] = headers
         compression = datasetinfo.get("compression")
         if compression:
             kwargs["compression"] = compression
@@ -302,7 +305,7 @@ class Read(Retrieve):
                 f"Using saved datasets in {filename}_n.json in {self.saved_dir}"
             )
             datasets = []
-            for file_path in glob.glob(f"{saved_path}_*.json"):
+            for file_path in sorted(glob.glob(f"{saved_path}_*.json")):
                 datasets.append(Dataset.load_from_json(file_path))
         else:
             datasets = Dataset.search_in_hdx(

{hdx_python_scraper-2.3.8.dist-info → hdx_python_scraper-2.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-scraper
-Version: 2.3.8
+Version: 2.4.0
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -27,8 +27,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
 Requires-Dist: hdx-python-api>=6.3.1
-Requires-Dist: hdx-python-country>=3.7.2
-Requires-Dist: hdx-python-utilities>=3.7.0
+Requires-Dist: hdx-python-country>=3.7.6
+Requires-Dist: hdx-python-utilities>=3.7.2
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_scraper-2.3.8.dist-info → hdx_python_scraper-2.4.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 hdx/scraper/__init__.py,sha256=11ozJKiUsqDCZ3_mcAHhGYUyGK_Unl54djVSBBExFB4,59
-hdx/scraper/_version.py,sha256=F6Kqj0YPILNxwibWkU1tRl9QTGLZQ0ppBHxok1TGI5I,411
+hdx/scraper/_version.py,sha256=NXpAHvzuYHxlLDJV0489874frLu4dA2joFw1iHLLrOg,411
 hdx/scraper/base_scraper.py,sha256=2eJifpb8G_KtEb9Z273suDCiMPteJsCBHwDEk3o0wA8,15433
 hdx/scraper/runner.py,sha256=v5ToiTBOvFbkMOcBAoWGmDyO5bhGooTL8pPIt3BIQ8Y,53550
 hdx/scraper/configurable/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdx/scraper/configurable/aggregator.py,sha256=xC7bOF-wrQ17LlvdjSZUnUGuZHlNMH5jlmLSgyz5pe0,14976
 hdx/scraper/configurable/resource_downloader.py,sha256=lCIQpNZtcCTRc3z0FFM2_JxRtoua9GEq2XiKRZ9fqZk,1549
-hdx/scraper/configurable/rowparser.py,sha256=GS2KYn3Q-r9OOd_PIs5ebA_30pSkwBOju4ZXiAEOqnU,14643
+hdx/scraper/configurable/rowparser.py,sha256=bH05JUqViIVes9T7gWp0D2778BlFiJuNHmdovSFdFoI,15614
 hdx/scraper/configurable/scraper.py,sha256=4f4kNbG0HCIfPe1ft93T247s841rk1fP4cIpkFQ6NWU,20594
 hdx/scraper/configurable/timeseries.py,sha256=oAby_sGL6NmRoKnDG_fMB952W9zvzujPIsXkbqcXv-o,3027
 hdx/scraper/outputs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,11 +15,11 @@ hdx/scraper/outputs/googlesheets.py,sha256=gPjzikxP4wmMBGL5LW50MXUcDq5nwCRMW74G1
 hdx/scraper/outputs/json.py,sha256=uw9_yAVpHVPWQ8LtMUZKTH88okyrHQs_SVjT6HJOxZ4,9498
 hdx/scraper/utilities/__init__.py,sha256=dvbp0qTV-kLvN4Xp0GQf8LjN3IqlytW1eaTmDjlyZy0,2391
 hdx/scraper/utilities/fallbacks.py,sha256=08tvqVFuFV_gsvS7jqEiJUr7gqNILKCakDa8xMuIMpI,6186
-hdx/scraper/utilities/reader.py,sha256=03S53U1GylPaeRoqEj3TT5UgiKTwVODUx3IETwCb9ps,26364
+hdx/scraper/utilities/reader.py,sha256=A8GeMAie9swvydouBeD3hPi2YuH8liBsLJsEiqUqqfw,26500
 hdx/scraper/utilities/region_lookup.py,sha256=VSfIoBGmhS0lNgwe4kKIhHqP7k0DlJYI2JDdABAAmoM,3917
 hdx/scraper/utilities/sources.py,sha256=KuhaTvvGzjuw0dbhWpmPFvSq5RWP9cY83nl687O3CSs,11513
 hdx/scraper/utilities/writer.py,sha256=x-3xnOjvZEMUR2Op42eiBbaSmtNM6MY86adnL_Cob9s,16726
-hdx_python_scraper-2.3.8.dist-info/METADATA,sha256=4AHZxyFFH8srfK4eIRZ3FUhxp5zG-nBy5ucPaFnFQiQ,3361
-hdx_python_scraper-2.3.8.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-hdx_python_scraper-2.3.8.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
-hdx_python_scraper-2.3.8.dist-info/RECORD,,
+hdx_python_scraper-2.4.0.dist-info/METADATA,sha256=TZmLTpIowY1ta86DPlupiocA_uLH4Vda7uANNh9j38E,3361
+hdx_python_scraper-2.4.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+hdx_python_scraper-2.4.0.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_scraper-2.4.0.dist-info/RECORD,,

{hdx_python_scraper-2.3.8.dist-info → hdx_python_scraper-2.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any