hdx-python-scraper 2.3.8__tar.gz → 2.4.0__tar.gz

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-scraper
-Version: 2.3.8
+Version: 2.4.0
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -27,8 +27,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
 Requires-Dist: hdx-python-api>=6.3.1
-Requires-Dist: hdx-python-country>=3.7.2
-Requires-Dist: hdx-python-utilities>=3.7.0
+Requires-Dist: hdx-python-country>=3.7.6
+Requires-Dist: hdx-python-utilities>=3.7.2
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/pyproject.toml

@@ -35,8 +35,8 @@ requires-python = ">=3.8"
 
 dependencies = [
     "hdx-python-api>=6.3.1",
-    "hdx-python-country>=3.7.2",
-    "hdx-python-utilities>=3.7.0",
+    "hdx-python-country>=3.7.6",
+    "hdx-python-utilities>=3.7.2",
     "gspread",
     "regex",
 ]

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/requirements.txt

@@ -10,7 +10,7 @@ attrs==23.2.0
     #   referencing
 cachetools==5.3.3
     # via google-auth
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -24,7 +24,7 @@ ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-coverage==7.5.3
+coverage==7.6.0
     # via pytest-cov
 cryptography==42.0.8
     # via pyopenssl
@@ -40,36 +40,36 @@ docopt==0.6.2
     #   num2words
 docutils==0.21.2
     # via defopt
-email-validator==2.1.2
+email-validator==2.2.0
     # via hdx-python-api
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.15.3
+filelock==3.15.4
     # via virtualenv
 frictionless==5.17.0
     # via hdx-python-utilities
-google-auth==2.30.0
+google-auth==2.32.0
     # via
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==1.2.0
+google-auth-oauthlib==1.2.1
     # via gspread
 gspread==6.1.2
     # via hdx-python-scraper (pyproject.toml)
 hdx-python-api==6.3.1
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.7.2
+hdx-python-country==3.7.6
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.7.0
+hdx-python-utilities==3.7.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
-humanize==4.9.0
+humanize==4.10.0
     # via frictionless
-identify==2.5.36
+identify==2.6.0
     # via pre-commit
 idna==3.7
     # via
@@ -77,7 +77,7 @@ idna==3.7
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.2.1
+inflect==7.3.1
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -89,7 +89,7 @@ jsonlines==4.0.0
     # via hdx-python-utilities
 jsonpath-ng==1.6.1
     # via libhxl
-jsonschema==4.22.0
+jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
@@ -101,11 +101,11 @@ libhxl==5.2.1
     #   hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
-makefun==1.15.2
+makefun==1.15.4
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
-marko==2.1.1
+marko==2.1.2
     # via frictionless
 markupsafe==2.1.5
     # via jinja2
@@ -123,7 +123,7 @@ numpy==2.0.0
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
-openpyxl==3.1.4
+openpyxl==3.1.5
     # via hdx-python-utilities
 packaging==24.1
     # via pytest
@@ -153,9 +153,9 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.7.4
+pydantic==2.8.2
     # via frictionless
-pydantic-core==2.18.4
+pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
@@ -190,7 +190,7 @@ pyyaml==6.0.1
     #   frictionless
     #   pre-commit
     #   tableschema-to-template
-quantulum3==0.9.1
+quantulum3==0.9.2
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
@@ -216,7 +216,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.7.1
     # via typer
-rpds-py==0.18.1
+rpds-py==0.19.0
     # via
     #   jsonschema
     #   referencing
@@ -226,7 +226,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==70.1.0
+setuptools==70.3.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -260,7 +260,6 @@ typer==0.12.3
 typing-extensions==4.12.2
     # via
     #   frictionless
-    #   inflect
     #   pydantic
     #   pydantic-core
     #   typeguard
@@ -275,9 +274,9 @@ urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.28.3
+validators==0.32.0
     # via frictionless
-virtualenv==20.26.2
+virtualenv==20.26.3
     # via pre-commit
 wheel==0.43.0
     # via libhxl

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/src/hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.8'
-__version_tuple__ = version_tuple = (2, 3, 8)
+__version__ = version = '2.4.0'
+__version_tuple__ = version_tuple = (2, 4, 0)

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/src/hdx/scraper/configurable/rowparser.py

@@ -185,20 +185,14 @@ class RowParser:
         Returns:
             Iterator[Dict]: Input data with prefilter applied if specified and sorted if specified or deemed necessary
         """
-        rows = []
-        for row in iterator:
-            if self.header_to_hxltag:
-                newrow = {}
-                for header in row:
-                    newrow[self.header_to_hxltag[header]] = row[header]
-                row = newrow
-            if self.stop_row:
-                if all(
-                    row[key] == value for key, value in self.stop_row.items()
-                ):
-                    break
-            for newrow in self.flatten(row):
-                rows.append(newrow)
+        if self.header_to_hxltag:
+            iterator = self.header_to_hxltag_rows(iterator)
+        if self.stop_row:
+            iterator = self.stop_rows(iterator)
+        if self.flatteninfo:
+            iterator = self.flatten_rows(iterator)
+        if self.prefilter:
+            iterator = (row for row in iterator if eval(self.prefilter))
         if not self.sort:
             if self.datecol:
                 for subset in self.subsets:
@@ -212,15 +206,59 @@ class RowParser:
                         )
                         self.sort = {"keys": [self.datecol], "reverse": True}
                         break
-        if self.prefilter:
-            rows = [row for row in rows if eval(self.prefilter)]
         if self.sort:
             keys = self.sort["keys"]
             reverse = self.sort.get("reverse", False)
-            rows = sorted(rows, key=itemgetter(*keys), reverse=reverse)
-        return rows
+            iterator = sorted(iterator, key=itemgetter(*keys), reverse=reverse)
+        return iterator
+
+    def header_to_hxltag_rows(
+        self, iterator: Iterator[Dict]
+    ) -> Generator[Dict, None, None]:
+        """Convert headers to HXL tags in keys
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Rows where keys are HXL tags
+        """
+        for row in iterator:
+            newrow = {}
+            for header in row:
+                newrow[self.header_to_hxltag[header]] = row[header]
+            yield newrow
+
+    def stop_rows(
+        self, iterator: Iterator[Dict]
+    ) -> Generator[Dict, None, None]:
+        """Stop processing rows after condition met
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Rows up to stop condition
+        """
+        for row in iterator:
+            if all(row[key] == value for key, value in self.stop_row.items()):
+                break
+            yield row
+
+    def flatten_rows(self, iterator: Iterator[Dict]) -> Iterator[Dict]:
+        """Flatten rows
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Flattened rows
+        """
+        for row in iterator:
+            for newrow in self.flatten_row(row):
+                yield newrow
 
-    def flatten(self, row: Dict) -> Generator[Dict, None, None]:
+    def flatten_row(self, row: Dict) -> Generator[Dict, None, None]:
         """Flatten a wide spreadsheet format into a long one
 
         Args:
@@ -229,9 +267,6 @@ class RowParser:
         Returns:
             Generator[Dict]: Flattened row(s)
         """
-        if not self.flatteninfo:
-            yield row
-            return
         counters = [-1 for _ in self.flatteninfo]
         while True:
             newrow = copy.deepcopy(row)

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/src/hdx/scraper/utilities/reader.py

@@ -206,15 +206,18 @@ class Read(Retrieve):
         if headers is None:
             headers = 1
             datasetinfo["headers"] = 1
-        kwargs["headers"] = headers
-        if isinstance(headers, list):
-            kwargs["fill_merged_cells"] = True
         format = datasetinfo["format"]
         kwargs["format"] = format
-        if not sheet and format in ("xls", "xlsx"):
-            sheet = 1
+        if format in ("xls", "xlsx"):
+            if not sheet:
+                sheet = 1
+            if isinstance(headers, list):
+                kwargs["fill_merged_cells"] = True
+            elif "fill_merged_cells" not in kwargs:
+                kwargs["fill_merged_cells"] = False
         if sheet:
             kwargs["sheet"] = sheet
+        kwargs["headers"] = headers
         compression = datasetinfo.get("compression")
         if compression:
             kwargs["compression"] = compression
@@ -302,7 +305,7 @@ class Read(Retrieve):
                 f"Using saved datasets in {filename}_n.json in {self.saved_dir}"
             )
             datasets = []
-            for file_path in glob.glob(f"{saved_path}_*.json"):
+            for file_path in sorted(glob.glob(f"{saved_path}_*.json")):
                 datasets.append(Dataset.load_from_json(file_path))
         else:
             datasets = Dataset.search_in_hdx(

{hdx_python_scraper-2.3.8 → hdx_python_scraper-2.4.0}/tests/hdx/scraper/test_readers.py

@@ -30,7 +30,7 @@ class TestReaders:
                     assert getattr(clone_reader, property) == value
 
     def test_read_dataset(self, configuration, monkeypatch):
-        def test_read_from_hdx(dataset_name, _):
+        def read_from_hdx(dataset_name, _):
             if dataset_name == "None":
                 return None
             dataset = Dataset({"name": dataset_name})
@@ -58,7 +58,7 @@ class TestReaders:
                     today=parse_date("2021-02-01"),
                 ) as reader:
                     monkeypatch.setattr(
-                        Dataset, "read_from_hdx", test_read_from_hdx
+                        Dataset, "read_from_hdx", read_from_hdx
                     )
                     dataset_name = "None"
                     dataset = reader.read_dataset(dataset_name)
@@ -89,7 +89,7 @@ class TestReaders:
     def test_search_datasets(self, configuration, monkeypatch):
         filename = "TestDataset"
 
-        def test_search_in_hdx(*args, **kwargs):
+        def search_in_hdx(*args, **kwargs):
             datasets = []
             for i in range(2):
                 dataset = Dataset({"name": f"{filename}_{i}"})
@@ -117,7 +117,7 @@ class TestReaders:
                     today=parse_date("2021-02-01"),
                 ) as reader:
                     monkeypatch.setattr(
-                        Dataset, "search_in_hdx", test_search_in_hdx
+                        Dataset, "search_in_hdx", search_in_hdx
                     )
                     datasets = reader.search_datasets(filename)
                     assert len(datasets) == 2