hdx-python-scraper 2.3.7__tar.gz → 2.3.9__tar.gz

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/.config/pre-commit-config.yaml

@@ -1,15 +1,15 @@
 default_language_version:
-    python: python3.11
+    python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.4
+    rev: v0.4.8
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.1.44
+    rev: 0.2.10
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/.github/workflows/publish.yaml

@@ -21,7 +21,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install --upgrade hatch
+    - name: Install Hatch
+      uses: pypa/hatch@install
     - name: Build with hatch
       run: |
         hatch build

hdx_python_scraper-2.3.9/.github/workflows/run-python-tests.yaml

@@ -0,0 +1,51 @@
+# This workflow will install Python dependencies, lint and run tests
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Run tests
+
+on:
+  workflow_dispatch: # add run button in github
+  push:
+    branches-ignore:
+      - gh-pages
+      - 'dependabot/**'
+  pull_request:
+    branches-ignore:
+      - gh-pages
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install Hatch
+      uses: pypa/hatch@install
+    - name: Test with hatch/pytest
+      env:
+        GSHEET_AUTH: ${{ secrets.GSHEET_AUTH }}
+      run: |
+        hatch test
+    - name: Check styling
+      if: always()
+      run: |
+        hatch fmt --check
+    - name: Publish Unit Test Results
+      uses: EnricoMi/publish-unit-test-result-action@v2
+      if: always()
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        junit_files: test-results.xml
+    - name: Publish in Coveralls
+      uses: coverallsapp/github-action@v2
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        flag-name: tests
+        format: lcov

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-scraper
-Version: 2.3.7
+Version: 2.3.9
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,14 +26,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.2.8
-Requires-Dist: hdx-python-country>=3.7.0
-Requires-Dist: hdx-python-utilities>=3.6.8
+Requires-Dist: hdx-python-api>=6.3.1
+Requires-Dist: hdx-python-country>=3.7.6
+Requires-Dist: hdx-python-utilities>=3.7.2
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: pandas
-Requires-Dist: pandas>=2.1.3; extra == 'pandas'
+Requires-Dist: pandas>=2.2.2; extra == 'pandas'
 Provides-Extra: test
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.2.8",
-    "hdx-python-country>=3.7.0",
-    "hdx-python-utilities>=3.6.8",
+    "hdx-python-api>=6.3.1",
+    "hdx-python-country>=3.7.6",
+    "hdx-python-utilities>=3.7.2",
     "gspread",
     "regex",
 ]
@@ -50,7 +50,7 @@ content-type = "text/markdown"
 Homepage = "https://github.com/OCHA-DAP/hdx-python-scraper"
 
 [project.optional-dependencies]
-pandas = ["pandas>=2.1.3"]
+pandas = ["pandas>=2.2.2"]
 test = ["pytest", "pytest-cov"]
 dev = ["pre-commit"]
 
@@ -78,25 +78,21 @@ version_scheme = "python-simplified-semver"
 
 # Tests
 
-[tool.hatch.envs.test]
+[tool.hatch.envs.hatch-test]
 features = ["pandas", "test"]
 
-[tool.hatch.envs.test.scripts]
-test = """
+[[tool.hatch.envs.hatch-test.matrix]]
+python = ["3.12"]
+
+[tool.hatch.envs.hatch-test.scripts]
+run = """
        pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \
        --cov --cov-config=.config/coveragerc --no-cov-on-fail \
        --cov-report=lcov --cov-report=term-missing
        """
 
-[[tool.hatch.envs.test.matrix]]
-python = ["3.12"]
-
-[tool.hatch.envs.lint]
-detached = true
-dependencies = ["ruff"]
-
-[tool.hatch.envs.lint.scripts]
-style = [
-  "ruff check --config .config/ruff.toml --diff {args:.}",
-  "ruff format --config .config/ruff.toml --diff {args:.}",
-]
+[tool.hatch.envs.hatch-static-analysis.scripts]
+format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
+format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
+lint-check = ["ruff check --config .config/ruff.toml {args:.}",]
+lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",]

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/requirements.txt

@@ -1,6 +1,6 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
-annotated-types==0.6.0
+annotated-types==0.7.0
     # via pydantic
 attrs==23.2.0
     # via
@@ -10,7 +10,7 @@ attrs==23.2.0
     #   referencing
 cachetools==5.3.3
     # via google-auth
-certifi==2024.2.2
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -24,9 +24,9 @@ ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-coverage==7.5.1
+coverage==7.6.0
     # via pytest-cov
-cryptography==42.0.7
+cryptography==42.0.8
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
@@ -40,44 +40,44 @@ docopt==0.6.2
     #   num2words
 docutils==0.21.2
     # via defopt
-email-validator==2.1.1
+email-validator==2.2.0
     # via hdx-python-api
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.14.0
+filelock==3.15.4
     # via virtualenv
 frictionless==5.17.0
     # via hdx-python-utilities
-google-auth==2.29.0
+google-auth==2.32.0
     # via
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==1.2.0
+google-auth-oauthlib==1.2.1
     # via gspread
-gspread==6.1.1
+gspread==6.1.2
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.2.8
+hdx-python-api==6.3.1
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.7.0
+hdx-python-country==3.7.6
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.6.8
+hdx-python-utilities==3.7.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
-humanize==4.9.0
+humanize==4.10.0
     # via frictionless
-identify==2.5.36
+identify==2.6.0
     # via pre-commit
 idna==3.7
     # via
     #   email-validator
     #   requests
-ijson==3.2.3
+ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.2.1
+inflect==7.3.1
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -89,7 +89,7 @@ jsonlines==4.0.0
     # via hdx-python-utilities
 jsonpath-ng==1.6.1
     # via libhxl
-jsonschema==4.22.0
+jsonschema==4.23.0
     # via
     #   frictionless
     #   tableschema-to-template
@@ -101,31 +101,31 @@ libhxl==5.2.1
     #   hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
-makefun==1.15.2
+makefun==1.15.3
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
-marko==2.0.3
+marko==2.1.2
     # via frictionless
 markupsafe==2.1.5
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
-more-itertools==10.2.0
+more-itertools==10.3.0
     # via inflect
 ndg-httpsclient==0.5.1
     # via hdx-python-api
-nodeenv==1.8.0
+nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.13
     # via quantulum3
-numpy==1.26.4
+numpy==2.0.0
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
-openpyxl==3.1.2
+openpyxl==3.1.5
     # via hdx-python-utilities
-packaging==24.0
+packaging==24.1
     # via pytest
 pandas==2.2.2
     # via hdx-python-scraper (pyproject.toml)
@@ -153,9 +153,9 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.7.1
+pydantic==2.8.2
     # via frictionless
-pydantic-core==2.18.2
+pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
@@ -165,13 +165,13 @@ pyopenssl==24.1.0
     #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pytest==8.2.0
+pytest==8.2.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
 pytest-cov==5.0.0
     # via hdx-python-scraper (pyproject.toml)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
     #   frictionless
     #   hdx-python-utilities
@@ -190,7 +190,7 @@ pyyaml==6.0.1
     #   frictionless
     #   pre-commit
     #   tableschema-to-template
-quantulum3==0.9.1
+quantulum3==0.9.2
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
@@ -200,7 +200,7 @@ referencing==0.35.1
     #   jsonschema-specifications
 regex==2024.5.15
     # via hdx-python-scraper (pyproject.toml)
-requests==2.31.0
+requests==2.32.3
     # via
     #   ckanapi
     #   frictionless
@@ -208,7 +208,7 @@ requests==2.31.0
     #   libhxl
     #   requests-file
     #   requests-oauthlib
-requests-file==2.0.0
+requests-file==2.1.0
     # via hdx-python-utilities
 requests-oauthlib==2.0.0
     # via google-auth-oauthlib
@@ -216,7 +216,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.7.1
     # via typer
-rpds-py==0.18.1
+rpds-py==0.19.0
     # via
     #   jsonschema
     #   referencing
@@ -226,10 +226,8 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==69.5.1
-    # via
-    #   ckanapi
-    #   nodeenv
+setuptools==70.3.0
+    # via ckanapi
 shellingham==1.5.4
     # via typer
 simpleeval==0.9.13
@@ -247,7 +245,7 @@ sphinxcontrib-napoleon==0.7
     # via defopt
 stringcase==1.2.0
     # via frictionless
-structlog==24.1.0
+structlog==24.2.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -255,14 +253,13 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typeguard==4.2.1
+typeguard==4.3.0
     # via inflect
 typer==0.12.3
     # via frictionless
-typing-extensions==4.11.0
+typing-extensions==4.12.2
     # via
     #   frictionless
-    #   inflect
     #   pydantic
     #   pydantic-core
     #   typeguard
@@ -273,13 +270,13 @@ unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.1
+urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.28.1
+validators==0.32.0
     # via frictionless
-virtualenv==20.26.2
+virtualenv==20.26.3
     # via pre-commit
 wheel==0.43.0
     # via libhxl

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/src/hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.7'
-__version_tuple__ = version_tuple = (2, 3, 7)
+__version__ = version = '2.3.9'
+__version_tuple__ = version_tuple = (2, 3, 9)

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/src/hdx/scraper/configurable/rowparser.py

@@ -185,20 +185,14 @@ class RowParser:
         Returns:
             Iterator[Dict]: Input data with prefilter applied if specified and sorted if specified or deemed necessary
         """
-        rows = []
-        for row in iterator:
-            if self.header_to_hxltag:
-                newrow = {}
-                for header in row:
-                    newrow[self.header_to_hxltag[header]] = row[header]
-                row = newrow
-            if self.stop_row:
-                if all(
-                    row[key] == value for key, value in self.stop_row.items()
-                ):
-                    break
-            for newrow in self.flatten(row):
-                rows.append(newrow)
+        if self.header_to_hxltag:
+            iterator = self.header_to_hxltag_rows(iterator)
+        if self.stop_row:
+            iterator = self.stop_rows(iterator)
+        if self.flatteninfo:
+            iterator = self.flatten_rows(iterator)
+        if self.prefilter:
+            iterator = (row for row in iterator if eval(self.prefilter))
         if not self.sort:
             if self.datecol:
                 for subset in self.subsets:
@@ -212,15 +206,59 @@ class RowParser:
                         )
                         self.sort = {"keys": [self.datecol], "reverse": True}
                         break
-        if self.prefilter:
-            rows = [row for row in rows if eval(self.prefilter)]
         if self.sort:
             keys = self.sort["keys"]
             reverse = self.sort.get("reverse", False)
-            rows = sorted(rows, key=itemgetter(*keys), reverse=reverse)
-        return rows
+            iterator = sorted(iterator, key=itemgetter(*keys), reverse=reverse)
+        return iterator
+
+    def header_to_hxltag_rows(
+        self, iterator: Iterator[Dict]
+    ) -> Generator[Dict, None, None]:
+        """Convert headers to HXL tags in keys
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Rows where keys are HXL tags
+        """
+        for row in iterator:
+            newrow = {}
+            for header in row:
+                newrow[self.header_to_hxltag[header]] = row[header]
+            yield newrow
+
+    def stop_rows(
+        self, iterator: Iterator[Dict]
+    ) -> Generator[Dict, None, None]:
+        """Stop processing rows after condition met
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Rows up to stop condition
+        """
+        for row in iterator:
+            if all(row[key] == value for key, value in self.stop_row.items()):
+                break
+            yield row
+
+    def flatten_rows(self, iterator: Iterator[Dict]) -> Iterator[Dict]:
+        """Flatten rows
+
+        Args:
+            iterator (Iterator[Dict]): Input data
+
+        Returns:
+            Generator[Dict]: Flattened rows
+        """
+        for row in iterator:
+            for newrow in self.flatten_row(row):
+                yield newrow
 
-    def flatten(self, row: Dict) -> Generator[Dict, None, None]:
+    def flatten_row(self, row: Dict) -> Generator[Dict, None, None]:
         """Flatten a wide spreadsheet format into a long one
 
         Args:
@@ -229,9 +267,6 @@ class RowParser:
         Returns:
             Generator[Dict]: Flattened row(s)
         """
-        if not self.flatteninfo:
-            yield row
-            return
         counters = [-1 for _ in self.flatteninfo]
         while True:
             newrow = copy.deepcopy(row)
@@ -314,7 +349,7 @@ class RowParser:
                     adms[i], exact = Country.get_iso3_country_code_fuzzy(adm)
                 elif i == 1:
                     adms[i], exact = self.adminlevel.get_pcode(
-                        adms[0], adm, self.name
+                        adms[0], adm, logname=self.name
                     )
                 if adms[i] not in self.adms[i]:
                     adms[i] = None

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/src/hdx/scraper/utilities/reader.py

@@ -302,7 +302,7 @@ class Read(Retrieve):
                 f"Using saved datasets in {filename}_n.json in {self.saved_dir}"
             )
             datasets = []
-            for file_path in glob.glob(f"{saved_path}_*.json"):
+            for file_path in sorted(glob.glob(f"{saved_path}_*.json")):
                 datasets.append(Dataset.load_from_json(file_path))
         else:
             datasets = Dataset.search_in_hdx(

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/tests/hdx/scraper/test_output.py

@@ -130,7 +130,7 @@ class TestOutput:
                 list(hxltags.keys()),
                 list(hxltags.values()),
                 ["AFG", "Afghanistan", 38041754],
-                ["BDI", "Burundi", np.NaN],
+                ["BDI", "Burundi", np.nan],
                 ["PAK", "Pakistan", -np.inf],
             ]
             df = pandas.DataFrame(output[1:], columns=output[0])
@@ -138,7 +138,7 @@ class TestOutput:
             result = googletab.get("A1:C5")
             result[2][2] = int(result[2][2])
             assert result[3][2] == "NaN"
-            result[3][2] = np.NaN
+            result[3][2] = np.nan
             assert result[4][2] == "-inf"
             result[4][2] = -np.inf
             assert result == output

{hdx_python_scraper-2.3.7 → hdx_python_scraper-2.3.9}/tests/hdx/scraper/test_readers.py

@@ -30,7 +30,7 @@ class TestReaders:
                     assert getattr(clone_reader, property) == value
 
     def test_read_dataset(self, configuration, monkeypatch):
-        def test_read_from_hdx(dataset_name, _):
+        def read_from_hdx(dataset_name, _):
             if dataset_name == "None":
                 return None
             dataset = Dataset({"name": dataset_name})
@@ -58,7 +58,7 @@ class TestReaders:
                     today=parse_date("2021-02-01"),
                 ) as reader:
                     monkeypatch.setattr(
-                        Dataset, "read_from_hdx", test_read_from_hdx
+                        Dataset, "read_from_hdx", read_from_hdx
                     )
                     dataset_name = "None"
                     dataset = reader.read_dataset(dataset_name)
@@ -89,7 +89,7 @@ class TestReaders:
     def test_search_datasets(self, configuration, monkeypatch):
         filename = "TestDataset"
 
-        def test_search_in_hdx(*args, **kwargs):
+        def search_in_hdx(*args, **kwargs):
             datasets = []
             for i in range(2):
                 dataset = Dataset({"name": f"{filename}_{i}"})
@@ -117,7 +117,7 @@ class TestReaders:
                     today=parse_date("2021-02-01"),
                 ) as reader:
                     monkeypatch.setattr(
-                        Dataset, "search_in_hdx", test_search_in_hdx
+                        Dataset, "search_in_hdx", search_in_hdx
                     )
                     datasets = reader.search_datasets(filename)
                     assert len(datasets) == 2

hdx_python_scraper-2.3.7/.github/workflows/run-python-tests.yaml

@@ -1,66 +0,0 @@
-# This workflow will install Python dependencies, lint and run tests
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: Run tests
-
-on:
-  workflow_dispatch: # add run button in github
-  push:
-    branches-ignore:
-      - gh-pages
-      - 'dependabot/**'
-  pull_request:
-    branches-ignore:
-      - gh-pages
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres:14-alpine
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: hapitest
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-          --name postgres-container
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade hatch
-      - name: Test with hatch/pytest
-        env:
-          GSHEET_AUTH: ${{ secrets.GSHEET_AUTH }}
-        run: |
-          hatch run test:test
-      - name: Check styling
-        if: always()
-        run: |
-          hatch run lint:style
-      - name: Publish Unit Test Results
-        uses: EnricoMi/publish-unit-test-result-action@v2
-        if: always()
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          junit_files: test-results.xml
-      - name: Publish in Coveralls
-        uses: coverallsapp/github-action@v2
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          flag-name: tests
-          format: lcov