hdx-python-scraper 2.3.5__tar.gz → 2.3.7__tar.gz

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/.config/pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.4.4
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,10 +17,11 @@ repos:
       # Run the formatter.
       - id: ruff-format
         args: [--config, .config/ruff.toml]
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.3.0
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.1.44
     hooks:
+      # Run the pip compile
       - id: pip-compile
         name: pip-compile requirements.txt
         files: pyproject.toml
-        args: [pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt]
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/.config/ruff.toml

@@ -1,14 +1,16 @@
 line-length = 79
 exclude = ["_version.py"]
-ignore = [
-    "E501" # Line too long
-]
+
+[lint]
 # List of rules: https://docs.astral.sh/ruff/rules/
 select = [
   "E",   # pycodestyle - default
   "F",   # pyflakes - default
   "I"    # isort
 ]
+ignore = [
+    "E501" # Line too long
+]
 
-[isort]
+[lint.isort]
 known-local-folder = ["hdx"]

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/.github/workflows/publish.yaml

@@ -9,13 +9,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get history and tags for versioning to work
       run: |
         git fetch --prune --unshallow
         git fetch --depth=1 origin +refs/tags/*:refs/tags/*
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies

hdx_python_scraper-2.3.7/.github/workflows/run-python-tests.yaml

@@ -0,0 +1,66 @@
+# This workflow will install Python dependencies, lint and run tests
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Run tests
+
+on:
+  workflow_dispatch: # add run button in github
+  push:
+    branches-ignore:
+      - gh-pages
+      - 'dependabot/**'
+  pull_request:
+    branches-ignore:
+      - gh-pages
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: hapitest
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --name postgres-container
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade hatch
+      - name: Test with hatch/pytest
+        env:
+          GSHEET_AUTH: ${{ secrets.GSHEET_AUTH }}
+        run: |
+          hatch run test:test
+      - name: Check styling
+        if: always()
+        run: |
+          hatch run lint:style
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          junit_files: test-results.xml
+      - name: Publish in Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          flag-name: tests
+          format: lcov

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/CONTRIBUTING.md

@@ -21,14 +21,13 @@ you make a git commit:
     pre-commit install
 
 The configuration file for this project is in a
-non-start location. Thus, you will need to edit your
+non-standard location. Thus, you will need to edit your
 `.git/hooks/pre-commit` file to reflect this. Change
 the line that begins with `ARGS` to:
 
     ARGS=(hook-impl --config=.config/pre-commit-config.yaml --hook-type=pre-commit)
 
 With pre-commit, all code is formatted according to
-[black]("https://github.com/psf/black") and
 [ruff]("https://github.com/charliermarsh/ruff") guidelines.
 
 To check if your changes pass pre-commit without committing, run:
@@ -41,7 +40,7 @@ To run the tests and view coverage, execute:
 
     pytest -c .config/pytest.ini --cov hdx --cov-config .config/coveragerc
 
-Follow the example set out already in ``api.rst`` as you write the documentation.
+Follow the example set out already in ``documentation/main.md`` as you write the documentation.
 
 ## Packages
 

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: hdx-python-scraper
-Version: 2.3.5
+Version: 2.3.7
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,8 +26,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.2.1
-Requires-Dist: hdx-python-country>=3.6.4
+Requires-Dist: hdx-python-api>=6.2.8
+Requires-Dist: hdx-python-country>=3.7.0
+Requires-Dist: hdx-python-utilities>=3.6.8
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/documentation/.readthedocs.yaml

@@ -11,5 +11,6 @@ build:
   jobs:
     pre_build:
       - pip install --upgrade mkdocs
+      - pip install mkdocs-material
       - pip install pydoc-markdown
       - pydoc-markdown documentation/pydoc-markdown.yaml

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/documentation/pydoc-markdown.yaml

@@ -9,7 +9,7 @@ renderer:
   output_directory: docs
   mkdocs_config:
     site_name: HDX Python Scraper
-    theme: mkdocs
+    theme: material
     repo_url: "https://github.com/OCHA-DAP/hdx-python-scraper"
   markdown:
     source_linker:

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/pyproject.toml

@@ -34,8 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.2.1",
-    "hdx-python-country>=3.6.4",
+    "hdx-python-api>=6.2.8",
+    "hdx-python-country>=3.7.0",
+    "hdx-python-utilities>=3.6.8",
     "gspread",
     "regex",
 ]

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/requirements.txt

@@ -1,9 +1,5 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
-#
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.6.0
     # via pydantic
 attrs==23.2.0
@@ -11,9 +7,10 @@ attrs==23.2.0
     #   frictionless
     #   jsonlines
     #   jsonschema
-cachetools==5.3.2
+    #   referencing
+cachetools==5.3.3
     # via google-auth
-certifi==2023.11.17
+certifi==2024.2.2
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -23,83 +20,82 @@ chardet==5.2.0
     # via frictionless
 charset-normalizer==3.3.2
     # via requests
-ckanapi==4.7
+ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-colorama==0.4.6
-    # via typer
-coverage[toml]==7.4.1
-    # via
-    #   coverage
-    #   pytest-cov
-cryptography==42.0.2
+coverage==7.5.1
+    # via pytest-cov
+cryptography==42.0.7
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
 distlib==0.3.8
     # via virtualenv
-dnspython==2.5.0
+dnspython==2.6.1
     # via email-validator
 docopt==0.6.2
     # via
     #   ckanapi
     #   num2words
-docutils==0.20.1
+docutils==0.21.2
     # via defopt
-email-validator==2.1.0.post1
+email-validator==2.1.1
     # via hdx-python-api
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.13.1
+filelock==3.14.0
     # via virtualenv
-frictionless==5.16.1
+frictionless==5.17.0
     # via hdx-python-utilities
-google-auth==2.27.0
+google-auth==2.29.0
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
-gspread==6.0.0
+gspread==6.1.1
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.2.1
+hdx-python-api==6.2.8
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.6.4
+hdx-python-country==3.7.0
     # via
-    #   hdx-python-api
     #   hdx-python-scraper (pyproject.toml)
-hdx-python-utilities==3.6.4
+    #   hdx-python-api
+hdx-python-utilities==3.6.8
     # via
+    #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
 humanize==4.9.0
     # via frictionless
-identify==2.5.33
+identify==2.5.36
     # via pre-commit
-idna==3.6
+idna==3.7
     # via
     #   email-validator
     #   requests
 ijson==3.2.3
     # via hdx-python-utilities
-inflect==7.0.0
+inflect==7.2.1
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
 isodate==0.6.1
     # via frictionless
-jinja2==3.1.3
+jinja2==3.1.4
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities
 jsonpath-ng==1.6.1
     # via libhxl
-jsonschema==4.17.3
+jsonschema==4.22.0
     # via
     #   frictionless
     #   tableschema-to-template
-libhxl==5.2
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+libhxl==5.2.1
     # via
     #   hdx-python-api
     #   hdx-python-country
@@ -109,33 +105,35 @@ makefun==1.15.2
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
-marko==2.0.2
+marko==2.0.3
     # via frictionless
-markupsafe==2.1.4
+markupsafe==2.1.5
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
+more-itertools==10.2.0
+    # via inflect
 ndg-httpsclient==0.5.1
     # via hdx-python-api
 nodeenv==1.8.0
     # via pre-commit
 num2words==0.5.13
     # via quantulum3
-numpy==1.26.3
+numpy==1.26.4
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
 openpyxl==3.1.2
     # via hdx-python-utilities
-packaging==23.2
+packaging==24.0
     # via pytest
-pandas==2.2.0
+pandas==2.2.2
     # via hdx-python-scraper (pyproject.toml)
-petl==1.7.14
+petl==1.7.15
     # via frictionless
-platformdirs==4.2.0
+platformdirs==4.2.2
     # via virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
 ply==3.11
     # via
@@ -143,39 +141,35 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.6.0
+pre-commit==3.7.1
     # via hdx-python-scraper (pyproject.toml)
-pyasn1==0.5.1
+pyasn1==0.6.0
     # via
     #   hdx-python-api
     #   ndg-httpsclient
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.0
     # via google-auth
-pycparser==2.21
+pycparser==2.22
     # via cffi
-pydantic==2.6.0
-    # via
-    #   frictionless
-    #   inflect
-pydantic-core==2.16.1
+pydantic==2.7.1
+    # via frictionless
+pydantic-core==2.18.2
     # via pydantic
-pygments==2.17.2
+pygments==2.18.0
     # via rich
-pyopenssl==24.0.0
+pyopenssl==24.1.0
     # via
     #   hdx-python-api
     #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pyrsistent==0.20.0
-    # via jsonschema
-pytest==8.0.0
+pytest==8.2.0
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
-pytest-cov==4.1.0
+pytest-cov==5.0.0
     # via hdx-python-scraper (pyproject.toml)
 python-dateutil==2.8.2
     # via
@@ -185,22 +179,26 @@ python-dateutil==2.8.2
     #   pandas
 python-io-wrapper==0.3.1
     # via libhxl
-python-slugify==8.0.3
+python-slugify==8.0.4
     # via
     #   ckanapi
     #   frictionless
-pytz==2023.4
+pytz==2024.1
     # via pandas
 pyyaml==6.0.1
     # via
     #   frictionless
     #   pre-commit
     #   tableschema-to-template
-quantulum3==0.9.0
+quantulum3==0.9.1
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
-regex==2023.12.25
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+regex==2024.5.15
     # via hdx-python-scraper (pyproject.toml)
 requests==2.31.0
     # via
@@ -212,22 +210,32 @@ requests==2.31.0
     #   requests-oauthlib
 requests-file==2.0.0
     # via hdx-python-utilities
-requests-oauthlib==1.3.1
+requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.7.0
+rich==13.7.1
     # via typer
+rpds-py==0.18.1
+    # via
+    #   jsonschema
+    #   referencing
 rsa==4.9
     # via google-auth
-ruamel-yaml==0.18.5
+ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
+setuptools==69.5.1
+    # via
+    #   ckanapi
+    #   nodeenv
 shellingham==1.5.4
     # via typer
 simpleeval==0.9.13
     # via frictionless
+simplejson==3.19.2
+    # via ckanapi
 six==1.16.0
     # via
     #   ckanapi
@@ -237,8 +245,6 @@ six==1.16.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-strenum==0.4.15
-    # via gspread
 stringcase==1.2.0
     # via frictionless
 structlog==24.1.0
@@ -249,41 +255,39 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer[all]==0.9.0
-    # via
-    #   frictionless
-    #   typer
-typing-extensions==4.9.0
+typeguard==4.2.1
+    # via inflect
+typer==0.12.3
+    # via frictionless
+typing-extensions==4.11.0
     # via
     #   frictionless
     #   inflect
     #   pydantic
     #   pydantic-core
+    #   typeguard
     #   typer
-tzdata==2023.4
+tzdata==2024.1
     # via pandas
 unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.0
+urllib3==2.2.1
     # via
     #   libhxl
     #   requests
-validators==0.22.0
+validators==0.28.1
     # via frictionless
-virtualenv==20.25.0
+virtualenv==20.26.2
     # via pre-commit
-wheel==0.42.0
+wheel==0.43.0
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
-xlsxwriter==3.1.9
+xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/src/hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.5'
-__version_tuple__ = version_tuple = (2, 3, 5)
+__version__ = version = '2.3.7'
+__version_tuple__ = version_tuple = (2, 3, 7)

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.3.7}/src/hdx/scraper/base_scraper.py

@@ -141,9 +141,9 @@ class BaseScraper(ABC):
             "should_overwrite_sources"
         )
         if should_overwrite_sources is not None:
-            self.source_configuration[
-                "should_overwrite_sources"
-            ] = should_overwrite_sources
+            self.source_configuration["should_overwrite_sources"] = (
+                should_overwrite_sources
+            )
         source = self.datasetinfo["source"]
         if isinstance(source, str):
             source = {"default_source": source}