hdx-python-scraper 2.3.5__tar.gz → 2.5.3__tar.gz

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/.config/pre-commit-config.yaml

@@ -1,15 +1,15 @@
 default_language_version:
-    python: python3.11
+    python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.8.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,10 +17,11 @@ repos:
       # Run the formatter.
       - id: ruff-format
         args: [--config, .config/ruff.toml]
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.3.0
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.5.15
     hooks:
+      # Run the pip compile
       - id: pip-compile
         name: pip-compile requirements.txt
         files: pyproject.toml
-        args: [pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt]
+        args: [ pyproject.toml, --resolver=backtracking, --all-extras, --upgrade, -q, -o, requirements.txt ]

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/.config/ruff.toml

@@ -1,14 +1,16 @@
 line-length = 79
 exclude = ["_version.py"]
-ignore = [
-    "E501" # Line too long
-]
+
+[lint]
 # List of rules: https://docs.astral.sh/ruff/rules/
 select = [
   "E",   # pycodestyle - default
   "F",   # pyflakes - default
   "I"    # isort
 ]
+ignore = [
+    "E501" # Line too long
+]
 
-[isort]
+[lint.isort]
 known-local-folder = ["hdx"]

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/.github/workflows/publish.yaml

@@ -9,19 +9,20 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get history and tags for versioning to work
       run: |
         git fetch --prune --unshallow
         git fetch --depth=1 origin +refs/tags/*:refs/tags/*
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install --upgrade hatch
+    - name: Install Hatch
+      uses: pypa/hatch@install
     - name: Build with hatch
       run: |
         hatch build

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/.github/workflows/run-python-tests.yaml

@@ -18,24 +18,25 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install --upgrade hatch
+    - name: Install Hatch
+      uses: pypa/hatch@install
     - name: Test with hatch/pytest
       env:
         GSHEET_AUTH: ${{ secrets.GSHEET_AUTH }}
       run: |
-        hatch run test:test
+        hatch test
     - name: Check styling
       if: always()
       run: |
-        hatch run lint:style
+        hatch fmt --check
     - name: Publish Unit Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2
       if: always()

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/CONTRIBUTING.md

@@ -2,10 +2,10 @@
 
 ## Environment
 
-Development is currently done using Python 3.11. We recommend using a virtual
+Development is currently done using Python 3.12. We recommend using a virtual
 environment such as ``venv``:
 
-    python3.11 -m venv venv
+    python3.12 -m venv venv
     source venv/bin/activate
 
 In your virtual environment, please install all packages for
@@ -21,14 +21,13 @@ you make a git commit:
     pre-commit install
 
 The configuration file for this project is in a
-non-start location. Thus, you will need to edit your
+non-standard location. Thus, you will need to edit your
 `.git/hooks/pre-commit` file to reflect this. Change
 the line that begins with `ARGS` to:
 
     ARGS=(hook-impl --config=.config/pre-commit-config.yaml --hook-type=pre-commit)
 
 With pre-commit, all code is formatted according to
-[black]("https://github.com/psf/black") and
 [ruff]("https://github.com/charliermarsh/ruff") guidelines.
 
 To check if your changes pass pre-commit without committing, run:
@@ -41,7 +40,7 @@ To run the tests and view coverage, execute:
 
     pytest -c .config/pytest.ini --cov hdx --cov-config .config/coveragerc
 
-Follow the example set out already in ``api.rst`` as you write the documentation.
+Follow the example set out already in ``documentation/main.md`` as you write the documentation.
 
 ## Packages
 

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.3.5
+Version: 2.5.3
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,13 +26,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.2.1
-Requires-Dist: hdx-python-country>=3.6.4
+Requires-Dist: hdx-python-api>=6.3.7
+Requires-Dist: hdx-python-country>=3.8.6
+Requires-Dist: hdx-python-utilities>=3.8.2
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: pandas
-Requires-Dist: pandas>=2.1.3; extra == 'pandas'
+Requires-Dist: pandas>=2.2.2; extra == 'pandas'
 Provides-Extra: test
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/documentation/.readthedocs.yaml

@@ -11,5 +11,6 @@ build:
   jobs:
     pre_build:
       - pip install --upgrade mkdocs
+      - pip install mkdocs-material
       - pip install pydoc-markdown
       - pydoc-markdown documentation/pydoc-markdown.yaml

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/documentation/main.md

@@ -26,6 +26,8 @@ install with:
     pip install hdx-python-scraper[pandas]
 
 ## Breaking Changes
+From 2.5.0, package names have changed to avoid name space clashes
+
 From 2.3.0, resource name is used when available instead of creating name from
 url so tests that use saved data from the Read class may break. file_type
 parameters in various Read methods renamed to format.
@@ -81,6 +83,7 @@ The library is set up broadly as follows:
             hdx_auth=configuration.get_api_key(),
             header_auths=header_auths,
             basic_auths=basic_auths,
+            bearer_tokens=bearer_tokens,
             param_auths=param_auths,
             today=today,
         )
@@ -335,7 +338,7 @@ default is:
 ### Custom Scrapers
 
 It is possible to define custom scrapers written in Python which must inherit
-[BaseScraper](https://github.com/OCHA-DAP/hdx-python-scraper/blob/main/src/hdx/scraper/base_scraper.py),
+[BaseScraper](https://github.com/OCHA-DAP/hdx-python-scraper/blob/main/src/hdx/scraper/framework/base_scraper.py),
 calling its constructor and providing a `run` method. Other methods where a default
 implementation has been provided can be overridden such as `add_sources` and
 `add_population`. There are also two hooks for running steps at particular points.
@@ -392,7 +395,7 @@ from regions to values. In this case, since national and regional each have only
 header and HXL hashtag, there is only one dictionary to populate for each.
 
 An example of a custom scraper can be seen
-[here](https://github.com/OCHA-DAP/hdx-python-scraper/blob/main/tests/hdx/scraper/education_closures.py).
+[here](https://github.com/OCHA-DAP/hdx-python-scraper/blob/main/tests/hdx/scraper/framework/scrapers/education_closures.py).
 
 An example of overriding `add_sources` to customise the source information that is
 output is as follows:
@@ -1089,6 +1092,30 @@ same HXL hashtags.
         use_hxl: True
         should_overwrite_sources: True
 
+The xlsx2csv option below converts the xlsx to csv before processing.
+
+      idps_somalia:
+        dataset: "somalia-internally-displaced-persons-idps"
+        format: "xlsx"
+        xlsx2csv: True
+        filter_cols:
+          - "Reason"
+          - "Year"
+        prefilter: "'drought' in Reason.lower() and int(Year) in (self.today.year - 1, self.today.year)"
+        admin:
+          - value: "SOM"
+          - "Current (Arrival) District"
+        input:
+          - "Number of Individuals"
+        sum:
+          - formula: "int(Number of Individuals)"
+            mustbepopulated: True
+        output:
+          - "IDPs"
+        output_hxl:
+          - "#affected+idps+ind"
+        source_date_format: "%Y-%m-%d"
+
 ## Population Data
 
 Population data is treated as a special class of data. By default, configurable and

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/documentation/pydoc-markdown.yaml

@@ -3,13 +3,13 @@ loaders:
     search_path:
       - ../src
     packages:
-      - hdx.scraper
+      - hdx.scraper.framework
 renderer:
   type: mkdocs
   output_directory: docs
   mkdocs_config:
     site_name: HDX Python Scraper
-    theme: mkdocs
+    theme: material
     repo_url: "https://github.com/OCHA-DAP/hdx-python-scraper"
   markdown:
     source_linker:
@@ -21,21 +21,18 @@ renderer:
       source: main.md
     - title: API Documentation
       children:
-        - title: Source Readers
+        - title: Runner
           contents:
-            - hdx.scraper.utilities.readers.*
-        - title: Configurable Scrapers
+            - hdx.scraper.framework.runner.Runner.*
+        - title: Scraper Base Class
           contents:
-            - hdx.scraper.configurable.*
-        - title: Base Class for Scrapers
+            - hdx.scraper.framework.base_scraper.BaseScraper.*
+        - title: Scrapers
           contents:
-            - hdx.scraper.base_scraper.BaseScraper.*
-        - title: Fallbacks
-          contents:
-            - hdx.scraper.utilities.fallbacks.Fallbacks
-        - title: Run Scrapers
-          contents:
-            - hdx.scraper.runner.Runner.*
+            - hdx.scraper.framework.scrapers.*
         - title: Outputs
           contents:
-            - hdx.scraper.outputs.*
+            - hdx.scraper.framework.outputs.*
+        - title: Utilities
+          contents:
+            - hdx.scraper.framework.utilities.*

{hdx_python_scraper-2.3.5 → hdx_python_scraper-2.5.3}/pyproject.toml

@@ -34,8 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.2.1",
-    "hdx-python-country>=3.6.4",
+    "hdx-python-api>=6.3.7",
+    "hdx-python-country>=3.8.6",
+    "hdx-python-utilities>=3.8.2",
     "gspread",
     "regex",
 ]
@@ -49,7 +50,7 @@ content-type = "text/markdown"
 Homepage = "https://github.com/OCHA-DAP/hdx-python-scraper"
 
 [project.optional-dependencies]
-pandas = ["pandas>=2.1.3"]
+pandas = ["pandas>=2.2.2"]
 test = ["pytest", "pytest-cov"]
 dev = ["pre-commit"]
 
@@ -64,7 +65,10 @@ dev = ["pre-commit"]
 packages = ["src/hdx"]
 
 [tool.hatch.build.hooks.vcs]
-version-file = "src/hdx/scraper/_version.py"
+version-file = "src/hdx/scraper/framework/_version.py"
+
+[tool.hatch.metadata]
+allow-direct-references = true
 
 # Versioning
 
@@ -77,25 +81,24 @@ version_scheme = "python-simplified-semver"
 
 # Tests
 
-[tool.hatch.envs.test]
+[tool.hatch.envs.hatch-test]
 features = ["pandas", "test"]
 
-[tool.hatch.envs.test.scripts]
-test = """
+[[tool.hatch.envs.hatch-test.matrix]]
+python = ["3.12"]
+
+[tool.hatch.envs.hatch-test.scripts]
+run = """
        pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \
        --cov --cov-config=.config/coveragerc --no-cov-on-fail \
        --cov-report=lcov --cov-report=term-missing
        """
 
-[[tool.hatch.envs.test.matrix]]
-python = ["3.12"]
-
-[tool.hatch.envs.lint]
-detached = true
-dependencies = ["ruff"]
+[tool.hatch.envs.hatch-static-analysis]
+dependencies = ["ruff==0.8.6"]
 
-[tool.hatch.envs.lint.scripts]
-style = [
-  "ruff check --config .config/ruff.toml --diff {args:.}",
-  "ruff format --config .config/ruff.toml --diff {args:.}",
-]
+[tool.hatch.envs.hatch-static-analysis.scripts]
+format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
+format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
+lint-check = ["ruff check --config .config/ruff.toml {args:.}",]
+lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",]