hdx-python-scraper 2.3.9__tar.gz → 2.4.1__tar.gz

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/.config/pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.8
+    rev: v0.5.1
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.2.10
+    rev: 0.2.13
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-scraper
-Version: 2.3.9
+Version: 2.4.1
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,9 +26,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.3.1
+Requires-Dist: hdx-python-api>=6.3.2
 Requires-Dist: hdx-python-country>=3.7.6
-Requires-Dist: hdx-python-utilities>=3.7.2
+Requires-Dist: hdx-python-utilities>=3.7.3
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/documentation/main.md

@@ -1089,6 +1089,30 @@ same HXL hashtags.
         use_hxl: True
         should_overwrite_sources: True
 
+The xlsx2csv option below converts the xlsx to csv before processing.
+
+      idps_somalia:
+        dataset: "somalia-internally-displaced-persons-idps"
+        format: "xlsx"
+        xlsx2csv: True
+        filter_cols:
+          - "Reason"
+          - "Year"
+        prefilter: "'drought' in Reason.lower() and int(Year) in (self.today.year - 1, self.today.year)"
+        admin:
+          - value: "SOM"
+          - "Current (Arrival) District"
+        input:
+          - "Number of Individuals"
+        sum:
+          - formula: "int(Number of Individuals)"
+            mustbepopulated: True
+        output:
+          - "IDPs"
+        output_hxl:
+          - "#affected+idps+ind"
+        source_date_format: "%Y-%m-%d"
+
 ## Population Data
 
 Population data is treated as a special class of data. By default, configurable and

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.3.1",
+    "hdx-python-api>=6.3.2",
     "hdx-python-country>=3.7.6",
-    "hdx-python-utilities>=3.7.2",
+    "hdx-python-utilities>=3.7.3",
     "gspread",
     "regex",
 ]

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/requirements.txt

@@ -8,7 +8,7 @@ attrs==23.2.0
     #   jsonlines
     #   jsonschema
     #   referencing
-cachetools==5.3.3
+cachetools==5.4.0
     # via google-auth
 certifi==2024.7.4
     # via requests
@@ -26,7 +26,7 @@ click==8.1.7
     # via typer
 coverage==7.6.0
     # via pytest-cov
-cryptography==42.0.8
+cryptography==43.0.0
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
@@ -56,13 +56,13 @@ google-auth-oauthlib==1.2.1
     # via gspread
 gspread==6.1.2
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.3.1
+hdx-python-api==6.3.2
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.7.6
+hdx-python-country==3.7.7
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.7.2
+hdx-python-utilities==3.7.3
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
@@ -101,7 +101,7 @@ libhxl==5.2.1
     #   hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
-makefun==1.15.3
+makefun==1.15.4
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
@@ -119,7 +119,7 @@ nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.13
     # via quantulum3
-numpy==2.0.0
+numpy==2.0.1
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -141,7 +141,7 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.7.1
+pre-commit==3.8.0
     # via hdx-python-scraper (pyproject.toml)
 pyasn1==0.6.0
     # via
@@ -159,13 +159,13 @@ pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
-pyopenssl==24.1.0
+pyopenssl==24.2.1
     # via
     #   hdx-python-api
     #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pytest==8.2.2
+pytest==8.3.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
@@ -198,7 +198,7 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2024.5.15
+regex==2024.7.24
     # via hdx-python-scraper (pyproject.toml)
 requests==2.32.3
     # via
@@ -216,7 +216,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.7.1
     # via typer
-rpds-py==0.19.0
+rpds-py==0.19.1
     # via
     #   jsonschema
     #   referencing
@@ -226,7 +226,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==70.3.0
+setuptools==72.1.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -245,7 +245,7 @@ sphinxcontrib-napoleon==0.7
     # via defopt
 stringcase==1.2.0
     # via frictionless
-structlog==24.2.0
+structlog==24.4.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -274,7 +274,7 @@ urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.32.0
+validators==0.33.0
     # via frictionless
 virtualenv==20.26.3
     # via pre-commit
@@ -284,6 +284,8 @@ xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
+xlsx2csv==0.8.3
+    # via hdx-python-utilities
 xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/src/hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.3.9'
-__version_tuple__ = version_tuple = (2, 3, 9)
+__version__ = version = '2.4.1'
+__version_tuple__ = version_tuple = (2, 4, 1)

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/src/hdx/scraper/outputs/googlesheets.py

@@ -91,4 +91,4 @@ class GoogleSheets(BaseOutput):
                 df.fillna("NaN", inplace=True)
                 rows.extend(df.values.tolist())
                 values = rows
-            tab.update("A1", values)
+            tab.update(values, "A1")

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/src/hdx/scraper/utilities/reader.py

@@ -206,15 +206,19 @@ class Read(Retrieve):
         if headers is None:
             headers = 1
             datasetinfo["headers"] = 1
-        kwargs["headers"] = headers
-        if isinstance(headers, list):
-            kwargs["fill_merged_cells"] = True
         format = datasetinfo["format"]
         kwargs["format"] = format
-        if not sheet and format in ("xls", "xlsx"):
-            sheet = 1
+        if format in ("xls", "xlsx"):
+            if not sheet:
+                sheet = 1
+            if isinstance(headers, list):
+                kwargs["fill_merged_cells"] = True
+            elif "fill_merged_cells" not in kwargs:
+                kwargs["fill_merged_cells"] = False
+            kwargs["xlsx2csv"] = datasetinfo.get("xlsx2csv", False)
         if sheet:
             kwargs["sheet"] = sheet
+        kwargs["headers"] = headers
         compression = datasetinfo.get("compression")
         if compression:
             kwargs["compression"] = compression

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/tests/config/project_configuration.yaml

@@ -1049,10 +1049,11 @@ scraper_subnational:
   idps_somalia:
     dataset: "somalia-internally-displaced-persons-idps"
     format: "xlsx"
+    xlsx2csv: True
     filter_cols:
       - "Reason"
       - "Year"
-    prefilter: "'drought' in Reason.lower() and Year in (self.today.year - 1, self.today.year)"
+    prefilter: "'drought' in Reason.lower() and int(Year) in (self.today.year - 1, self.today.year)"
     admin:
       - value: "SOM"
       - "Current (Arrival) District"

{hdx_python_scraper-2.3.9 → hdx_python_scraper-2.4.1}/tests/hdx/scraper/test_scrapers_subnational.py

@@ -353,10 +353,11 @@ class TestScrapersSubnational:
     def test_fixed_country(self, configuration):
         BaseScraper.population_lookup = {}
         today = parse_date("2020-10-01")
-        adminlevel = AdminLevel(configuration["admin1"])
-        adminlevel.setup_from_admin_info(configuration["admin1"]["admin_info"])
         level = "subnational"
         scraper_configuration = configuration[f"scraper_{level}"]
+
+        adminlevel = AdminLevel(configuration["admin1"])
+        adminlevel.setup_from_admin_info(configuration["admin1"]["admin_info"])
         runner = Runner(("SOM",), today)
         runner.add_configurables(
             scraper_configuration, level, adminlevel=adminlevel