hdx-python-scraper 2.2.3__tar.gz → 2.3.0__tar.gz

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/.config/coveragerc

@@ -14,4 +14,4 @@ exclude_also =
     if 0:
     if __name__ == .__main__.:
     if TYPE_CHECKING:
-    @(abc\.)?abstractmethod
+    @(abc\.)?abstractmethod

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/CONTRIBUTING.md

@@ -20,7 +20,7 @@ you make a git commit:
 
     pre-commit install
 
-The configuration file for this project is in a 
+The configuration file for this project is in a
 non-start location. Thus, you will need to edit your
 `.git/hooks/pre-commit` file to reflect this. Change
 the line that begins with `ARGS` to:
@@ -29,7 +29,7 @@ the line that begins with `ARGS` to:
 
 With pre-commit, all code is formatted according to
 [black]("https://github.com/psf/black") and
-[ruff]("https://github.com/charliermarsh/ruff") guidelines. 
+[ruff]("https://github.com/charliermarsh/ruff") guidelines.
 
 To check if your changes pass pre-commit without committing, run:
 
@@ -46,8 +46,8 @@ Follow the example set out already in ``api.rst`` as you write the documentation
 ## Packages
 
 [pip-tools](https://github.com/jazzband/pip-tools) is used for
-package management.  If you’ve introduced a new package to the 
-source code (i.e.anywhere in `src/`), please add it to the 
+package management.  If you’ve introduced a new package to the
+source code (i.e.anywhere in `src/`), please add it to the
 `project.dependencies` section of
 `pyproject.toml` with any known version constraints.
 

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hdx-python-scraper
-Version: 2.2.3
+Version: 2.3.0
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,12 +26,13 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.1.3
+Requires-Dist: hdx-python-api>=6.1.4
+Requires-Dist: hdx-python-country>=3.6.3
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: pandas
-Requires-Dist: pandas>=2.1.1; extra == 'pandas'
+Requires-Dist: pandas>=2.1.3; extra == 'pandas'
 Provides-Extra: test
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'
@@ -43,19 +44,19 @@ Description-Content-Type: text/markdown
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
 [![Downloads](https://img.shields.io/pypi/dm/hdx-python-scraper.svg)](https://pypistats.org/packages/hdx-python-scraper)
 
-The HDX Python Scraper Library is designed to enable you to easily develop code that 
-assembles data from one or more tabular sources that can be csv, xls, xlsx or JSON. It 
-uses a YAML file that specifies for each source what needs to be read and allows some 
-transformations to be performed on the data. The output is written to JSON, Google sheets 
-and/or Excel and includes the addition of 
-[Humanitarian Exchange Language (HXL)](https://hxlstandard.org/) hashtags specified in 
-the YAML file. Custom Python scrapers can also be written that conform to a defined 
-specification and the framework handles the execution of both configurable and custom 
+The HDX Python Scraper Library is designed to enable you to easily develop code that
+assembles data from one or more tabular sources that can be csv, xls, xlsx or JSON. It
+uses a YAML file that specifies for each source what needs to be read and allows some
+transformations to be performed on the data. The output is written to JSON, Google sheets
+and/or Excel and includes the addition of
+[Humanitarian Exchange Language (HXL)](https://hxlstandard.org/) hashtags specified in
+the YAML file. Custom Python scrapers can also be written that conform to a defined
+specification and the framework handles the execution of both configurable and custom
 scrapers.
 
-For more information, please read the 
-[documentation](https://hdx-python-scraper.readthedocs.io/en/latest/). 
+For more information, please read the
+[documentation](https://hdx-python-scraper.readthedocs.io/en/latest/).
 
-This library is part of the 
-[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have 
+This library is part of the
+[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
 humanitarian related data, please upload your datasets to HDX.

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/README.md

@@ -4,19 +4,19 @@
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
 [![Downloads](https://img.shields.io/pypi/dm/hdx-python-scraper.svg)](https://pypistats.org/packages/hdx-python-scraper)
 
-The HDX Python Scraper Library is designed to enable you to easily develop code that 
-assembles data from one or more tabular sources that can be csv, xls, xlsx or JSON. It 
-uses a YAML file that specifies for each source what needs to be read and allows some 
-transformations to be performed on the data. The output is written to JSON, Google sheets 
-and/or Excel and includes the addition of 
-[Humanitarian Exchange Language (HXL)](https://hxlstandard.org/) hashtags specified in 
-the YAML file. Custom Python scrapers can also be written that conform to a defined 
-specification and the framework handles the execution of both configurable and custom 
+The HDX Python Scraper Library is designed to enable you to easily develop code that
+assembles data from one or more tabular sources that can be csv, xls, xlsx or JSON. It
+uses a YAML file that specifies for each source what needs to be read and allows some
+transformations to be performed on the data. The output is written to JSON, Google sheets
+and/or Excel and includes the addition of
+[Humanitarian Exchange Language (HXL)](https://hxlstandard.org/) hashtags specified in
+the YAML file. Custom Python scrapers can also be written that conform to a defined
+specification and the framework handles the execution of both configurable and custom
 scrapers.
 
-For more information, please read the 
-[documentation](https://hdx-python-scraper.readthedocs.io/en/latest/). 
+For more information, please read the
+[documentation](https://hdx-python-scraper.readthedocs.io/en/latest/).
 
-This library is part of the 
-[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have 
+This library is part of the
+[Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
 humanitarian related data, please upload your datasets to HDX.

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/documentation/main.md

@@ -26,6 +26,10 @@ install with:
     pip install hdx-python-scraper[pandas]
 
 ## Breaking Changes
+From 2.3.0, resource name is used when available instead of creating name from
+url so tests that use saved data from the Read class may break. file_type
+parameters in various Read methods renamed to format.
+
 From 2.1.2, Python 3.7 no longer supported
 
 From 2.0.1, all functions in outputs.update_tabs are methods in the new Writer class

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/pyproject.toml

@@ -34,7 +34,8 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.1.3",
+    "hdx-python-api>=6.1.4",
+    "hdx-python-country>=3.6.3",
     "gspread",
     "regex",
 ]
@@ -48,7 +49,7 @@ content-type = "text/markdown"
 Homepage = "https://github.com/OCHA-DAP/hdx-python-scraper"
 
 [project.optional-dependencies]
-pandas = ["pandas>=2.1.1"]
+pandas = ["pandas>=2.1.3"]
 test = ["pytest", "pytest-cov"]
 dev = ["pre-commit"]
 
@@ -87,7 +88,7 @@ test = """
        """
 
 [[tool.hatch.envs.test.matrix]]
-python = ["3.11"]
+python = ["3.12"]
 
 [tool.hatch.envs.lint]
 detached = true

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/requirements.txt

@@ -4,16 +4,16 @@
 #
 #    pip-compile --all-extras --output-file=requirements.txt --resolver=backtracking pyproject.toml
 #
-annotated-types==0.5.0
+annotated-types==0.6.0
     # via pydantic
 attrs==23.1.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
-cachetools==5.3.1
+cachetools==5.3.2
     # via google-auth
-certifi==2023.7.22
+certifi==2023.11.17
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -21,7 +21,7 @@ cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
     # via requests
 ckanapi==4.7
     # via hdx-python-api
@@ -31,7 +31,7 @@ colorama==0.4.6
     # via typer
 coverage[toml]==7.3.2
     # via pytest-cov
-cryptography==41.0.4
+cryptography==41.0.7
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
@@ -45,33 +45,37 @@ docopt==0.6.2
     #   num2words
 docutils==0.20.1
     # via defopt
-email-validator==2.0.0.post2
+email-validator==2.1.0.post1
     # via hdx-python-api
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.12.4
+filelock==3.13.1
     # via virtualenv
 frictionless==5.16.0
     # via hdx-python-utilities
-google-auth==2.23.2
+google-auth==2.25.2
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.1.0
     # via gspread
-gspread==5.11.3
+gspread==5.12.2
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.1.3
+hdx-python-api==6.1.4
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.5.7
-    # via hdx-python-api
+hdx-python-country==3.6.3
+    # via
+    #   hdx-python-api
+    #   hdx-python-scraper (pyproject.toml)
 hdx-python-utilities==3.6.2
-    # via hdx-python-country
-humanize==4.8.0
+    # via
+    #   hdx-python-api
+    #   hdx-python-country
+humanize==4.9.0
     # via frictionless
-identify==2.5.30
+identify==2.5.33
     # via pre-commit
-idna==3.4
+idna==3.6
     # via
     #   email-validator
     #   requests
@@ -94,14 +98,16 @@ jsonschema==4.17.3
     #   frictionless
     #   tableschema-to-template
 libhxl==5.1
-    # via hdx-python-country
+    # via
+    #   hdx-python-api
+    #   hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
-makefun==1.15.1
+makefun==1.15.2
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
-marko==2.0.0
+marko==2.0.2
     # via frictionless
 markupsafe==2.1.3
     # via jinja2
@@ -111,9 +117,9 @@ ndg-httpsclient==0.5.1
     # via hdx-python-api
 nodeenv==1.8.0
     # via pre-commit
-num2words==0.5.12
+num2words==0.5.13
     # via quantulum3
-numpy==1.26.0
+numpy==1.26.2
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -121,11 +127,11 @@ openpyxl==3.1.2
     # via hdx-python-utilities
 packaging==23.2
     # via pytest
-pandas==2.1.1
+pandas==2.1.4
     # via hdx-python-scraper (pyproject.toml)
 petl==1.7.14
     # via frictionless
-platformdirs==3.11.0
+platformdirs==4.1.0
     # via virtualenv
 pluggy==1.3.0
     # via pytest
@@ -135,9 +141,9 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.4.0
+pre-commit==3.6.0
     # via hdx-python-scraper (pyproject.toml)
-pyasn1==0.5.0
+pyasn1==0.5.1
     # via
     #   hdx-python-api
     #   ndg-httpsclient
@@ -147,23 +153,23 @@ pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pydantic==2.4.2
+pydantic==2.5.2
     # via
     #   frictionless
     #   inflect
-pydantic-core==2.10.1
+pydantic-core==2.14.5
     # via pydantic
-pygments==2.16.1
+pygments==2.17.2
     # via rich
-pyopenssl==23.2.0
+pyopenssl==23.3.0
     # via
     #   hdx-python-api
     #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pyrsistent==0.19.3
+pyrsistent==0.20.0
     # via jsonschema
-pytest==7.4.2
+pytest==7.4.3
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
@@ -198,6 +204,7 @@ requests==2.31.0
     # via
     #   ckanapi
     #   frictionless
+    #   hdx-python-api
     #   libhxl
     #   requests-file
     #   requests-oauthlib
@@ -207,15 +214,15 @@ requests-oauthlib==1.3.1
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.6.0
+rich==13.7.0
     # via typer
 rsa==4.9
     # via google-auth
-ruamel-yaml==0.17.35
+ruamel-yaml==0.18.5
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-shellingham==1.5.3
+shellingham==1.5.4
     # via typer
 simpleeval==0.9.13
     # via frictionless
@@ -231,7 +238,7 @@ sphinxcontrib-napoleon==0.7
     # via defopt
 stringcase==1.2.0
     # via frictionless
-structlog==23.1.0
+structlog==23.2.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -241,7 +248,7 @@ text-unidecode==1.3
     # via python-slugify
 typer[all]==0.9.0
     # via frictionless
-typing-extensions==4.8.0
+typing-extensions==4.9.0
     # via
     #   frictionless
     #   inflect
@@ -254,21 +261,21 @@ unidecode==1.3.7
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.0.6
+urllib3==2.1.0
     # via
     #   libhxl
     #   requests
 validators==0.22.0
     # via frictionless
-virtualenv==20.24.5
+virtualenv==20.25.0
     # via pre-commit
-wheel==0.41.2
+wheel==0.42.0
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
-xlsxwriter==3.1.6
+xlsxwriter==3.1.9
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/src/hdx/scraper/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.2.3'
-__version_tuple__ = version_tuple = (2, 2, 3)
+__version__ = version = '2.3.0'
+__version_tuple__ = version_tuple = (2, 3, 0)

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/src/hdx/scraper/base_scraper.py

@@ -81,15 +81,12 @@ class BaseScraper(ABC):
         self.sources: Dict[str, List] = {level: [] for level in self.headers}
         self.source_configuration = deepcopy(source_configuration)
 
-    def get_reader(
-        self, name: Optional[str] = None, prefix: Optional[str] = None
-    ):
-        """Get reader given name if provided or using name member variable if not.
-        Set reader prefix to given prefix or name if not provided.
+    def get_reader(self, name: Optional[str] = None):
+        """Get reader given name if provided or using name member variable if
+        not.
 
         Args:
             name (str): Name of scraper
-            prefix (Optional[str]): Prefix to use. Defaults to None (use scraper name).
 
         Returns:
              None
@@ -97,9 +94,6 @@ class BaseScraper(ABC):
         if not name:
             name = self.name
         reader = Read.get_reader(name)
-        if not prefix:
-            prefix = name
-        reader.prefix = prefix
         return reader
 
     def get_headers(self, level: str) -> Optional[Tuple[Tuple]]:
@@ -362,6 +356,24 @@ class BaseScraper(ABC):
         Returns:
             Optional[Dict]: HAPI resource metadata
         """
+        hapi_resource_metadata = self.datasetinfo.get("hapi_resource_metadata")
+        if not hapi_resource_metadata:
+            return None
+        if "is_hxl" in hapi_resource_metadata:
+            return hapi_resource_metadata
+        reader = self.get_reader(self.name)
+        filename = self.datasetinfo.get("filename")
+        hxl_info = reader.hxl_info_hapi_resource_metadata(
+            hapi_resource_metadata, filename=filename, file_prefix=self.name
+        )
+        is_hxl = False
+        if hxl_info:
+            for sheet in hxl_info.get("sheets", ()):
+                if sheet["is_hxlated"]:
+                    is_hxl = True
+                    break
+        hapi_resource_metadata["is_hxl"] = is_hxl
+
         return self.datasetinfo.get("hapi_resource_metadata")
 
     def add_population(self) -> None:

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/src/hdx/scraper/configurable/resource_downloader.py

@@ -33,7 +33,7 @@ class ResourceDownloader(BaseScraper):
         """
         reader = self.get_reader("hdx")
         resource = reader.read_hdx_metadata(self.datasetinfo)
-        url, path = reader.download_resource(self.name, resource)
+        url, path = reader.download_resource(resource, file_prefix=self.name)
         logger.info(f"Downloading {url} to {path}")
         copy2(path, join(self.folder, self.datasetinfo["filename"]))
 

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/src/hdx/scraper/configurable/scraper.py

@@ -147,6 +147,7 @@ class ConfigurableScraper(BaseScraper):
         """
         return self.get_reader(name).read(
             self.datasetinfo,
+            file_prefix=name,
             **self.variables,
         )
 

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/src/hdx/scraper/configurable/timeseries.py

@@ -50,7 +50,9 @@ class TimeSeries(BaseScraper):
             "output_hxl"
         ]
         rows = [headers, hxltags]
-        file_headers, iterator = self.get_reader().read(self.datasetinfo)
+        file_headers, iterator = self.get_reader(self.name).read(
+            self.datasetinfo, file_prefix=self.name
+        )
         for inrow in iterator:
             if isinstance(datecol, list):
                 dates = [str(inrow[x]) for x in datecol]

{hdx_python_scraper-2.2.3 → hdx_python_scraper-2.3.0}/src/hdx/scraper/runner.py

@@ -1145,7 +1145,9 @@ class Runner:
         return sorted(source_urls)
 
     def get_hapi_metadata(
-        self, names: Optional[ListTuple[str]] = None
+        self,
+        names: Optional[ListTuple[str]] = None,
+        has_run: bool = True,
     ) -> Dict:
         """Get HAPI metadata for all datasets. A dictionary is returned that
         maps from dataset ids to a dictionary. The dictionary has keys for
@@ -1154,6 +1156,7 @@ class Runner:
 
         Args:
             names (Optional[ListTuple[str]]): Names of scrapers
+            has_run (bool): Only get results for scrapers marked as having run. Defaults to True.
 
         Returns:
             Dict: HAPI metadata for all datasets
@@ -1163,7 +1166,7 @@ class Runner:
         results = {}
         for name in names:
             scraper = self.get_scraper(name)
-            if not scraper.has_run:
+            if has_run and not scraper.has_run:
                 continue
             hapi_dataset_metadata = scraper.get_hapi_dataset_metadata()
             hapi_resource_metadata = scraper.get_hapi_resource_metadata()