owid-datautils 0.6.1__tar.gz

owid_datautils-0.6.1/.bumpversion.cfg

@@ -0,0 +1,12 @@
+[bumpversion]
+current_version = 0.5.3
+commit = True
+tag = True
+
+[bumpversion:file:pyproject.toml]
+
+[bumpversion:file:README.md]
+
+[bumpversion:file:owid/datautils/__init__.py]
+
+[bumpversion:file:docs/conf.py]

owid_datautils-0.6.1/.codecov.yml

@@ -0,0 +1,13 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        # adjust accordingly based on how flaky your tests are
+        # this allows a 10% drop from the previous base commit coverage
+        threshold: 3%
+    patch:
+      default:
+        target: auto
+        threshold: 3%
+        base: auto

owid_datautils-0.6.1/.github/workflows/python-package.yml

@@ -0,0 +1,52 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install packages
+      run: |
+        make .venv
+
+    - name: Check formatting
+      run: |
+        make check-formatting
+
+    - name: Lint
+      run: |
+        make lint
+
+    - name: Check typing
+      run: |
+        make check-typing
+
+    - name: Unit test
+      run: |
+        make report-coverage
+
+    - name: "Coverage: Upload to codecov.io"
+      uses: codecov/codecov-action@v2
+      with:
+        fail_ci_if_error: true
+        verbose: true
+        files: coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}

owid_datautils-0.6.1/.gitignore

@@ -0,0 +1,141 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# PyCharm config
+.idea
+
+# Mac hidden files
+.DS_Store
+
+# reports (coverage, linting)
+.reports/
+
+# Google
+client_secret*

owid_datautils-0.6.1/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+- repo: local
+  hooks:
+    - id: format
+      name: format
+      entry: make format
+      language: system
+    - id: lint
+      name: lint
+      entry: make lint
+      language: system
+    - id: check-typing
+      name: check-typing
+      entry: make check-typing
+      language: system

owid_datautils-0.6.1/.readthedocs.yaml

@@ -0,0 +1,23 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools: {python: "3.8"}
+  jobs:
+    pre_create_environment:
+      - asdf plugin add poetry
+      - asdf install poetry latest
+      - asdf global poetry latest
+      - poetry config virtualenvs.create false
+    post_install:
+      - poetry install
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true

owid_datautils-0.6.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Global Change Data Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

owid_datautils-0.6.1/Makefile

@@ -0,0 +1,8 @@
+#
+#  Makefile
+#
+
+
+include ../../default.mk
+
+SRC = owid tests

owid_datautils-0.6.1/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: owid-datautils
+Version: 0.6.1
+Summary: Data utils library by the Data Team at Our World in Data
+Author-email: Our World in Data <tech@ourworldindata.org>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: data cleaning,data processing,data utils,our world in data
+Requires-Python: <4.0,>=3.10
+Requires-Dist: boto3>=1.38.23
+Requires-Dist: click>=8.1.7
+Requires-Dist: colorama>=0.4.4
+Requires-Dist: gdown>=4.5.2
+Requires-Dist: gsheets>=0.6.1
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: py7zr>=0.22.0
+Requires-Dist: pyarrow>=18.0.0
+Requires-Dist: pydrive2>=1.15.0
+Requires-Dist: structlog>=21.5.0
+Requires-Dist: urllib3<2

owid_datautils-0.6.1/README.md

@@ -0,0 +1,85 @@
+# owid-datautils
+
+![version](https://img.shields.io/badge/version-0.6.0-blue)
+![version](https://img.shields.io/badge/python-3.10|3.11|3.12|3.13-blue.svg?&logo=python&logoColor=yellow)
+[![Build status](https://badge.buildkite.com/caba621fb64f2c7dcc692a474e68f4ead21e6ba6ee151fe3b6.svg)](https://buildkite.com/our-world-in-data/owid-datautils-unit-tests)
+[![Documentation Status](https://readthedocs.org/projects/owid-datautils/badge/?version=latest)](https://docs.owid.io/projects/owid-datautils/en/latest/?badge=latest)
+
+**owid-datautils** is a library to support the work of the Data Team at Our World in Data.
+
+## Install
+
+```
+pip install owid-datautils
+```
+
+Or install the latest development version directly from GitHub:
+
+```
+pip install git+https://github.com/owid/etl.git#subdirectory=lib/datautils
+```
+
+## Development
+
+### Pre-requisites
+
+You need Python 3.10+, `UV` and `make` installed.
+
+### Install in development mode
+
+```
+make .venv
+```
+
+### Test the code
+
+Run:
+
+```
+# run all unit tests and CI checks
+make test
+```
+
+### Other useful commands
+
+#### Code Quality
+
+```
+make check
+```
+
+Format, lint, and typecheck changed files from master branch using `ruff` and `pyright`.
+
+```
+make format
+```
+
+Format code using `ruff`.
+
+```
+make lint
+```
+
+Lint code using `ruff`.
+
+```
+make check-typing
+```
+
+Run type checking using `pyright`.
+
+#### Coverage
+
+```
+make coverage
+```
+
+Run unit tests with coverage reporting.
+
+#### Versioning
+
+```
+make bump [part]
+```
+
+Upgrade version in all files where it appears. `[part]` can be `patch`, `minor` ad `major`.

owid_datautils-0.6.1/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

owid_datautils-0.6.1/docs/apidoc-templates/module.rst_t

@@ -0,0 +1,8 @@
+{%- if show_headings %}
+{{- ["owid.", basename] | join('') | e | heading }}
+
+{% endif -%}
+.. automodule:: {{ qualname }}
+{%- for option in automodule_options %}
+   :{{ option }}:
+{%- endfor %}

owid_datautils-0.6.1/docs/apidoc-templates/package.rst_t

@@ -0,0 +1,52 @@
+{%- macro automodule(modname, options) -%}
+.. automodule:: {{ modname }}
+{%- for option in options %}
+   :{{ option }}:
+{%- endfor %}
+{%- endmacro %}
+
+{%- macro toctree(docnames) -%}
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}
+{%- endmacro %}
+
+{%- if is_namespace %}
+{{- [pkgname, "namespace"] | join(" ") | e | heading }}
+{% else %}
+{{- pkgname | e | heading }}
+{% endif %}
+
+{%- if is_namespace %}
+.. py:module:: {{ pkgname }}
+{% endif %}
+
+{%- if modulefirst and not is_namespace %}
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}
+
+{%- if subpackages %}
+
+
+{{ toctree(subpackages) }}
+{% endif %}
+
+{%- if submodules %}
+{% if separatemodules %}
+{{ toctree(submodules) }}
+{% else %}
+{%- for submodule in submodules %}
+{% if show_headings %}
+{{- submodule | e | heading(2) }}
+{% endif %}
+{{ automodule(submodule, automodule_options) }}
+{% endfor %}
+{%- endif %}
+{%- endif %}
+
+{%- if not modulefirst and not is_namespace %}
+
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}

owid_datautils-0.6.1/docs/apidoc-templates/toc.rst_t

@@ -0,0 +1,7 @@
+{{ header | heading }}
+
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}

owid_datautils-0.6.1/docs/conf.py

@@ -0,0 +1,90 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join("../owid/")))
+
+# -- Project information -----------------------------------------------------
+
+project = "owid-datautils"
+copyright = "2022, Our World in Data"
+author = "Our World in Data"
+
+# The full version, including alpha/beta/rc tags
+release = "0.5.3"
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "api/modules.rst"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "furo"
+
+html_theme_options = {
+    "sidebar_hide_name": True,
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+
+html_logo = "_static/owid.png"
+html_favicon = "_static/favicon.ico"
+
+autodoc_default_flags = [
+    "members",
+    "undoc-members",
+    "private-members",
+    "special-members",
+    "inherited-members",
+    "show-inheritance",
+]
+
+html_context = {
+    "display_github": True,  # Integrate GitHub
+    "github_user": "owid",  # Username
+    "github_repo": "owid-datautils-py",  # Repo name
+    "github_version": "main",  # Version
+    "conf_py_path": "docs/",  # Path in the checkout to the docs root
+}
+
+## API docs
+from sphinx.ext.apidoc import main
+
+# uv run sphinx-apidoc --help
+main(["-f", "-e", "-t", "apidoc-templates", "-P", "-o", "api", "../owid/"])

owid_datautils-0.6.1/docs/index.rst

@@ -0,0 +1,12 @@
+.. owid-datautils documentation master file, created by
+   sphinx-quickstart on Wed Jun  1 23:00:03 2022.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+owid-datautils
+==========================================
+
+.. toctree::
+   :maxdepth: 2
+
+   owid.datautils <api/datautils>

owid_datautils-0.6.1/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

owid_datautils-0.6.1/owid/datautils/__init__.py

@@ -0,0 +1,3 @@
+"""Library to support the work of the Data Team at Our World in Data."""
+
+__version__ = "0.5.3"

owid_datautils-0.6.1/owid/datautils/checks.py

@@ -0,0 +1 @@
+"""Utils related to sanity checks of datasets."""

owid_datautils-0.6.1/owid/datautils/common.py

@@ -0,0 +1,44 @@
+"""Common objects shared by other modules."""
+
+import warnings
+from typing import Any, List, Set, Union, cast
+
+
+class ExceptionFromDocstring(Exception):
+    """Exception that returns its own docstring, if no message is explicitly given."""
+
+    def __init__(self, exception_message: Union[str, None] = None, *args: Any):
+        super().__init__(exception_message or self.__doc__, *args)
+
+
+class ExceptionFromDocstringWithKwargs(Exception):
+    """Exception that returns its own docstring, if no message is explicitly given."""
+
+    def __init__(self, exception_message: Union[str, None] = None, *args: Any, **kwargs: Any):
+        text = cast(str, exception_message or self.__doc__)
+        if kwargs:
+            additional_text = ", ".join([f"{key}: {value}" for key, value in kwargs.items()])
+            if additional_text is not None:
+                text += " " + additional_text
+        super().__init__(text, *args)
+
+
+def warn_on_list_of_entities(
+    list_of_entities: Union[List[Any], Set[Any]], warning_message: str, show_list: bool
+) -> None:
+    """Raise a warning with a custom message, and optionally print a list of affected elements.
+
+    Parameters
+    ----------
+    list_of_entities : list or set
+        Elements to optionally print one by one (only relevant if show_list is True).
+    warning_message : str
+        Warning message.
+    show_list : bool
+        True to print a list of affected entities.
+
+    """
+    warnings.warn(warning_message)
+    if show_list:
+        print(warning_message)
+        print("\n".join(["* " + str(entity) for entity in list_of_entities]))