owid-datautils 0.6.1__py3-none-any.whl

owid/datautils/decorators.py

@@ -0,0 +1,91 @@
+"""Library decorators."""
+
+import functools
+import tempfile
+from typing import Any, Callable, Optional
+
+from owid.datautils.web import download_file_from_url
+
+
+def enable_file_download(path_arg_name: Optional[str] = None) -> Callable[[Any], Any]:
+    """Decorator that allows functions expecting local file paths to accept URLs and S3 paths.
+
+    This decorator automatically downloads remote files to temporary storage before calling
+    the decorated function, making any file-processing function work transparently with:
+    - Local file paths (unchanged behavior)
+    - HTTP/HTTPS URLs (downloaded via web request)
+    - S3 paths (downloaded via S3 client)
+
+    Args:
+        path_arg_name: Name of the parameter containing the file path. If None,
+                      uses the first positional argument.
+
+    Example:
+        @enable_file_download(path_arg_name="file_path")
+        def load_data(file_path):
+            with open(file_path, 'r') as f:
+                return f.read()
+
+        # Now works with all of these:
+        load_data("/local/file.txt")                    # Local file
+        load_data("https://example.com/data.txt")       # HTTP download
+        load_data("s3://bucket/data.txt")               # S3 download
+
+    Warning:
+        Downloads entire files to temporary storage on every call. For large files
+        or frequent access, consider explicit caching or streaming approaches.
+    """
+    # Download options, add them as needed (value: str, key: Tuple[str])
+    prefixes = {
+        "url": (
+            "http://",
+            "https://",
+        ),
+        "s3": ("s3://",),
+    }
+    # Get list of prefixes as a flat tuple
+    prefixes_flat = tuple(prefix for prefixes_list in prefixes.values() for prefix in prefixes_list)
+
+    def _enable_file_download(func: Callable[[Any], Any]) -> Callable[[Any], Any]:
+        @functools.wraps(func)
+        def wrapper_download(*args: Any, **kwargs: Any) -> Any:
+            # Get path to file
+            _used_args = False
+            if args:
+                args = list(args)  # type: ignore
+                path = args[0]
+                _used_args = True
+            else:
+                path = kwargs.get(path_arg_name)  # type: ignore
+                if path is None:
+                    raise ValueError(f"Filename was not found in args or kwargs ({path_arg_name}!")
+            # Check if download is needed and download
+            path = str(path)
+            if path.startswith(prefixes_flat):  # Download from URL and run function
+                with tempfile.NamedTemporaryFile() as temp_file:
+                    # Download file from URL
+                    if path.startswith(prefixes["url"]):
+                        download_file_from_url(path, temp_file.name)  # TODO: Add custom args here
+                    # Download file from S3 (need credentials)
+                    elif path.startswith(prefixes["s3"]):
+                        try:
+                            # Import here to avoid circular dependency
+                            from owid.catalog import s3_utils  # type: ignore
+                        except ImportError as e:
+                            raise ImportError("owid-catalog is required for S3 downloads. ") from e
+
+                        s3_utils.download(path, temp_file.name, quiet=True)  # TODO: Add custom args here
+
+                    # Modify args/kwargs
+                    if _used_args:
+                        args[0] = temp_file.name  # type: ignore
+                    else:
+                        kwargs[path_arg_name] = temp_file.name  # type: ignore
+                    # Call function
+                    return func(*args, **kwargs)
+            else:  # Run function on local file
+                return func(*args, **kwargs)
+
+        return wrapper_download
+
+    return _enable_file_download

owid/datautils/format/__init__.py

@@ -0,0 +1,7 @@
+"""Utils for the processing of different data formats."""
+
+from owid.datautils.format.numbers import format_number
+
+__all__ = [
+    "format_number",
+]

owid/datautils/format/numbers.py

@@ -0,0 +1,304 @@
+"""Numeric formatting."""
+
+import re
+from typing import Any, Dict, Set, Union
+
+
+class IntegerNumber:
+    """Wrapper around integer numbers."""
+
+    def __init__(self, number: Union[str, int]) -> None:
+        self.number_raw = number
+        self.number = self.init_clean(number)
+
+    @classmethod
+    def init_clean(cls, number: Union[int, str]) -> str:
+        """First number cleaning steps.
+
+        Parameters
+        ----------
+        number : Union[int, str]
+            Raw number as given.
+
+        Returns
+        -------
+        int
+            Number as a string, wihtout repeated whitespaces.
+        """
+        # Force string
+        number = num_to_str(number)
+        # Remove multiple spaces
+        number = remove_multiple_whitespaces(number)
+        return number
+
+    def clean(self) -> int:
+        """Clean number."""
+        if self.number.isnumeric():  # already a number
+            return int(self.number)
+        elif IntegerNumberWithWords.is_valid(self.number):  # contains words
+            return IntegerNumberWithWords(self.number).clean()
+        elif IntegerNumberWithSeparators.is_valid(self.number):  # has zero-separator
+            return IntegerNumberWithSeparators(self.number).clean()
+        else:  # error
+            raise ValueError(f"Invalid number format {self.number}")
+
+
+class IntegerNumberWithSeparators:
+    """Class for numbers with separators.
+
+    Accepted separators are: '.', ',' and ' '.
+    """
+
+    accepted_separators = [r"\.", ",", r"\s"]
+
+    def __init__(self, number_raw: str) -> None:
+        self.number_raw = number_raw
+
+    @classmethod
+    def regex_number_with_separator(cls) -> str:
+        """Regex expression for number with separators."""
+        accepted_separators_str = "|".join(rf"(({sep}\d{{3}})+)" for sep in cls.accepted_separators)
+        regex_number_with_separator: str = rf"\d{{1,3}}({accepted_separators_str})"
+        return regex_number_with_separator
+
+    @classmethod
+    def is_valid(cls, number: str) -> bool:
+        """Check if given number has valid separators.
+
+        Parameters
+        ----------
+        number : str
+            Raw number.
+
+        Returns
+        -------
+        bool:
+            True if valid syntax.
+        """
+        return bool(re.fullmatch(cls.regex_number_with_separator(), number))
+
+    def clean(self) -> int:
+        """Clean number.
+
+        Returns
+        -------
+        int
+            Cleaned number (i.e. without separators).
+
+        Raises
+        ------
+        ValueError:
+            If provided number was not correct (e.g. does not contain a separator).
+        """
+        if self.is_valid(self.number_raw):
+            n = re.sub(r"[^0-9]", "", str(self.number_raw))
+            return int(n)
+        raise ValueError(f"Given number {self.number_raw} is not valid!")
+
+
+class IntegerNumberWithWords:
+    """Class for numbers with words.
+
+    Words suported can be found in class attribute `numeric_words`.
+
+    Examples of accepted formats:
+
+    - 1 million ten thousand
+    - 20 thousand 1 hondred
+
+    Examples of not supported formats:
+
+    - 1 hundred thousand
+    - one hundred
+    - 1,22 thousand
+
+    Use with caution!
+    """
+
+    numeric_words: Dict[str, Dict[str, Any]] = {
+        "million": {
+            "words": [
+                "million",
+                "millió",
+                "millón",
+                "millones",
+                "millions",
+                "millionen",
+                "milioni",
+                "milione",
+                "miljoen",
+                "milhão",
+                "milhões",
+            ],
+            "factor": 1e6,
+        },
+        "ten_thousand": {
+            "words": ["万"],
+            "factor": 1e4,
+        },
+        "thousand": {
+            "words": [
+                "thousand",
+                "ezren",
+                "mil",
+                "duizend",
+                "mila",
+                "mille",
+                "tausend",
+            ],
+            "factor": 1e3,
+        },
+        "hundred": {
+            "words": [
+                "hundred",
+                "cien",
+                "cientos",
+                "cent",
+                "hundert",
+                "honderd",
+                "cem",
+                "cento",
+            ],
+            "factor": 1e2,
+        },
+        "one": {"words": [""], "factor": 1},
+    }
+    regex_number_verbose_template: str = r"(?:(?P<{}>[1-9]\d{{,2}})\s?(?:{}))?"
+
+    def __init__(self, number_raw: str) -> None:
+        self.number_raw = number_raw
+        self.number = self.init_clean(number_raw)
+
+    @classmethod
+    def init_clean(cls, number_raw: str) -> str:
+        """Clean raw number."""
+        return number_raw.replace(" and ", " ")
+
+    @classmethod
+    def regex_number_verbose(cls) -> str:
+        """Build regex for a number with words.
+
+        Returns
+        -------
+        str
+            Regex.
+        """
+        regex = [
+            cls.regex_number_verbose_template.format(k, "|".join(v["words"])) for k, v in cls.numeric_words.items()
+        ]
+        return r"\s?".join(regex)
+
+    @classmethod
+    def numeric_words_list(cls) -> Set[str]:
+        """Return list of all numeric words (flattened)."""
+        words = set(word for value in cls.numeric_words.values() for word in value["words"] if word != "")
+        return words
+
+    def _match_numeric_words(self) -> Dict[str, Union[str, int]]:
+        """Match number with words."""
+        match = re.search(self.regex_number_verbose(), self.number)
+        if match:
+            numbers = match.groupdict(default=0)
+            return numbers
+        else:
+            raise ValueError("Number may not contain numeric words. Please review!")
+
+    @classmethod
+    def _build_IntegerNumber(cls, numbers: Dict[str, Union[str, int]]) -> int:
+        """Build number from dictionary."""
+        value = 0
+        for k, v in numbers.items():
+            value += float(v) * cls.numeric_words[k]["factor"]
+        return int(value)
+
+    @classmethod
+    def is_valid(cls, number: str) -> bool:
+        """Check if number contains numeric words.
+
+        Parameters
+        ----------
+        number : str
+            Candidate number.
+
+        Returns
+        -------
+        bool
+            True if number contains numeric valid words.
+        """
+        # return any(word in number for word in cls.numeric_words_list())
+        return bool(re.fullmatch(cls.regex_number_verbose(), number))
+
+    def clean(self) -> int:
+        """Clean number.
+
+        Returns:
+        --------
+        int:
+            Cleaned number.
+        """
+        if self.is_valid(self.number_raw):
+            number_dix = self._match_numeric_words()
+            number = self._build_IntegerNumber(number_dix)
+            return number
+        raise ValueError(f"Given number {self.number_raw} is not valid!")
+
+
+def num_to_str(num_as_str: Union[int, str]) -> str:
+    """Convert number to string.
+
+    Parameters
+    ----------
+    num_as_str: Union[int, str]
+        Raw number.
+
+    Returns:
+    --------
+    str:
+        Raw number as string.
+    """
+    if not isinstance(num_as_str, str):
+        return str(num_as_str)
+    return num_as_str
+
+
+def remove_multiple_whitespaces(text: str) -> str:
+    """Remove excess of whitespaces.
+
+    '  ' -> ' '.
+
+    Parameters
+    ----------
+    text : str
+        Raw text.
+
+    Returns
+    -------
+    str
+        String without duplicated whitespaces.
+    """
+    return re.sub(r"\s+", " ", text)
+
+
+def format_number(number: Union[int, str]) -> int:
+    """Format number.
+
+    Only supports integer conversion.
+
+    Some examples of transformations are:
+
+        - '1 000 000' -> 1000000
+        - '1 million 1 hundred' -> 1000100
+        - '2,000' -> 2000
+
+    Parameters
+    ----------
+    number : Union[int, str]
+        Input raw number to be formatted.
+
+    Returns
+    -------
+    int
+        Formatted number.
+    """
+    number_ = IntegerNumber(number)
+    return number_.clean()

owid/datautils/google/__init__.py

@@ -0,0 +1,7 @@
+"""Google utils."""
+
+from owid.datautils.google.api import GoogleApi
+
+__all__ = [
+    "GoogleApi",
+]

owid/datautils/google/api.py

@@ -0,0 +1,136 @@
+"""Google API class."""
+
+from typing import Any, Optional
+
+import gdown
+from pydrive2.auth import GoogleAuth
+from pydrive2.drive import GoogleDrive
+from pydrive2.files import GoogleDriveFileList
+
+from owid.datautils.google.config import (
+    CLIENT_SECRETS_PATH,
+    CREDENTIALS_PATH,
+    SETTINGS_PATH,
+    google_config_init,
+    is_google_config_init,
+)
+from owid.datautils.google.sheets import GSheetsApi
+
+
+class GoogleApi:
+    """API for Google Drive."""
+
+    def __init__(self, clients_secrets_file: Optional[str] = None) -> None:
+        """Initialise Google API.
+
+        To obtain `client_secrets_file`, follow the instructions from:
+        https://medium.com/analytics-vidhya/how-to-connect-google-drive-to-python-using-pydrive-9681b2a14f20
+
+        IMPORTANT:
+            - Additionally, make sure to add yourself in Test users, as noted in:
+              https://stackoverflow.com/questions/65980758/pydrive-quickstart-and-error-403-access-denied
+            - Select Desktop App instead of Web Application as the application type.
+
+        Parameters
+        ----------
+        clients_secrets_file : str, optional
+            Path to client_secrets file.
+
+        Examples
+        --------
+        First time calling the function should look similar to:
+
+        >>> from owid.datautils.google.api import GoogleApi
+        >>> api = GoogleApi("path/to/credentials.json")
+
+        New calls can then be made as follows:
+
+        >>> api = GoogleApi()
+        """
+        if not is_google_config_init():
+            if not clients_secrets_file:
+                # No clients_secrets, can't initialize Google configuration!
+                raise ValueError("No value for `clients_secrets_file` was provided!")
+            else:
+                google_config_init(clients_secrets_file)
+
+        self.sheets = GSheetsApi(CLIENT_SECRETS_PATH, CREDENTIALS_PATH)
+
+    @classmethod
+    def download_folder(cls, url: str, output: str, quiet: bool = True, **kwargs: Any) -> None:
+        """Download a folder from Google Drive.
+
+        The folderm must be public, otherwise this function won't work.
+
+        Parameters
+        ----------
+        url : str
+            URL to the folder on Google Drive (must be public).
+        output : str
+            Local path to save the downloaded folder.
+        quiet: bool, optional
+            Suppress terminal output. Default is False.
+        """
+        gdown.download_folder(url, output=output, quiet=quiet, use_cookies=False, **kwargs)
+
+    @classmethod
+    def download_file(
+        cls,
+        output: str,
+        url: Optional[str] = None,
+        file_id: Optional[str] = None,
+        quiet: bool = True,
+        **kwargs: Any,
+    ) -> None:
+        """Download a file from Google Drive.
+
+        The file must be public, otherwise this function won't work.
+
+        Parameters
+        ----------
+        output : str
+            Local path to save the downloaded file.
+        url : str, optional
+            URL to the file on Google Drive (it must be public), by default None
+        file_id : str, optional
+            ID of the file on Google Drive (the file must be public), by default None.
+        quiet: bool, optional
+            Suppress terminal output. Default is False.
+
+        Raises
+        ------
+        ValueError
+            If neither `url` nor `id` are provided.
+        """
+        if url:
+            gdown.download(url=url, output=output, fuzzy=True, quiet=quiet, **kwargs)
+        elif file_id:
+            gdown.download(id=file_id, output=output, quiet=quiet, **kwargs)
+        else:
+            raise ValueError("You must provide a `url` or `file_id`")
+
+    @property
+    def drive(self) -> GoogleDrive:
+        """Google Drive object."""
+        gauth = GoogleAuth(settings_file=SETTINGS_PATH)
+        # gauth.LocalWebserverAuth()
+        drive = GoogleDrive(gauth)
+        return drive
+
+    def list_files(self, parent_id: str) -> GoogleDriveFileList:
+        """List files in a Google Drive folder.
+
+        Parameters
+        ----------
+        parent_id : str
+            Google Drive folder ID.
+
+        Returns
+        -------
+        List
+            Files
+        """
+        request = f"'{parent_id}' in parents and trashed=false"
+        # Get list of files
+        files = self.drive.ListFile({"q": request}).GetList()
+        return files  # type: ignore[reportReturnType]

owid/datautils/google/config.py

@@ -0,0 +1,117 @@
+"""Google configuration functions."""
+
+import os
+from pathlib import Path
+from shutil import copyfile
+from typing import Union
+
+import yaml
+from pydrive2.auth import GoogleAuth
+
+# PATHS
+CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".config", "owid")
+CLIENT_SECRETS_PATH = os.path.join(CONFIG_DIR, "google_client_secrets.json")
+SETTINGS_PATH = os.path.join(CONFIG_DIR, "google_settings.yaml")
+CREDENTIALS_PATH = os.path.join(CONFIG_DIR, "google_credentials.json")
+
+
+def google_config_init(client_secrets_file: Union[str, Path], encoding: str = "utf8") -> GoogleAuth:
+    """Initialise Google configuration files.
+
+    Uses `clients_secrets` to generate all the required Google configuration files: `SETTINGS_PATH`, `CREDENTIALS_PATH`.
+    Also, it creates a copy of `clients_secrets` in `CLIENT_SECRETS_PATH`, so the original file can be safely deleted
+    after running this method.
+
+    This should only run once before reading a file from Google Drive the first time. Subsequent executions should run
+    seamlessly.
+
+    To obtain `client_secrets_file`, follow the instructions from:
+    https://medium.com/analytics-vidhya/how-to-connect-google-drive-to-python-using-pydrive-9681b2a14f20
+
+    IMPORTANT:
+        - Additionally, make sure to add yourself in Test users, as noted in:
+          https://stackoverflow.com/questions/65980758/pydrive-quickstart-and-error-403-access-denied
+        - Select Desktop App instead of Web Application as the application type.
+
+    Method partly sourced from:
+    https://github.com/lucasrodes/whatstk/blob/bcb9cf7c256df1c9e270aab810b74ab0f7329436/whatstk/utils/gdrive.py#L38
+
+    Parameters
+    ----------
+    client_secrets_file : str
+        Path to client_secrets file.
+    encoding : str, optional
+        Encoding of the text in `client_secrets` file, by default "utf8"
+
+    Returns
+    -------
+    GoogleAuth
+        Google authenticator object.
+    """
+    # Check client_secrets
+    if not os.path.isfile(client_secrets_file):
+        raise ValueError(f"Credentials not found at {client_secrets_file}. Please provide a valid" " path!")
+    # Check or create config directory
+    if not os.path.isdir(CONFIG_DIR):
+        os.makedirs(CONFIG_DIR, exist_ok=True)
+
+    # Copy credentials to config folder
+    copyfile(client_secrets_file, CLIENT_SECRETS_PATH)
+
+    # Create settings.yaml file
+    dix = {
+        "client_config_backend": "file",
+        "client_config_file": CLIENT_SECRETS_PATH,
+        "save_credentials": True,
+        "save_credentials_backend": "file",
+        "save_credentials_file": CREDENTIALS_PATH,
+        "get_refresh_token": True,
+        "oauth_scope": [
+            "https://www.googleapis.com/auth/drive",
+            "https://www.googleapis.com/auth/drive.install",
+        ],
+    }
+    with open(SETTINGS_PATH, "w", encoding=encoding) as f:
+        yaml.dump(dix, f)
+
+    # credentials.json
+    gauth = GoogleAuth(settings_file=SETTINGS_PATH)
+    gauth.CommandLineAuth()
+    return gauth
+
+
+def _check_google_config() -> None:
+    """Run checks agains Google configuration files.
+
+    Raises
+    ------
+    FileNotFoundError
+        If owid.datautils.google.config.CONFIG_DIR directory does not exist.
+    FileNotFoundError
+        If any of the Google configuration files is missing. These include:
+            - owid.datautils.google.config.CLIENT_SECRETS_PATH
+            - owid.datautils.google.config.SETTINGS_PATH
+            - owid.datautils.google.config.CREDENTIALS_PATH
+    """
+    if not os.path.isdir(CONFIG_DIR):
+        raise FileNotFoundError(
+            f"{CONFIG_DIR} folder is not created. Please check you have run" " `google_config_init`!"
+        )
+    for f in [CLIENT_SECRETS_PATH, CREDENTIALS_PATH, SETTINGS_PATH]:
+        if not os.path.isfile(f):
+            raise FileNotFoundError(f"{f} file was not found. Please check you have run" " `google_config_init`!")
+
+
+def is_google_config_init() -> bool:
+    """Check if Google configuration files were created.
+
+    Returns
+    -------
+    bool
+        True if Google configuration files were created, False otherwise.
+    """
+    try:
+        _check_google_config()
+    except FileNotFoundError:
+        return False
+    return True