hestia-earth-utils 0.16.8__py3-none-any.whl → 0.16.10__py3-none-any.whl

hestia_earth/utils/request.py

@@ -3,18 +3,29 @@ import os
 from .tools import non_empty_value
 
 
-def api_url() -> str: return os.getenv('API_URL', 'https://api.hestia.earth')
+def api_url() -> str:
+    return os.getenv("API_URL", "https://api.hestia.earth")
 
 
-def api_access_token() -> str: return os.getenv('API_ACCESS_TOKEN')
+def api_access_token() -> str:
+    return os.getenv("API_ACCESS_TOKEN")
 
 
-def web_url() -> str: return os.getenv('WEB_URL', 'https://www.hestia.earth')
+def web_url() -> str:
+    return os.getenv("WEB_URL", "https://www.hestia.earth")
 
 
-def join_args(values) -> str: return '&'.join(list(filter(non_empty_value, values))).strip()
+def join_args(values) -> str:
+    return "&".join(list(filter(non_empty_value, values))).strip()
 
 
 def request_url(base_url: str, **kwargs) -> str:
-    args = list(map(lambda key: '='.join([key, str(kwargs.get(key))]) if kwargs.get(key) else None, kwargs.keys()))
-    return '?'.join(list(filter(non_empty_value, [base_url, join_args(args)]))).strip()
+    args = list(
+        map(
+            lambda key: (
+                "=".join([key, str(kwargs.get(key))]) if kwargs.get(key) else None
+            ),
+            kwargs.keys(),
+        )
+    )
+    return "?".join(list(filter(non_empty_value, [base_url, join_args(args)]))).strip()

hestia_earth/utils/stats.py

@@ -1,11 +1,28 @@
 """
 Based on code by Cool Farm Tool: https://gitlab.com/MethodsCFT/coolfarm-soc/-/blob/main/src/cfasoc/builders.py
 """
+
 import hashlib
 from functools import reduce
 from numpy import (
-    cumsum, dot, full, linalg, hstack, random, mean, vstack, abs, array, concatenate, exp, float64, inf, pi, prod, sign,
-    sqrt
+    cumsum,
+    dot,
+    full,
+    linalg,
+    hstack,
+    random,
+    mean,
+    vstack,
+    abs,
+    array,
+    concatenate,
+    exp,
+    float64,
+    inf,
+    pi,
+    prod,
+    sign,
+    sqrt,
 )
 from numpy.typing import NDArray, DTypeLike
 from typing import Union
@@ -90,7 +107,10 @@ def repeat_1d_array_as_columns(n_columns: int, column: NDArray) -> NDArray:
 
 
 def discrete_uniform_1d(
-    shape: tuple, low: float, high: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    low: float,
+    high: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a discrete uniform distribution and produce an array of a specified shape.
@@ -116,14 +136,14 @@ def discrete_uniform_1d(
     """
     n_rows, n_columns = shape
     rng = random.default_rng(seed)
-    return repeat_array_as_rows(
-        n_rows,
-        rng.uniform(low=low, high=high, size=n_columns)
-    )
+    return repeat_array_as_rows(n_rows, rng.uniform(low=low, high=high, size=n_columns))
 
 
 def discrete_uniform_2d(
-    shape: tuple, low: float, high: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    low: float,
+    high: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a discrete uniform distribution and produce an array of a specified shape.
@@ -151,7 +171,11 @@ def discrete_uniform_2d(
 
 
 def triangular_1d(
-    shape: tuple, low: float, high: float, mode: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    low: float,
+    high: float,
+    mode: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a triangular distribution and produce an array of a specified shape.
@@ -180,13 +204,16 @@ def triangular_1d(
     n_rows, n_columns = shape
     rng = random.default_rng(seed)
     return repeat_array_as_rows(
-        n_rows,
-        rng.triangular(left=low, mode=mode, right=high, size=n_columns)
+        n_rows, rng.triangular(left=low, mode=mode, right=high, size=n_columns)
     )
 
 
 def triangular_2d(
-    shape: tuple, low: float, high: float, mode: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    low: float,
+    high: float,
+    mode: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a triangular distribution and produce an array of a specified shape.
@@ -216,7 +243,10 @@ def triangular_2d(
 
 
 def normal_1d(
-    shape: tuple, mu: float, sigma: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    mu: float,
+    sigma: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a normal distribution and produce an array of a specified shape.
@@ -242,14 +272,14 @@ def normal_1d(
     """
     n_rows, n_columns = shape
     rng = random.default_rng(seed)
-    return repeat_array_as_rows(
-        n_rows,
-        rng.normal(loc=mu, scale=sigma, size=n_columns)
-    )
+    return repeat_array_as_rows(n_rows, rng.normal(loc=mu, scale=sigma, size=n_columns))
 
 
 def normal_2d(
-    shape: tuple, mu: float, sigma: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    mu: float,
+    sigma: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a normal distribution and produce an array of a specified shape.
@@ -277,7 +307,12 @@ def normal_2d(
 
 
 def truncated_normal_1d(
-    shape: tuple, mu: float, sigma: float, low: float, high: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    mu: float,
+    sigma: float,
+    low: float,
+    high: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a truncated normal distribution and produce an array of a specified shape.
@@ -308,12 +343,17 @@ def truncated_normal_1d(
     n_rows, n_columns = shape
     return repeat_array_as_rows(
         n_rows,
-        truncnorm_rvs(a=low, b=high, loc=mu, scale=sigma, shape=n_columns, seed=seed)
+        truncnorm_rvs(a=low, b=high, loc=mu, scale=sigma, shape=n_columns, seed=seed),
     )
 
 
 def truncated_normal_2d(
-    shape: tuple, mu: float, sigma: float, low: float, high: float, seed: Union[int, random.Generator, None] = None
+    shape: tuple,
+    mu: float,
+    sigma: float,
+    low: float,
+    high: float,
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Sample from a truncated normal distribution and produce an array of a specified shape.
@@ -348,7 +388,7 @@ def plus_minus_uncertainty_to_normal_1d(
     value: float,
     uncertainty: float,
     confidence_interval: float = 95,
-    seed: Union[int, random.Generator, None] = None
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Return a normally distributed sample given a value and uncertainty expressed as +/- a percentage.
@@ -390,8 +430,7 @@ def plus_minus_uncertainty_to_normal_1d(
     n_sds = calc_z_critical(confidence_interval)
     sigma = (value * (uncertainty / 100)) / n_sds
     return repeat_array_as_rows(
-        n_rows,
-        normal_1d(shape=(1, n_columns), mu=value, sigma=sigma, seed=seed)
+        n_rows, normal_1d(shape=(1, n_columns), mu=value, sigma=sigma, seed=seed)
     )
 
 
@@ -400,7 +439,7 @@ def plus_minus_uncertainty_to_normal_2d(
     value: float,
     uncertainty: float,
     confidence_interval: float = 95,
-    seed: Union[int, random.Generator, None] = None
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Return a normally distributed sample given a value and uncertainty expressed as +/- a percentage.
@@ -443,7 +482,7 @@ def plus_minus_uncertainty_to_normal_2d(
 
 
 def grouped_avg(arr: NDArray, n: int = 12) -> NDArray:
-    """ Row-wise averaging of numpy arrays. For example:
+    """Row-wise averaging of numpy arrays. For example:
     1   2   3
     4   5   6
     7   8   9
@@ -482,7 +521,7 @@ def grouped_avg(arr: NDArray, n: int = 12) -> NDArray:
     NDArray
         Output array
     """
-    result = cumsum(arr, 0)[n-1::n] / float(n)
+    result = cumsum(arr, 0)[n - 1 :: n] / float(n)
     result[1:] = result[1:] - result[:-1]
     return result
 
@@ -582,19 +621,14 @@ def correlated_normal_2d(
     correlated_samples = dot(cholesky_decomp, independent_samples)
 
     # Scale by standard deviations and shift by means
-    scaled_samples = (
-        correlated_samples
-        * repeat_1d_array_as_columns(n_iterations, sds)
-        + repeat_1d_array_as_columns(n_iterations, means)
-    )
+    scaled_samples = correlated_samples * repeat_1d_array_as_columns(
+        n_iterations, sds
+    ) + repeat_1d_array_as_columns(n_iterations, means)
 
     return scaled_samples
 
 
-def calc_z_critical(
-    confidence_interval: float,
-    n_sided: int = 2
-) -> float64:
+def calc_z_critical(confidence_interval: float, n_sided: int = 2) -> float64:
     """
     Calculate the z-critical value from the confidence interval.
 
@@ -640,9 +674,13 @@ def _normal_ppf(q: float64, tol: float64 = 1e-10) -> float64:
         return x_new if abs(x_new - x) >= tol else x
 
     return (
-        inf if q == 1 else
-        -inf if q == 0 else
-        reduce(lambda x, _: step(x), range(MAX_ITER), INITIAL_GUESS)
+        inf
+        if q == 1
+        else (
+            -inf
+            if q == 0
+            else reduce(lambda x, _: step(x), range(MAX_ITER), INITIAL_GUESS)
+        )
     )
 
 
@@ -715,10 +753,7 @@ def _normal_pdf(x: float64) -> float64:
     return 1 / sqrt(2 * pi) * exp(-0.5 * x**2)
 
 
-def _calc_confidence_level(
-    z_critical: float64,
-    n_sided: int = 2
-) -> float64:
+def _calc_confidence_level(z_critical: float64, n_sided: int = 2) -> float64:
     """
     Calculate the confidence interval from the z-critical value.
 
@@ -739,9 +774,7 @@ def _calc_confidence_level(
 
 
 def calc_required_iterations_monte_carlo(
-    confidence_level: float,
-    precision: float,
-    sd: float
+    confidence_level: float, precision: float, sd: float
 ) -> int:
     """
     Calculate the number of iterations required for a Monte Carlo simulation to have a desired precision, subject to a
@@ -770,9 +803,7 @@ def calc_required_iterations_monte_carlo(
 
 
 def calc_confidence_level_monte_carlo(
-    n_iterations: int,
-    precision: float,
-    sd: float
+    n_iterations: int, precision: float, sd: float
 ) -> float:
     """
     Calculate the confidence level that the sample mean calculated by the Monte Carlo simulation deviates from the
@@ -794,13 +825,11 @@ def calc_confidence_level_monte_carlo(
         The confidence level, as a percentage out of 100, that the precision should be subject too (i.e., we are x%
         sure that the sample mean deviates from the true populatation mean by less than the desired precision).
     """
-    return _calc_confidence_level(precision*sqrt(n_iterations)/sd)
+    return _calc_confidence_level(precision * sqrt(n_iterations) / sd)
 
 
 def calc_precision_monte_carlo(
-    confidence_level: float,
-    n_iterations: int,
-    sd: float
+    confidence_level: float, n_iterations: int, sd: float
 ) -> float:
     """
     Calculate the +/- precision of a Monte Carlo simulation for a desired confidence level.
@@ -822,7 +851,7 @@ def calc_precision_monte_carlo(
         units as the estimated mean.
     """
     z_critical = calc_z_critical(confidence_level)
-    return (sd*z_critical)/sqrt(n_iterations)
+    return (sd * z_critical) / sqrt(n_iterations)
 
 
 def truncnorm_rvs(
@@ -831,7 +860,7 @@ def truncnorm_rvs(
     loc: float,
     scale: float,
     shape: Union[int, tuple[int, ...]],
-    seed: Union[int, random.Generator, None] = None
+    seed: Union[int, random.Generator, None] = None,
 ) -> NDArray:
     """
     Generate random samples from a truncated normal distribution. Unlike the `scipy` equivalent, the `a` and `b` values
@@ -908,11 +937,7 @@ def add_normal_distributions(
         **Z = X<sub>1</sub> + X<sub>2</sub>**.
     """
     mu_sum = mu_1 + mu_2
-    sigma_sum = sqrt(
-        sigma_1 ** 2
-        + sigma_2 ** 2
-        + 2 * rho * sigma_1 * sigma_2
-    )
+    sigma_sum = sqrt(sigma_1**2 + sigma_2**2 + 2 * rho * sigma_1 * sigma_2)
     return mu_sum, sigma_sum
 
 
@@ -953,11 +978,7 @@ def subtract_normal_distributions(
         **Z = X<sub>1</sub> - X<sub>2</sub>**.
     """
     mu_sum = mu_1 - mu_2
-    sigma_sum = sqrt(
-        sigma_1 ** 2
-        + sigma_2 ** 2
-        - 2 * rho * sigma_1 * sigma_2
-    )
+    sigma_sum = sqrt(sigma_1**2 + sigma_2**2 - 2 * rho * sigma_1 * sigma_2)
     return mu_sum, sigma_sum
 
 
@@ -967,7 +988,7 @@ def lerp_normal_distributions(
     mu_2: float,
     sigma_2: float,
     alpha: float,
-    rho: float = 0
+    rho: float = 0,
 ) -> tuple[float, float]:
     """
     Linearly interpolate between two normal distributions, with optional correlation.
@@ -1008,8 +1029,8 @@ def lerp_normal_distributions(
     """
     mu_Z = (1 - alpha) * mu_1 + alpha * mu_2
     var_Z = (
-        ((1 - alpha) ** 2) * sigma_1 ** 2
-        + (alpha ** 2) * sigma_2 ** 2
+        ((1 - alpha) ** 2) * sigma_1**2
+        + (alpha**2) * sigma_2**2
         + 2 * alpha * (1 - alpha) * rho * sigma_1 * sigma_2
     )
     sigma_Z = sqrt(var_Z)

hestia_earth/utils/storage/_azure_client.py

@@ -1,8 +1,8 @@
 import os
 
-CONN_STRING = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
-CONTAINER = os.getenv('AZURE_STORAGE_CONTAINER')
-CONTAINER_GLOSSARY = os.getenv('AZURE_STORAGE_CONTAINER_GLOSSARY')
+CONN_STRING = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+CONTAINER = os.getenv("AZURE_STORAGE_CONTAINER")
+CONTAINER_GLOSSARY = os.getenv("AZURE_STORAGE_CONTAINER_GLOSSARY")
 _blob_service = None  # noqa: F824
 
 
@@ -10,7 +10,12 @@ _blob_service = None  # noqa: F824
 def _get_blob_service_client():
     global _blob_service
     from azure.storage.blob import BlobServiceClient
-    _blob_service = BlobServiceClient.from_connection_string(CONN_STRING) if _blob_service is None else _blob_service
+
+    _blob_service = (
+        BlobServiceClient.from_connection_string(CONN_STRING)
+        if _blob_service is None
+        else _blob_service
+    )
     return _blob_service
 
 
@@ -20,8 +25,11 @@ def _get_container(glossary: bool = False) -> str:
 
 def _load_from_container(container: str, key: str):
     from azure.core.exceptions import ResourceNotFoundError
+
     try:
-        blob_client = _get_blob_service_client().get_blob_client(container=container, blob=key)
+        blob_client = _get_blob_service_client().get_blob_client(
+            container=container, blob=key
+        )
         return blob_client.download_blob().readall()
     except ResourceNotFoundError:
         return None
@@ -29,8 +37,11 @@ def _load_from_container(container: str, key: str):
 
 def _exists_in_container(container: str, key: str):
     from azure.core.exceptions import ResourceNotFoundError
+
     try:
-        blob_client = _get_blob_service_client().get_blob_client(container=container, blob=key)
+        blob_client = _get_blob_service_client().get_blob_client(
+            container=container, blob=key
+        )
         return blob_client.exists()
     except ResourceNotFoundError:
         return False

hestia_earth/utils/storage/_local_client.py

@@ -2,16 +2,21 @@ import os
 
 
 def _get_folder(glossary: bool = False) -> str:
-    return os.getenv('DOWNLOAD_FOLDER_GLOSSARY') if glossary else os.getenv('DOWNLOAD_FOLDER')
+    return (
+        os.getenv("DOWNLOAD_FOLDER_GLOSSARY")
+        if glossary
+        else os.getenv("DOWNLOAD_FOLDER")
+    )
 
 
 def _load_from_folder(folder: str, key: str):
     try:
         with open(os.path.join(folder, key)) as f:
-            return f.read().encode('utf-8')
+            return f.read().encode("utf-8")
     except Exception:
         # in case the file does not exist, should simply return None
         return None
 
 
-def _exists_in_folder(folder: str, key: str): return os.path.exists(os.path.join(folder, key))
+def _exists_in_folder(folder: str, key: str):
+    return os.path.exists(os.path.join(folder, key))

hestia_earth/utils/storage/_s3_client.py

@@ -1,7 +1,7 @@
 import os
 
-BUCKET = os.getenv('AWS_BUCKET')
-BUCKET_GLOSSARY = os.getenv('AWS_BUCKET_GLOSSARY')
+BUCKET = os.getenv("AWS_BUCKET")
+BUCKET_GLOSSARY = os.getenv("AWS_BUCKET_GLOSSARY")
 _s3_client = None  # noqa: F824
 
 
@@ -9,7 +9,10 @@ _s3_client = None  # noqa: F824
 def _get_s3_client():
     global _s3_client
     import boto3
-    _s3_client = boto3.session.Session().client('s3') if _s3_client is None else _s3_client
+
+    _s3_client = (
+        boto3.session.Session().client("s3") if _s3_client is None else _s3_client
+    )
     return _s3_client
 
 
@@ -19,14 +22,16 @@ def _get_bucket(glossary: bool = False) -> str:
 
 def _load_from_bucket(bucket: str, key: str):
     from botocore.exceptions import ClientError
+
     try:
-        return _get_s3_client().get_object(Bucket=bucket, Key=key)['Body'].read()
+        return _get_s3_client().get_object(Bucket=bucket, Key=key)["Body"].read()
     except ClientError:
         return None
 
 
 def _exists_in_bucket(bucket: str, key: str):
     from botocore.exceptions import ClientError
+
     try:
         _get_s3_client().head_object(Bucket=bucket, Key=key)
         return True
@@ -36,14 +41,18 @@ def _exists_in_bucket(bucket: str, key: str):
 
 def _read_size(bucket: str, key: str):
     try:
-        return _get_s3_client().head_object(Bucket=bucket, Key=key).get('ContentLength')
+        return _get_s3_client().head_object(Bucket=bucket, Key=key).get("ContentLength")
     except Exception:
         return 0
 
 
 def _read_metadata(bucket_name: str, key: str):
     try:
-        return _get_s3_client().head_object(Bucket=bucket_name, Key=key).get('Metadata', {})
+        return (
+            _get_s3_client()
+            .head_object(Bucket=bucket_name, Key=key)
+            .get("Metadata", {})
+        )
     except Exception:
         return {}
 
@@ -55,9 +64,9 @@ def _update_metadata(bucket: str, key: str, data: dict = {}):
         _get_s3_client().copy_object(
             Bucket=bucket,
             Key=key,
-            CopySource={'Bucket': bucket, 'Key': key},
+            CopySource={"Bucket": bucket, "Key": key},
             Metadata=metadata,
-            MetadataDirective='REPLACE'
+            MetadataDirective="REPLACE",
         )
     except Exception:
         pass
@@ -65,33 +74,32 @@ def _update_metadata(bucket: str, key: str, data: dict = {}):
 
 def _last_modified(bucket: str, key: str):
     try:
-        return _get_s3_client().head_object(Bucket=bucket, Key=key).get('LastModified')
+        return _get_s3_client().head_object(Bucket=bucket, Key=key).get("LastModified")
     except Exception:
         return None
 
 
 def _upload_to_bucket(bucket: str, key: str, body, content_type: str):
     from botocore.exceptions import ClientError
+
     try:
         return _get_s3_client().put_object(
-            Bucket=bucket,
-            Key=key,
-            Body=body,
-            ContentType=content_type
+            Bucket=bucket, Key=key, Body=body, ContentType=content_type
         )
     except ClientError:
         return None
 
 
-def _list_bucket_objects(bucket: str, folder: str = ''):
+def _list_bucket_objects(bucket: str, folder: str = ""):
     from botocore.exceptions import ClientError
+
     try:
-        paginator = _get_s3_client().get_paginator('list_objects_v2')
+        paginator = _get_s3_client().get_paginator("list_objects_v2")
         pages = paginator.paginate(Bucket=bucket, Prefix=folder)
 
         contents = []
         for page in pages:
-            contents.extend(page.get('Contents', []))
+            contents.extend(page.get("Contents", []))
         return contents
     except ClientError:
         return []
@@ -99,17 +107,14 @@ def _list_bucket_objects(bucket: str, folder: str = ''):
 
 def _delete_objects(bucket: str, objects: list):
     from botocore.exceptions import ClientError
+
     try:
         # delete in batch of 1000 max allowed
         batch_size = 1000
         for i in range(0, len(objects), batch_size):
-            batch_objects = objects[i:i + batch_size]
+            batch_objects = objects[i : i + batch_size]
             _get_s3_client().delete_objects(
-                Bucket=bucket,
-                Delete={
-                    'Objects': batch_objects,
-                    'Quiet': True
-                }
+                Bucket=bucket, Delete={"Objects": batch_objects, "Quiet": True}
             )
     except ClientError:
         return None

hestia_earth/utils/storage/_sns_client.py

@@ -7,6 +7,11 @@ _sns_client = None  # noqa: F824
 def _get_sns_client():
     global _sns_client
     import boto3
-    region_name = os.getenv('AWS_REGION')
-    _sns_client = boto3.session.Session().client('sns', region_name=region_name) if _sns_client is None else _sns_client
+
+    region_name = os.getenv("AWS_REGION")
+    _sns_client = (
+        boto3.session.Session().client("sns", region_name=region_name)
+        if _sns_client is None
+        else _sns_client
+    )
     return _sns_client

hestia_earth/utils/term.py

@@ -12,7 +12,7 @@ def _load_term_file(term_type: str):
     try:
         filepath = f"glossary/{term_type}.json"
         nodes = json.loads(_load_from_storage(filepath, glossary=True))
-        return {node.get('@id'): node for node in nodes}
+        return {node.get("@id"): node for node in nodes}
     except Exception:
         return {}
 
@@ -21,11 +21,11 @@ def download_term(term: Union[str, dict], termType: Union[str, TermTermType] = N
     """
     Download a Term, using the glossary file if available, or default to the standard download.
     """
-    term_id = term.get('@id', term.get('id')) if isinstance(term, dict) else term
+    term_id = term.get("@id", term.get("id")) if isinstance(term, dict) else term
     term_type = (
-        termType if isinstance(termType, str) else termType.value
-    ) if termType else (
-        term.get('termType') if isinstance(term, dict) else None
+        (termType if isinstance(termType, str) else termType.value)
+        if termType
+        else (term.get("termType") if isinstance(term, dict) else None)
     )
     cached_nodes = _load_term_file(term_type) if term_type else {}
     return cached_nodes.get(term_id) or download_hestia(term_id)