hestia-earth-utils 0.16.6__tar.gz → 0.16.7__tar.gz

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.6
+Version: 0.16.7
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
@@ -13,12 +13,8 @@ Requires-Dist: hestia-earth-schema>=35.0.1
 Requires-Dist: requests>=2.24.0
 Requires-Dist: urllib3~=1.26.0
 Requires-Dist: python-dateutil>=2.8.1
-Requires-Dist: numpy>=2
+Requires-Dist: pandas>=2
 Requires-Dist: flatten_json
-Provides-Extra: pivot-csv
-Requires-Dist: pandas>=2; extra == "pivot-csv"
-Provides-Extra: table
-Requires-Dist: pandas>=2; extra == "table"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -26,7 +22,6 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
-Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/hestia_earth/utils/blank_node.py

@@ -5,7 +5,7 @@ from functools import reduce
 from statistics import mode, mean
 from hestia_earth.schema import TermTermType
 
-from .lookup import download_lookup, get_table_value, column_name
+from .lookup import download_lookup, get_table_value
 from .tools import non_empty_list, non_empty_value, flatten
 from .emission import cycle_emissions_in_system_boundary
 from .model import filter_list_term_type
@@ -15,7 +15,7 @@ def get_lookup_value(blank_node: dict, column: str):
     term = blank_node.get('term', {})
     table_name = f"{term.get('termType')}.csv" if term else None
     value = get_table_value(
-        download_lookup(table_name), 'termid', term.get('@id'), column_name(column)
+        download_lookup(table_name), 'term.id', term.get('@id'), column
     ) if table_name else None
     return value
 

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/hestia_earth/utils/lookup.py

@@ -1,15 +1,14 @@
 from functools import reduce
-from typing import Union
+from typing import Any
 import requests
-import numpy
+from io import StringIO
+import pandas as pd
 
 from .storage import _load_from_storage
 from .request import request_url, web_url
-from .csv_utils import csv_str_to_recarray, csv_file_to_recarray, is_missing_value, _replace_chars
 
 _GLOSSARY_FOLDER = 'glossary/lookups'
 _memory = {}
-_INDEX_COL = 'termid'
 
 
 def _memory_wrapper(key: str, func):
@@ -18,6 +17,18 @@ def _memory_wrapper(key: str, func):
     return _memory[key]
 
 
+def _read_csv(value: str) -> pd.DataFrame:
+    return pd.read_csv(value, na_values=['-', ''])
+
+
+def _read_csv_from_string(data: str) -> pd.DataFrame:
+    return _read_csv(StringIO(data))
+
+
+def is_missing_value(value):
+    return pd.isna(value) or value is None or value == '' or value == '-'
+
+
 def load_lookup(filepath: str, keep_in_memory: bool = False):
     """
     Import local lookup table as csv file into a `numpy.recarray`.
@@ -34,7 +45,7 @@ def load_lookup(filepath: str, keep_in_memory: bool = False):
     numpy.recarray
         The `numpy.recarray` converted from the csv content.
     """
-    def load(): return csv_file_to_recarray(filepath)
+    def load(): return _read_csv(filepath)
     return _memory_wrapper(filepath, load) if keep_in_memory else load()
 
 
@@ -53,20 +64,7 @@ def _download_lookup_data(filename: str):
         return fallback()
 
 
-def _build_index(array: numpy.recarray):
-    columns = list(array.dtype.names)
-    try:
-        return {
-            row[_INDEX_COL]: {col: row[col] for col in columns}
-            for row in array
-        } if _INDEX_COL in columns else array
-    except TypeError:
-        return {
-            array[_INDEX_COL].item(): {col: array[col].item() for col in columns}
-        } if _INDEX_COL in columns else array
-
-
-def download_lookup(filename: str, keep_in_memory: bool = True, build_index: bool = False):
+def download_lookup(filename: str, keep_in_memory: bool = True):
     """
     Download lookup table from HESTIA as csv into a `numpy.recarray`.
 
@@ -86,8 +84,7 @@ def download_lookup(filename: str, keep_in_memory: bool = True, build_index: boo
     """
     def load():
         data = _download_lookup_data(filename)
-        rec = csv_str_to_recarray(data) if data else None
-        return (_build_index(rec) if build_index else rec) if data else None
+        return _read_csv_from_string(data) if data else None
 
     try:
         return _memory_wrapper(filename, load) if keep_in_memory else load()
@@ -97,19 +94,9 @@ def download_lookup(filename: str, keep_in_memory: bool = True, build_index: boo
 
 def column_name(key: str):
     """
-    Convert the column name to a usable key on a `numpy.recarray`.
-
-    Parameters
-    ----------
-    key : str
-        The column name.
-
-    Returns
-    -------
-    str
-        The column name that can be used in `get_table_value`.
+    Deprecated. Columns are no longer renamed.
     """
-    return _replace_chars(key) if key else ''
+    return key
 
 
 def _parse_value(value: str):
@@ -124,36 +111,24 @@ def _parse_value(value: str):
         return value
 
 
-def _get_single_table_value(data: Union[dict, numpy.recarray], col_match: str, col_match_with, col_val):
-    return (
-        data.get(col_match_with, {})[col_val] if isinstance(data, dict) else
-        data[data[col_match] == col_match_with][col_val][0]
-    )
-
-
-def _get_multiple_table_values(data: Union[dict, numpy.recarray], col_match: str, col_match_with, col_val):
-    def reducer(x, values):
-        col = values[1]
-        value = col_match_with[values[0]]
-        return x.get(value) if isinstance(x, dict) else x[x[col] == value]
+def _get_single_table_value(df: pd.DataFrame, col_match: str, col_match_with, col_val):
+    filtered_df = df[df[col_match] == col_match_with]
+    return None if filtered_df.empty else filtered_df[col_val].iloc[0]
 
-    return reduce(reducer, enumerate(col_match), data)[col_val][0]
 
-
-def get_table_value(lookup: Union[dict, numpy.recarray], col_match: str, col_match_with, col_val, default_value=''):
+def get_table_value(lookup: pd.DataFrame, col_match: str, col_match_with: str, col_val: Any, default_value=''):
     """
     Get a value matched by one or more columns from a `numpy.recarray`.
 
     Parameters
     ----------
-    lookup : dict | numpy.recarray
+    lookup : DataFrame
         The value returned by the `download_lookup` function.
     col_match : str
         Which `column` should be used to find data in. This will restrict the rows to search for.
         Can be a single `str` or a list of `str`. If a list is used, must be the same length as `col_match_with`.
-    col_match_with
+    col_match_with: str
         Which column `value` should be used to find data in. This will restrict the rows to search for.
-        Can be a single `str` or a list of `str`. If a list is used, must be the same length as `col_match`.
     col_val: str
         The column which contains the value to look for.
     default_value : Any
@@ -164,44 +139,35 @@ def get_table_value(lookup: Union[dict, numpy.recarray], col_match: str, col_mat
     str
         The value found or `None` if no match.
     """
-    single = isinstance(col_match, str) and isinstance(col_match_with, str)
     try:
-        value = (
-            _get_single_table_value(lookup, col_match, col_match_with, col_val) if single else
-            _get_multiple_table_values(lookup, col_match, col_match_with, col_val)
-        )
+        value = _get_single_table_value(lookup, col_match, col_match_with, col_val)
+        print(value, type(value))
         return default_value if is_missing_value(value) else _parse_value(value)
     except Exception:
         return None
 
 
-def find_term_ids_by(lookup: Union[dict, numpy.recarray], col_match: str, col_match_with):
+def find_term_ids_by(lookup: pd.DataFrame, col_match: str, col_match_with: str):
     """
     Find `term.id` values where a column matches a specific value.
 
     Parameters
     ----------
-    lookup : dict | numpy.recarray
+    lookup : DataFrame
         The value returned by the `download_lookup` function.
     col_match : str
         Which `column` should be used to find data in. This will restrict the rows to search for.
         Can be a single `str` or a list of `str`. If a list is used, must be the same length as `col_match_with`.
-    col_match_with
+    col_match_with: str
         Which column `value` should be used to find data in. This will restrict the rows to search for.
-        Can be a single `str` or a list of `str`. If a list is used, must be the same length as `col_match`.
 
     Returns
     -------
     list[str]
         The list of `term.id` that matched the expected column value.
     """
-    term_ids = (
-        set([
-            key
-            for key, value in lookup.items()
-            if value.get(col_match) == col_match_with
-        ])
-    ) if isinstance(lookup, dict) else set(list(lookup[lookup[col_match] == col_match_with].termid))
+    filtered_df = lookup[lookup[col_match] == col_match_with]
+    term_ids = filtered_df['term.id'].unique().tolist() if 'term.id' in filtered_df.columns else []
     return list(map(str, term_ids))
 
 
@@ -266,13 +232,13 @@ def extract_grouped_data_closest_date(data: str, year: int) -> str:
     return None if closest_year is None else _parse_value(data_by_date.get(closest_year))
 
 
-def lookup_term_ids(lookup: Union[dict, numpy.recarray]):
+def lookup_term_ids(lookup: pd.DataFrame):
     """
     Get the `term.id` values from a lookup.
 
     Parameters
     ----------
-    lookup : dict | numpy.recarray
+    lookup : DataFrame
         The value returned by the `download_lookup` function.
 
     Returns
@@ -280,16 +246,16 @@ def lookup_term_ids(lookup: Union[dict, numpy.recarray]):
     list[str]
         The `term.id` values from the lookup.
     """
-    return lookup.keys() if isinstance(lookup, dict) else list(lookup.termid)
+    return list(map(str, lookup['term.id'].tolist())) if 'term.id' in lookup.columns else []
 
 
-def lookup_columns(lookup: Union[dict, numpy.recarray]):
+def lookup_columns(lookup: pd.DataFrame):
     """
     Get the columns from a lookup.
 
     Parameters
     ----------
-    lookup : dict | numpy.recarray
+    lookup : DataFrame
         The value returned by the `download_lookup` function.
 
     Returns
@@ -297,4 +263,4 @@ def lookup_columns(lookup: Union[dict, numpy.recarray]):
     list[str]
         The columns from the lookup.
     """
-    return list(list(lookup.values())[0].keys()) if isinstance(lookup, dict) else list(lookup.dtype.names)
+    return list(lookup.columns)

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/hestia_earth/utils/lookup_utils.py

@@ -2,7 +2,7 @@ from functools import lru_cache
 import json
 from hestia_earth.schema import SchemaType
 
-from .lookup import _download_lookup_data, download_lookup, get_table_value, column_name
+from .lookup import _download_lookup_data, download_lookup, get_table_value
 from .api import download_hestia
 from .tools import non_empty_list, flatten
 
@@ -45,7 +45,7 @@ def _allowed_model_mapping(model: str, term_id: str, column: str):
     mapping = _allowed_mapping_data()
     value = mapping.get(term_id, {}).get(model, {}).get(column) if mapping else get_table_value(
         download_lookup(f"{(download_hestia(term_id) or {}).get('termType')}-model-{column}.csv"),
-        'termid', term_id, column_name(column)
+        'term.id', term_id, column
     )
     return (value or _ALLOW_ALL).split(';') if isinstance(value, str) else _ALLOW_ALL
 
@@ -78,7 +78,7 @@ def _allowed_mapping(term_id: str, column: str):
     mapping = _allowed_mapping_data()
     value = mapping.get(term_id, {}).get(column) if mapping else get_table_value(
         download_lookup(f"{(download_hestia(term_id) or {}).get('termType')}.csv"),
-        'termid', term_id, column_name(column)
+        'term.id', term_id, column
     )
     return (value or _ALLOW_ALL).split(';') if isinstance(value, str) else _ALLOW_ALL
 
@@ -174,7 +174,7 @@ def is_in_system_boundary(term_id: str) -> bool:
     column = 'inHestiaDefaultSystemBoundary'
     value = mapping.get(term_id, {}).get(column) if mapping else get_table_value(
         download_lookup(f"{(download_hestia(term_id) or {}).get('termType')}.csv"),
-        'termid', term_id, column_name(column)
+        'term.id', term_id, column
     )
     # handle numpy bool from table value
     return not (not value)

hestia_earth_utils-0.16.7/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.16.7'

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/hestia_earth_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.6
+Version: 0.16.7
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
@@ -13,12 +13,8 @@ Requires-Dist: hestia-earth-schema>=35.0.1
 Requires-Dist: requests>=2.24.0
 Requires-Dist: urllib3~=1.26.0
 Requires-Dist: python-dateutil>=2.8.1
-Requires-Dist: numpy>=2
+Requires-Dist: pandas>=2
 Requires-Dist: flatten_json
-Provides-Extra: pivot-csv
-Requires-Dist: pandas>=2; extra == "pivot-csv"
-Provides-Extra: table
-Requires-Dist: pandas>=2; extra == "table"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -26,7 +22,6 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
-Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/hestia_earth_utils.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ hestia_earth/utils/__init__.py
 hestia_earth/utils/api.py
 hestia_earth/utils/blank_node.py
 hestia_earth/utils/calculation_status.py
-hestia_earth/utils/csv_utils.py
 hestia_earth/utils/cycle.py
 hestia_earth/utils/date.py
 hestia_earth/utils/descriptive_stats.py

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/hestia_earth_utils.egg-info/requires.txt

@@ -2,11 +2,5 @@ hestia-earth-schema>=35.0.1
 requests>=2.24.0
 urllib3~=1.26.0
 python-dateutil>=2.8.1
-numpy>=2
-flatten_json
-
-[pivot-csv]
-pandas>=2
-
-[table]
 pandas>=2
+flatten_json

{hestia_earth_utils-0.16.6 → hestia_earth_utils-0.16.7}/setup.py

@@ -32,9 +32,5 @@ setup(
     scripts=[
         'bin/hestia-pivot-csv',
         'bin/hestia-format-upload'
-    ],
-    extras_require={
-        'pivot-csv': ['pandas>=2'],
-        'table': ['pandas>=2'],
-    }
+    ]
 )

hestia_earth_utils-0.16.7/tests/test_lookup.py

@@ -0,0 +1,128 @@
+import pytest
+import pandas as pd
+
+from .utils import fixtures_path
+from hestia_earth.utils.lookup import (
+    load_lookup,
+    get_table_value,
+    find_term_ids_by,
+    download_lookup,
+    extract_grouped_data,
+    extract_grouped_data_closest_date,
+    _get_single_table_value,
+    lookup_term_ids,
+    lookup_columns
+)
+
+
+def test_load_lookup_type():
+    lookup = load_lookup(f"{fixtures_path}/lookup.csv")
+    assert isinstance(lookup, pd.DataFrame)
+
+
+@pytest.mark.parametrize(
+    'col_match,col_match_with,col_val,expected',
+    [
+        ('Col1', 'val10', 'Col3', 'val30'),
+        ('Col1', 'val10', 'Col5', None),
+        ('Col1', 'val10', 'Col4', ''),
+    ]
+)
+def test_get_table_value(col_match, col_match_with, col_val, expected):
+    lookup = load_lookup(f"{fixtures_path}/lookup.csv")
+    assert get_table_value(lookup, col_match, col_match_with, col_val) == expected
+
+
+def test_get_table_value_no_lookup():
+    assert not get_table_value(None, 'Col10', 'val10', 'Col3')
+
+
+def test_get_table_value_default_value():
+    lookup = load_lookup(f"{fixtures_path}/lookup.csv")
+    assert get_table_value(lookup, 'Col2', 'val22', 'Col1') == ''
+
+    lookup = download_lookup('crop.csv')
+    assert get_table_value(lookup, 'term.id', 'genericCropSeed', 'Plantation_density') == ''
+    assert get_table_value(lookup, 'term.id', 'fixedNitrogen', 'Combustion_Factor_crop_residue') == ''
+
+
+def test_find_term_ids_by():
+    lookup = download_lookup('crop.csv')
+    assert 'wheatGrain' in find_term_ids_by(lookup, 'cropGroupingFAO', 'Temporary crops')
+
+
+def test_handle_missing_float_value():
+    filename = 'measurement.csv'
+    lookup = download_lookup(filename)
+    assert get_table_value(lookup, 'term.id', 'rainfallPeriod', 'maximum') == ''
+
+
+def test_handle_missing_lookup_value():
+    filename = 'region-crop-cropGroupingFaostatProduction-price.csv'
+    lookup = download_lookup(filename)
+    assert get_table_value(lookup, 'term.id', 'GADM-CYP', 'Sugar crops nes') is None
+
+
+def test_extract_grouped_data_no_data():
+    assert not extract_grouped_data('', '2000')
+    assert not extract_grouped_data('-', '2000')
+
+
+def test_extract_grouped_data():
+    data = 'Average_price_per_tonne:106950.5556;1991:-;1992:-'
+    assert extract_grouped_data(data, 'Average_price_per_tonne') == 106950.5556
+    assert extract_grouped_data(data, '2010') is None
+
+
+def test_extract_grouped_data_lookup():
+    filename = 'region-crop-cropGroupingFaostatProduction-price.csv'
+    lookup = download_lookup(filename)
+    data = get_table_value(lookup, 'term.id', 'GADM-NPL', 'Chick peas, dry')
+    assert extract_grouped_data(data, '2000') is None
+    assert extract_grouped_data(data, '2012') is not None
+
+    filename = 'region-animalProduct-animalProductGroupingFAO-price.csv'
+    lookup = download_lookup(filename)
+    data = get_table_value(lookup, 'term.id', 'GADM-NPL', 'Eggs from other birds in shell, fresh, n.e.c.')
+    assert extract_grouped_data(data, '2000') is None
+    assert extract_grouped_data(data, '2012') is not None
+
+
+def test_get_single_table_value_float_values():
+    filename = 'ecoClimateZone.csv'
+    lookup = download_lookup(filename)
+    column = 'STEHFEST_BOUWMAN_2006_N2O-N_FACTOR'
+    assert _get_single_table_value(lookup, 'ecoClimateZone', 11, column) == -0.3022
+
+
+def test_extract_grouped_data_closest_date_no_data():
+    assert not extract_grouped_data_closest_date('', 2000)
+    assert not extract_grouped_data_closest_date('-', 2000)
+
+
+def test_extract_grouped_data_closest_date():
+    data = '2000:-;2001:0.1;2002:0.2;2003:0.3;2004:0.4;2005:0.5'
+    assert extract_grouped_data_closest_date(data, 2000) == 0.1
+    assert extract_grouped_data_closest_date(data, 2001) == 0.1
+    assert extract_grouped_data_closest_date(data, 2020) == 0.5
+
+
+def test_lookup_term_ids():
+    assert 'wheatGrain' in lookup_term_ids(download_lookup('crop.csv'))
+
+
+def test_lookup_columns():
+    assert 'term.id' in lookup_columns(download_lookup('crop.csv'))
+
+
+def test_get_data_advanced():
+    lookup = download_lookup('liveAnimal.csv')
+    value = get_table_value(lookup, 'term.id', 'sheepRam', 'ratioCPregnancyNetEnergyPregnancyIpcc2019')
+    assert value == ''
+
+
+def test_grouping_with_comma():
+    lookup = download_lookup('animalProduct.csv')
+    term_id = 'meatChickenReadyToCookWeight'
+    value = get_table_value(lookup, 'term.id', term_id, 'animalProductGroupingFAO')
+    assert value == 'Meat of chickens, fresh or chilled'

hestia_earth_utils-0.16.6/hestia_earth/utils/csv_utils.py

@@ -1,84 +0,0 @@
-import io
-import csv
-import re
-import numpy as np
-
-_MISSING_VALUE = '-'
-_MISSING = -99999
-_DELIMITER = ','
-_QUOTE_CHAR = '"'
-ENCODING = 'ISO-8859-1'
-# default: " !#$%&'()*+,-./:;<=>?@[\\]^{|}~"
-_DELETE_CHARS = " !#$%&'()*,./:;<=>?@^{|}~"
-
-
-def is_missing_value(value): return value == _MISSING_VALUE or value == _MISSING or value == str(_MISSING)
-
-
-def _replace_missing_values(value: str): return str(_MISSING) if str(value) == _MISSING_VALUE else value
-
-
-def _replace_chars(value: str): return re.sub(f'[{re.escape(_DELETE_CHARS)}]', '', value.replace(' ', '_'))
-
-
-def _text_to_csv(csv_content: str):
-    return csv.reader(io.StringIO(csv_content.strip()), delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
-
-
-def _get_columns(csv_content: str):
-    try:
-        reader = _text_to_csv(csv_content)
-        names = next(reader)
-        return list(map(_replace_chars, names))
-    except StopIteration:
-        return []
-
-
-def _get_rows(csv_content: str):
-    string_io = io.StringIO(csv_content.strip())
-    try:
-        next(string_io)
-    except StopIteration:
-        return
-
-    return csv.reader(string_io, delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
-
-
-def _csv_str_to_recarray_chunks_numpy(csv_content: str, chunk_size: int = 5):
-    names = _get_columns(csv_content)
-    num_cols = len(names)
-
-    max_size = 1000
-    dtype = [(name, f"U{max_size}") for name in names]
-
-    reader = _get_rows(csv_content)
-
-    # 4. Process the file in batches
-    chunk_rows = []
-    for row in reader:
-        if not row:
-            continue
-        if len(row) != num_cols:
-            continue
-
-        # replace missing values
-        processed_row = tuple(_replace_missing_values(field) for field in row)
-        chunk_rows.append(processed_row)
-
-        if len(chunk_rows) >= chunk_size:
-            yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
-            chunk_rows = []
-
-    if chunk_rows:
-        yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
-
-
-def csv_str_to_recarray(csv_content: str) -> np.recarray:
-    array_rows = list(_csv_str_to_recarray_chunks_numpy(csv_content))
-    return np.hstack(array_rows).view(np.recarray)
-
-
-def csv_file_to_recarray(filepath: str):
-    with open(filepath, 'r', encoding=ENCODING) as f:
-        content = f.read()
-    return csv_str_to_recarray(content)

hestia_earth_utils-0.16.6/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = '0.16.6'

hestia_earth_utils-0.16.6/tests/test_lookup.py

@@ -1,163 +0,0 @@
-import numpy
-
-from .utils import fixtures_path
-from hestia_earth.utils.lookup import (
-    load_lookup,
-    column_name,
-    get_table_value,
-    find_term_ids_by,
-    download_lookup,
-    extract_grouped_data,
-    extract_grouped_data_closest_date,
-    _get_single_table_value,
-    lookup_term_ids,
-    lookup_columns
-)
-
-
-def test_load_lookup_numpy_array():
-    lookup = load_lookup(f"{fixtures_path}/lookup.csv")
-    assert isinstance(lookup, numpy.recarray)
-
-
-def test_column_name():
-    assert column_name('Maize (corn)') == 'Maize_corn'
-    assert column_name('grassland/pasture/meadow') == 'grasslandpasturemeadow'
-
-
-def test_get_table_value():
-    lookup = load_lookup(f"{fixtures_path}/lookup.csv")
-
-    # single column match
-    assert get_table_value(lookup, column_name('Col1'), 'val10', column_name('Col3')) == 'val30'
-    # multiple column match
-    assert get_table_value(lookup, [
-        column_name('Col1'),
-        column_name('Col2'),
-    ], [
-        'val10',
-        'val21'
-    ], column_name('Col3')) == 'val31'
-    # no match
-    assert not get_table_value(lookup, column_name('Col10'), 'val10', column_name('Col3'))
-
-    # column does not exist
-    assert not get_table_value(lookup, [
-        column_name('Col1'),
-        column_name('Col2'),
-    ], [
-        'random',
-        'val21'
-    ], column_name('random'))
-
-    # table does not exist
-    assert not get_table_value(None, column_name('Col10'), 'val10', column_name('Col3'))
-
-
-def test_get_table_value_empty():
-    lookup = load_lookup(f"{fixtures_path}/lookup.csv")
-    assert get_table_value(lookup, column_name('Col1'), 'val10', column_name('Col4'), default_value=None) is None
-    assert get_table_value(lookup, column_name('Col2'), 'val22', column_name('Col1')) == ''
-
-    lookup = download_lookup('crop.csv')
-    assert get_table_value(lookup, 'termid', 'genericCropSeed', column_name('Plantation_density')) == ''
-
-
-def test_find_term_ids_by():
-    lookup = download_lookup('crop.csv')
-    assert 'wheatGrain' in find_term_ids_by(lookup, column_name('cropGroupingFAO'), 'Temporary crops')
-
-
-def test_download_lookup_with_index():
-    filename = 'crop.csv'
-    lookup = download_lookup(filename, keep_in_memory=False, build_index=True)
-    assert isinstance(lookup, dict) is True
-
-
-def test_download_lookup_without_index():
-    filename = 'crop.csv'
-    lookup = download_lookup(filename, keep_in_memory=False, build_index=False)
-    assert isinstance(lookup, numpy.recarray) is True
-
-
-def test_handle_missing_float_value():
-    filename = 'measurement.csv'
-    lookup = download_lookup(filename)
-    assert get_table_value(lookup, 'termid', 'rainfallPeriod', 'maximum') == ''
-
-
-def test_handle_missing_string_value():
-    filename = 'crop.csv'
-    lookup = download_lookup(filename)
-    assert get_table_value(lookup, 'termid', 'fixedNitrogen', 'combustion_factor_crop_residue') is None
-
-
-def test_handle_missing_lookup_value():
-    filename = 'region-crop-cropGroupingFaostatProduction-price.csv'
-    lookup = download_lookup(filename)
-    assert get_table_value(lookup, 'termid', 'GADM-CYP', column_name('Sugar crops nes')) is None
-
-
-def test_extract_grouped_data_no_data():
-    assert not extract_grouped_data('', '2000')
-    assert not extract_grouped_data('-', '2000')
-
-
-def test_extract_grouped_data():
-    data = 'Average_price_per_tonne:106950.5556;1991:-;1992:-'
-    assert extract_grouped_data(data, 'Average_price_per_tonne') == 106950.5556
-    assert extract_grouped_data(data, '2010') is None
-
-
-def test_extract_grouped_data_lookup():
-    filename = 'region-crop-cropGroupingFaostatProduction-price.csv'
-    lookup = download_lookup(filename)
-    data = get_table_value(lookup, 'termid', 'GADM-NPL', column_name('Chick peas, dry'))
-    assert extract_grouped_data(data, '2000') is None
-    assert extract_grouped_data(data, '2012') is not None
-
-    filename = 'region-animalProduct-animalProductGroupingFAO-price.csv'
-    lookup = download_lookup(filename)
-    data = get_table_value(lookup, 'termid', 'GADM-NPL', column_name('Eggs from other birds in shell, fresh, n.e.c.'))
-    assert extract_grouped_data(data, '2000') is None
-    assert extract_grouped_data(data, '2012') is not None
-
-
-def test_get_single_table_value_float_values():
-    filename = 'ecoClimateZone.csv'
-    lookup = download_lookup(filename)
-    column = column_name('STEHFEST_BOUWMAN_2006_N2O-N_FACTOR')
-    assert _get_single_table_value(lookup, column_name('ecoClimateZone'), '11', column) == '-0.3022'
-
-
-def test_extract_grouped_data_closest_date_no_data():
-    assert not extract_grouped_data_closest_date('', 2000)
-    assert not extract_grouped_data_closest_date('-', 2000)
-
-
-def test_extract_grouped_data_closest_date():
-    data = '2000:-;2001:0.1;2002:0.2;2003:0.3;2004:0.4;2005:0.5'
-    assert extract_grouped_data_closest_date(data, 2000) == 0.1
-    assert extract_grouped_data_closest_date(data, 2001) == 0.1
-    assert extract_grouped_data_closest_date(data, 2020) == 0.5
-
-
-def test_lookup_term_ids():
-    assert 'wheatGrain' in lookup_term_ids(download_lookup('crop.csv'))
-
-
-def test_lookup_columns():
-    assert 'termid' in lookup_columns(download_lookup('crop.csv'))
-
-
-def test_get_data_advanced():
-    lookup = download_lookup('liveAnimal.csv')
-    value = get_table_value(lookup, 'termid', 'sheepRam', column_name('ratioCPregnancyNetEnergyPregnancyIpcc2019'))
-    assert value == ''
-
-
-def test_grouping_with_comma():
-    lookup = download_lookup('animalProduct.csv')
-    term_id = 'meatChickenReadyToCookWeight'
-    value = get_table_value(lookup, 'termid', term_id, column_name('animalProductGroupingFAO'))
-    assert value == 'Meat of chickens, fresh or chilled'