hestia-earth-utils 0.15.15__tar.gz → 0.16.1__tar.gz

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/PKG-INFO

@@ -1,16 +1,30 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.15.15
+Version: 0.16.1
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
 Author-email: guillaumeroyer.mail@gmail.com
 License: MIT
-Platform: UNKNOWN
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: Programming Language :: Python :: 3.9
-Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
+Requires-Dist: hestia-earth-schema>=35.0.1
+Requires-Dist: requests>=2.24.0
+Requires-Dist: urllib3~=1.26.0
+Requires-Dist: python-dateutil>=2.8.1
+Requires-Dist: numpy<2,>=1.25.0
+Requires-Dist: flatten_json
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # HESTIA Utils
 
@@ -60,5 +74,3 @@ from hestia_earth.utils.lookup import download_lookup
 
 df = download_lookup('crop.csv')
 ```
-
-

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/hestia_earth/utils/lookup_utils.py

@@ -64,11 +64,15 @@ def is_model_siteType_allowed(model: str, term_id: str, data: dict):
 
 
 def is_model_product_id_allowed(model: str, term_id: str, data: dict):
-    products = data.get('products', [])
-    values = _blank_node_term_values(products)
+    values = _blank_node_term_values(data.get('products', []))
     return _is_model_value_allowed(model, term_id, values, 'productTermIdsAllowed')
 
 
+def is_model_measurement_id_allowed(model: str, term_id: str, data: dict):
+    values = _blank_node_term_values(data.get('measurements', []))
+    return _is_model_value_allowed(model, term_id, values, 'measurementIdsAllowed')
+
+
 @lru_cache()
 def _allowed_mapping(term_id: str, column: str):
     mapping = _allowed_mapping_data()

hestia_earth_utils-0.16.1/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.16.1'

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/hestia_earth_utils.egg-info/PKG-INFO

@@ -1,16 +1,30 @@
-Metadata-Version: 2.1
-Name: hestia-earth-utils
-Version: 0.15.15
+Metadata-Version: 2.4
+Name: hestia_earth_utils
+Version: 0.16.1
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
 Author-email: guillaumeroyer.mail@gmail.com
 License: MIT
-Platform: UNKNOWN
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: Programming Language :: Python :: 3.9
-Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
+Requires-Dist: hestia-earth-schema>=35.0.1
+Requires-Dist: requests>=2.24.0
+Requires-Dist: urllib3~=1.26.0
+Requires-Dist: python-dateutil>=2.8.1
+Requires-Dist: numpy<2,>=1.25.0
+Requires-Dist: flatten_json
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # HESTIA Utils
 
@@ -60,5 +74,3 @@ from hestia_earth.utils.lookup import download_lookup
 
 df = download_lookup('crop.csv')
 ```
-
-

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/hestia_earth_utils.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ README.md
 setup.py
 bin/hestia-format-upload
 bin/hestia-pivot-csv
-hestia_earth/__init__.py
 hestia_earth/utils/__init__.py
 hestia_earth/utils/api.py
 hestia_earth/utils/blank_node.py
@@ -36,6 +35,19 @@ hestia_earth_utils.egg-info/SOURCES.txt
 hestia_earth_utils.egg-info/dependency_links.txt
 hestia_earth_utils.egg-info/requires.txt
 hestia_earth_utils.egg-info/top_level.txt
-tests/pivot/__init__.py
-tests/pivot/test_pivot_csv.py
-tests/pivot/test_pivot_json.py
+tests/test_api.py
+tests/test_blank_node.py
+tests/test_calculation_status.py
+tests/test_cycle.py
+tests/test_date.py
+tests/test_descriptive_stats.py
+tests/test_emission.py
+tests/test_lookup.py
+tests/test_lookup_utils.py
+tests/test_model.py
+tests/test_pipeline.py
+tests/test_request.py
+tests/test_stats.py
+tests/test_table.py
+tests/test_term.py
+tests/test_tools.py

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/hestia_earth_utils.egg-info/requires.txt

@@ -1,4 +1,4 @@
-hestia_earth.schema>=24.3.0
+hestia-earth-schema>=35.0.1
 requests>=2.24.0
 urllib3~=1.26.0
 python-dateutil>=2.8.1

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/hestia_earth_utils.egg-info/top_level.txt

@@ -1,2 +1 @@
 hestia_earth
-tests

{hestia_earth_utils-0.15.15 → hestia_earth_utils-0.16.1}/setup.py

@@ -1,5 +1,5 @@
 import pathlib
-from setuptools import find_packages, setup
+from setuptools import find_namespace_packages, setup
 
 from hestia_earth.utils.version import VERSION
 
@@ -23,13 +23,12 @@ setup(
     author_email='guillaumeroyer.mail@gmail.com',
     license='MIT',
     classifiers=[
-        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
-        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.12',
     ],
-    packages=find_packages(exclude=('tests', 'scripts')),
+    packages=find_namespace_packages(include=['hestia_earth.*']),
+    python_requires='>=3.12',
     include_package_data=True,
     install_requires=REQUIRES,
-    python_requires='>=3.9',
     scripts=[
         'bin/hestia-pivot-csv',
         'bin/hestia-format-upload'

hestia_earth_utils-0.16.1/tests/test_api.py

@@ -0,0 +1,169 @@
+from unittest.mock import patch
+import os
+import requests
+import json
+from hestia_earth.schema import SchemaType
+import pytest
+
+from .utils import fixtures_path
+from hestia_earth.utils.request import api_url
+from hestia_earth.utils.api import (
+    search,
+    find_related,
+    download_hestia,
+    node_exists,
+    find_node,
+    find_node_exact,
+    find_term_ids_by_names,
+)
+
+
+fake_related_response = {'results': [[{'@id': 'related_id'}]]}
+fake_download_response = {'@id': 'id', '@type': 'type'}
+
+
+class FakeFindRelatedSuccess():
+    def json():
+        return fake_related_response
+
+
+class FakeFindRelatedError():
+    def json():
+        return {}
+
+
+class FakeFindRelatedException():
+    def json():
+        raise requests.exceptions.RequestException('error')
+
+
+class FakeDownloadSuccess():
+    def json():
+        return fake_download_response
+
+
+class FakeDownloadError():
+    def json():
+        raise requests.exceptions.RequestException('error')
+
+
+class FakeNodeExistSuccess():
+    def json():
+        return fake_download_response
+
+
+class FakeNodeExistError():
+    def json():
+        return {"message": "not-found", "details": {}}
+
+
+class FakeElasticSearchEmptyResult:
+    def json():
+        return {"results": []}
+
+
+@patch('requests.get', return_value=FakeFindRelatedSuccess)
+def test_find_related_success(mock_get):
+    res = find_related(SchemaType.CYCLE, 'id', SchemaType.SOURCE)
+    assert res == fake_related_response.get('results')
+    mock_get.assert_called_once_with(
+        f"{api_url()}/cycles/id/sources?limit=100", headers={'Content-Type': 'application/json'}
+    )
+
+
+@patch('requests.get', return_value=FakeFindRelatedError)
+def test_find_related_error(*args):
+    res = find_related(SchemaType.CYCLE, 'id', SchemaType.SOURCE)
+    assert not res
+
+
+@patch('requests.get', return_value=FakeFindRelatedException)
+def test_find_related_exception(*args):
+    res = find_related(SchemaType.CYCLE, 'id', SchemaType.SOURCE)
+    assert not res
+
+
+@patch('requests.get', return_value=FakeDownloadError)  # make sure fallback is not enabled
+def test_download_hestia_local_file(*args):
+    id = 'sandContent'
+    with open(f"{fixtures_path}/Term/{id}.jsonld", encoding='utf-8') as f:
+        expected = json.load(f)
+    os.environ['DOWNLOAD_FOLDER'] = fixtures_path
+    res = download_hestia(id, SchemaType.TERM)
+    assert res == expected
+    del os.environ['DOWNLOAD_FOLDER']
+
+
+@patch('requests.get', return_value=FakeDownloadSuccess)
+def test_download_hestia_success(mock_get):
+    res = download_hestia('id', SchemaType.SOURCE)
+    assert res == fake_download_response
+    mock_get.assert_called_once_with(
+        f"{api_url()}/sources/id", headers={'Content-Type': 'application/json'})
+
+
+@patch('requests.get', return_value=FakeDownloadError)
+def test_download_hestia_error(*args):
+    res = download_hestia('id', SchemaType.SOURCE)
+    assert not res
+
+
+@patch('requests.get', return_value=FakeNodeExistError)  # make sure fallback is not enabled
+def test_node_exists_local_file(*args):
+    os.environ['DOWNLOAD_FOLDER'] = fixtures_path
+    id = 'sandContent'
+    assert node_exists(id, SchemaType.TERM)
+    del os.environ['DOWNLOAD_FOLDER']
+
+
+@patch('requests.get', return_value=FakeNodeExistSuccess)
+def test_node_exists_true(*args):
+    assert node_exists('id', SchemaType.SOURCE)
+
+
+@patch('requests.get', return_value=FakeNodeExistError)
+def test_node_exists_false(*args):
+    assert not node_exists('id', SchemaType.SOURCE)
+
+
+def test_search():
+    name = 'Wheat'
+    res = search(query={
+        'bool': {
+            'must': [{'match': {'name': name}}]
+        }
+    }, limit=2)
+    assert res[0].get('name').startswith(name)
+
+
+def test_find_node():
+    name = 'Wheat'
+    res = find_node(SchemaType.TERM, {'name': name}, 2)
+    assert res[0].get('name').startswith(name)
+
+
+def test_find_node_exact():
+    name = 'Wheat'
+    res = find_node_exact(SchemaType.TERM, {'name': name})
+    assert not res
+
+    name = 'Wheat, grain'
+    res = find_node_exact(SchemaType.TERM, {'name': name})
+    assert res.get('name') == name
+
+
+def test_find_term_ids_by_names():
+    names = ["Harris Termite Powder", "Wheat, grain", "Urea (kg N)"]
+    res = find_term_ids_by_names(names, 2)
+    assert res == {
+        "Wheat, grain": "wheatGrain",
+        "Harris Termite Powder": "harrisTermitePowder",
+        "Urea (kg N)": "ureaKgN",
+    }
+
+
+@patch("requests.post", return_value=FakeElasticSearchEmptyResult)
+def test_find_term_ids_by_names_error(mock):
+    names = ["id_not_found_name"]
+    with pytest.raises(Exception, match=names[0]):
+        find_term_ids_by_names(names)

hestia_earth_utils-0.16.1/tests/test_blank_node.py

@@ -0,0 +1,59 @@
+# import os
+# import json
+# from pytest import mark
+# from hestia_earth.schema import TermTermType
+
+# from tests.utils import fixtures_path
+# from hestia_earth.utils.blank_node import get_node_value, ArrayTreatment, get_blank_nodes_calculation_status
+
+# fixtures_folder = os.path.join(fixtures_path, 'blank_node')
+# calculation_status_folder = os.path.join(fixtures_folder, 'calculation_status')
+
+
+# def test_get_node_value():
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': [10]}
+#     assert get_node_value(blank_node, 'value', default=None) == 10
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': [0]}
+#     assert get_node_value(blank_node, 'value', default=None) == 0
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}}
+#     assert get_node_value(blank_node, 'value', default=None) is None
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': [10, 20]}
+#     assert get_node_value(blank_node, 'value', default=None) == 15
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': True}
+#     assert get_node_value(blank_node, 'value', default=None) is True
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': 10}
+#     assert get_node_value(blank_node, 'value', default=None) == 10
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': None}
+#     assert get_node_value(blank_node, 'value', default=None) is None
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': None}
+#     assert get_node_value(blank_node, 'value', default=0) == 0
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': [10, None, 20]}
+#     assert get_node_value(blank_node, 'value', default=None) == 15
+
+#     blank_node = {'term': {'termType': 'crop', '@id': 'wheatGrain'}, 'value': [10, None, 20]}
+#     assert get_node_value(blank_node, 'value', array_treatment=ArrayTreatment.SUM) == 30
+
+
+# @mark.parametrize(
+#     'folder,list_key,termType',
+#     [
+#         ('cycle', 'emissions', TermTermType.EMISSION),
+#     ]
+# )
+# def test_get_blank_nodes_calculation_status(folder: str, list_key: str, termType: TermTermType):
+#     with open(f"{calculation_status_folder}/{folder}/node.jsonld", encoding='utf-8') as f:
+#         node = json.load(f)
+
+#     with open(f"{calculation_status_folder}/{folder}/{list_key}-{termType.value}.json", encoding='utf-8') as f:
+#         expected = json.load(f)
+
+#     result = get_blank_nodes_calculation_status(node, list_key=list_key, termType=termType)
+#     assert result == expected, folder

hestia_earth_utils-0.16.1/tests/test_calculation_status.py

@@ -0,0 +1,40 @@
+# import os
+# import json
+
+# from tests.utils import fixtures_path
+# from hestia_earth.utils.calculation_status import _emissions_with_status, get_nodes_calculations_status_dataframe
+
+# fixtures_folder = os.path.join(fixtures_path, 'calculation_status')
+
+
+# def test_emissions_with_status():
+#     with open(os.path.join(fixtures_folder, 'nodes.json')) as f:
+#         nodes = json.load(f)
+
+#     result = _emissions_with_status(nodes[0])
+#     assert result == {
+#         'emissions-total': 193,
+#         'emissions-complete': 56,
+#         'emissions-incomplete': 1,
+#         'emissions-missing': 136,
+#         'emissions': result['emissions']  # ignore
+#     }
+
+#     result = _emissions_with_status(nodes[1])
+#     assert result == {
+#         'emissions-total': 193,
+#         'emissions-complete': 0,
+#         'emissions-incomplete': 13,
+#         'emissions-missing': 180,
+#         'emissions': result['emissions']  # ignore
+#     }
+
+
+# def test_get_nodes_calculations_status_dataframe():
+#     with open(os.path.join(fixtures_folder, 'nodes.json')) as f:
+#         nodes = json.load(f)
+
+#     expected = open(os.path.join(fixtures_folder, 'result.csv'), 'r').read()
+
+#     df = get_nodes_calculations_status_dataframe(nodes, file_format='csv')
+#     assert df.to_csv(None, index=None) == expected

hestia_earth_utils-0.16.1/tests/test_cycle.py

@@ -0,0 +1,18 @@
+# import os
+# import json
+
+# from tests.utils import fixtures_path
+# from hestia_earth.utils.cycle import get_cycle_emissions_calculation_status
+
+
+# def test_get_cycle_emissions_calculation_status():
+#     folder = os.path.join(fixtures_path, 'blank_node', 'calculation_status', 'cycle')
+
+#     with open(f"{folder}/node.jsonld", encoding='utf-8') as f:
+#         cycle = json.load(f)
+
+#     with open(f"{folder}/emissions-emission-with-missing-inputs.json", encoding='utf-8') as f:
+#         expected = json.load(f)
+
+#     result = get_cycle_emissions_calculation_status(cycle)
+#     assert result == expected

hestia_earth_utils-0.16.1/tests/test_date.py

@@ -0,0 +1,17 @@
+from hestia_earth.utils.date import diff_in_years, is_in_days, is_in_months
+
+
+def test_diff_in_years():
+    assert diff_in_years('1990-01-01', '1999-02-01') == 9.1
+
+
+def test_is_in_days():
+    assert not is_in_days('2000')
+    assert not is_in_days('2000-01')
+    assert is_in_days('2000-01-01')
+
+
+def test_is_in_months():
+    assert not is_in_months('2000')
+    assert is_in_months('2000-01')
+    assert not is_in_months('2000-01-01')

hestia_earth_utils-0.16.1/tests/test_descriptive_stats.py

@@ -0,0 +1,49 @@
+from numpy import array
+from pytest import mark
+from hestia_earth.schema import MeasurementStatsDefinition
+
+from hestia_earth.utils.descriptive_stats import calc_descriptive_stats
+
+EXPECTED_FLATTENED = {
+    "value": [5],
+    "sd": [2.581989],
+    "min": [1],
+    "max": [9],
+    "statsDefinition": "simulated",
+    "observations": [9]
+}
+
+EXPECTED_COLUMNWISE = {
+    "value": [4, 5, 6],
+    "sd": [2.44949, 2.44949, 2.44949],
+    "min": [1, 2, 3],
+    "max": [7, 8, 9],
+    "statsDefinition": "simulated",
+    "observations": [3, 3, 3]
+}
+
+EXPECTED_ROWWISE = {
+    "value": [2, 5, 8],
+    "sd": [0.816497, 0.816497, 0.816497],
+    "min": [1, 4, 7],
+    "max": [3, 6, 9],
+    "statsDefinition": "simulated",
+    "observations": [3, 3, 3]
+}
+
+
+@mark.parametrize(
+    "axis, expected",
+    [(None, EXPECTED_FLATTENED), (0, EXPECTED_COLUMNWISE), (1, EXPECTED_ROWWISE)],
+    ids=["flattened", "columnwise", "rowwise"]
+)
+@mark.parametrize("stats_definition", [MeasurementStatsDefinition.SIMULATED, "simulated"], ids=["Enum", "str"])
+def test_calc_descriptive_stats(stats_definition, axis, expected):
+    ARR = array([
+        [1, 2, 3],
+        [4, 5, 6],
+        [7, 8, 9]
+    ])
+
+    result = calc_descriptive_stats(ARR, stats_definition, axis=axis)
+    assert result == expected

hestia_earth_utils-0.16.1/tests/test_emission.py

@@ -0,0 +1,62 @@
+from hestia_earth.schema import SiteSiteType, TermTermType
+
+from hestia_earth.utils.emission import cycle_emissions_in_system_boundary, emissions_in_system_boundary
+
+class_path = 'hestia_earth.utils.emission'
+
+
+def test_cycle_emissions_in_system_boundary_cropland():
+    cycle = {'site': {}}
+
+    cycle['products'] = [
+        {
+            'term': {
+                '@id': 'wheatGrain',
+                'termType': TermTermType.CROP.value
+            }
+        }
+    ]
+    cycle['site']['siteType'] = SiteSiteType.CROPLAND.value
+    term_ids = cycle_emissions_in_system_boundary(cycle)
+    assert len(term_ids) > 50
+
+    cycle['products'] = [
+        {
+            'term': {
+                '@id': 'ricePlantFlooded',
+                'termType': TermTermType.CROP.value
+            }
+        }
+    ]
+    cycle['site']['siteType'] = SiteSiteType.CROPLAND.value
+    term_ids = cycle_emissions_in_system_boundary(cycle)
+    assert len(term_ids) > 50
+
+    # with inputs restriction, we should have less emissions
+    cycle['inputs'] = [
+        {
+            'term': {'termType': 'crop'}
+        }
+    ]
+    cycle['site']['siteType'] = SiteSiteType.CROPLAND.value
+    assert len(cycle_emissions_in_system_boundary(cycle)) < len(term_ids)
+
+
+def test_cycle_emissions_in_system_boundary_animal_housing():
+    cycle = {'site': {}}
+    cycle['products'] = [
+        {
+            'term': {
+                '@id': 'meatBeefCattleLiveweight',
+                'termType': TermTermType.ANIMALPRODUCT.value
+            }
+        }
+    ]
+    cycle['site']['siteType'] = SiteSiteType.ANIMAL_HOUSING.value
+    term_ids = cycle_emissions_in_system_boundary(cycle)
+    assert len(term_ids) > 20
+
+
+def test_emissions_in_system_boundary():
+    term_ids = emissions_in_system_boundary()
+    assert len(term_ids) > 50