hestia-earth-models 0.57.2__py3-none-any.whl → 0.59.0__py3-none-any.whl

hestia_earth/models/utils/blank_node.py

@@ -8,6 +8,7 @@ from functools import reduce
 from statistics import mode, mean
 from typing import (
     Any,
+    List,
     Callable,
     NamedTuple,
     Optional,
@@ -23,7 +24,7 @@ from hestia_earth.utils.tools import (
 )
 
 from ..log import debugValues, log_as_table
-from . import _filter_list_term_unit
+from . import is_from_model, _filter_list_term_unit
 from .constant import Units
 from .property import get_node_property, get_node_property_value
 from .lookup import (
@@ -108,7 +109,7 @@ def run_if_required(model: str, term_id: str, data: dict, module):
     return getattr(module, 'run')(data) if is_run_required(model, _module_term_id(term_id, module), data) else []
 
 
-def find_terms_value(nodes: list, term_id: str):
+def find_terms_value(nodes: list, term_id: str, default: Union[int, None] = 0):
     """
     Returns the sum of all blank nodes in the list which match the `Term` with the given `@id`.
 
@@ -124,7 +125,17 @@ def find_terms_value(nodes: list, term_id: str):
     float
         The total `value` as a number.
     """
-    return list_sum(get_total_value(filter(lambda node: node.get('term', {}).get('@id') == term_id, nodes)))
+    return list_sum(get_total_value(filter(lambda node: node.get('term', {}).get('@id') == term_id, nodes)), default)
+
+
+def has_gap_filled_by_ids(nodes: list, term_ids: List[str]):
+    nodes = [n for n in nodes if n.get('term', {}).get('@id') in term_ids]
+    return any([is_from_model(n) for n in nodes])
+
+
+def has_original_by_ids(nodes: list, term_ids: List[str]):
+    nodes = [n for n in nodes if n.get('term', {}).get('@id') in term_ids]
+    return any([not is_from_model(n) for n in nodes])
 
 
 def get_total_value(nodes: list):
@@ -294,32 +305,35 @@ def get_P2O5_total(nodes: list) -> list:
     return get_total_value(kg_P_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP2O5')
 
 
-def convert_to_nitrogen(node: dict, model: str, term_id: str, blank_nodes: list):
+def convert_to_nitrogen(node: dict, model: str, term_id: str, blank_nodes: list, **log_args):
     def prop_value(input: dict):
-        value = get_node_property_value(model, input, 'nitrogenContent')
-        return value or get_node_property_value(model, input, 'crudeProteinContent', default=0) / 6.25
+        value = get_node_property_value(model, input, 'nitrogenContent', default=None, **log_args)
+        return value or get_node_property_value(model, input, 'crudeProteinContent', default=0, **log_args) / 6.25
 
     values = [(i, prop_value(i)) for i in blank_nodes]
     missing_nitrogen_property = [i.get('term', {}).get('@id') for i, p_value in values if not p_value]
 
     debugValues(node, model=model, term=term_id,
-                missing_nitrogen_property=';'.join(set(missing_nitrogen_property)))
+                missing_nitrogen_property=';'.join(set(missing_nitrogen_property)),
+                **log_args)
 
     return list_sum([
         list_sum(i.get('value', [])) * p_value for i, p_value in values if p_value is not None
     ]) if len(missing_nitrogen_property) == 0 else None
 
 
-def convert_to_carbon(node: dict, model: str, term_id: str, blank_nodes: list):
+def convert_to_carbon(node: dict, model: str, term_id: str, blank_nodes: list, **log_args):
     def prop_value(input: dict):
-        value = get_node_property_value(model, input, 'carbonContent')
-        return value or get_node_property_value(model, input, 'energyContentHigherHeatingValue', default=0) * 0.021
+        value = get_node_property_value(model, input, 'carbonContent', default=None, **log_args)
+        return value or \
+            get_node_property_value(model, input, 'energyContentHigherHeatingValue', default=0, **log_args) * 0.021
 
     values = [(i, prop_value(i)) for i in blank_nodes]
     missing_carbon_property = [i.get('term', {}).get('@id') for i, p_value in values if not p_value]
 
     debugValues(node, model=model, term=term_id,
-                missing_carbon_property=';'.join(missing_carbon_property))
+                missing_carbon_property=';'.join(missing_carbon_property),
+                **log_args)
 
     return list_sum([
         list_sum(i.get('value', [])) * p_value for i, p_value in values if p_value is not None
@@ -399,7 +413,8 @@ def _retrieve_array_treatment(
 def get_node_value(
     node: dict,
     is_larger_unit: bool = False,
-    array_treatment: Optional[ArrayTreatment] = None
+    array_treatment: Optional[ArrayTreatment] = None,
+    default: Any = 0
 ) -> Union[float, bool]:
     """
     Get the value from the dictionary representing the node,
@@ -427,7 +442,7 @@ def get_node_value(
         array_treatment or _retrieve_array_treatment(node, is_larger_unit=is_larger_unit)
     )] if isinstance(value, list) and len(value) > 0 else None
 
-    return reducer(value) if reducer else value if isinstance(value, bool) else value or 0
+    return reducer(value) if reducer else value if isinstance(value, bool) else value or default
 
 
 def _convert_to_set(
@@ -653,7 +668,7 @@ def cumulative_nodes_lookup_match(
     )
 
 
-# --- Group nodes by year ---
+# --- Blank Node date utils ---
 
 
 class DatestrFormat(Enum):
@@ -670,6 +685,16 @@ class DatestrFormat(Enum):
     MONTH_DAY = r"--%m-%d"
 
 
+DATESTR_FORMAT_TO_EXPECTED_LENGTH = {
+    DatestrFormat.YEAR: len("2001"),
+    DatestrFormat.YEAR_MONTH: len("2001-01"),
+    DatestrFormat.YEAR_MONTH_DAY: len("2001-01-01"),
+    DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND: len("2001-01-01T00:00:00"),
+    DatestrFormat.MONTH: len("--01"),
+    DatestrFormat.MONTH_DAY: len("--01-01")
+}
+
+
 DatestrGapfillMode = Enum("DatestrGapfillMode", [
     "START",
     "END"
@@ -698,16 +723,15 @@ end : datetime
 """
 
 
-def _check_datestr_format(datestr: str, format: Union[DatestrFormat, str]) -> bool:
+def _check_datestr_format(datestr: str, format: DatestrFormat) -> bool:
     """
     Use `datetime.strptime` to determine if a datestr is in a particular ISO format.
     """
     try:
-        date_format_str = (
-            format.value if isinstance(format, DatestrFormat)
-            else str(format)
-        )
-        return bool(datetime.strptime(str(datestr), date_format_str))
+        expected_length = DATESTR_FORMAT_TO_EXPECTED_LENGTH.get(format, 0)
+        format_str = format.value
+        parsed_datetime = datetime.strptime(datestr, format_str)
+        return bool(parsed_datetime) and len(datestr) == expected_length
     except ValueError:
         return False
 
@@ -771,21 +795,25 @@ def _datetime_within_range(datetime: datetime, range: DatetimeRange) -> bool:
     """
     Determine whether or not a `datetime` falls within a `DatetimeRange`.
     """
-    return range.start < datetime < range.end
+    return range.start <= datetime <= range.end
 
 
-def _datetime_range_duration(range: DatetimeRange) -> float:
+def _datetime_range_duration(range: DatetimeRange, add_second=False) -> float:
     """
     Determine the length of a `DatetimeRange` in seconds.
+
+    Option to `add_second` to account for 1 second between 23:59:59 and 00:00:00)
     """
-    return (range.end - range.start).total_seconds()
+    return (range.end - range.start).total_seconds() + int(add_second)
 
 
 def _calc_datetime_range_intersection_duration(
-    range_a: DatetimeRange, range_b: DatetimeRange
+    range_a: DatetimeRange, range_b: DatetimeRange, add_second=False
 ) -> float:
     """
     Determine the length of a `DatetimeRange` in seconds.
+
+    Option to `add_second` to account for 1 second between 23:59:59 and 00:00:00)
     """
     latest_start = max(range_a.start, range_b.start)
     earliest_end = min(range_a.end, range_b.end)
@@ -795,14 +823,156 @@ def _calc_datetime_range_intersection_duration(
         end=earliest_end
     )
 
+    duration = _datetime_range_duration(intersection_range)
+
     # if less than 0 the ranges do not intersect, so return 0.
-    return max(0, _datetime_range_duration(intersection_range))
+    return (
+        _datetime_range_duration(intersection_range) + int(add_second)
+        if duration > 0 else 0
+    )
+
+
+# --- Group nodes by year ---
+
+
+VALID_DATE_FORMATS_GROUP_NODES_BY_YEAR = {
+    DatestrFormat.YEAR,
+    DatestrFormat.YEAR_MONTH,
+    DatestrFormat.YEAR_MONTH_DAY,
+    DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND
+}
+
+
+GroupNodesByYearMode = Enum("GroupNodesByYearMode", [
+    "START_AND_END_DATE",
+    "DATES"
+])
+"""
+Enum representing modes of grouping nodes by year.
+
+Members
+-------
+START_AND_END_DATE
+    Use the `startDate` and `endDate` fields of the node.
+DATES
+    Use the `dates` field of the node.
+"""
+
+
+def _should_run_node_by_end_date(node: dict) -> bool:
+    """
+    Validate nodes for `group_nodes_by_year` using the "startDate" and "endDate" fields.
+    """
+    return _get_datestr_format(node.get("endDate")) in VALID_DATE_FORMATS_GROUP_NODES_BY_YEAR
 
 
-def _validate_intersection_threshold(
-    fraction_of_year: float,
-    fraction_of_node_duration: float,
-    is_final_year: bool
+def _should_run_node_by_dates(node: dict) -> bool:
+    """
+    Validate nodes for `group_nodes_by_year` using the "dates" field.
+    """
+    value = node.get("value")
+    dates = node.get("dates")
+    return (
+        value and dates and len(dates) > 0 and len(value) == len(dates)
+        and all(_get_datestr_format(datestr) in VALID_DATE_FORMATS_GROUP_NODES_BY_YEAR for datestr in node.get("dates"))
+    )
+
+
+GROUP_NODES_BY_YEAR_MODE_TO_SHOULD_RUN_NODE_FUNCTION = {
+    GroupNodesByYearMode.START_AND_END_DATE: _should_run_node_by_end_date,
+    GroupNodesByYearMode.DATES: _should_run_node_by_dates
+}
+
+
+def _get_node_datetime_range_from_start_and_end_date(
+    node: dict, default_node_duration: int = 1
+) -> Union[DatetimeRange, None]:
+    """
+    Get the datetime range from a node's "startDate" and "endDate" fields.
+
+    If "startDate" field is not available, a start date is calculated using the end date
+    and `default_node_duration`.
+    """
+    end = safe_parse_date(_gapfill_datestr(node.get("endDate"), DatestrGapfillMode.END))
+    start = (
+        safe_parse_date(_gapfill_datestr(node.get("startDate"), DatestrGapfillMode.START))
+        or end - relativedelta(years=default_node_duration, seconds=-1) if end else None
+    )
+
+    valid = isinstance(start, datetime) and isinstance(end, datetime)
+    return DatetimeRange(start, end) if valid else None
+
+
+def _get_node_datetime_range_from_dates(
+    node: dict, **_
+) -> Union[DatetimeRange, None]:
+    """
+    Get the datetime range from a node's "dates" field.
+    """
+    dates = node.get("dates")
+    end = max(
+        non_empty_list(
+            safe_parse_date(_gapfill_datestr(datestr, DatestrGapfillMode.END)) for datestr in dates
+        ), default=None
+    )
+    start = min(
+        non_empty_list(
+            safe_parse_date(_gapfill_datestr(datestr, DatestrGapfillMode.START)) for datestr in dates
+        ), default=None
+    )
+
+    valid = isinstance(start, datetime) and isinstance(end, datetime)
+    return DatetimeRange(start, end) if valid else None
+
+
+GROUP_NODES_BY_YEAR_MODE_TO_GET_DATETIME_RANGE_FUNCTION = {
+    GroupNodesByYearMode.START_AND_END_DATE: _get_node_datetime_range_from_start_and_end_date,
+    GroupNodesByYearMode.DATES: _get_node_datetime_range_from_dates
+}
+
+
+def _build_time_fraction_dict(
+    group_datetime_range: DatetimeRange,
+    node_datetime_range: DatetimeRange
+) -> dict:
+    """
+    Build a dictionary containing fractions of the year and node duration based on datetime ranges.
+
+    This function calculates the duration of the group or year, the duration of the node, and the intersection
+    duration between the two. It then computes the fractions of the year and node duration represented by the
+    intersection. The results are returned in a dictionary.
+
+    Parameters
+    ----------
+    group_datetime_range : DatetimeRange
+        The datetime range representing the entire group or year.
+    node_datetime_range : DatetimeRange
+        The datetime range representing the node.
+
+    Returns
+    -------
+    dict
+        A dictionary containing "fraction_of_group_duration" and "fraction_of_node_duration".
+    """
+    group_duration = _datetime_range_duration(group_datetime_range, add_second=True)
+    node_duration = _datetime_range_duration(node_datetime_range, add_second=True)
+
+    intersection_duration = _calc_datetime_range_intersection_duration(
+        node_datetime_range, group_datetime_range, add_second=True
+    )
+
+    fraction_of_group_duration = intersection_duration / group_duration
+    fraction_of_node_duration = intersection_duration / node_duration
+
+    return {
+        "fraction_of_group_duration": fraction_of_group_duration,
+        "fraction_of_node_duration": fraction_of_node_duration
+    }
+
+
+def _validate_time_fraction_dict(
+    time_fraction_dict: dict,
+    is_final_group: bool
 ) -> bool:
     """
     Return `True` if the the node intersections with a year group by
@@ -814,24 +984,54 @@ def _validate_intersection_threshold(
     be counted in the year group if the majority of that node takes place in
     that year.
     """
-    FRACTION_OF_YEAR_THRESHOLD = 0.3
+    FRACTION_OF_GROUP_DURATION_THRESHOLD = 0.3
     FRACTION_OF_NODE_DURATION_THRESHOLD = 0.5
 
-    return (
-        fraction_of_year > FRACTION_OF_YEAR_THRESHOLD
-        or fraction_of_node_duration > FRACTION_OF_NODE_DURATION_THRESHOLD
-        or (is_final_year and fraction_of_node_duration == FRACTION_OF_NODE_DURATION_THRESHOLD)
-    )
+    return any([
+        time_fraction_dict["fraction_of_group_duration"] > FRACTION_OF_GROUP_DURATION_THRESHOLD,
+        time_fraction_dict["fraction_of_node_duration"] > FRACTION_OF_NODE_DURATION_THRESHOLD,
+        is_final_group and time_fraction_dict["fraction_of_node_duration"] == FRACTION_OF_NODE_DURATION_THRESHOLD
+    ])
+
+
+def _build_update_dict(node: dict, years: list, target_year: int) -> dict:
+    """
+    Build an update dictionary containing values and dates from a node that fall within a given year.
+
+    This is only required if when `group_nodes_by_year` `mode = GroupNodesByYearMode.DATES`
+
+    Parameters
+    ----------
+    node : dict
+        The node containing values and dates.
+    year : int
+        The year to be matched.
+
+    Returns
+    -------
+    dict
+        An update dictionary containing "value" and "dates" keys.
+    """
+    valid_indices = {
+        i for i, y in enumerate(years) if y == target_year
+    }
+    return {
+        "value": [node.get("value")[i] for i in valid_indices],
+        "dates": [node.get("dates")[i] for i in valid_indices]
+    }
 
 
 def group_nodes_by_year(
     nodes: list[dict],
     default_node_duration: int = 1,
-    sort_result: bool = True
+    sort_result: bool = True,
+    inner_key: Union[Any, None] = None,
+    mode: GroupNodesByYearMode = GroupNodesByYearMode.START_AND_END_DATE
 ) -> dict[int, list[dict]]:
     """
-    Group nodes by year based on their start and end dates. Incomplete date strings are gap-filled automatically
-    using `_gapfill_datestr` function.
+    Group nodes by year based on either their "startDate" and "endDate" fields or their
+    "dates" field. Incomplete date strings are gap-filled automatically using `_gapfill_datestr`
+    function.
 
     Parameters
     ----------
@@ -841,32 +1041,38 @@ def group_nodes_by_year(
         Default duration of a node years if start date is not available, by default 1.
     sort_result : bool, optional
         Flag to sort the result by year, by default True.
+    inner_key: Any | None
+        An optional inner dictionary key for the outputted annualised groups (can be used to merge annualised
+        dictionaries together), default value: `None`.
+    mode : GroupNodesByYearMode, optional
+        The mode to determine how nodes are grouped by year. Options are defined in `GroupNodesByYearMode`.
 
     Returns
     -------
     dict[int, list[dict]]
         A dictionary where keys are years and values are lists of nodes.
     """
-    def group_node(groups: dict, index: int):
-        node = nodes[index]
 
-        end_datestr = _gapfill_datestr(node.get("endDate"), DatestrGapfillMode.END)
-        start_datestr = _gapfill_datestr(node.get("startDate"), DatestrGapfillMode.START)
+    should_run_node = GROUP_NODES_BY_YEAR_MODE_TO_SHOULD_RUN_NODE_FUNCTION[mode]
+    get_node_datetime_range = GROUP_NODES_BY_YEAR_MODE_TO_GET_DATETIME_RANGE_FUNCTION[mode]
+
+    valid_nodes = [node for node in nodes if should_run_node(node)]
 
-        end = safe_parse_date(end_datestr)
-        start = (
-            safe_parse_date(start_datestr)
-            or end - relativedelta(years=default_node_duration, seconds=-1)
-            if bool(end) else None
+    def group_node(groups: dict, index: int):
+        node = valid_nodes[index]
+
+        node_datetime_range = get_node_datetime_range(
+            node, default_node_duration=default_node_duration
         )
 
-        node_datetime_range = DatetimeRange(
-            start=start,
-            end=end
+        # pre-parse the "dates" field so it doesn't get re-calculated in each iteration of the for-loop
+        years = (
+            [safe_parse_date(datestr).year for datestr in node.get("dates", [])]
+            if mode == GroupNodesByYearMode.DATES else []
         )
 
-        range_end = end.year + 1 if end else 0
-        range_start = start.year if start else 0
+        range_start = node_datetime_range.start.year if node_datetime_range else 0
+        range_end = node_datetime_range.end.year + 1 if node_datetime_range else 0
 
         for year in range(range_start, range_end):
 
@@ -877,32 +1083,148 @@ def group_nodes_by_year(
 
             is_final_year = _datetime_within_range(node_datetime_range.end, group_datetime_range)
 
-            # add 1 to durations if datestrs gap filled (to account for 1 second between 23:59:59 and 00:00:00)
-            year_duration = _datetime_range_duration(group_datetime_range) + 1
-            node_duration = _datetime_range_duration(node_datetime_range) + 1
-            intersection_duration = (
-                _calc_datetime_range_intersection_duration(node_datetime_range, group_datetime_range) + 1
+            time_fraction_dict = _build_time_fraction_dict(group_datetime_range, node_datetime_range)
+            update_dict = _build_update_dict(node, years, year) if mode == GroupNodesByYearMode.DATES else {}
+
+            should_run = (
+                mode == GroupNodesByYearMode.DATES
+                or _validate_time_fraction_dict(
+                    time_fraction_dict,
+                    is_final_year
+                )
             )
 
-            fraction_of_year = intersection_duration / year_duration
-            fraction_of_node_duration = intersection_duration / node_duration
+            should_run and groups[year].append(
+                node | time_fraction_dict | update_dict
+            )
 
-            time_fraction_dict = {
-                "fraction_of_year": fraction_of_year,
-                "fraction_of_node_duration": fraction_of_node_duration
-            }
+        return groups
 
-            _node = node | time_fraction_dict
+    grouped = reduce(group_node, range(len(valid_nodes)), defaultdict(list))
 
-            should_run = _validate_intersection_threshold(
-                fraction_of_year,
-                fraction_of_node_duration,
-                is_final_year
-            )
+    iterated = {
+        year: {inner_key: group} if inner_key else group
+        for year, group in grouped.items()
+    }
+
+    return dict(sorted(iterated.items())) if sort_result else iterated
+
+
+def group_nodes_by_year_and_month(
+    nodes: list[dict],
+    default_node_duration: int = 1,
+    sort_result: bool = True,
+    inner_key: Union[Any, None] = None
+) -> dict[int, list[dict]]:
+    """
+    Group nodes by year based on either their "startDate" and "endDate" fields. Incomplete date strings are gap-filled
+    automatically using `_gapfill_datestr` function.
+
+    Returns a dict in the shape:
+    ```
+    {
+        year (int): {
+            month (int): nodes (list[dict])  # for each month 1 - 12
+        }
+    }
+    ```
+
+    Parameters
+    ----------
+    nodes : list[dict]
+        A list of nodes with start and end date information.
+    default_node_duration : int, optional
+        Default duration of a node years if start date is not available, by default 1.
+    sort_result : bool, optional
+        Flag to sort the result by year, by default True.
+    inner_key: Any | None
+        An optional inner dictionary key for the outputted annualised groups (can be used to merge annualised
+        dictionaries together), default value: `None`.
+
+    Returns
+    -------
+    dict[int, list[dict]]
+        A dictionary where keys are years and values are lists of nodes.
+    """
+    valid_nodes = [node for node in nodes if _should_run_node_by_end_date(node)]
+
+    def group_node(groups: dict, index: int):
+        node = valid_nodes[index]
+
+        node_datetime_range = _get_node_datetime_range_from_start_and_end_date(
+            node, default_node_duration=default_node_duration
+        )
+
+        range_start = node_datetime_range.start.year if node_datetime_range else 0
+        range_end = node_datetime_range.end.year + 1 if node_datetime_range else 0
+
+        for year in range(range_start, range_end):
+            for month in range(1, 13):
 
-            should_run and groups[year].append(_node)
+                group_datetime_range = DatetimeRange(
+                    start=safe_parse_date(_gapfill_datestr(f"{year}-{month:02}", DatestrGapfillMode.START)),
+                    end=safe_parse_date(_gapfill_datestr(f"{year}-{month}", DatestrGapfillMode.END))
+                )
+
+                is_final_month = _datetime_within_range(node_datetime_range.end, group_datetime_range)
+                time_fraction_dict = _build_time_fraction_dict(group_datetime_range, node_datetime_range)
+                should_run = _validate_time_fraction_dict(time_fraction_dict, is_final_month)
+
+                should_run and groups[year][month].append(node)
 
         return groups
 
-    grouped = reduce(group_node, range(len(nodes)), defaultdict(list))
-    return dict(sorted(grouped.items())) if sort_result else grouped
+    grouped = reduce(group_node, range(len(valid_nodes)), defaultdict(lambda: defaultdict(list)))
+
+    iterated = {
+        year: {inner_key: dict(group)} if inner_key else dict(group)
+        for year, group in grouped.items()
+    }
+
+    return dict(sorted(iterated.items())) if sort_result else iterated
+
+
+# --- Group nodes by last date ---
+
+
+def _get_last_date(datestrs: list[str]) -> Optional[str]:
+    """
+    Reduce a datestrs down to a single datestr by selecting the last one.
+
+    Parameters
+    ----------
+    datestrs : list
+        A list of datestrings, e.g. the value of a node's `dates` field.
+
+    Returns
+    -------
+    str | None
+        Returns the latest datestr or `None` if no valid datestr in list.
+
+    """
+    return sorted(datestrs)[-1] if len(datestrs) > 0 else None
+
+
+def group_nodes_by_last_date(nodes: list) -> dict[str, list[dict]]:
+    """
+    Group a list of nodes by the last date of their `dates` field. Nodes with no `dates` field will be sorted into
+    the `no-dates` group.
+
+    Parameters
+    ----------
+    nodes : list[dict]
+        A list of Hestia format nodes.
+
+    Return
+    ------
+    dict
+        A dictionary of nodes grouped by latest date, in the format `{date: list[node]}`.
+    """
+    DEFAULT_KEY = 'no-dates'
+
+    def group_by(group: dict, node: dict):
+        dates = node.get('dates', [])
+        key = _get_last_date(dates) or DEFAULT_KEY
+        return group | {key: group.get(key, []) + [node]}
+
+    return reduce(group_by, nodes, {})

hestia_earth/models/utils/cropResidue.py

@@ -0,0 +1,13 @@
+PRODUCT_ID_TO_PRACTICES_ID = [
+    {'product': 'aboveGroundCropResidueRemoved', 'practices': ['residueRemoved']},
+    {'product': 'aboveGroundCropResidueIncorporated', 'practices': [
+        'residueIncorporated',
+        'residueIncorporatedLessThan30DaysBeforeCultivation',
+        'residueIncorporatedMoreThan30DaysBeforeCultivation'
+    ]},
+    {'product': 'aboveGroundCropResidueBurnt', 'practices': ['residueBurnt']},
+    {'product': 'aboveGroundCropResidueLeftOnField', 'practices': ['residueLeftOnField']}
+]
+
+
+def crop_residue_product_ids(): return [v.get('product') for v in PRODUCT_ID_TO_PRACTICES_ID]