data-disaggregation 0.10.0__tar.gz → 0.11.0__tar.gz

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: data-disaggregation
-Version: 0.10.0
+Version: 0.11.0
 Author: Christian Winger
 License: MIT
 Project-URL: Documentation, https://wingechr.github.io/data-disaggregation
@@ -11,13 +11,22 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: platform
+Dynamic: project-url
+Dynamic: requires-dist
 
-# DataDisAggregation
+# README
 
-![](https://img.shields.io/github/license/wingechr/data-disaggregation)
-![](https://img.shields.io/pypi/pyversions/data-disaggregation)
+[![](https://img.shields.io/github/license/wingechr/data-disaggregation)](./LICENSE)
+[![](https://img.shields.io/pypi/pyversions/data-disaggregation)](https://github.com/wingechr/data-disaggregation)
 [![](https://img.shields.io/pypi/v/data-disaggregation)](https://pypi.org/project/data-disaggregation/)
-![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/test.yml?label=tests)
+[![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/test.yml?label=tests)](https://github.com/wingechr/data-disaggregation/actions/workflows/test.yml)
 [![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/publish.yml?label=docs)](https://wingechr.github.io/data-disaggregation/)
 
 ## Install

data_disaggregation-0.11.0/README.md

@@ -0,0 +1,13 @@
+# README
+
+[![](https://img.shields.io/github/license/wingechr/data-disaggregation)](./LICENSE)
+[![](https://img.shields.io/pypi/pyversions/data-disaggregation)](https://github.com/wingechr/data-disaggregation)
+[![](https://img.shields.io/pypi/v/data-disaggregation)](https://pypi.org/project/data-disaggregation/)
+[![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/test.yml?label=tests)](https://github.com/wingechr/data-disaggregation/actions/workflows/test.yml)
+[![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/publish.yml?label=docs)](https://wingechr.github.io/data-disaggregation/)
+
+## Install
+
+```bash
+pip install data-disaggregation
+```

data_disaggregation-0.11.0/data_disaggregation/__init__.py

@@ -0,0 +1,5 @@
+__version__ = "0.11.0"
+
+from . import actions, vtypes
+
+__all__ = ["actions", "vtypes"]

data_disaggregation-0.11.0/data_disaggregation/actions.py

@@ -0,0 +1,7 @@
+"""Functions to perform data transformations.
+"""
+
+from .base import transform
+from .ext import transform_pandas
+
+__all__ = ["transform", "transform_pandas"]

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/data_disaggregation/base.py

@@ -45,10 +45,8 @@ Helper to create the mapping
 
 """
 
-
 from typing import Mapping, Tuple
 
-from .classes import VT, F, T, V, VT_NumericExt
 from .utils import (
     as_set,
     group_idx_first,
@@ -60,42 +58,74 @@ from .utils import (
     is_unique,
     iter_values,
 )
+from .vtypes import F, T, V, VariableType, VT_NumericExt
 
 VALIDATE_EQ_REL_TOLERANCE = 1e-10
 
 
 def transform(
-    vtype: VT,
+    vtype: VariableType,
     data: Mapping[F, V],
     weight_map: Mapping[Tuple[F, T], float],
-    size_in: Mapping[F, float] = None,
-    size_out: Mapping[T, float] = None,
-    threshold: float = 0.0,
-    validate=True,
+    weights_from: Mapping[F, float] = None,
+    weights_to: Mapping[T, float] = None,
+    weight_rel_threshold: float = 0.0,
+    validate: bool = True,
 ) -> Mapping[T, V]:
-    if size_in is None:
-        size_in = group_idx_first(weight_map)
-
-    if size_out is None:
-        size_out = group_idx_second(weight_map)
+    """(dis-)aggregate data.
+
+    Parameters
+    ----------
+    vtype: VariableType
+        Variable type of input data, determines the aggregation method.
+    data: Mapping[F, V]
+        Input data: mapping (usually dict) from any keys (any hashable) to values.
+    weight_map: Mapping[Tuple[F, T], float]
+        weights for combinations of input and output elements (must be positive).
+        Keys must tuples from input/output key pairs.
+    weights_from: Mapping[F, float]
+        optional weights of input elements (must be positive).
+        If not specified, this will be calculated as a sum from `weight_map`.
+    weights_to: Mapping[T, float]
+        optional weights of output elements (must be positive).
+        If not specified, this will be calculated as a sum from `weight_map`.
+    weight_rel_threshold: float
+        optional value between 0 and 1: all mappings are dropped
+        if the sum of input weights / output weight is smaller than this threshold.
+        For example, you may want to set it to 0.5 for geographical mappings with
+        extensive data.
+    validate bool:
+        if True: run additional (but costly) validations of weights and data.
+
+    Returns
+    -------
+    : Mapping[T, V]
+        output data as a mapping from output keys (any hashable) to values.
+
+    """
+    if weights_from is None:
+        weights_from = group_idx_first(weight_map)
+
+    if weights_to is None:
+        weights_to = group_idx_second(weight_map)
 
     if validate:
         # validate size_f
-        assert is_mapping(size_in)
-        assert is_unique(size_in)
-        assert all(v > 0 for v in iter_values(size_in))
+        assert is_mapping(weights_from)
+        assert is_unique(weights_from)
+        assert all(v > 0 for v in iter_values(weights_from))
 
         # validate size_t
-        assert is_mapping(size_out)
-        assert is_unique(size_out)
-        assert all(v > 0 for v in iter_values(size_out))
+        assert is_mapping(weights_to)
+        assert is_unique(weights_to)
+        assert all(v > 0 for v in iter_values(weights_to))
 
         # validate var
         assert is_mapping(data)
         assert is_unique(data)
 
-        if not is_subset(data, size_in):
-            err = as_set(data) - as_set(size_in)
+        if not is_subset(data, weights_from):
+            err = as_set(data) - as_set(weights_from)
             raise Exception(
                 f"Variable index is not a subset of input dimension subset: {err}"
             )
@@ -104,8 +134,8 @@ def transform(
         assert is_map(weight_map)
         assert is_unique(weight_map)
         assert all(v >= 0 for v in iter_values(weight_map))
-        assert is_subset([x[0] for x in weight_map.keys()], size_in)
-        assert is_subset([x[1] for x in weight_map.keys()], size_out)
+        assert is_subset([x[0] for x in weight_map.keys()], weights_from)
+        assert is_subset([x[1] for x in weight_map.keys()], weights_to)
         # assert all(isinstance(v, (float, int)) for v in iter_values(weight_map))
 
     # filter nan in data
@@ -113,22 +143,19 @@ def transform(
 
     #  scale extensive => intensive
     if vtype == VT_NumericExt:
-        try:
-            data = dict((f, v / size_in[f]) for f, v in data.items())
-        except TypeError:
-            raise Exception(data)
+        data = dict((f, v / weights_from[f]) for f, v in data.items())
 
     # filter unused in weight_map: input:
     weight_map = dict(((f, t), w) for (f, t), w in weight_map.items() if f in data)
 
     # filter unused in weight_map: output
     weight_map = dict(
-        ((f, t), w) for (f, t), w in weight_map.items() if size_out.get(t, 0) > 0
+        ((f, t), w) for (f, t), w in weight_map.items() if weights_to.get(t, 0) > 0
     )
 
     # init groups
     groups = dict((t, []) for t in set(_t for (_, _t) in weight_map.keys()))
-    # group data
+    # group data by output keys
     for (f, t), w in weight_map.items():
         v = data[f]
         groups[t].append((v, w))
@@ -136,18 +163,12 @@ def transform(
     # create weight sums
     group_sumw = dict((t, sum(w for _, w in vws)) for t, vws in groups.items())
 
-    if validate or threshold:
-        sumw_rel = dict((t, sumw / size_out[t]) for t, sumw in group_sumw.items())
-
-    # sumw always <= size_out ==> sumw_rel <= 1
-    # TODO: maybe drop this check, this should never really happen
-    # if validate:
-    #    scomp = 1 + VALIDATE_EQ_REL_TOLERANCE
-    #    assert all(s <= scomp for s in sumw_rel.values()), sumw_rel
-
     # drop groups under threshold
-    if threshold:
-        groups = dict((t, vws) for t, vws in groups.items() if sumw_rel[t] >= threshold)
+    if weight_rel_threshold:
+        sumw_rel = dict((t, sumw / weights_to[t]) for t, sumw in group_sumw.items())
+        groups = dict(
+            (t, vws) for t, vws in groups.items() if sumw_rel[t] >= weight_rel_threshold
+        )
 
     result = {}
     for t, vws in groups.items():
@@ -159,6 +180,6 @@ def transform(
     #  re-scale intensive => extensive
     if vtype == VT_NumericExt:
         for t, v in result.items():
-            result[t] = v * size_out[t]
+            result[t] = v * weights_to[t]
 
     return result

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/data_disaggregation/ext.py

@@ -1,16 +1,17 @@
 """extended functions, especially for pandas Series
 """
+
 from typing import List, Tuple, Union
 
 import numpy as np
 from pandas import DataFrame, Index, MultiIndex, Series
 
 from .base import transform
-from .classes import SCALAR_DIM_NAME, SCALAR_INDEX_KEY, VT
 from .utils import is_scalar
+from .vtypes import SCALAR_DIM_NAME, SCALAR_INDEX_KEY, VariableType
 
 IDX_SCALAR = MultiIndex.from_product([Index([SCALAR_INDEX_KEY], name=SCALAR_DIM_NAME)])
-COL_VAL = "__WEIGHT__"
+COL_WEIGHT = "__WEIGHT__"
 COL_FROM = "__FROM__"
 COL_TO = "__TO__"
 
@@ -20,31 +21,29 @@ def harmonize_input_data(data: Union[DataFrame, Series, float]) -> DataFrame:
     return DataFrame with MultiIndex
     """
     if is_scalar(data):
-        return DataFrame({SCALAR_INDEX_KEY: data}, index=IDX_SCALAR)
-    if isinstance(data, Series):
+        data = DataFrame({SCALAR_INDEX_KEY: data}, index=IDX_SCALAR)
+    elif isinstance(data, Series):
         data = data.to_frame()
     # ensure multiindex
     data = ensure_multiindex(data)
     return data
 
 
-def as_multiindex(item: Index) -> MultiIndex:
-    if not isinstance(item, MultiIndex):
-        item = MultiIndex.from_product([item])
-
-    assert isinstance(item, MultiIndex)
-    return item
+def as_multiindex(index: Index) -> MultiIndex:
+    if not isinstance(index, MultiIndex):
+        index = MultiIndex.from_product([index])
+    return index
 
 
 def ensure_multiindex(item: Union[DataFrame, Series]) -> Union[DataFrame, Series]:
     if not isinstance(item.index, MultiIndex):
-        item = item.copy()
-        item.index = as_multiindex(item.index)
-    assert isinstance(item.index, MultiIndex)
+        index = as_multiindex(item.index)
+        item = item.copy()  # TODO: can we replace index without copy?
+        item.index = index
     return item
 
 
-def as_list_of_series(
+def as_list_of_series_w_multiindex(
     items: Union[Index, Series, Tuple[Union[Index, Series]]]
 ) -> List[Series]:
     # make sure we have a list/tuple
@@ -52,14 +51,13 @@ def as_list_of_series(
         items = [items]
     # make sure we have series:
     items = [it if isinstance(it, Series) else Series(1, index=it) for it in items]
-    for it in items:
-        valdidate_index(it)
+    items = [ensure_multiindex(it) for it in items]
     return items
 
 
 def merge_indices(items: List[Union[Series, Index]]) -> MultiIndex:
     """Create product of unions of indices"""
-    # ensure items are indices
+    # ensure items are multiindices
     items = [it if isinstance(it, Index) else it.index for it in items]
     items = [as_multiindex(it) for it in items]
     indices = {}
@@ -83,16 +81,20 @@ def combine_weights(
         Series with MultiIndex
     """
     # make sure we have series:
-    weights = as_list_of_series(weights)
+    weights = as_list_of_series_w_multiindex(weights)
+
     # merge indices
     idx = merge_indices(weights)
+
     # multiply all and drop nan
     result = Series(1, index=idx)
     for w in weights:
-        # IMPORTANT: `result *= w` gives a different result,so DONT Use it
+        # IMPORTANT: `result *= w` gives a different result, so DONT use it
         result = result * w
+
     # multiplications implicit join can change order:
-    result.index = result.index.reorder_levels(idx.names)
+    if isinstance(result.index, MultiIndex):
+        result.index = result.index.reorder_levels(idx.names)
 
     # drop nan
     result.dropna(inplace=True)
@@ -142,6 +144,7 @@ def remap_series_to_frame(s: Series, idx: MultiIndex, colname: str) -> DataFrame
     # convert index into columns
     df_result.reset_index(inplace=True)
     # set index to same levels as `s`, but keep columns
+
     df_result.set_index(s.index.names, drop=False, inplace=True)
     df_result = ensure_multiindex(df_result)
 
@@ -163,7 +166,7 @@ def create_weight_map(
     """
     idx_all = merge_indices([idx_in, idx_out])
     # expand index (TODO: check if weights are dropped??)
-    df = remap_series_to_frame(ds_weights, idx_all, COL_VAL)
+    df = remap_series_to_frame(ds_weights, idx_all, COL_WEIGHT)
     df[COL_FROM] = list(zip(*[df[n] for n in idx_in.names]))
     df[COL_TO] = list(zip(*[df[n] for n in idx_out.names]))
 
@@ -171,34 +174,64 @@ def create_weight_map(
     df = df.loc[df[COL_FROM].isin(idx_in)]
     df = df.loc[df[COL_TO].isin(idx_out)]
 
-    ds_weight_map = df.set_index([COL_FROM, COL_TO])[COL_VAL]
+    ds_weight_map = df.set_index([COL_FROM, COL_TO])[COL_WEIGHT]
     ds_weight_map = ds_weight_map.fillna(0)
 
     return ds_weight_map
 
 
-def valdidate_index(item: Union[Index, Series, DataFrame]):
+def validate_multiindex(item: Union[Index, Series, DataFrame]):
     if isinstance(item, (Series, DataFrame)):
         item = item.index
-
-    assert len(set(item.names)) == len(item.names)
-    assert item.is_unique
+    assert isinstance(item, MultiIndex)
+    assert all(item.names)  # no empty names
+    assert len(set(item.names)) == len(item.names)  # unique names
+    assert item.is_unique  # unique values
 
 
 def transform_pandas(
-    vtype: VT,
+    vtype: VariableType,
     data: Union[DataFrame, Series, float],
     weights: Union[Index, Series, Tuple[Union[Index, Series]]],
     dim_in: Union[Index, Series] = None,
     dim_out: Union[Index, Series] = None,
+    validate: bool = True,
 ) -> Union[DataFrame, Series, float]:
+    """(dis-)aggregate data (pandas).
+
+    Parameters
+    ----------
+    vtype : VariableType
+        Variable type of input data, determines the aggregation method.
+    data : Union[DataFrame, Series, float]
+        Input data. Indices must be unique and have unique level names
+    weights : Union[Index, Series, Tuple[Union[Index, Series]]]
+        weights for combinations of input and output elements (must be positive).
+    dim_in : Union[Index, Series], optional
+        TODO
+    dim_out : Union[Index, Series], optional
+        TODO
+    validate : bool, optional
+        if True: run additional (but costly) validations of weights and data.
+
+    Returns
+    -------
+    Union[DataFrame, Series, float]
+        output data
+
+    """
+
     # ensure data is DataFrame with MultiIndex
     df_data = harmonize_input_data(data)
-    valdidate_index(df_data)
+
+    if validate:
+        validate_multiindex(df_data)
 
     # combine weights into single Series with MultiIndex
     ds_weights = combine_weights(weights)
-    valdidate_index(ds_weights)
+
+    if validate:
+        validate_multiindex(ds_weights)
 
     # determine input index
     if dim_in is None:
@@ -222,16 +255,17 @@ def transform_pandas(
         ds_size_out = ensure_multiindex(dim_out)
         idx_out = ds_size_out.index
 
-    valdidate_index(idx_in)
-    valdidate_index(idx_out)
+    if validate:
+        validate_multiindex(idx_in)
+        validate_multiindex(idx_out)
 
     # create weight map
     ds_weight_map = create_weight_map(ds_weights, idx_in, idx_out)
 
-    valdidate_index(ds_weight_map)
-
     if ds_size_in is None:
-        ds_size_in = ds_weight_map.reset_index().groupby(COL_FROM).sum(COL_VAL)[COL_VAL]
+        ds_size_in = (
+            ds_weight_map.reset_index().groupby(COL_FROM).sum(COL_WEIGHT)[COL_WEIGHT]
+        )
         # fix index
         ds_size_in.index = MultiIndex.from_tuples(
             ds_size_in.index.values, names=idx_in.names
@@ -239,28 +273,20 @@ def transform_pandas(
         ds_size_in = ds_size_in.loc[ds_size_in > 0]
 
     if ds_size_out is None:
-        ds_size_out = ds_weight_map.reset_index().groupby(COL_TO).sum(COL_VAL)[COL_VAL]
+        ds_size_out = (
+            ds_weight_map.reset_index().groupby(COL_TO).sum(COL_WEIGHT)[COL_WEIGHT]
+        )
         # fix index
         ds_size_out.index = MultiIndex.from_tuples(
             ds_size_out.index.values, names=idx_out.names
         )
         ds_size_out = ds_size_out.loc[ds_size_out > 0]
 
-    valdidate_index(ds_size_in)
-    valdidate_index(ds_size_out)
-
-    assert ds_size_in.index.isin(idx_in).all()
-    assert ds_size_out.index.isin(idx_out).all()
-
-    # TODO drop from ds_weight_map
-    # TODO: drop from data
-
-    # df_data = df_data[idx_in]
-
-    # raise Exception(idx_in[idx_in.isin(ds_size_in.index)])
-
-    # assert ds_size_in.index.equals(idx_in), (ds_size_in.index, idx_in)
-    # assert ds_size_out.index.equals(idx_out), (ds_size_out.index, idx_out)
+    if validate:
+        validate_multiindex(ds_size_in)
+        validate_multiindex(ds_size_out)
+        assert ds_size_in.index.isin(idx_in).all()
+        assert ds_size_out.index.isin(idx_out).all()
 
     idx_in = ds_size_in.index
     idx_out = ds_size_out.index
@@ -275,9 +301,9 @@ def transform_pandas(
             vtype=vtype,
             data=s_col,
             weight_map=ds_weight_map,
-            size_in=ds_size_in,
-            size_out=ds_size_out,
-            threshold=0.0,
+            weights_from=ds_size_in,
+            weights_to=ds_size_out,
+            weight_rel_threshold=0.0,
             validate=False,  # we do validation in pandas
         )
 

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/data_disaggregation/utils.py

@@ -1,22 +1,27 @@
 """utility functions
 """
+
 import math
 from typing import Callable, List, Mapping, Tuple
 
 from pandas import DataFrame, Index, Series
 
-from . import classes
+from . import vtypes
 
 
 def group_sum(key_vals: Mapping, get_key: Callable = None) -> Mapping:
-    """simple group sum
+    """simple group sum.
 
-    Args:
-        key_vals (list): non empty list of (key, value) pairs
-          * keys can be anything hashable,
-          * values must be numerical
+    Parameters
+    ----------
+    key_vals: Mapping
+        * keys can be anything hashable,
+        * values must be numerical
+    get_key: Callable = None
 
     Returns
+    -------
+    : Mapping
         list of (unique key, sum of values) pairs
     """
 
@@ -34,15 +39,19 @@ def group_sum(key_vals: Mapping, get_key: Callable = None) -> Mapping:
 
 
 def weighted_sum(value_normweights: Tuple[float]) -> float:
-    """get sum product
+    """get sum product.
 
-    Args:
-        value_normweights (list): non empty list of (value, weight) pairs
-          * values must be numerical
-          * weights must be numerical, positive, and sum up to 1.0
+    Parameters
+    ----------
+    value_normweights : list
+        non empty list of (value, weight) pairs.
+        * values must be numerical.
+        * weights must be numerical, positive, and sum up to 1.0.
 
     Returns
-        key from original list with highest weight
+    -------
+    : float
+
     """
     # TODO faster methods with numpy or ?
     return sum(v * w for v, w in value_normweights)
@@ -51,13 +60,16 @@ def weighted_sum(value_normweights: Tuple[float]) -> float:
 def weighted_mode(value_normweights: Tuple):
     """get most common value (but by weight)
 
-    Args:
-        value_normweights (list): non empty list of (value, weight) pairs
-          * values can be anything sortable
-          * weights must be numerical, positive, and sum up to 1.0
+    Parameters
+    ----------
+    value_normweights : list
+        non empty list of (value, weight) pairs.
+        * values must be anything sortable.
+        * weights must be numerical, positive, and sum up to 1.0.
 
     Returns
-        key from original list with highest weight
+    -------
+    Any
     """
     # make values unique (sum weights)
     value_normweights = group_sum(value_normweights).items()
@@ -68,13 +80,16 @@ def weighted_mode(value_normweights: Tuple):
 def weighted_percentile(value_normweights: Tuple, p=0.5):
     """get most median (but by weight)
 
-    Args:
-        value_normweights (list): non empty list of (value, weight) pairs
-          * values can be anything sortable
-          * weights must be numerical, positive, and sum up to 1.0
+    Parameters
+    ----------
+    value_normweights : list
+        non empty list of (value, weight) pairs.
+        * values must be anything sortable.
+        * weights must be numerical, positive, and sum up to 1.0.
 
     Returns
-        key from original list with highest weight
+    -------
+    Any
     """
     # make values unique (sum weights)
     value_normweights = group_sum(value_normweights).items()
@@ -90,13 +105,16 @@ def weighted_percentile(value_normweights: Tuple, p=0.5):
 def weighted_median(value_normweights: Tuple):
     """get most median (but by weight)
 
-    Args:
-        value_normweights (list): non empty list of (value, weight) pairs
-          * values can be anything sortable
-          * weights must be numerical, positive, and sum up to 1.0
+    Parameters
+    ----------
+    value_normweights : list
+        non empty list of (value, weight) pairs.
+        * values must be anything sortable.
+        * weights must be numerical, positive, and sum up to 1.0.
 
     Returns
-        key from original list with highest weight
+    -------
+    Any
     """
     return weighted_percentile(value_normweights, p=0.5)
 
@@ -160,7 +178,7 @@ def as_mapping(x, default_val=1) -> Mapping:
     elif is_list(x):
         return dict((k, default_val) for k in x)
     elif is_scalar(x):
-        return {classes.SCALAR_INDEX_KEY: x}
+        return {vtypes.SCALAR_INDEX_KEY: x}
     raise TypeError(x)
 
 
@@ -168,8 +186,8 @@ def as_scalar(x):
     if as_scalar(x):
         return x
     elif is_mapping(x):
-        assert set(x.keys()) == set([classes.SCALAR_INDEX_KEY])
-        return x[classes.SCALAR_INDEX_KEY]
+        assert set(x.keys()) == set([vtypes.SCALAR_INDEX_KEY])
+        return x[vtypes.SCALAR_INDEX_KEY]
     raise TypeError(x)
 
 

data_disaggregation-0.11.0/data_disaggregation/vtypes.py

@@ -0,0 +1,88 @@
+"""Type classes for data.
+"""
+
+from abc import ABC
+from typing import TypeVar
+
+from . import utils
+
+F = TypeVar("F")
+T = TypeVar("T")
+V = TypeVar("V")
+
+SCALAR_DIM_NAME = "__SCALAR__"
+# TODO: using None in pandas causes problems with autoconvert to nan
+SCALAR_INDEX_KEY = "__SCALAR__"
+
+
+class VariableType(ABC):
+    @classmethod
+    def weighted_aggregate(cls, data):
+        """aggregate data
+
+        Parameters
+        ----------
+        data : Iterable
+            non empty list of (value, weight) pairs.
+            weights must be numerical, positive, and sum up to 1.0.
+
+        Returns
+        -------
+        aggregated value
+        """
+        raise NotImplementedError()
+
+
+class VT_Nominal(VariableType):
+    """Type class for nominal (categorical) data.
+
+    - Aggregation method: mode (most commonly used)
+    - Disaggregation method: keep value
+    - Examples: regional codes
+    """
+
+    @classmethod
+    def weighted_aggregate(cls, data):
+        return utils.weighted_mode(data)
+
+
+class VT_Ordinal(VT_Nominal):
+    """Type class for ordinal data (ranked categorical).
+
+    - Aggregation method: median
+    - Disaggregation method: keep value
+    - Examples: Level of agreement
+    """
+
+    @classmethod
+    def weighted_aggregate(cls, data):
+        return utils.weighted_median(data)
+
+
+class VT_Numeric(VariableType):
+    """Type class for numerical, intensive data
+
+    An intensive variable is one which does not scale with the system size.
+
+    - Aggregation method: weighted average
+    - Disaggregation method: keep value
+    - Examples: temperature, density, pressure
+    """
+
+    @classmethod
+    def weighted_aggregate(cls, data):
+        return utils.weighted_sum(data)
+
+
+class VT_NumericExt(VT_Numeric):
+    """Type class for numerical, extensive data.
+
+    An extensive variable is one which does scale with the system size
+    (assuming an equal distribution).
+
+    - Aggregation method: sum
+    - Disaggregation method: distribute by weights
+    - Examples: population, energy, total cost
+    """
+
+    pass

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/data_disaggregation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: data-disaggregation
-Version: 0.10.0
+Version: 0.11.0
 Author: Christian Winger
 License: MIT
 Project-URL: Documentation, https://wingechr.github.io/data-disaggregation
@@ -11,13 +11,22 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: platform
+Dynamic: project-url
+Dynamic: requires-dist
 
-# DataDisAggregation
+# README
 
-![](https://img.shields.io/github/license/wingechr/data-disaggregation)
-![](https://img.shields.io/pypi/pyversions/data-disaggregation)
+[![](https://img.shields.io/github/license/wingechr/data-disaggregation)](./LICENSE)
+[![](https://img.shields.io/pypi/pyversions/data-disaggregation)](https://github.com/wingechr/data-disaggregation)
 [![](https://img.shields.io/pypi/v/data-disaggregation)](https://pypi.org/project/data-disaggregation/)
-![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/test.yml?label=tests)
+[![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/test.yml?label=tests)](https://github.com/wingechr/data-disaggregation/actions/workflows/test.yml)
 [![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/publish.yml?label=docs)](https://wingechr.github.io/data-disaggregation/)
 
 ## Install

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/data_disaggregation.egg-info/SOURCES.txt

@@ -2,10 +2,11 @@ LICENSE
 README.md
 setup.py
 data_disaggregation/__init__.py
+data_disaggregation/actions.py
 data_disaggregation/base.py
-data_disaggregation/classes.py
 data_disaggregation/ext.py
 data_disaggregation/utils.py
+data_disaggregation/vtypes.py
 data_disaggregation.egg-info/PKG-INFO
 data_disaggregation.egg-info/SOURCES.txt
 data_disaggregation.egg-info/dependency_links.txt

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/setup.py

@@ -12,7 +12,7 @@ if __name__ == "__main__":
         description="",
         long_description=long_description_md,
         long_description_content_type="text/markdown",
-        version="0.10.0",
+        version="0.11.0",
         author="Christian Winger",
         platforms=["any"],
         license="MIT",

{data-disaggregation-0.10.0 → data_disaggregation-0.11.0}/test/test.py

@@ -1,3 +1,4 @@
+import doctest
 import logging
 from functools import partial
 from unittest import TestCase
@@ -6,18 +7,13 @@ import numpy as np
 import pandas as pd
 from pandas import DataFrame, Index, MultiIndex, Series
 
+from data_disaggregation import ext
 from data_disaggregation.base import transform
-from data_disaggregation.classes import (
-    SCALAR_INDEX_KEY,
-    VT_Nominal,
-    VT_Numeric,
-    VT_NumericExt,
-    VT_Ordinal,
-)
 from data_disaggregation.ext import (
     COL_FROM,
     COL_TO,
-    COL_VAL,
+    COL_WEIGHT,
+    as_multiindex,
     combine_weights,
     create_weight_map,
     ensure_multiindex,
@@ -37,6 +33,13 @@ from data_disaggregation.utils import (
     weighted_mode,
     weighted_sum,
 )
+from data_disaggregation.vtypes import (
+    SCALAR_INDEX_KEY,
+    VT_Nominal,
+    VT_Numeric,
+    VT_NumericExt,
+    VT_Ordinal,
+)
 
 logging.basicConfig(
     format="[%(asctime)s %(levelname)7s] %(message)s",
@@ -78,7 +81,6 @@ class TestUtils(TestCase):
             (float("inf"), True),
             (float("nan"), True),
             (np.nan, True),
-            (np.NAN, True),
         ]:
             self.assertEqual(is_na(x), y)
 
@@ -124,8 +126,6 @@ class TestUtils(TestCase):
 
 
 class TestBase(TestCase):
-    """"""
-
     def get_example(self, vtype):
         """
         M | D  E  F | S | V
@@ -255,7 +255,7 @@ class TestBaseExamples(TestCase):
                 (2, SCALAR_INDEX_KEY): 20,
                 (3, SCALAR_INDEX_KEY): 30,
             },
-            threshold=0.5,
+            weight_rel_threshold=0.5,
         )
         self.assertAlmostEqual(res[SCALAR_INDEX_KEY], 1 + 1 + 10)
 
@@ -285,9 +285,9 @@ class TestBaseExamples(TestCase):
                 (SCALAR_INDEX_KEY, "11"): 0.99,  # cell will be dropped (size = 2)
                 (SCALAR_INDEX_KEY, "10"): 1.1,  # (size = 2)
             },
-            size_in={SCALAR_INDEX_KEY: 5},
-            size_out={"00": 1, "01": 1, "11": 2, "10": 2},
-            threshold=0.5,
+            weights_from={SCALAR_INDEX_KEY: 5},
+            weights_to={"00": 1, "01": 1, "11": 2, "10": 2},
+            weight_rel_threshold=0.5,
         )
 
         self.assertAlmostEqual(res["00"], 100 / 5)
@@ -337,6 +337,13 @@ class TestBaseExamples(TestCase):
 
 class TextExtPandas(TestCase):
     def assertPandasEqal(self, left, right):
+        if isinstance(left, Index):
+            left = left.sort_values()
+            right = right.sort_values()
+        else:
+            left = left.sort_index()
+            right = right.sort_index()
+
         if isinstance(left, Index):
             method = pd.testing.assert_index_equal
         elif isinstance(left, Series):
@@ -398,20 +405,19 @@ class TextExtPandas(TestCase):
 
     def test_combine_weights_1(self):
         # no overlap
-        s_res = Series(
-            [2, 3, 2, 3],
-            index=MultiIndex.from_tuples(
-                [(11, 21), (11, 22), (12, 21), (12, 22)], names=["i1", "i2"]
-            ),
-        )
         self.assertPandasEqal(
-            s_res,
             combine_weights(
                 [
                     Index([11, 12], name="i1"),
                     Series([2, 3], index=Index([21, 22], name="i2")),
                 ]
             ),
+            Series(
+                [2, 3, 2, 3],
+                index=MultiIndex.from_tuples(
+                    [(11, 21), (11, 22), (12, 21), (12, 22)], names=["i1", "i2"]
+                ),
+            ),
         )
 
     def test_combine_weights_2(self):
@@ -459,7 +465,7 @@ class TextExtPandas(TestCase):
                 ],
                 names=[COL_FROM, COL_TO],
             ),
-            name=COL_VAL,
+            name=COL_WEIGHT,
         )
 
         self.assertPandasEqal(ds_result, create_weight_map(ds_weights, idx_in, idx_out))
@@ -477,27 +483,36 @@ class TextExtPandas(TestCase):
             ensure_multiindex(DataFrame({"v": [1, 1]}, index=Index([1, 1], name="i1"))),
         )
 
-    def test_transform_pandas_1(self):
-        exp_res = Series(
-            [1 / (2 + 1), 2 / (2 + 1), 2.0],
-            index=MultiIndex.from_tuples([("a",), ("b",), ("c",)], names=["i2"]),
-            name="s1",
-        )
-        res = transform_pandas(
-            vtype=VT_NumericExt,
-            data=Series([1, 2], index=Index([0, 1], name="i1"), name="s1"),
-            weights=Series(
-                [1, 2, 1.5],
-                index=MultiIndex.from_tuples(
-                    [(0, "a"), (0, "b"), (1, "c")], names=["i1", "i2"]
+    def test_pandas_utils(self):
+        for idx in [Index(["b", "a"], name="d1")]:
+            idx2 = as_multiindex(idx)
+            self.assertEqual(idx.names, idx2.names)
+            self.assertEqual(tuple(tuple(x) for x in idx.values), tuple(idx2.values))
+
+    def test_transform_pandas(self):
+        self.assertPandasEqal(
+            transform_pandas(
+                vtype=VT_NumericExt,
+                data=Series([1, 2], index=Index([0, 1], name="i1"), name="s1"),
+                weights=Series(
+                    [1, 2, 1.5],
+                    index=MultiIndex.from_tuples(
+                        [(0, "a"), (0, "b"), (1, "c")], names=["i1", "i2"]
+                    ),
                 ),
             ),
+            Series(
+                [1 / (2 + 1), 2 / (2 + 1), 2.0],
+                index=MultiIndex.from_tuples([("a",), ("b",), ("c",)], names=["i2"]),
+                name="s1",
+            ),
         )
-        self.assertPandasEqal(
-            exp_res,
-            res,
-        )
-        self.assertPandasEqal(
-            exp_res.to_frame("V"),
-            res.to_frame("V"),
-        )
+
+
+class Doctests(TestCase):
+    def run_doctest(self, module):
+        report = doctest.testmod(module)
+        self.assertFalse(report.failed)
+
+    def test_modules(self):
+        self.run_doctest(ext)

data-disaggregation-0.10.0/README.md

@@ -1,13 +0,0 @@
-# DataDisAggregation
-
-![](https://img.shields.io/github/license/wingechr/data-disaggregation)
-![](https://img.shields.io/pypi/pyversions/data-disaggregation)
-[![](https://img.shields.io/pypi/v/data-disaggregation)](https://pypi.org/project/data-disaggregation/)
-![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/test.yml?label=tests)
-[![](https://img.shields.io/github/actions/workflow/status/wingechr/data-disaggregation/publish.yml?label=docs)](https://wingechr.github.io/data-disaggregation/)
-
-## Install
-
-```bash
-pip install data-disaggregation
-```

data-disaggregation-0.10.0/data_disaggregation/__init__.py

@@ -1,15 +0,0 @@
-__version__ = "0.10.0"
-
-# isort: skip_file -> keep order to prevent circular import
-from .ext import transform_pandas
-from .base import transform
-from .classes import VT_Nominal, VT_Numeric, VT_NumericExt, VT_Ordinal
-
-__all__ = [
-    "VT_Nominal",
-    "VT_Ordinal",
-    "VT_Numeric",
-    "VT_NumericExt",
-    "transform_pandas",
-    "transform",
-]

data-disaggregation-0.10.0/data_disaggregation/classes.py

@@ -1,76 +0,0 @@
-"""classes and types
-"""
-
-from abc import ABC
-from typing import TypeVar
-
-from . import utils
-
-F = TypeVar("F")
-T = TypeVar("T")
-V = TypeVar("V")
-
-SCALAR_DIM_NAME = ""
-# TODO: using None in pandas causes problems with autoconvert to nan
-SCALAR_INDEX_KEY = ""
-
-
-class VT(ABC):
-    @classmethod
-    def weighted_aggregate(cls, data):
-        """aggregation
-
-        Args:
-            data (list): non empty list of (value, weight) pairs
-
-        Returns
-            aggregated value
-        """
-        raise NotImplementedError()
-
-
-class VT_Nominal(VT):
-    """
-    Examples: Regional Codes
-    """
-
-    @classmethod
-    def weighted_aggregate(cls, data):
-        return utils.weighted_mode(data)
-
-
-class VT_Ordinal(VT_Nominal):
-    """
-    Values can be sorted in a meaningful way
-    Usually, that means using numerical codes that
-    do not represent a metric distance, liek a likert scale
-
-    Examples: [1 = "a little", 2 = "somewhat", 3 = "a lot"]
-
-    """
-
-    @classmethod
-    def weighted_aggregate(cls, data):
-        return utils.weighted_median(data)
-
-
-class VT_Numeric(VT):
-    """
-    * Values can be calculated by linear combinations
-    * Examples: height, temperature, density
-    """
-
-    @classmethod
-    def weighted_aggregate(cls, data):
-        return utils.weighted_sum(data)
-
-
-class VT_NumericExt(VT_Numeric):
-    """
-    Values are extensive, i.e. they are can be transformed into intensive
-    by dividing by domain size
-
-    * Examples: population, energy production
-    """
-
-    pass