openpls-engine 0.7.0a2__py3-none-any.whl

plspm/util.py

@@ -0,0 +1,164 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2019 Google Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import collections as c
+import math
+
+import numpy as np
+import pandas as pd
+
+
+def treat(data: pd.DataFrame, center: bool = True, scale: bool = True, scale_values=None) -> pd.DataFrame:
+    """Internal function that treats data in Pandas Dataframe format.
+
+    Args:
+        data: The data to treat
+        center: Whether to center the data
+        scale: Whether to scale the data
+        scale_values: The scaling to use
+
+    Returns:
+        The treated data
+    """
+    if center:
+        data = data.subtract(data.mean())
+    if scale:
+        if scale_values:
+            data = data.divide(scale_values)
+        else:
+            data = data.divide(data.std())
+    return data
+
+
+def treat_numpy(data: np.ndarray) -> np.ndarray:
+    """Internal function that centers and scales data in Numpy format.
+
+    Args:
+        data: The data to treat
+
+    Returns:
+        The treated data
+    """
+    data = data - np.nanmean(data)
+    return data / np.nanstd(data, axis=0, ddof=1)
+
+
+def sort_cols(data: pd.DataFrame) -> pd.DataFrame:
+    """Internal convenience function to sort data by column."""
+    return data.reindex(sorted(data.columns), axis=1)
+
+
+def impute(data: pd.DataFrame) -> pd.DataFrame:
+    """Internal function that imputes missing data using the mean value (only suitable for metric data)."""
+    imputed = pd.DataFrame(0, data.index, data.columns)
+    for column in list(data):
+        average = data[column].mean(skipna=True)
+        imputed[column] = data[column].fillna(average)
+        assert math.isclose(imputed[column].mean(), average, rel_tol=1e-09, abs_tol=0.0)
+    return imputed
+
+
+def list_to_dummy(data: dict) -> pd.DataFrame:
+    """Internal function used to create the outer design matrix."""
+    matrix = pd.DataFrame()
+    for col in data:
+        dummy = pd.DataFrame(1, index=data[col], columns=[col])
+        matrix = pd.concat([matrix, dummy], axis=1, sort=False)
+    return matrix.fillna(0)
+
+
+def rank(data: pd.Series) -> pd.Series:
+    """Internal function used to rank ordinal and nominal data."""
+    unique = pd.Series(data.unique())
+    ranked = unique.rank()
+    lookup = pd.concat([unique, ranked], axis=1)
+    lookup_series = pd.Series(lookup.iloc[:, 1].values, index=lookup.iloc[:, 0])
+    return data.replace(lookup_series.to_dict()).astype(float)
+
+
+def dummy(data: pd.Series) -> pd.DataFrame:
+    """Internal function used to create a dummy matrix used to perform calculations with ordinal and nominal data"""
+    unique = data.unique().size
+    dummy = pd.DataFrame(0, data.index, range(1, unique + 1))
+    for i in range(unique):
+        dummy.loc[data[data == i + 1].index, i + 1] = 1
+    return dummy
+
+
+def groupby_mean(data: np.ndarray) -> np.ndarray:
+    """Internal function which performs the Numpy equivalent of Pandas ``.groupby(...).mean()``"""
+    values = {}
+    reduced = 0
+    for i in range(data.shape[1]):
+        index = data[0, i]
+        if index not in values:
+            values[index] = []
+            reduced += 1
+        values[index].append(data[1, i])
+    means = np.zeros((2, reduced), dtype=np.float64)
+    for i, index in enumerate(sorted(values.keys())):
+        means[0, i] = index
+        means[1, i] = np.mean(values[index])
+    return means
+
+
+class Value:
+    """Internal class which models a value type"""
+    def __init__(self, val):
+        self.__value = val
+
+    def __eq__(self, other):
+        return self.__value == other.__value
+
+    def __ne__(self, other):
+        return self.__value != other.__value
+
+
+class TopoSort:
+    """Internal function which performs a topological sort using Kahn's algorithm"""
+
+    def __init__(self):
+        self.__indegree = c.Counter()
+        self.__children = {}
+        self.__edges = []
+
+    def append(self, src: str, dest: str):
+        self.__edges.append((src, dest))
+        self.__indegree[dest] += 1
+        self.__indegree[src] += 0
+        for vertex in [src, dest]:
+            if vertex not in self.__children:
+                self.__children[vertex] = []
+        self.__children[src].append(dest)
+
+    def order(self):
+        ordered = []
+        orphaned = c.deque([v for v in self.__indegree if self.__indegree[v] == 0])
+        while orphaned:
+            vertex = orphaned.pop()
+            ordered.append(vertex)
+            for child in self.__children[vertex]:
+                self.__indegree[child] -= 1
+                if self.__indegree[child] == 0:
+                    orphaned.append(child)
+        for v in self.__indegree:
+            if self.__indegree[v] != 0:
+                raise ValueError("Structural graph contains cycles.")
+        return ordered
+
+    def elements(self):
+        return self.__edges

plspm/weights.py

@@ -0,0 +1,193 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2019 Google Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import math
+
+import numpy as np
+import pandas as pd
+import statsmodels.api as sm
+
+import plspm.config as c
+import plspm.util as util
+from plspm.mode import Mode
+from plspm.scheme import Scheme
+
+pd.options.mode.chained_assignment = None  # default='warn'
+
+
+class _MetricWeights:
+    """Internal class that calculates weights and scores when using metric data."""
+    def __init__(self, data: pd.DataFrame, config: c.Config, correction: float, path: pd.DataFrame):
+        odm = config.odm(path)
+        weight_factors = correction / data.dot(odm).std(axis=0)
+        self.__mvs = list(odm.index)
+        wf_diag = pd.DataFrame(np.diag(weight_factors), index=weight_factors.index, columns=weight_factors.index)
+        weights = odm.dot(wf_diag)
+        self.__weights_old = weights.sum(axis=1).to_frame(name="weight")
+        self.__data = data
+        self.__config = config
+        self.__weights = weights
+        self.__correction = correction
+        self.__path = path
+
+    def iterate(self, inner_weight_calculator: Scheme) -> float:
+        lvs = list(self.__path)
+        scores = self.__data.dot(self.__weights).reindex(lvs, axis=1)
+        scores = util.treat(scores) / self.__correction
+        inner_weights = pd.DataFrame(inner_weight_calculator.value.calculate(self.__path, scores.values), index=lvs, columns=lvs)
+        Z = scores.dot(inner_weights)
+        for lv in list(lvs):
+            mvs = self.__config.mvs(lv)
+            weights = self.__config.mode(lv).value.outer_weights_metric(self.__data, Z, lv, mvs)
+            self.__weights.loc[mvs, [lv]] = weights
+        weights_new = self.__weights.sum(axis=1).to_frame(name="weight")
+        convergence = np.power(self.__weights_old.abs() - weights_new.abs(), 2).sum(axis=1).sum(axis=0)
+        self.__weights_old = weights_new
+        return convergence
+
+    def calculate(self) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+        weight_factors = 1 / (self.__data.dot(self.__weights).std(axis=0) / self.__correction)
+        wf_diag = pd.DataFrame(np.diag(weight_factors), index=weight_factors.index, columns=weight_factors.index)
+        weights = self.__weights.dot(wf_diag)
+        scores = self.__data.dot(weights)
+        cor = pd.concat([self.__data, scores], axis=1).corr().loc[list(self.__data), list(scores)]
+        odm = weights.apply(lambda x: x!= 0).astype(int)
+        sign = lambda x : math.copysign(1.0, x)
+        w_sign = (cor * odm).map(sign).sum(axis=0).apply(sign)
+        if -1 in w_sign.values:
+            w_sign = w_sign.apply(lambda x : -1 if x == 0 else x)
+            w_sign_matrix = pd.DataFrame(np.diag(w_sign), index=w_sign.index, columns=w_sign.index)
+            scores = scores.dot(w_sign_matrix)
+        weights = pd.DataFrame(weights.sum(axis=1), index=self.__mvs, columns=["weight"])
+        return self.__data, scores, weights
+
+
+class _NonmetricWeights:
+    """Internal class that calculates weights and scores when using nonmetric data."""
+    def __init__(self, data: pd.DataFrame, config: c.Config, correction: float, path: pd.DataFrame):
+        self.__mv_grouped_by_lv_initial = {}
+        self.__mvs = []
+        mv_grouped_by_lv = {}
+        self.__mv_grouped_by_lv_missing = {}
+        lvs = list(path)
+        scores = np.zeros((len(data.index), len(lvs)), dtype=np.float64)
+        for i, lv in enumerate(lvs):
+            mvs = config.mvs(lv)
+            self.__mvs.extend(mvs)
+            mv_grouped_by_lv[lv] = data.filter(config.mvs(lv)).values.astype(np.float64)
+            self.__mv_grouped_by_lv_initial[lv] = mv_grouped_by_lv[lv].copy()
+            sizes = mv_grouped_by_lv[lv].shape[1]
+            weight = [1 / np.sqrt(sizes)] * sizes
+            if np.isnan(np.sum(mv_grouped_by_lv[lv])):
+                self.__mv_grouped_by_lv_missing[lv] = 1 - np.isnan(mv_grouped_by_lv[lv])
+                for j in range(len(data.index)):
+                    numerator = np.nansum(mv_grouped_by_lv[lv][j, :] * weight)
+                    denominator = np.power(weight * self.__mv_grouped_by_lv_missing[lv][j, :], 2).sum()
+                    if denominator == 0:
+                        raise ValueError("All mvs for lv " + lv + " in row " + str(j) + " are NaN.")
+                    scores[j, i] = numerator / denominator
+            else:
+                scores[:, i] = np.dot(mv_grouped_by_lv[lv], weight)
+        self.__weights = {}
+        self.__scores = scores
+        self.__mv_grouped_by_lv = mv_grouped_by_lv
+        self.__config = config
+        self.__correction = correction
+        self.__index = data.index
+        self.__path = path
+
+    def iterate(self, inner_weight_calculator: Scheme) -> float:
+        self.__betas = {}
+        scores_old = self.__scores.copy()
+        inner_weights = inner_weight_calculator.value.calculate(self.__path, self.__scores)
+        Z = np.dot(self.__scores, inner_weights)
+        for i, lv in enumerate(list(self.__path)):
+            for j, mv in enumerate(list(self.__config.mvs(lv))):
+                self.__mv_grouped_by_lv[lv][:, j] = \
+                    self.__config.scale(mv).value.scale(lv, mv, Z[:, i], self)
+            self.__weights[lv], self.__scores[:, i] = \
+                self.__config.mode(lv).value.outer_weights_nonmetric(self.__mv_grouped_by_lv,
+                                                                     self.__mv_grouped_by_lv_missing, Z[:, i], lv,
+                                                                     self.__correction)
+        return np.power(np.abs(scores_old) - np.abs(self.__scores), 2).sum()
+
+    def calculate(self) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+        lvs = list(self.__path)
+        weights = pd.DataFrame(0.0, index=self.__mvs, columns=lvs)
+        data_new = pd.DataFrame(0.0, index=self.__index, columns=self.__mvs)
+        for lv in lvs:
+            mvs = self.__config.mvs(lv)
+            weights.loc[mvs, [lv]] = self.__weights[lv].reshape(self.__weights[lv].shape[0],1)
+            data_new.loc[:, mvs] = self.__mv_grouped_by_lv[lv]
+        weight_factors = 1 / (data_new.dot(weights).std(axis=0, skipna=True) / self.__correction)
+        wf_diag = pd.DataFrame(np.diag(weight_factors), index=lvs, columns=lvs)
+        weights = weights.dot(wf_diag).sum(axis=1).to_frame(name="weight")
+        return data_new, pd.DataFrame(self.__scores, index=self.__index, columns=lvs), weights
+
+    def get_Z_for_mode_b(self, lv, mv, z_by_lv):
+        mv_index = self.__config.mv_index(lv, mv)
+        if self.__config.mode(lv) != Mode.B or len(self.__config.mvs(lv)) == 1:
+            return z_by_lv
+        if lv not in self.__betas:
+            exogenous = sm.add_constant(self.__mv_grouped_by_lv[lv])
+            regression = sm.OLS(z_by_lv, exogenous).fit()
+            self.__betas[lv] = regression.params[1:]
+        correction = 1 / self.__betas[lv][mv_index]
+        return correction * (z_by_lv - np.delete(self.__mv_grouped_by_lv[lv], mv_index, axis=1).dot(
+            np.delete(self.__betas[lv], mv_index)))
+
+    def correction(self) -> float:
+        return self.__correction
+
+    def mv_grouped_by_lv(self, lv: str, mv: str):
+        return self.__mv_grouped_by_lv_initial[lv][:, self.__config.mv_index(lv, mv)]
+
+    def dummies(self, mv: str) -> pd.DataFrame:
+        return self.__config.dummies(mv)
+
+
+class WeightsCalculatorFactory:
+    """Internal class that is used to calculate weights and scores from the data using the model."""
+    def __init__(self, config: c.Config, iterations: int, tolerance: float, correction: float, scheme: Scheme):
+        self.__iterations = iterations
+        self.__tolerance = tolerance
+        self.__config = config
+        self.__correction = correction
+        self.__scheme = scheme
+
+    def clone(self):
+        return WeightsCalculatorFactory(self.__config.clone(), self.__iterations, self.__tolerance, self.__correction, self.__scheme)
+
+    def config(self):
+        return self.__config
+
+    def calculate(self, data: pd.DataFrame, path: pd.DataFrame):
+        """Internal method that performs the calculation to estimate weights and scores."""
+        if self.__config.metric():
+            calculator = _MetricWeights(data, self.__config, self.__correction, path)
+        else:
+            calculator = _NonmetricWeights(data, self.__config, self.__correction, path)
+
+        iteration = 0
+        while True:
+            iteration += 1
+            convergence = calculator.iterate(self.__scheme)
+            if (convergence < self.__tolerance) or (iteration > self.__iterations):
+                break
+        if iteration > self.__iterations:
+            raise Exception("Could not converge after " + str(iteration) + " iterations")
+        return calculator.calculate()