ararpy 0.1.199__py3-none-any.whl → 0.2.1__py3-none-any.whl

ararpy/calc/regression.py

@@ -16,7 +16,7 @@ import traceback
 import numpy as np
 import pandas as pd
 from scipy.stats import distributions
-from scipy.optimize import fsolve
+from scipy.optimize import minimize_scalar
 import warnings
 from scipy.optimize import minimize
 warnings.simplefilter(action="ignore", category=RuntimeWarning)
@@ -495,22 +495,25 @@ def linest(a0: list, a1: list, *args):
     # calculate Y values base on the fitted formula
     estimate_y = np.matmul(x, beta)
     resid = (estimate_y - y) ** 2
-    reg = (estimate_y - np.mean(estimate_y)) ** 2
-    ssresid = resid.sum()
-    ssreg = reg.sum()
-    sstotal = ssreg + ssresid
+    reg = (estimate_y - np.mean(y)) ** 2
+    ssresid = resid.sum()  # 残差平方和
+    ssreg = reg.sum()  # 回归平方和
+    sstotal = ((y - np.mean(y)) ** 2).sum()
+    r2 = ssreg / sstotal if sstotal != 0 else np.inf
+
     df = m - n
-    m_ssresid = ssresid / df
-    se_beta = (m_ssresid * np.diagonal(inv_xtx)) ** .5
-    beta = beta.transpose()[0]
+    m_ssresid = ssresid / df  # 均方残差，与加权平均中的MSWD对应
+    cov_beta = m_ssresid * inv_xtx
+    se_beta = np.diagonal(cov_beta) ** .5
+
+    beta = beta.flatten()
     rse_beta = se_beta / beta
-    r2 = ssreg / sstotal if sstotal != 0 else np.inf
 
     def get_adjusted_y(*args):
         args = [[1] * len(args[0]), *args]
         return [sum([beta[i] * args[i][j] for i in range(len(beta))]) for j in range(len(args[0]))]
 
-    return beta[0], se_beta[0], rse_beta[0] * 100, r2, m_ssresid, beta, se_beta, get_adjusted_y, m_ssresid
+    return beta[0], se_beta[0], abs(rse_beta[0]) * 100, r2, m_ssresid, beta, cov_beta, get_adjusted_y, m_ssresid
 
 
 def average(a0: list, a1=None):
@@ -535,12 +538,12 @@ def average(a0: list, a1=None):
     m_ssresid = ssresid / df
     r2 = ssreg / sstotal if sstotal != 0 else 1  # r2 = ssreg / sstotal
 
-    k1 = pow(sum([(i - k0) ** 2 for i in a0]) / df, 0.5)  # standard deviation
-    k2 = k1 / k0 * 100 if k0 != 0 else 0  # relative standard error
+    k6 = [[sum([(i - k0) ** 2 for i in a0]) / df]]
+    k1 = pow(k6[0][0], 0.5)  # standard deviation
+    k2 = k1 / abs(k0) * 100 if k0 != 0 else 0  # relative standard error
     k3 = r2  # determination coefficient
-    k4 = 'MSWD'
+    k4 = m_ssresid  # 'MSWD'
     k5 = [k0]
-    k6 = [k1]
     k8 = m_ssresid
 
     def get_adjusted_y(x: list):
@@ -737,7 +740,7 @@ def quadratic(a0: list, a1: list):
     """
     # y = b + m1 * x + m2 * x ^ 2
     k = list(linest(a0, a1, [i ** 2 for i in a1]))
-    b, seb, rseb, r2, mswd, [b, m1, m2], [seb, sem1, sem2] = k[0:7]
+    [b, m1, m2] = k[5]
 
     def get_adjusted_y(x: list):
         return [b + m1 * _x + m2 * _x ** 2 for _x in x]
@@ -756,7 +759,7 @@ def polynomial(a0: list, a1: list, degree: int = 5):
     """
     # y = b + m1 * x + m2 * x ^ 2 + ... + m[n] * x ^ n
     k = list(linest(a0, *[[j ** (i + 1) for j in a1] for i in range(degree)]))
-    b, seb, rseb, r2, mswd, beta, se_beta = k[0:7]
+    beta = k[5]
 
     def get_adjusted_y(x: list):
         return [sum([beta[i] * _x ** i for i in range(degree + 1)]) for _x in x]
@@ -766,6 +769,7 @@ def polynomial(a0: list, a1: list, degree: int = 5):
     return k
 
 
+### Deprecated
 def logest(a0: list, a1: list):
     """
     :param a0: known_y's, y = b * m ^ x
@@ -775,15 +779,19 @@ def logest(a0: list, a1: list):
     # y = b * m ^ x, Microsoft Excel LOGEST function, ln(y) = ln(b) + ln(m) * x
     a0 = [np.log(i) for i in a0]  # ln(y)
     linest_res = linest(a0, a1)
-    b, seb, rseb, r2, mswd, [b, lnm], [seb, selnm] = linest_res[0:7]
-    b = np.exp(b)
+    lnb, selnb, rseb, r2, mswd, beta, cov_beta = linest_res[0:7]
+    lnb, lnm = beta
+    selnb, selnm = np.diagonal(cov_beta) ** .5
+
+    b = np.exp(lnb)
     m = np.exp(lnm)
     sem = np.exp(lnm) * selnm
-    seb = b * seb  # Excel.Logest function do not consider the error propagation
-    rseb = seb / b * 100
+    seb = np.exp(lnb) * selnb  # Excel.Logest function do not consider the error propagation
+    rseb = seb / abs(b) * 100
     return b, seb, rseb, r2, mswd, m, sem
 
 
+### Deprecated
 def power(a0: list, a1: list):
     """
     :param a0: known_y's, y = a * x ^ b + c
@@ -794,39 +802,31 @@ def power(a0: list, a1: list):
     def _pow_func(x, a, b, c):
         return a * x ** b + c
 
-    def _solve_pow(params):
+    def _residuals(params):
         a, b, c = params
-        x, y = [0, 0, 0], [0, 0, 0]
-        x[0] = sum(a1[:3]) / 3
-        y[0] = sum(a0[:3]) / 3
-        x[1] = sum(a1) / len(a1)
-        y[1] = sum(a0) / len(a0)
-        x[2] = sum(a1[-3:]) / 3
-        y[2] = sum(a0[-3:]) / 3
-        return np.array([
-            _pow_func(x[0], a, b, c) - y[0],
-            _pow_func(x[1], a, b, c) - y[1],
-            _pow_func(x[2], a, b, c) - y[2],
-        ])
-
-    def _get_sum(a, b, c):
-        y_predicted = [_pow_func(_x, a, b, c) for _x in a1]
-        return sum([(y_predicted[i] - a0[i]) ** 2 for i in range(len(a0))])
+        return [_pow_func(xi, a, b, c) - yi for xi, yi in zip(a1, a0)]
+
+    def _sum_squared_error(params):
+        return sum(r**2 for r in _residuals(params))
 
     def _get_abc(b):  # Return a, b, c given b based on linest regression
         f = linest(a0, [_x ** b for _x in a1])
         return f[5][1], b, f[0]
 
+    def _get_init():
+        f = linest(np.log(np.array(a0)), np.log(np.array(a1)))
+        return np.exp(f[0]), f[5][1], 0
+
     try:
-        a, b, c = fsolve(func=_solve_pow, x0=np.array([1, 1, 1]))  # initial estimate
+        a, b, c = _get_init()  # initial estimate
         count = 0
         step = 0.01
         while count < 100:
             a, b, c = _get_abc(b)
-            s = _get_sum(a, b, c)
+            s = _sum_squared_error([a, b, c])
             b_left, b_right = b - step * b, b + step * b
-            s_left = _get_sum(*_get_abc(b_left))
-            s_right = _get_sum(*_get_abc(b_right))
+            s_left = _sum_squared_error(_get_abc(b_left))
+            s_right = _sum_squared_error(_get_abc(b_right))
             if s_left > s > s_right:
                 b = b_right
                 continue
@@ -849,7 +849,9 @@ def power(a0: list, a1: list):
         raise IndexError
 
     f = linest(a0, [_x ** b for _x in a1])
-    a, sea, c, sec = f[5][1], f[6][1], f[0], f[1]
+    beta, cov_beta = f[5:7]
+    c, a = beta
+    sec, sea = np.diagonal(cov_beta) ** .5
 
     calculated_y = [_pow_func(i, a, b, c) for i in a1]
     resid = [(calculated_y[i] - a0[i]) ** 2 for i in range(len(a0))]
@@ -870,111 +872,193 @@ def power(a0: list, a1: list):
     errfx = pow(sum([i ** 2 for i in a1]) / (dp * sum([i ** 2 for i in a1]) - sum(a1) ** 2), 0.5)
     # seb = errfz * sey = errfz * ssresid / df -> se_intercept = sey * errfx = seb / errfz * errfx
     se_intercept = sec / errfz * errfx
-    rse_intercept = se_intercept / intercept * 100
+    rse_intercept = se_intercept / abs(intercept) * 100
+
+
+    exp_beta = [a, b, c]
+    exp_cov_beta = [
+        [cov_beta[1][1], 0, cov_beta[1][0]],
+        [0,              0,              0],
+        [cov_beta[0][1], 0, cov_beta[0][0]],
+    ]
 
     return intercept, se_intercept, rse_intercept, r2, 'mswd', [a, b, c], 'se', \
            lambda x: [_pow_func(i, a, b, c) for i in x], m_ssresid
 
 
 def exponential(a0: list, a1: list):
-    """
-    :param a0: known_y's, y = a * b ^ x + c
-    :param a1: known_x's
-    :return: intercept | standard error of intercept | relative error | R2 | MSWD | [m, c, b] | [sem, sec, seb]
-    """
+    try:
+        k = _exponential(a0, a1, newton_minimize_1d)
+        if k[2] > 100:
+            raise ValueError
+        if abs(k[0] - min(a0)) > 5 * (max(a0) - min(a0)):
+            raise ValueError
+    except Exception as e:
+        print(f"newton_minimize_1d failed, using grad minimize")
+        k = _exponential(a0, a1, grad_minimize)
+    return k
 
-    def _exp_func(x, a, b, c):
-        return a * b ** x + c
 
-    def _solve_exp(params):
-        a, b, c = params
-        x, y = [0, 0, 0], [0, 0, 0]
-        x[0] = sum(a1[:3]) / 3
-        y[0] = sum(a0[:3]) / 3
-        x[1] = sum(a1) / len(a1)
-        y[1] = sum(a0) / len(a0)
-        x[2] = sum(a1[-3:]) / 3
-        y[2] = sum(a0[-3:]) / 3
-        return np.array([
-            _exp_func(x[0], a, b, c) - y[0],
-            _exp_func(x[1], a, b, c) - y[1],
-            _exp_func(x[2], a, b, c) - y[2],
-        ])
-
-    def _get_sum(a, b, c):
-        y_predicted = [_exp_func(_x, a, b, c) for _x in a1]
-        return sum([(y_predicted[i] - a0[i]) ** 2 for i in range(len(a0))])
-
-    def _get_ac(b):
-        f = linest(a0, [b ** _x for _x in a1])
-        return f[5][1], b, f[0]
+def _exponential(a0, a1, method):
 
-    try:
-        a, b, c = fsolve(_solve_exp, np.array([1, 1, 1]))
-        count = 0
-        step = 0.01
-        while count < 100:
-            a, b, c = _get_ac(b)
-            s = _get_sum(a, b, c)
-            b_left, b_right = b - step * b, b + step * b
-            s_left = _get_sum(*_get_ac(b_left))
-            s_right = _get_sum(*_get_ac(b_right))
-            if s_left > s > s_right:
-                b = b_right
-                continue
-            elif s_left < s < s_right:
-                b = b_left
-                continue
-            elif s_left < s_right:
-                b = (b + b_left) / 2
-            else:
-                b = (b + b_right) / 2
-            count += 1
-            step = step * 0.5
-            if step < 0.000001:
-                break
+    X = np.array(a1)
+    Y = np.array(a0)
+    y = np.array([a0]).transpose()
 
-    except RuntimeError:
-        raise RuntimeError
+    def _exp_func(xi, a, b, c):
+        return a * np.exp(b * xi) + c
+
+    @np.vectorize
+    def _get_s(b):
+        x = np.concatenate(([np.ones(len(a1))], [np.exp(b * np.array(a1))]), axis=0).transpose()
+        try:
+            inv_xtx = np.linalg.inv(np.matmul(x.transpose(), x))
+        except np.linalg.LinAlgError:
+            raise np.linalg.LinAlgError(f"The determinant of the given matrix must not be zero ")
+        beta = np.matmul(inv_xtx, np.matmul(x.transpose(), y))
+        c, a = beta.flatten()
+        reg_y = _exp_func(X, a, b, c)
+        resid = (reg_y - Y) ** 2
+        ssresid = sum(resid)
+        return ssresid
+
+    # 1阶
+    ini_f1 = linest(Y, X)
+    y_range = max(Y) - min(Y)
+    ini_b = ini_f1[5][1] / max(y_range, 1e-12)
+    # 2阶
+    ini_f2 = quadratic(Y, X)
+    ini_b *= np.sign(ini_f1[5][1]) * np.sign(ini_f2[5][2])
+
+    b, count = method(_get_s, ini_b)
+
+    x = np.concatenate(([np.ones(len(a1))], [np.exp(b * np.array(a1))]), axis=0).transpose()
+
+    m, n = x.shape  # number of data, number of unknown x
+    try:
+        inv_xtx = np.linalg.inv(np.matmul(x.transpose(), x))
     except np.linalg.LinAlgError:
-        raise np.linalg.LinAlgError
-    except TypeError:
-        raise TypeError
-    except IndexError:
-        raise IndexError
-
-    f = linest(a0, [b ** _x for _x in a1])
-    a, sea, c, sec = f[5][1], f[6][1], f[0], f[1]
-
-    calculated_y = [_exp_func(i, a, b, c) for i in a1]
-    resid = [(calculated_y[i] - a0[i]) ** 2 for i in range(len(a0))]
-    reg = [(i - sum(calculated_y) / len(calculated_y)) ** 2 for i in calculated_y]
-    ssresid = sum(resid)
-    ssreg = sum(reg)
-    sstotal = ssreg + ssresid
-    dp = len(a1)
-    df = dp - 1
-    m_ssresid = ssresid / df
+        raise np.linalg.LinAlgError(f"The determinant of the given matrix must not be zero ")
+    beta = np.matmul(inv_xtx, np.matmul(x.transpose(), y))
+    estimate_y = np.matmul(x, beta)
+    resid = (estimate_y - y) ** 2
+    reg = (estimate_y - np.mean(y)) ** 2
+    ssresid = resid.sum()  # 残差平方和
+    ssreg = reg.sum()  # 回归平方和
+    sstotal = ((y - np.mean(y)) ** 2).sum()
     r2 = ssreg / sstotal if sstotal != 0 else 1
 
-    z = [b ** i for i in a1]
+    df = m - n
+    m_ssresid = ssresid / df  # 均方残差，与加权平均中的MSWD对应
+    cov_beta = m_ssresid * inv_xtx
+
+    sc, sa = np.diagonal(cov_beta) ** .5
+    c, a = beta.flatten()
     intercept = a + c
-    se_intercept_1 = np.sqrt(sea ** 2 + sec ** 2)
-    # calculate error of intercept
-    errfz = pow(sum([i ** 2 for i in z]) / (dp * sum([i ** 2 for i in z]) - sum(z) ** 2), 0.5)
-    errfx = pow(sum([i ** 2 for i in a1]) / (dp * sum([i ** 2 for i in a1]) - sum(a1) ** 2), 0.5)
-    # seb = errfz * sey = errfz * ssresid / df -> se_intercept = sey * errfx = seb / errfz * errfx
-    se_intercept = sec / errfz * errfx
-    rse_intercept = se_intercept / intercept * 100
+    se_intercept = pow(sa ** 2 + sc ** 2 + 2 * cov_beta[0][1] , 0.5)
 
-    return intercept, se_intercept, rse_intercept, r2, 'mswd', [a, b, c], 'se', \
-           lambda x: [_exp_func(i, a, b, c) for i in x], m_ssresid
+    exp_beta = [a, b, c]
+    exp_cov_beta = [
+        [cov_beta[1][1], 0, cov_beta[1][0]],
+        [0,              0,              0],
+        [cov_beta[0][1], 0, cov_beta[0][0]],
+    ]
+
+    # print(f"{b = }, {a = }, {c = }, {count = }, {ssresid = }, {r2 = }")
+
+    return intercept, se_intercept, se_intercept / abs(intercept) * 100, r2, m_ssresid, exp_beta, exp_cov_beta, \
+           lambda x: [_exp_func(xi, a, b, c) for xi in x], m_ssresid
+
+
+def newton_minimize_1d(func, x0, h=1e-5, tol=1e-8, max_iter=30):
+    x = x0
+    for i in range(max_iter):
+        fx = func(x)
+        if not np.isfinite(fx):
+            raise ValueError("Newton Minimize: Function returned non-finite value at x={}".format(x))
+
+        # 一阶导数（中心差分）
+        f_plus = func(x + h)
+        f_minus = func(x - h)
+        if not (np.isfinite(f_plus) and np.isfinite(f_minus)):
+            raise ValueError("Newton Minimize: Non-finite values in derivative computation")
+        df = (f_plus - f_minus) / (2 * h)
+        d2f = (f_plus - 2 * fx + f_minus) / (h * h)
+
+        # 牛顿步长
+        if abs(d2f) < 1e-12:  # 避免除零或平坦区域
+            raise ValueError("Newton Minimize: Second derivative too small, stopping.")
+
+        step = df / d2f
+        x_new = x - step
+
+        if abs(step) < tol:
+            x = x_new
+            break
+
+        x = x_new
+
+    if i + 1 == max_iter:
+        raise ValueError("Newton Minimize: Over iteration max_iter={}".format(max_iter))
+
+    return x, i + 1
+
+
+def grad_minimize(func, x, lr=1e-5, tol=1e-8, max_iter=1000):
+
+    for i in range(max_iter):
+        f_plus = func(x + lr)
+        f_minus = func(x - lr)
+        if not (np.isfinite(f_plus) and np.isfinite(f_minus)):
+            raise ValueError("Newton Minimize: Non-finite values in derivative computation")
+        g = (f_plus - f_minus) / (2 * lr)
+        x_new = x - lr * g
+        if abs(func(x_new) - func(x)) < tol:
+            break
+        x = x_new
+    return x, i+1
 
 
 """ line functions """
 
 
-def linear_eq(x: list, beta: list):
+def linest_var(beta: list, cov: list, x: float):
+    """ y = b0 * x^0 + b1 * x^1 + ... + bn * x^n
+    Parameters
+    ----------
+
+    Returns
+    -------
+
+    """
+    beta = np.array(beta, dtype=np.float64)
+    cov_matrix = np.array(cov, dtype=np.float64)
+    g = np.array([x ** k for k in range(len(beta))]).reshape(-1, 1)
+    y = np.dot(g.T, beta.reshape(-1, 1))[0, 0]
+    var = np.matmul(np.matmul(g.T, cov_matrix), g)[0, 0]
+    return y, var
+
+
+def quadratic_var(beta: list, cov: list, x: float):
+    """ y = b0 * x^0 + b1 * x^1 + b2 * x^2
+    Parameters
+    ----------
+    beta : coefficients
+    x :
+
+    Returns
+    -------
+
+    """
+    beta = np.array(beta, dtype=np.float64)
+    cov_matrix = np.array(cov, dtype=np.float64)
+    g = np.array([x ** k for k in range(len(beta))]).reshape(-1, 1)
+    y = np.dot(g.T, beta.reshape(-1, 1))[0, 0]
+    var = np.matmul(np.matmul(g.T, cov_matrix), g)[0, 0]
+    return y, var
+
+
+def average_var(beta: list, cov: list, x: float):
     """ y = b0 * x^0 + b1 * x^1 + ... + bn * x^n
     Parameters
     ----------
@@ -985,21 +1069,29 @@ def linear_eq(x: list, beta: list):
     -------
 
     """
-    return [np.prod([beta, [_x ** i for i in range(len(beta))]], axis=0).sum() for _x in x]
+    y, = beta
+    var = np.array(cov, dtype=np.float64)[0, 0]
+    return y, var
 
 
-def exponential_eq(x: list, beta: list):
-    """ y = a * b ^ x + c
+def exponential_var(beta: list, cov: list, x: float):
+    """ y = a * exp(bx) + c
     Parameters
     ----------
     beta : coefficients, [a, b, c]
-    x :
+    cov : covariance [[], [], []]
 
     Returns
     -------
 
     """
-    return [beta[0] * beta[1] ** _x + beta[0] for _x in x]
+    cov_matrix = np.array(cov)
+    a, b, c = beta
+    k = np.exp(b*x)
+    g = np.array([k, a*x*k, 1]).reshape(-1, 1)
+    y = a * k + c
+    var = np.matmul(np.matmul(g.T, cov_matrix), g)[0, 0]
+    return y, var
 
 
 def power_eq(x: list, beta: list):

ararpy/files/arr_file.py

@@ -26,7 +26,8 @@ def save(file_path, sample):
     -------
     str, file name
     """
-    file_path = os.path.join(file_path, f"{sample.Info.sample.name}.arr")
+    if not file_path.endswith(".arr"):
+        file_path = os.path.join(file_path, f"{sample.Info.sample.name}.arr")
     with open(file_path, 'wb') as f:
         f.write(pickle.dumps(sample))
     # with open(file_path, 'w') as f:

ararpy/files/basic.py

@@ -12,31 +12,9 @@
 
 import os
 import pickle
-import traceback
 import json
 
 
-def upload(file, media_dir):
-    try:
-        name, suffix = os.path.splitext(file.name)
-        if suffix.lower() not in [
-            '.xls', '.age', '.xlsx', '.arr', '.jpg', '.png', '.txt',
-            '.log', '.seq', '.json', '.ahd', '.csv']:
-            raise TypeError(f"The imported file is not supported: {suffix}")
-        web_file_path = os.path.join(media_dir, file.name)
-        with open(web_file_path, 'wb') as f:
-            for chunk in file.chunks():
-                f.write(chunk)
-        print("File path on the server: %s" % web_file_path)
-    except PermissionError:
-        raise ValueError(f'Permission denied')
-    except (Exception, BaseException) as e:
-        print(traceback.format_exc())
-        raise ValueError(f'Error in opening file: {e}')
-    else:
-        return web_file_path, name, suffix
-
-
 def read(file_path):
     """ Read text files, default 'r', 'rb'
     Parameters