apatools 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
apatools/lib.py ADDED
@@ -0,0 +1,415 @@
1
+ import numpy as np
2
+ from scipy.stats import norm, t
3
+
4
+ from ._deps import missing_dependency
5
+
6
+ try:
7
+ import pandas as pd
8
+ from statsmodels.stats.inter_rater import aggregate_raters
9
+ except ImportError as e: # pragma: no cover
10
+ missing_dependency("lib", "lib", e)
11
+
12
+ from .calc import bootstrap # noqa: F401 (moved to apatools.calc; re-exported for backward compatibility)
13
+
14
+
15
+ def gini(values, drop_zeros=False, tolerance=10**-6):
16
+ """Calculate the Gini coefficient"""
17
+ # https://github.com/oliviaguest/gini
18
+ # based on bottom eq: http://www.statsdirect.com/help/content/image/stat0206_wmf.gif
19
+ # from: http://www.statsdirect.com/help/default.htm#nonparametric_methods/gini.htm
20
+
21
+ x = (
22
+ np.array(list(values)).astype("float64").flatten()
23
+ ) # all values are treated equally, arrays must be 1d
24
+ x_min = np.amin(x)
25
+ if x_min < tolerance:
26
+ x += np.abs(x_min) # values cannot be negative
27
+ if drop_zeros:
28
+ x = x[x >= tolerance]
29
+ else:
30
+ x += tolerance # values cannot be 0
31
+ n, x = x.shape[0], np.sort(x) # values must be sorted
32
+ idx = np.arange(1, n + 1) # index per array element
33
+ return np.sum((2 * idx - n - 1) * x) / (n * np.sum(x)) # Gini coefficient
34
+
35
+
36
+ def gini_from_percentiles(values, percentiles, values_are_shares=False):
37
+ """
38
+ Calculate the Gini coefficient from percentile-based cumulative data.
39
+ Parameters:
40
+ - percentiles: array of population percentiles (e.g., [10, 20, ..., 100])
41
+ - values: array of corresponding cumulative shares/proportion (e.g., [1.5, 5.0, ..., 100])
42
+ or cumulative values themselves (e.g., [100, 150, 350, ..., 1200])
43
+ """
44
+
45
+ x = np.array(list(percentiles)).astype("float64").flatten() / 100
46
+ y = np.array(list(values)).astype("float64").flatten()
47
+ if values_are_shares:
48
+ y = y / 100
49
+ else:
50
+ y = y / np.max(y)
51
+ if x[0] > 0: # the Lorenz curve always starts in the origin (0% of the population has 0% of the total)
52
+ x, y = np.insert(x, 0, 0.0), np.insert(y, 0, 0.0)
53
+ # Trapezoidal approximation of area under Lorenz curve
54
+ b = (y[1:] + y[:-1]) * np.diff(x) / 2
55
+
56
+ return 1 - 2 * b.sum() # Gini coefficient is 1 - 2 * area under Lorenz curve
57
+
58
+
59
+ def log_transform(values):
60
+ """Pseudo log-transformation, keeping sign"""
61
+
62
+ x = np.array(list(values)).astype("float64").flatten()
63
+ return np.sign(x) * np.log10(1 + np.abs(x))
64
+
65
+
66
+ def z_transform(values, return_params=False, ignore_errors=True):
67
+ """z-transformation or standardization"""
68
+
69
+ x = np.array(list(values)).astype("float64").flatten()
70
+ m, s = np.nanmean(x), np.nanstd(x)
71
+ if not s and not ignore_errors:
72
+ raise ValueError(
73
+ "Standard deviation is zero or undefined. Z-transformation is not defined."
74
+ )
75
+
76
+ z = (x - m) / s
77
+
78
+ if return_params:
79
+ return z, m, s
80
+ return z
81
+
82
+
83
+ def fleiss_kappa(df, return_per_item_agreement=False, method="two-tailed"):
84
+ """
85
+ Compute Fleiss' kappa, z-score, and p-value, per-item agreement
86
+ Null hypothesis: Agreement is due to chance (kappa = 0)
87
+ 0.01-0.02 Slight agreement
88
+ 0.21-0.40 Fair Agreement
89
+ 0.41-0.60 Moderate Agreement
90
+ 0.61-0.80 Substantial Agreement
91
+ 0.81-1.00 Almost Perfect Agreement
92
+ Negative (kappa < 0): Agreement less than that expected by chance
93
+ """
94
+ # https://en.wikipedia.org/wiki/Fleiss%27s_kappa
95
+ # https://www.statsmodels.org/dev/generated/statsmodels.stats.inter_rater.aggregate_raters.html
96
+
97
+ df = df.astype(str).where(
98
+ pd.notnull(df), "None"
99
+ ) # be careful about NaNs, they are transformed to "None" category
100
+ n_items, n_raters = df.shape
101
+ m = aggregate_raters(df, n_cat=None)[0]
102
+ P_i = (np.sum(m**2, axis=1) - n_raters) / (
103
+ n_raters * (n_raters - 1)
104
+ ) # Per-item agreement
105
+ P_j = np.sum(m, axis=0) / (n_items * n_raters) # Category proportions
106
+ P_e = np.sum(P_j**2) # Expected agreement
107
+ kappa = (
108
+ (np.mean(P_i) - P_e) / (1 - P_e) if (1 - P_e) != 0 else np.nan
109
+ ) # Fleiss' kappa
110
+
111
+ # variance of kappa (approximation)
112
+ term1 = np.sum(P_j**2 * (1 - P_j) ** 2)
113
+ term2 = (1 - P_e) * (np.sum(P_j**3) - P_e * np.sum(P_j**2))
114
+ var_kappa = (term1 - term2) / (n_items * n_raters * (n_raters - 1) * (1 - P_e) ** 2)
115
+
116
+ # z-score and p-value
117
+ z, p = np.nan, np.nan
118
+ if var_kappa > 0:
119
+ z = kappa / np.sqrt(var_kappa)
120
+ p = norm.sf(np.abs(z)) # one-tailed, agreement > chance
121
+ p = p * 2 if method == "two-tailed" else p
122
+
123
+ if return_per_item_agreement:
124
+ return (
125
+ kappa,
126
+ z,
127
+ p,
128
+ pd.Series(P_i, index=df.index, name="per_item_agreement"),
129
+ )
130
+
131
+ return kappa, z, p
132
+
133
+
134
+ def krippendorffs_alpha(
135
+ df,
136
+ measurement="nominal",
137
+ return_per_item_agreement=False,
138
+ return_bootstraped_z_score=True,
139
+ n_iter_for_bootstrap=100,
140
+ bootstrap_seed=13,
141
+ method="two-tailed",
142
+ ):
143
+ """
144
+ Compute Krippendorff' alpha, z-score, and p-value, per-item agreement
145
+ Level of measurement = "nominal", "ordinal", "interval", "ratio"
146
+ Null hypothesis: Agreement is due to chance
147
+ """
148
+ # https://en.wikipedia.org/wiki/Krippendorff's_Alpha
149
+ # https://github.com/pln-fing-udelar/fast-krippendorff
150
+ try:
151
+ import krippendorff # optional dependency (GPL-3.0-or-later): pip install "apatools[krippendorff]"
152
+ except ImportError as e:
153
+ raise ImportError(
154
+ "krippendorffs_alpha() needs the optional dependency krippendorff: "
155
+ 'pip install "apatools[krippendorff]"'
156
+ ) from e
157
+
158
+ df = df.astype(str).where(
159
+ pd.notnull(df), "None"
160
+ ) # be careful about NaNs, they are transformed to "None" category
161
+ m = aggregate_raters(df, n_cat=None)[0] # [:,:-1]
162
+ alpha = krippendorff.alpha(value_counts=m, level_of_measurement=measurement)
163
+ # z-score and p-value
164
+ z, p = np.nan, np.nan
165
+ if return_bootstraped_z_score:
166
+ np.random.seed(bootstrap_seed)
167
+ null_dist = []
168
+ for _ in range(n_iter_for_bootstrap):
169
+ m = aggregate_raters(df.apply(np.random.permutation, axis=0), n_cat=None)[
170
+ 0
171
+ ] # [:,:-1]
172
+ null_alpha = krippendorff.alpha(
173
+ value_counts=m, level_of_measurement=measurement
174
+ )
175
+ null_dist.append(null_alpha)
176
+ mu, sigma = np.mean(null_dist), np.std(null_dist)
177
+ if sigma > 0:
178
+ z = (alpha - mu) / sigma
179
+ p = norm.sf(np.abs(z)) # one-tailed, agreement > chance
180
+ p = p * 2 if method == "two-tailed" else p
181
+
182
+ if return_per_item_agreement:
183
+ agreements = {}
184
+ for idx, row in df.astype(str).iterrows():
185
+ row = row.dropna()
186
+ agreements[idx] = (
187
+ np.nan if len(row) <= 1 else row.value_counts().max() / len(row)
188
+ )
189
+ return (
190
+ alpha,
191
+ z,
192
+ p,
193
+ pd.Series(agreements, index=df.index, name="per_item_agreement"),
194
+ )
195
+
196
+ return alpha, z, p
197
+
198
+
199
+ def average_correlations(r, n, alpha=0.05, method="two-tailed"):
200
+ """
201
+ r : list/array of correlations
202
+ n : list/array of sample sizes
203
+ """
204
+ r = np.array(r, dtype=float)
205
+ n = np.array(n, dtype=float)
206
+ z = np.arctanh(r) # 1) Fisher z-transform
207
+ w = n - 3 # 2) Weights (inverse variance)
208
+ z_bar = np.sum(w * z) / np.sum(
209
+ w
210
+ ) # 3) Pooled effect (mean z weighted by sample size)
211
+ se = 1 / np.sqrt(np.sum(w)) # 4) Standard error of pooled effect
212
+ z = z_bar / se # 5) Test statistic
213
+ p = 1 - norm.cdf(np.abs(z)) # 6) Two-tailed p-value
214
+ p = p * 2 if method == "two-tailed" else p
215
+ r_bar = np.tanh(z_bar) # 7) Convert back to correlation
216
+ # 8) CI in z-space, then back-transform
217
+ # z-critical = stats.norm.ppf(1 - alpha) (use alpha = alpha/2 for two-sided)
218
+ a = alpha / 2 if method == "two-tailed" else alpha
219
+ z_crit = norm.ppf(1 - a)
220
+ z_ci_low = z_bar - z_crit * se
221
+ z_ci_high = z_bar + z_crit * se
222
+ r_ci_low = np.tanh(z_ci_low)
223
+ r_ci_high = np.tanh(z_ci_high)
224
+ return {
225
+ "r_mean": r_bar,
226
+ "z_mean": z_bar,
227
+ "se": se,
228
+ "z": z,
229
+ "p-value": p,
230
+ "cil": r_ci_low,
231
+ "cir": r_ci_high,
232
+ }
233
+
234
+
235
+ def average_morans_i(i, i_var, method="two-tailed"):
236
+ """
237
+ I: array of Moran's I values
238
+ var_I: array of variances of Moran's I
239
+ """
240
+ i = np.array(i, dtype=float)
241
+ var_i = np.array(i_var, dtype=float)
242
+ w = 1 / var_i # weights = inverse variance
243
+ i_bar = np.sum(w * i) / np.sum(w) # weighted mean
244
+ se = np.sqrt(1 / np.sum(w)) # standard error
245
+ z = i_bar / se # test statistic
246
+ p = 1 - norm.cdf(np.abs(z)) # p-value (two-tailed)
247
+ p = p * 2 if method == "two-tailed" else p
248
+ return {
249
+ "i_mean": i_bar,
250
+ "z": z,
251
+ "se": se,
252
+ "p-value": p,
253
+ }
254
+
255
+
256
+ """
257
+ Functions for calculating the statistical significant differences between two dependent or independent correlation
258
+ coefficients.
259
+ The Fisher and Steiger method is adopted from the R package http://personality-project.org/r/html/paired.r.html
260
+ and is described in detail in the book 'Statistical Methods for Psychology'
261
+ The Zou method is adopted from http://seriousstats.wordpress.com/2012/02/05/comparing-correlations/
262
+ Credit goes to the authors of above mentioned packages!
263
+
264
+ Author: Philipp Singer (www.philippsinger.info)
265
+ """
266
+
267
+
268
+ def rz_ci(r, n, conf_level=0.95):
269
+ zr_se = np.pow(1 / (n - 3), 0.5)
270
+ moe = norm.ppf(1 - (1 - conf_level) / float(2)) * zr_se
271
+ zu = np.atanh(r) + moe
272
+ zl = np.atanh(r) - moe
273
+ return np.tanh((zl, zu))
274
+
275
+
276
+ def rho_rxy_rxz(rxy, rxz, ryz):
277
+ num = (ryz - 1 / 2.0 * rxy * rxz) * (
278
+ 1 - np.pow(rxy, 2) - np.pow(rxz, 2) - np.pow(ryz, 2)
279
+ ) + np.pow(ryz, 3)
280
+ den = (1 - np.pow(rxy, 2)) * (1 - np.pow(rxz, 2))
281
+ return num / float(den)
282
+
283
+
284
+ def dependent_correlations(xy, xz, yz, n, method="steiger", twotailed=True, alpha=0.95):
285
+ """
286
+ Calculates the statistic significance between two dependent correlation coefficients
287
+ xy: correlation coefficient between x and y
288
+ xz: correlation coefficient between x and z
289
+ yz: correlation coefficient between y and z
290
+ n: number of elements in x, y and z
291
+ twotailed: whether to calculate a one or two tailed test, only works for 'steiger' method
292
+ alpha / conf_level: confidence level, only works for 'zou' method
293
+ method: defines the method uses, 'steiger' or 'zou'
294
+ t and p-val
295
+ """
296
+ if method == "steiger":
297
+ d = xy - xz
298
+ determin = 1 - xy * xy - xz * xz - yz * yz + 2 * xy * xz * yz
299
+ av = (xy + xz) / 2
300
+ cube = (1 - yz) * (1 - yz) * (1 - yz)
301
+ t2 = d * np.sqrt(
302
+ (n - 1) * (1 + yz) / ((2 * (n - 1) / (n - 3)) * determin + av * av * cube)
303
+ )
304
+ p = 1 - t.cdf(abs(t2), n - 3)
305
+ if twotailed:
306
+ p *= 2
307
+ return t2, p
308
+ elif method == "zou":
309
+ L1 = rz_ci(xy, n, conf_level=alpha)[0]
310
+ U1 = rz_ci(xy, n, conf_level=alpha)[1]
311
+ L2 = rz_ci(xz, n, conf_level=alpha)[0]
312
+ U2 = rz_ci(xz, n, conf_level=alpha)[1]
313
+ rho_r12_r13 = rho_rxy_rxz(xy, xz, yz)
314
+ lower = (
315
+ xy
316
+ - xz
317
+ - np.pow(
318
+ (
319
+ np.pow((xy - L1), 2)
320
+ + np.pow((U2 - xz), 2)
321
+ - 2 * rho_r12_r13 * (xy - L1) * (U2 - xz)
322
+ ),
323
+ 0.5,
324
+ )
325
+ )
326
+ upper = (
327
+ xy
328
+ - xz
329
+ + np.pow(
330
+ (
331
+ np.pow((U1 - xy), 2)
332
+ + np.pow((xz - L2), 2)
333
+ - 2 * rho_r12_r13 * (U1 - xy) * (xz - L2)
334
+ ),
335
+ 0.5,
336
+ )
337
+ )
338
+ return lower, upper
339
+ else:
340
+ raise ValueError(f"Wrong method {method!r}: use 'steiger' or 'zou'.")
341
+
342
+
343
+ def independent_correlations(
344
+ xy, ab, n1, n2=None, method="fisher", twotailed=True, alpha=0.95
345
+ ):
346
+ """
347
+ Calculates the statistic significance between two independent correlation coefficients
348
+ @param xy: correlation coefficient between x and y
349
+ @param xz: correlation coefficient between a and b
350
+ @param n1: number of elements in xy
351
+ @param n2: number of elements in ab (if distinct from n1)
352
+ @param twotailed: whether to calculate a one or two tailed test, only works for 'fisher' method
353
+ @param alpha: confidence level, only works for 'zou' method
354
+ @param method: defines the method uses, 'fisher' or 'zou'
355
+ @return: z and p-val
356
+ """
357
+
358
+ if method == "fisher":
359
+ xy_z = 0.5 * np.log((1 + xy) / (1 - xy))
360
+ ab_z = 0.5 * np.log((1 + ab) / (1 - ab))
361
+ if n2 is None:
362
+ n2 = n1
363
+ se_diff_r = np.sqrt(1 / (n1 - 3) + 1 / (n2 - 3))
364
+ diff = xy_z - ab_z
365
+ z = np.abs(diff / se_diff_r)
366
+ p = 1 - norm.cdf(z)
367
+ if twotailed:
368
+ p *= 2
369
+ return z, p
370
+ elif method == "zou":
371
+ L1 = rz_ci(xy, n1, conf_level=alpha)[0]
372
+ U1 = rz_ci(xy, n1, conf_level=alpha)[1]
373
+ L2 = rz_ci(ab, n2, conf_level=alpha)[0]
374
+ U2 = rz_ci(ab, n2, conf_level=alpha)[1]
375
+ lower = xy - ab - np.pow((np.pow((xy - L1), 2) + np.pow((U2 - ab), 2)), 0.5)
376
+ upper = xy - ab + np.pow((np.pow((U1 - xy), 2) + np.pow((ab - L2), 2)), 0.5)
377
+ return lower, upper
378
+ else:
379
+ raise ValueError(f"Wrong method {method!r}: use 'fisher' or 'zou'.")
380
+
381
+
382
+ def htmt_ratio(data, construct_a, construct_b, method="spearman"):
383
+ """
384
+ The Heterotrait-Monotrait (HTMT) ratio
385
+ measures the similarity between different constructs (heterotrait) compared to
386
+ the similarity of items measuring the exact same construct (monotrait)
387
+ Conservative Threshold: < 0.85 (typically used for strict discriminant validity).
388
+ Liberal Threshold: < 0.90 (widely accepted in most social and business sciences).
389
+ Note: If an HTMT value is above the threshold (e.g., 0.95), it suggests that the two constructs are too similar.
390
+ data: DataFrame with item scores
391
+ construct_a: list of columns for latent variable A
392
+ construct_b: list of columns for latent variable B
393
+ method: correlation method to use ("spearman" or "pearson")
394
+ """
395
+ corr = (
396
+ data[construct_a + construct_b]
397
+ .dropna()
398
+ .astype("float64")
399
+ .corr(method=method)
400
+ .abs()
401
+ )
402
+ hetero = [
403
+ corr.loc[i, j] for i in construct_a for j in construct_b
404
+ ] # Heterotrait correlations
405
+ mono_a = [ # Monotrait correlations within A
406
+ corr.loc[construct_a[i], construct_a[j]]
407
+ for i in range(len(construct_a))
408
+ for j in range(i + 1, len(construct_a))
409
+ ]
410
+ mono_b = [ # Monotrait correlations within B
411
+ corr.loc[construct_b[i], construct_b[j]]
412
+ for i in range(len(construct_b))
413
+ for j in range(i + 1, len(construct_b))
414
+ ]
415
+ return np.mean(hetero) / np.sqrt(np.mean(mono_a) * np.mean(mono_b))