ErisPulse-ChatSonar 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- erispulse_chatsonar-1.1.0/ChatSonar/Analyzer.py +380 -0
- erispulse_chatsonar-1.1.0/ChatSonar/Collector.py +246 -0
- erispulse_chatsonar-1.1.0/ChatSonar/Commands.py +524 -0
- erispulse_chatsonar-1.1.0/ChatSonar/Core.py +163 -0
- erispulse_chatsonar-1.1.0/ChatSonar/Visualizer.py +505 -0
- erispulse_chatsonar-1.1.0/ChatSonar/__init__.py +1 -0
- erispulse_chatsonar-1.1.0/ChatSonar/fonts/SourceHanSansSC-Regular.otf +0 -0
- erispulse_chatsonar-1.1.0/ChatSonar/fonts/__init__.py +0 -0
- erispulse_chatsonar-1.1.0/ErisPulse_ChatSonar.egg-info/PKG-INFO +184 -0
- erispulse_chatsonar-1.1.0/ErisPulse_ChatSonar.egg-info/SOURCES.txt +18 -0
- erispulse_chatsonar-1.1.0/ErisPulse_ChatSonar.egg-info/dependency_links.txt +1 -0
- erispulse_chatsonar-1.1.0/ErisPulse_ChatSonar.egg-info/entry_points.txt +2 -0
- erispulse_chatsonar-1.1.0/ErisPulse_ChatSonar.egg-info/requires.txt +3 -0
- erispulse_chatsonar-1.1.0/ErisPulse_ChatSonar.egg-info/top_level.txt +1 -0
- erispulse_chatsonar-1.1.0/LICENSE +205 -0
- erispulse_chatsonar-1.1.0/MANIFEST.in +1 -0
- erispulse_chatsonar-1.1.0/PKG-INFO +184 -0
- erispulse_chatsonar-1.1.0/README.md +159 -0
- erispulse_chatsonar-1.1.0/pyproject.toml +42 -0
- erispulse_chatsonar-1.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import time
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Analyzer:
|
|
7
|
+
def __init__(self, sdk, config):
|
|
8
|
+
self.sdk = sdk
|
|
9
|
+
self.logger = sdk.logger.get_child("ChatSonar.Analyzer")
|
|
10
|
+
self.storage = sdk.storage
|
|
11
|
+
self.config = config
|
|
12
|
+
|
|
13
|
+
def _default_weights(self):
|
|
14
|
+
return self.config.get("weights", {
|
|
15
|
+
"timing": 0.20,
|
|
16
|
+
"emoji": 0.15,
|
|
17
|
+
"vocab": 0.20,
|
|
18
|
+
"interaction": 0.30,
|
|
19
|
+
"cooccurrence": 0.15,
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
def _min_messages(self):
|
|
23
|
+
return self.config.get("min_messages", 5)
|
|
24
|
+
|
|
25
|
+
def _distance_threshold(self):
|
|
26
|
+
return self.config.get("distance_threshold", 0.6)
|
|
27
|
+
|
|
28
|
+
def _get_users(self, scope):
|
|
29
|
+
return list(self.storage.get(f"{scope}:users", {}).keys())
|
|
30
|
+
|
|
31
|
+
def _get_message_count(self, scope, uid):
|
|
32
|
+
length_data = self.storage.get(f"{scope}:length:{uid}", {"count": 0})
|
|
33
|
+
return length_data.get("count", 0)
|
|
34
|
+
|
|
35
|
+
def _eligible_users(self, scope):
|
|
36
|
+
users = self._get_users(scope)
|
|
37
|
+
min_msg = self._min_messages()
|
|
38
|
+
return [u for u in users if self._get_message_count(scope, u) >= min_msg]
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def jaccard(set_a, set_b):
|
|
42
|
+
if not set_a and not set_b:
|
|
43
|
+
return 1.0
|
|
44
|
+
if not set_a or not set_b:
|
|
45
|
+
return 0.0
|
|
46
|
+
a = set(set_a)
|
|
47
|
+
b = set(set_b)
|
|
48
|
+
intersection = len(a & b)
|
|
49
|
+
union = len(a | b)
|
|
50
|
+
return intersection / union if union > 0 else 0.0
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def cosine_sim(dict_a, dict_b):
|
|
54
|
+
if not dict_a or not dict_b:
|
|
55
|
+
return 0.0
|
|
56
|
+
all_keys = set(dict_a.keys()) | set(dict_b.keys())
|
|
57
|
+
dot = sum(dict_a.get(k, 0) * dict_b.get(k, 0) for k in all_keys)
|
|
58
|
+
norm_a = math.sqrt(sum(v ** 2 for v in dict_a.values()))
|
|
59
|
+
norm_b = math.sqrt(sum(v ** 2 for v in dict_b.values()))
|
|
60
|
+
if norm_a == 0 or norm_b == 0:
|
|
61
|
+
return 0.0
|
|
62
|
+
return dot / (norm_a * norm_b)
|
|
63
|
+
|
|
64
|
+
def timing_similarity(self, scope, uid_a, uid_b):
|
|
65
|
+
ta = self.storage.get(f"{scope}:timing:{uid_a}", {})
|
|
66
|
+
tb = self.storage.get(f"{scope}:timing:{uid_b}", {})
|
|
67
|
+
hours_a = set(ta.keys())
|
|
68
|
+
hours_b = set(tb.keys())
|
|
69
|
+
if not hours_a and not hours_b:
|
|
70
|
+
return 0.5
|
|
71
|
+
all_hours = set(range(24))
|
|
72
|
+
vec_a = {h: ta.get(str(h), 0) for h in all_hours}
|
|
73
|
+
vec_b = {h: tb.get(str(h), 0) for h in all_hours}
|
|
74
|
+
return self.cosine_sim(vec_a, vec_b)
|
|
75
|
+
|
|
76
|
+
def emoji_similarity(self, scope, uid_a, uid_b):
|
|
77
|
+
ea = self.storage.get(f"{scope}:emoji:{uid_a}", {})
|
|
78
|
+
eb = self.storage.get(f"{scope}:emoji:{uid_b}", {})
|
|
79
|
+
if not ea and not eb:
|
|
80
|
+
return 0.5
|
|
81
|
+
return self.cosine_sim(ea, eb)
|
|
82
|
+
|
|
83
|
+
def vocab_similarity(self, scope, uid_a, uid_b):
|
|
84
|
+
va = self.storage.get(f"{scope}:vocab:{uid_a}", {})
|
|
85
|
+
vb = self.storage.get(f"{scope}:vocab:{uid_b}", {})
|
|
86
|
+
if not va and not vb:
|
|
87
|
+
return 0.5
|
|
88
|
+
return self.cosine_sim(va, vb)
|
|
89
|
+
|
|
90
|
+
def interaction_score(self, scope, uid_a, uid_b, max_interact=100.0):
|
|
91
|
+
ia = self.storage.get(f"{scope}:interact:{uid_a}", {})
|
|
92
|
+
ib = self.storage.get(f"{scope}:interact:{uid_b}", {})
|
|
93
|
+
a_to_b = ia.get(uid_b, 0)
|
|
94
|
+
b_to_a = ib.get(uid_a, 0)
|
|
95
|
+
total = a_to_b + b_to_a
|
|
96
|
+
if total == 0:
|
|
97
|
+
return 0.0
|
|
98
|
+
return min(total / max_interact, 1.0)
|
|
99
|
+
|
|
100
|
+
def cooccurrence_score(self, scope, uid_a, uid_b, max_cooccur=50.0):
|
|
101
|
+
cooccur = self.storage.get(f"{scope}:cooccur", {})
|
|
102
|
+
pair = f"{uid_a}|{uid_b}"
|
|
103
|
+
val = cooccur.get(pair, 0)
|
|
104
|
+
if val == 0:
|
|
105
|
+
return 0.0
|
|
106
|
+
return min(val / max_cooccur, 1.0)
|
|
107
|
+
|
|
108
|
+
def _compute_max_interact(self, scope, users):
|
|
109
|
+
max_val = 0
|
|
110
|
+
for uid in users:
|
|
111
|
+
interact = self.storage.get(f"{scope}:interact:{uid}", {})
|
|
112
|
+
for count in interact.values():
|
|
113
|
+
max_val = max(max_val, count)
|
|
114
|
+
return max(max_val, 1.0)
|
|
115
|
+
|
|
116
|
+
def _compute_max_cooccur(self, scope, users):
|
|
117
|
+
cooccur = self.storage.get(f"{scope}:cooccur", {})
|
|
118
|
+
max_val = 0
|
|
119
|
+
for count in cooccur.values():
|
|
120
|
+
max_val = max(max_val, count)
|
|
121
|
+
return max(max_val, 1.0)
|
|
122
|
+
|
|
123
|
+
def compute_pair_distance(self, scope, uid_a, uid_b, weights=None, max_interact=100.0, max_cooccur=50.0):
|
|
124
|
+
if weights is None:
|
|
125
|
+
weights = self._default_weights()
|
|
126
|
+
|
|
127
|
+
scores = {
|
|
128
|
+
"timing": self.timing_similarity(scope, uid_a, uid_b),
|
|
129
|
+
"emoji": self.emoji_similarity(scope, uid_a, uid_b),
|
|
130
|
+
"vocab": self.vocab_similarity(scope, uid_a, uid_b),
|
|
131
|
+
"interaction": self.interaction_score(scope, uid_a, uid_b, max_interact),
|
|
132
|
+
"cooccurrence": self.cooccurrence_score(scope, uid_a, uid_b, max_cooccur),
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
total_w = sum(weights.values())
|
|
136
|
+
distance = 0.0
|
|
137
|
+
for dim, w in weights.items():
|
|
138
|
+
nw = w / total_w if total_w > 0 else 0
|
|
139
|
+
distance += nw * (1.0 - scores.get(dim, 0))
|
|
140
|
+
|
|
141
|
+
return distance, scores
|
|
142
|
+
|
|
143
|
+
def compute_distance_matrix(self, scope, force=False):
|
|
144
|
+
cache_key = f"{scope}:cache"
|
|
145
|
+
cached = self.storage.get(cache_key)
|
|
146
|
+
if cached and not force:
|
|
147
|
+
cached_time = cached.get("timestamp", 0)
|
|
148
|
+
interval = self.config.get("update_interval", 3600)
|
|
149
|
+
if time.time() - cached_time < interval:
|
|
150
|
+
return cached
|
|
151
|
+
|
|
152
|
+
users = self._eligible_users(scope)
|
|
153
|
+
if len(users) < 2:
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
weights = self._default_weights()
|
|
157
|
+
max_interact = self._compute_max_interact(scope, users)
|
|
158
|
+
max_cooccur = self._compute_max_cooccur(scope, users)
|
|
159
|
+
matrix = {}
|
|
160
|
+
scores_map = {}
|
|
161
|
+
|
|
162
|
+
for i in range(len(users)):
|
|
163
|
+
for j in range(i + 1, len(users)):
|
|
164
|
+
dist, scores = self.compute_pair_distance(scope, users[i], users[j], weights, max_interact, max_cooccur)
|
|
165
|
+
matrix[f"{users[i]}|{users[j]}"] = dist
|
|
166
|
+
matrix[f"{users[j]}|{users[i]}"] = dist
|
|
167
|
+
scores_map[f"{users[i]}|{users[j]}"] = scores
|
|
168
|
+
scores_map[f"{users[j]}|{users[i]}"] = scores
|
|
169
|
+
|
|
170
|
+
result = {
|
|
171
|
+
"timestamp": time.time(),
|
|
172
|
+
"users": users,
|
|
173
|
+
"matrix": matrix,
|
|
174
|
+
"scores": scores_map,
|
|
175
|
+
}
|
|
176
|
+
self.storage.set(cache_key, result)
|
|
177
|
+
return result
|
|
178
|
+
|
|
179
|
+
def detect_islands(self, scope, data=None, threshold=None):
|
|
180
|
+
if threshold is None:
|
|
181
|
+
threshold = self._distance_threshold()
|
|
182
|
+
if data is None:
|
|
183
|
+
data = self.compute_distance_matrix(scope)
|
|
184
|
+
if not data:
|
|
185
|
+
return []
|
|
186
|
+
|
|
187
|
+
users = data["users"]
|
|
188
|
+
matrix = data["matrix"]
|
|
189
|
+
|
|
190
|
+
parent = {u: u for u in users}
|
|
191
|
+
|
|
192
|
+
def find(x):
|
|
193
|
+
while parent[x] != x:
|
|
194
|
+
parent[x] = parent[parent[x]]
|
|
195
|
+
x = parent[x]
|
|
196
|
+
return x
|
|
197
|
+
|
|
198
|
+
def union(a, b):
|
|
199
|
+
ra, rb = find(a), find(b)
|
|
200
|
+
if ra != rb:
|
|
201
|
+
parent[ra] = rb
|
|
202
|
+
|
|
203
|
+
for i in range(len(users)):
|
|
204
|
+
for j in range(i + 1, len(users)):
|
|
205
|
+
key = f"{users[i]}|{users[j]}"
|
|
206
|
+
dist = matrix.get(key, 1.0)
|
|
207
|
+
if dist < threshold:
|
|
208
|
+
union(users[i], users[j])
|
|
209
|
+
|
|
210
|
+
groups = defaultdict(list)
|
|
211
|
+
for u in users:
|
|
212
|
+
groups[find(u)].append(u)
|
|
213
|
+
|
|
214
|
+
islands = []
|
|
215
|
+
for root, members in groups.items():
|
|
216
|
+
avg_dist = 0.0
|
|
217
|
+
count = 0
|
|
218
|
+
for mi in range(len(members)):
|
|
219
|
+
for mj in range(mi + 1, len(members)):
|
|
220
|
+
key = f"{members[mi]}|{members[mj]}"
|
|
221
|
+
avg_dist += matrix.get(key, 1.0)
|
|
222
|
+
count += 1
|
|
223
|
+
avg_dist = avg_dist / count if count > 0 else 0.0
|
|
224
|
+
|
|
225
|
+
islands.append({
|
|
226
|
+
"members": members,
|
|
227
|
+
"avg_distance": round(avg_dist, 3),
|
|
228
|
+
"size": len(members),
|
|
229
|
+
})
|
|
230
|
+
|
|
231
|
+
islands.sort(key=lambda x: x["size"], reverse=True)
|
|
232
|
+
return islands
|
|
233
|
+
|
|
234
|
+
def find_parallel_universe(self, scope, user_id):
|
|
235
|
+
result_local = self._find_parallel_in_scope(scope, user_id)
|
|
236
|
+
|
|
237
|
+
results_cross = []
|
|
238
|
+
all_keys = self.storage.keys()
|
|
239
|
+
all_scopes = set()
|
|
240
|
+
for k in all_keys:
|
|
241
|
+
if k.startswith("sonar:") and k.endswith(":users"):
|
|
242
|
+
parts = k.rsplit(":users", 1)[0]
|
|
243
|
+
if parts != scope:
|
|
244
|
+
all_scopes.add(parts)
|
|
245
|
+
|
|
246
|
+
for other_scope in all_scopes:
|
|
247
|
+
cross = self._find_parallel_in_scope(other_scope, user_id)
|
|
248
|
+
if cross:
|
|
249
|
+
cross["scope"] = other_scope
|
|
250
|
+
results_cross.append(cross)
|
|
251
|
+
|
|
252
|
+
results_cross.sort(key=lambda x: x["similarity"], reverse=True)
|
|
253
|
+
top_cross = results_cross[:3] if results_cross else []
|
|
254
|
+
|
|
255
|
+
return {
|
|
256
|
+
"local": result_local,
|
|
257
|
+
"cross_scope": top_cross,
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
def _find_parallel_in_scope(self, scope, user_id):
|
|
261
|
+
data = self.compute_distance_matrix(scope)
|
|
262
|
+
if not data or user_id not in data["users"]:
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
users = data["users"]
|
|
266
|
+
matrix = data["matrix"]
|
|
267
|
+
scores_map = data["scores"]
|
|
268
|
+
|
|
269
|
+
interact = self.storage.get(f"{scope}:interact:{user_id}", {})
|
|
270
|
+
cooccur = self.storage.get(f"{scope}:cooccur", {})
|
|
271
|
+
|
|
272
|
+
best = None
|
|
273
|
+
best_behavior_sim = -1.0
|
|
274
|
+
|
|
275
|
+
for other in users:
|
|
276
|
+
if other == user_id:
|
|
277
|
+
continue
|
|
278
|
+
pair_key = f"{user_id}|{other}"
|
|
279
|
+
|
|
280
|
+
has_interaction = (
|
|
281
|
+
interact.get(other, 0) > 0
|
|
282
|
+
or self.storage.get(f"{scope}:interact:{other}", {}).get(user_id, 0) > 0
|
|
283
|
+
)
|
|
284
|
+
has_cooccur = (
|
|
285
|
+
cooccur.get(pair_key, 0) > 0
|
|
286
|
+
or cooccur.get(f"{other}|{user_id}", 0) > 0
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
if has_interaction or has_cooccur:
|
|
290
|
+
continue
|
|
291
|
+
|
|
292
|
+
scores = scores_map.get(pair_key, {})
|
|
293
|
+
behavior_sim = (
|
|
294
|
+
scores.get("timing", 0) * 0.35
|
|
295
|
+
+ scores.get("emoji", 0) * 0.25
|
|
296
|
+
+ scores.get("vocab", 0) * 0.40
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
if behavior_sim > best_behavior_sim:
|
|
300
|
+
best_behavior_sim = behavior_sim
|
|
301
|
+
best = {
|
|
302
|
+
"user_id": other,
|
|
303
|
+
"similarity": round(behavior_sim, 3),
|
|
304
|
+
"scores": scores,
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
return best
|
|
308
|
+
|
|
309
|
+
def get_user_distances(self, scope, user_id):
|
|
310
|
+
data = self.compute_distance_matrix(scope)
|
|
311
|
+
if not data or user_id not in data["users"]:
|
|
312
|
+
return None
|
|
313
|
+
|
|
314
|
+
users = data["users"]
|
|
315
|
+
matrix = data["matrix"]
|
|
316
|
+
scores_map = data["scores"]
|
|
317
|
+
|
|
318
|
+
distances = {}
|
|
319
|
+
for other in users:
|
|
320
|
+
if other == user_id:
|
|
321
|
+
continue
|
|
322
|
+
key = f"{user_id}|{other}"
|
|
323
|
+
distances[other] = {
|
|
324
|
+
"distance": round(matrix.get(key, 1.0), 3),
|
|
325
|
+
"scores": scores_map.get(key, {}),
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
sorted_distances = dict(
|
|
329
|
+
sorted(distances.items(), key=lambda x: x[1]["distance"])
|
|
330
|
+
)
|
|
331
|
+
return sorted_distances
|
|
332
|
+
|
|
333
|
+
def get_user_detail(self, scope, user_id):
|
|
334
|
+
timing = self.storage.get(f"{scope}:timing:{user_id}", {})
|
|
335
|
+
emoji = self.storage.get(f"{scope}:emoji:{user_id}", {})
|
|
336
|
+
vocab = self.storage.get(f"{scope}:vocab:{user_id}", {})
|
|
337
|
+
length = self.storage.get(f"{scope}:length:{user_id}", {"total": 0, "count": 0})
|
|
338
|
+
interact = self.storage.get(f"{scope}:interact:{user_id}", {})
|
|
339
|
+
|
|
340
|
+
active_hours = sorted(timing.items(), key=lambda x: x[1], reverse=True)
|
|
341
|
+
peak_hours = [h for h, _ in active_hours[:3]] if active_hours else []
|
|
342
|
+
|
|
343
|
+
top_emoji = sorted(emoji.items(), key=lambda x: x[1], reverse=True)[:5]
|
|
344
|
+
top_vocab = sorted(vocab.items(), key=lambda x: x[1], reverse=True)[:5]
|
|
345
|
+
|
|
346
|
+
avg_len = 0
|
|
347
|
+
if length.get("count", 0) > 0:
|
|
348
|
+
avg_len = round(length["total"] / length["count"], 1)
|
|
349
|
+
|
|
350
|
+
return {
|
|
351
|
+
"message_count": length.get("count", 0),
|
|
352
|
+
"avg_length": avg_len,
|
|
353
|
+
"peak_hours": peak_hours,
|
|
354
|
+
"top_emoji": top_emoji,
|
|
355
|
+
"top_vocab": top_vocab,
|
|
356
|
+
"interact_count": sum(interact.values()),
|
|
357
|
+
"interact_targets": len(interact),
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
def get_interact_detail(self, scope, uid_a, uid_b):
|
|
361
|
+
ia = self.storage.get(f"{scope}:interact:{uid_a}", {})
|
|
362
|
+
ib = self.storage.get(f"{scope}:interact:{uid_b}", {})
|
|
363
|
+
a_to_b = ia.get(uid_b, 0)
|
|
364
|
+
b_to_a = ib.get(uid_a, 0)
|
|
365
|
+
|
|
366
|
+
timing_a = self.storage.get(f"{scope}:timing:{uid_a}", {})
|
|
367
|
+
timing_b = self.storage.get(f"{scope}:timing:{uid_b}", {})
|
|
368
|
+
overlap = [h for h in timing_a if h in timing_b and timing_a[h] > 0 and timing_b[h] > 0]
|
|
369
|
+
overlap.sort()
|
|
370
|
+
|
|
371
|
+
return {
|
|
372
|
+
"a_to_b": a_to_b,
|
|
373
|
+
"b_to_a": b_to_a,
|
|
374
|
+
"reply_total": a_to_b + b_to_a,
|
|
375
|
+
"shared_hours": overlap,
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
def get_cooccur_count(self, scope, uid_a, uid_b):
|
|
379
|
+
cooccur = self.storage.get(f"{scope}:cooccur", {})
|
|
380
|
+
return cooccur.get(f"{uid_a}|{uid_b}", 0)
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import time
|
|
3
|
+
import asyncio
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
import emoji as emoji_lib
|
|
9
|
+
HAS_EMOJI_LIB = True
|
|
10
|
+
except ImportError:
|
|
11
|
+
HAS_EMOJI_LIB = False
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Collector:
|
|
15
|
+
def __init__(self, sdk, config):
|
|
16
|
+
self.sdk = sdk
|
|
17
|
+
self.logger = sdk.logger.get_child("ChatSonar.Collector")
|
|
18
|
+
self.storage = sdk.storage
|
|
19
|
+
self.config = config
|
|
20
|
+
self._last_active = {}
|
|
21
|
+
|
|
22
|
+
async def start(self):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
async def stop(self):
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
def _scope(self, platform, group_id):
|
|
29
|
+
return f"sonar:{platform}:{group_id}"
|
|
30
|
+
|
|
31
|
+
def _is_optout(self, scope, user_id):
|
|
32
|
+
optout = self.storage.get(f"{scope}:optout", [])
|
|
33
|
+
return user_id in optout
|
|
34
|
+
|
|
35
|
+
async def on_message(self, event):
|
|
36
|
+
platform = event.get_platform()
|
|
37
|
+
detail_type = event.get_detail_type()
|
|
38
|
+
if detail_type not in ("group", "private"):
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
group_id = event.get_group_id() if detail_type == "group" else f"dm_{event.get_user_id()}"
|
|
42
|
+
if not group_id:
|
|
43
|
+
return
|
|
44
|
+
|
|
45
|
+
scope = self._scope(platform, group_id)
|
|
46
|
+
user_id = event.get_user_id()
|
|
47
|
+
if not user_id:
|
|
48
|
+
return
|
|
49
|
+
|
|
50
|
+
if self._is_optout(scope, user_id):
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
text = event.get_text() or ""
|
|
54
|
+
msg_len = len(text)
|
|
55
|
+
event_time = event.get_time()
|
|
56
|
+
if event_time and event_time > 0:
|
|
57
|
+
if event_time > 1e12:
|
|
58
|
+
event_time = int(event_time / 1000)
|
|
59
|
+
utc_offset = self.config.get("utc_offset", 8)
|
|
60
|
+
hour = int((event_time / 3600 + utc_offset) % 24)
|
|
61
|
+
else:
|
|
62
|
+
hour = datetime.now().hour
|
|
63
|
+
emojis = self._extract_emojis(text)
|
|
64
|
+
words = self._extract_words(text)
|
|
65
|
+
mentions = event.get_mentions() or []
|
|
66
|
+
timestamp = time.time()
|
|
67
|
+
nickname = event.get_user_nickname() or ""
|
|
68
|
+
|
|
69
|
+
record = {
|
|
70
|
+
"scope": scope,
|
|
71
|
+
"user_id": user_id,
|
|
72
|
+
"nickname": nickname,
|
|
73
|
+
"hour": hour,
|
|
74
|
+
"msg_len": msg_len,
|
|
75
|
+
"emojis": emojis,
|
|
76
|
+
"words": words,
|
|
77
|
+
"mentions": mentions,
|
|
78
|
+
"timestamp": timestamp,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
self._persist(record)
|
|
83
|
+
except Exception as e:
|
|
84
|
+
self.logger.error(f"Persist error: {e}")
|
|
85
|
+
|
|
86
|
+
self._last_active.setdefault(scope, {})
|
|
87
|
+
self._last_active[scope][user_id] = time.time()
|
|
88
|
+
if len(self._last_active.get(scope, {})) >= 2:
|
|
89
|
+
try:
|
|
90
|
+
self._update_cooccurrence(scope)
|
|
91
|
+
except Exception as e:
|
|
92
|
+
self.logger.error(f"Cooccurrence error: {e}")
|
|
93
|
+
|
|
94
|
+
def _extract_emojis(self, text):
|
|
95
|
+
found = []
|
|
96
|
+
if HAS_EMOJI_LIB:
|
|
97
|
+
found = [c for c in text if c in emoji_lib.EMOJI_DATA]
|
|
98
|
+
else:
|
|
99
|
+
emoji_pattern = re.compile(
|
|
100
|
+
"["
|
|
101
|
+
"\U0001F600-\U0001F64F"
|
|
102
|
+
"\U0001F300-\U0001F5FF"
|
|
103
|
+
"\U0001F680-\U0001F6FF"
|
|
104
|
+
"\U0001F1E0-\U0001F1FF"
|
|
105
|
+
"\U00002702-\U000027B0"
|
|
106
|
+
"\U000024C2-\U0001F251"
|
|
107
|
+
"\U0001f926-\U0001f937"
|
|
108
|
+
"\U00010000-\U0010ffff"
|
|
109
|
+
"\u2640-\u2642"
|
|
110
|
+
"\u2600-\u2B55"
|
|
111
|
+
"\u200d"
|
|
112
|
+
"\u23cf"
|
|
113
|
+
"\u23e9"
|
|
114
|
+
"\u231a"
|
|
115
|
+
"\ufe0f"
|
|
116
|
+
"\u3030"
|
|
117
|
+
"]+",
|
|
118
|
+
flags=re.UNICODE,
|
|
119
|
+
)
|
|
120
|
+
for m in emoji_pattern.finditer(text):
|
|
121
|
+
found.append(m.group())
|
|
122
|
+
return found
|
|
123
|
+
|
|
124
|
+
def _extract_words(self, text):
|
|
125
|
+
words = []
|
|
126
|
+
cn_chars = re.findall(r"[\u4e00-\u9fff]{2,}", text)
|
|
127
|
+
for seg in cn_chars:
|
|
128
|
+
for i in range(len(seg) - 1):
|
|
129
|
+
words.append(seg[i : i + 2])
|
|
130
|
+
en_words = re.findall(r"[a-zA-Z]{2,}", text)
|
|
131
|
+
words.extend(w.lower() for w in en_words)
|
|
132
|
+
return words
|
|
133
|
+
|
|
134
|
+
def _persist(self, rec):
|
|
135
|
+
scope = rec["scope"]
|
|
136
|
+
uid = rec["user_id"]
|
|
137
|
+
ts = rec["timestamp"]
|
|
138
|
+
|
|
139
|
+
timing_key = f"{scope}:timing:{uid}"
|
|
140
|
+
timing = self.storage.get(timing_key, {})
|
|
141
|
+
hour_str = str(rec["hour"])
|
|
142
|
+
timing[hour_str] = timing.get(hour_str, 0) + 1
|
|
143
|
+
self.storage.set(timing_key, timing)
|
|
144
|
+
|
|
145
|
+
if rec["emojis"]:
|
|
146
|
+
emoji_key = f"{scope}:emoji:{uid}"
|
|
147
|
+
emoji_data = self.storage.get(emoji_key, {})
|
|
148
|
+
for e in rec["emojis"]:
|
|
149
|
+
emoji_data[e] = emoji_data.get(e, 0) + 1
|
|
150
|
+
self.storage.set(emoji_key, emoji_data)
|
|
151
|
+
|
|
152
|
+
if rec["words"]:
|
|
153
|
+
vocab_key = f"{scope}:vocab:{uid}"
|
|
154
|
+
vocab_data = self.storage.get(vocab_key, {})
|
|
155
|
+
for w in rec["words"]:
|
|
156
|
+
vocab_data[w] = vocab_data.get(w, 0) + 1
|
|
157
|
+
if len(vocab_data) > 200:
|
|
158
|
+
sorted_items = sorted(vocab_data.items(), key=lambda x: x[1], reverse=True)
|
|
159
|
+
vocab_data = dict(sorted_items[:100])
|
|
160
|
+
self.storage.set(vocab_key, vocab_data)
|
|
161
|
+
|
|
162
|
+
length_key = f"{scope}:length:{uid}"
|
|
163
|
+
length_data = self.storage.get(length_key, {"total": 0, "count": 0})
|
|
164
|
+
length_data["total"] += rec["msg_len"]
|
|
165
|
+
length_data["count"] += 1
|
|
166
|
+
self.storage.set(length_key, length_data)
|
|
167
|
+
|
|
168
|
+
if rec["mentions"]:
|
|
169
|
+
interact_key = f"{scope}:interact:{uid}"
|
|
170
|
+
interact_data = self.storage.get(interact_key, {})
|
|
171
|
+
for mentioned_id in rec["mentions"]:
|
|
172
|
+
if mentioned_id and mentioned_id != uid:
|
|
173
|
+
interact_data[mentioned_id] = interact_data.get(mentioned_id, 0) + 1
|
|
174
|
+
self.storage.set(interact_key, interact_data)
|
|
175
|
+
|
|
176
|
+
users_key = f"{scope}:users"
|
|
177
|
+
users = self.storage.get(users_key, {})
|
|
178
|
+
for uid_key in list(users.keys()):
|
|
179
|
+
if not isinstance(users[uid_key], dict):
|
|
180
|
+
users[uid_key] = {"nickname": "", "first_seen": users[uid_key]}
|
|
181
|
+
if uid not in users:
|
|
182
|
+
users[uid] = {"nickname": rec.get("nickname", ""), "first_seen": ts}
|
|
183
|
+
elif rec.get("nickname"):
|
|
184
|
+
if not users[uid].get("nickname"):
|
|
185
|
+
users[uid]["nickname"] = rec["nickname"]
|
|
186
|
+
self.storage.set(users_key, users)
|
|
187
|
+
|
|
188
|
+
def _update_cooccurrence(self, scope):
|
|
189
|
+
now = time.time()
|
|
190
|
+
window = self.config.get("cooccur_window", 300)
|
|
191
|
+
active = {}
|
|
192
|
+
for uid, last_t in self._last_active.get(scope, {}).items():
|
|
193
|
+
if now - last_t < window:
|
|
194
|
+
active[uid] = last_t
|
|
195
|
+
if len(active) < 2:
|
|
196
|
+
return
|
|
197
|
+
|
|
198
|
+
cooccur_key = f"{scope}:cooccur"
|
|
199
|
+
cooccur = self.storage.get(cooccur_key, {})
|
|
200
|
+
uids = list(active.keys())
|
|
201
|
+
for i in range(len(uids)):
|
|
202
|
+
for j in range(i + 1, len(uids)):
|
|
203
|
+
pair = f"{uids[i]}|{uids[j]}"
|
|
204
|
+
rpair = f"{uids[j]}|{uids[i]}"
|
|
205
|
+
cooccur[pair] = cooccur.get(pair, 0) + 1
|
|
206
|
+
cooccur[rpair] = cooccur.get(rpair, 0) + 1
|
|
207
|
+
self.storage.set(cooccur_key, cooccur)
|
|
208
|
+
|
|
209
|
+
def delete_user_data(self, scope, user_id):
|
|
210
|
+
suffixes = ["timing", "emoji", "vocab", "length", "interact"]
|
|
211
|
+
for s in suffixes:
|
|
212
|
+
self.storage.delete(f"{scope}:{s}:{user_id}")
|
|
213
|
+
|
|
214
|
+
users = self.storage.get(f"{scope}:users", {})
|
|
215
|
+
users.pop(user_id, None)
|
|
216
|
+
self.storage.set(f"{scope}:users", users)
|
|
217
|
+
|
|
218
|
+
cooccur = self.storage.get(f"{scope}:cooccur", {})
|
|
219
|
+
to_del = [k for k in cooccur if user_id in k.split("|")]
|
|
220
|
+
for k in to_del:
|
|
221
|
+
del cooccur[k]
|
|
222
|
+
self.storage.set(f"{scope}:cooccur", cooccur)
|
|
223
|
+
|
|
224
|
+
optout = self.storage.get(f"{scope}:optout", [])
|
|
225
|
+
if user_id not in optout:
|
|
226
|
+
optout.append(user_id)
|
|
227
|
+
self.storage.set(f"{scope}:optout", optout)
|
|
228
|
+
|
|
229
|
+
cache_key = f"{scope}:cache"
|
|
230
|
+
self.storage.delete(cache_key)
|
|
231
|
+
|
|
232
|
+
def rejoin_user(self, scope, user_id):
|
|
233
|
+
optout = self.storage.get(f"{scope}:optout", [])
|
|
234
|
+
if user_id in optout:
|
|
235
|
+
optout.remove(user_id)
|
|
236
|
+
self.storage.set(f"{scope}:optout", optout)
|
|
237
|
+
|
|
238
|
+
def get_all_scopes(self):
|
|
239
|
+
all_keys = self.storage.keys()
|
|
240
|
+
scopes = set()
|
|
241
|
+
for k in all_keys:
|
|
242
|
+
if k.startswith("sonar:") and ":users" in k:
|
|
243
|
+
parts = k.split(":")
|
|
244
|
+
if len(parts) >= 3:
|
|
245
|
+
scopes.add(f"{parts[1]}:{parts[2]}")
|
|
246
|
+
return scopes
|