PerSent 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of PerSent might be problematic. Click here for more details.
- {persent-0.1.0 → persent-0.1.2}/PKG-INFO +1 -1
- {persent-0.1.0 → persent-0.1.2}/PerSent/__init__.py +1 -1
- {persent-0.1.0 → persent-0.1.2}/PerSent/analyzer.py +41 -30
- {persent-0.1.0 → persent-0.1.2}/PerSent.egg-info/PKG-INFO +1 -1
- {persent-0.1.0 → persent-0.1.2}/setup.py +1 -1
- {persent-0.1.0 → persent-0.1.2}/LICENSE.txt +0 -0
- {persent-0.1.0 → persent-0.1.2}/PerSent.egg-info/SOURCES.txt +0 -0
- {persent-0.1.0 → persent-0.1.2}/PerSent.egg-info/dependency_links.txt +0 -0
- {persent-0.1.0 → persent-0.1.2}/PerSent.egg-info/requires.txt +0 -0
- {persent-0.1.0 → persent-0.1.2}/PerSent.egg-info/top_level.txt +0 -0
- {persent-0.1.0 → persent-0.1.2}/README.md +0 -0
- {persent-0.1.0 → persent-0.1.2}/setup.cfg +0 -0
- {persent-0.1.0 → persent-0.1.2}/tests/test_basic.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
2
|
from hazm import Normalizer, word_tokenize, Stemmer, stopwords_list
|
|
3
3
|
import re
|
|
4
|
-
from tqdm import tqdm
|
|
4
|
+
from tqdm import tqdm
|
|
5
5
|
from gensim.models import Word2Vec
|
|
6
6
|
import numpy as np
|
|
7
7
|
from sklearn.linear_model import LogisticRegression
|
|
@@ -17,31 +17,31 @@ class PersianSentimentAnalyzer:
|
|
|
17
17
|
self.model_dir = model_dir
|
|
18
18
|
self.vectorizer = None
|
|
19
19
|
self.classifier = None
|
|
20
|
+
self.emotion_keywords = {
|
|
21
|
+
'Fear': ['ترس', 'وحشت', 'هراس'],
|
|
22
|
+
'Surprise': ['تعجب', 'شگفت', 'حیرت'],
|
|
23
|
+
'Anger': ['خشم', 'عصبانیت', 'خشونت'],
|
|
24
|
+
'Contempt': ['تحقیر', 'توهین', 'حقارت'],
|
|
25
|
+
'Joy': ['شادی', 'خوشحالی', 'خنده'],
|
|
26
|
+
'Sadness': ['غم', 'اندوه', 'ناراحتی'],
|
|
27
|
+
'Disgust': ['انزجار', 'تهوع', 'نفرت']
|
|
28
|
+
}
|
|
20
29
|
|
|
21
|
-
# ایجاد پوشه مدل اگر وجود نداشته باشد
|
|
22
30
|
os.makedirs(self.model_dir, exist_ok=True)
|
|
23
31
|
|
|
24
32
|
def _preprocess_text(self, text):
|
|
25
|
-
"""پیشپردازش متن فارسی"""
|
|
26
|
-
# نرمالسازی
|
|
27
33
|
text = self.normalizer.normalize(str(text))
|
|
28
|
-
|
|
29
|
-
# حذف اعداد و علائم
|
|
30
34
|
text = re.sub(r'[!()-\[\]{};:\'",؟<>./?@#$%^&*_~۰-۹\d]+', ' ', text)
|
|
31
35
|
text = re.sub(r'\s+', ' ', text).strip()
|
|
32
|
-
|
|
33
|
-
# توکنسازی و ریشهیابی
|
|
34
36
|
tokens = word_tokenize(text)
|
|
35
37
|
processed_tokens = [
|
|
36
38
|
self.stemmer.stem(token)
|
|
37
39
|
for token in tokens
|
|
38
40
|
if token not in self.stopwords and len(token) > 1
|
|
39
41
|
]
|
|
40
|
-
|
|
41
42
|
return processed_tokens
|
|
42
43
|
|
|
43
44
|
def _sentence_vector(self, sentence, model):
|
|
44
|
-
"""تبدیل جمله به بردار با استفاده از مدل Word2Vec"""
|
|
45
45
|
vectors = []
|
|
46
46
|
for word in sentence:
|
|
47
47
|
try:
|
|
@@ -50,13 +50,19 @@ class PersianSentimentAnalyzer:
|
|
|
50
50
|
vectors.append(np.zeros(100))
|
|
51
51
|
return np.mean(vectors, axis=0) if vectors else np.zeros(100)
|
|
52
52
|
|
|
53
|
+
def _emotion_analysis(self, tokens):
|
|
54
|
+
"""تحلیل احساسات بر اساس کلیدواژههای از پیش تعریف شده"""
|
|
55
|
+
emotions = {emotion: 0 for emotion in self.emotion_keywords.keys()}
|
|
56
|
+
for word in tokens:
|
|
57
|
+
for emotion, keywords in self.emotion_keywords.items():
|
|
58
|
+
if word in keywords:
|
|
59
|
+
emotions[emotion] += 1
|
|
60
|
+
return emotions
|
|
61
|
+
|
|
53
62
|
def train(self, train_csv, test_size=0.2, vector_size=100, window=5):
|
|
54
|
-
"""آموزش مدل از دادههای CSV"""
|
|
55
|
-
# خواندن دادهها
|
|
56
63
|
df = pd.read_csv(train_csv)
|
|
57
64
|
df['tokens'] = df['body'].apply(self._preprocess_text)
|
|
58
65
|
|
|
59
|
-
# آموزش مدل Word2Vec
|
|
60
66
|
self.vectorizer = Word2Vec(
|
|
61
67
|
sentences=df['tokens'],
|
|
62
68
|
vector_size=vector_size,
|
|
@@ -65,7 +71,6 @@ class PersianSentimentAnalyzer:
|
|
|
65
71
|
workers=4
|
|
66
72
|
)
|
|
67
73
|
|
|
68
|
-
# تبدیل جملات به بردار
|
|
69
74
|
X = np.array([self._sentence_vector(s, self.vectorizer) for s in df['tokens']])
|
|
70
75
|
y = df['recommendation_status'].map({
|
|
71
76
|
"no_idea": 2,
|
|
@@ -73,41 +78,47 @@ class PersianSentimentAnalyzer:
|
|
|
73
78
|
"not_recommended": 0
|
|
74
79
|
}).values
|
|
75
80
|
|
|
76
|
-
# تقسیم دادهها
|
|
77
81
|
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)
|
|
78
82
|
|
|
79
|
-
# آموزش طبقهبند
|
|
80
83
|
self.classifier = LogisticRegression(max_iter=1000)
|
|
81
84
|
self.classifier.fit(X_train, y_train)
|
|
82
85
|
|
|
83
|
-
# ذخیره مدل
|
|
84
86
|
self.save_model()
|
|
85
|
-
|
|
86
|
-
# ارزیابی
|
|
87
87
|
accuracy = self.classifier.score(X_test, y_test)
|
|
88
88
|
return accuracy
|
|
89
89
|
|
|
90
|
-
def predict(self, text):
|
|
91
|
-
"""پیشبینی احساس متن
|
|
90
|
+
def predict(self, text, mode=0):
|
|
91
|
+
"""پیشبینی احساس متن با قابلیت انتخاب حالت تحلیل
|
|
92
|
+
|
|
93
|
+
پارامترها:
|
|
94
|
+
mode (int):
|
|
95
|
+
0 = تحلیل توصیه (پیشفرض)
|
|
96
|
+
1 = تحلیل احساسات (Fear, Joy, etc.)
|
|
97
|
+
"""
|
|
92
98
|
if not self.classifier or not self.vectorizer:
|
|
93
99
|
raise Exception("Model not trained! Call train() first or load a pretrained model.")
|
|
94
100
|
|
|
95
101
|
tokens = self._preprocess_text(text)
|
|
96
|
-
vector = self._sentence_vector(tokens, self.vectorizer)
|
|
97
|
-
prediction = self.classifier.predict([vector])[0]
|
|
98
102
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
if mode == 0:
|
|
104
|
+
vector = self._sentence_vector(tokens, self.vectorizer)
|
|
105
|
+
prediction = self.classifier.predict([vector])[0]
|
|
106
|
+
return {
|
|
107
|
+
0: "not_recommended",
|
|
108
|
+
1: "recommended",
|
|
109
|
+
2: "no_idea"
|
|
110
|
+
}[prediction]
|
|
111
|
+
|
|
112
|
+
elif mode == 1:
|
|
113
|
+
return self._emotion_analysis(tokens)
|
|
114
|
+
|
|
115
|
+
else:
|
|
116
|
+
raise ValueError("Invalid mode. Use 0 (recommendation analysis) or 1 (emotion analysis)")
|
|
104
117
|
|
|
105
118
|
def save_model(self):
|
|
106
|
-
"""ذخیره مدل آموزش دیده"""
|
|
107
119
|
joblib.dump(self.classifier, os.path.join(self.model_dir, 'classifier.joblib'))
|
|
108
120
|
self.vectorizer.save(os.path.join(self.model_dir, 'word2vec.model'))
|
|
109
121
|
|
|
110
122
|
def load_model(self):
|
|
111
|
-
"""بارگذاری مدل از فایل"""
|
|
112
123
|
self.classifier = joblib.load(os.path.join(self.model_dir, 'classifier.joblib'))
|
|
113
124
|
self.vectorizer = Word2Vec.load(os.path.join(self.model_dir, 'word2vec.model'))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|