PerSent 0.1.6__tar.gz → 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of PerSent might be problematic. Click here for more details.
- persent-0.1.8/PKG-INFO +28 -0
- persent-0.1.8/PerSent.egg-info/PKG-INFO +28 -0
- {persent-0.1.6 → persent-0.1.8}/PerSent.egg-info/SOURCES.txt +1 -5
- persent-0.1.8/PerSent.egg-info/top_level.txt +1 -0
- {persent-0.1.6 → persent-0.1.8}/setup.py +2 -2
- persent-0.1.6/LICENSE.txt +0 -21
- persent-0.1.6/PKG-INFO +0 -162
- persent-0.1.6/PerSent/__init__.py +0 -4
- persent-0.1.6/PerSent/analyzer.py +0 -251
- persent-0.1.6/PerSent.egg-info/PKG-INFO +0 -162
- persent-0.1.6/PerSent.egg-info/top_level.txt +0 -1
- persent-0.1.6/README.md +0 -130
- {persent-0.1.6 → persent-0.1.8}/PerSent.egg-info/dependency_links.txt +0 -0
- {persent-0.1.6 → persent-0.1.8}/PerSent.egg-info/requires.txt +0 -0
- /persent-0.1.6/tests/test_basic.py → /persent-0.1.8/README.md +0 -0
- {persent-0.1.6 → persent-0.1.8}/setup.cfg +0 -0
persent-0.1.8/PKG-INFO
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: PerSent
|
|
3
|
+
Version: 0.1.8
|
|
4
|
+
Summary: Persian Sentiment Analysis Toolkit
|
|
5
|
+
Home-page: https://github.com/RezaGooner/PerSent
|
|
6
|
+
Author: RezaGooner
|
|
7
|
+
Author-email: RezaAsadiProgrammer@Gmail.com
|
|
8
|
+
Keywords: persian sentiment analysis nlp
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
Requires-Dist: hazm>=0.7.0
|
|
15
|
+
Requires-Dist: gensim>=4.0.0
|
|
16
|
+
Requires-Dist: scikit-learn>=1.0.0
|
|
17
|
+
Requires-Dist: pandas>=1.3.0
|
|
18
|
+
Requires-Dist: tqdm>=4.62.0
|
|
19
|
+
Requires-Dist: joblib>=1.1.0
|
|
20
|
+
Dynamic: author
|
|
21
|
+
Dynamic: author-email
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: description-content-type
|
|
24
|
+
Dynamic: home-page
|
|
25
|
+
Dynamic: keywords
|
|
26
|
+
Dynamic: requires-dist
|
|
27
|
+
Dynamic: requires-python
|
|
28
|
+
Dynamic: summary
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: PerSent
|
|
3
|
+
Version: 0.1.8
|
|
4
|
+
Summary: Persian Sentiment Analysis Toolkit
|
|
5
|
+
Home-page: https://github.com/RezaGooner/PerSent
|
|
6
|
+
Author: RezaGooner
|
|
7
|
+
Author-email: RezaAsadiProgrammer@Gmail.com
|
|
8
|
+
Keywords: persian sentiment analysis nlp
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
Requires-Dist: hazm>=0.7.0
|
|
15
|
+
Requires-Dist: gensim>=4.0.0
|
|
16
|
+
Requires-Dist: scikit-learn>=1.0.0
|
|
17
|
+
Requires-Dist: pandas>=1.3.0
|
|
18
|
+
Requires-Dist: tqdm>=4.62.0
|
|
19
|
+
Requires-Dist: joblib>=1.1.0
|
|
20
|
+
Dynamic: author
|
|
21
|
+
Dynamic: author-email
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: description-content-type
|
|
24
|
+
Dynamic: home-page
|
|
25
|
+
Dynamic: keywords
|
|
26
|
+
Dynamic: requires-dist
|
|
27
|
+
Dynamic: requires-python
|
|
28
|
+
Dynamic: summary
|
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
LICENSE.txt
|
|
2
1
|
README.md
|
|
3
2
|
setup.py
|
|
4
|
-
PerSent/__init__.py
|
|
5
|
-
PerSent/analyzer.py
|
|
6
3
|
PerSent.egg-info/PKG-INFO
|
|
7
4
|
PerSent.egg-info/SOURCES.txt
|
|
8
5
|
PerSent.egg-info/dependency_links.txt
|
|
9
6
|
PerSent.egg-info/requires.txt
|
|
10
|
-
PerSent.egg-info/top_level.txt
|
|
11
|
-
tests/test_basic.py
|
|
7
|
+
PerSent.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name="PerSent",
|
|
8
|
-
version="0.1.
|
|
8
|
+
version="0.1.8",
|
|
9
9
|
author="RezaGooner",
|
|
10
10
|
author_email="RezaAsadiProgrammer@Gmail.com",
|
|
11
11
|
description="Persian Sentiment Analysis Toolkit",
|
|
@@ -31,4 +31,4 @@ setup(
|
|
|
31
31
|
],
|
|
32
32
|
python_requires='>=3.8',
|
|
33
33
|
keywords='persian sentiment analysis nlp',
|
|
34
|
-
)
|
|
34
|
+
)
|
persent-0.1.6/LICENSE.txt
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025 𝐑𝐞𝐳𝐚 𝐀𝐬𝐚𝐝𝐢
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
persent-0.1.6/PKG-INFO
DELETED
|
@@ -1,162 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: PerSent
|
|
3
|
-
Version: 0.1.6
|
|
4
|
-
Summary: Persian Sentiment Analysis Toolkit
|
|
5
|
-
Home-page: https://github.com/RezaGooner/PerSent
|
|
6
|
-
Author: RezaGooner
|
|
7
|
-
Author-email: RezaAsadiProgrammer@Gmail.com
|
|
8
|
-
Keywords: persian sentiment analysis nlp
|
|
9
|
-
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
-
Classifier: Operating System :: OS Independent
|
|
12
|
-
Requires-Python: >=3.8
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
License-File: LICENSE.txt
|
|
15
|
-
Requires-Dist: hazm>=0.7.0
|
|
16
|
-
Requires-Dist: gensim>=4.0.0
|
|
17
|
-
Requires-Dist: scikit-learn>=1.0.0
|
|
18
|
-
Requires-Dist: pandas>=1.3.0
|
|
19
|
-
Requires-Dist: tqdm>=4.62.0
|
|
20
|
-
Requires-Dist: joblib>=1.1.0
|
|
21
|
-
Dynamic: author
|
|
22
|
-
Dynamic: author-email
|
|
23
|
-
Dynamic: classifier
|
|
24
|
-
Dynamic: description
|
|
25
|
-
Dynamic: description-content-type
|
|
26
|
-
Dynamic: home-page
|
|
27
|
-
Dynamic: keywords
|
|
28
|
-
Dynamic: license-file
|
|
29
|
-
Dynamic: requires-dist
|
|
30
|
-
Dynamic: requires-python
|
|
31
|
-
Dynamic: summary
|
|
32
|
-
|
|
33
|
-
# Persian Sentiment Analyzer
|
|
34
|
-
|
|
35
|
-
[](https://www.python.org/)
|
|
36
|
-
[](LICENSE)
|
|
37
|
-
|
|
38
|
-
A Python library for sentiment analysis of Persian (Farsi) text, capable of classifying opinions as "recommended", "not_recommended", or "no_idea".
|
|
39
|
-
|
|
40
|
-
## Features
|
|
41
|
-
|
|
42
|
-
- **Text Preprocessing**: Normalization, tokenization, stemming, and stopword removal for Persian text
|
|
43
|
-
- **Word Embeddings**: Built-in Word2Vec implementation for Persian language
|
|
44
|
-
- **Sentiment Classification**: Logistic Regression classifier trained on Persian sentiment data
|
|
45
|
-
- **Model Persistence**: Save and load trained models for future use
|
|
46
|
-
- **Batch Processing**: Analyze sentiment for multiple texts at once
|
|
47
|
-
|
|
48
|
-
## Installation
|
|
49
|
-
|
|
50
|
-
```bash
|
|
51
|
-
pip install persian-sentiment-analyzer
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
## Dependencies
|
|
55
|
-
- Python 3.6+
|
|
56
|
-
|
|
57
|
-
- hazm
|
|
58
|
-
|
|
59
|
-
- gensim
|
|
60
|
-
|
|
61
|
-
- scikit-learn
|
|
62
|
-
|
|
63
|
-
- numpy
|
|
64
|
-
|
|
65
|
-
- pandas
|
|
66
|
-
|
|
67
|
-
## Usage
|
|
68
|
-
|
|
69
|
-
### Basic Usage
|
|
70
|
-
|
|
71
|
-
``` bash
|
|
72
|
-
from persian_sentiment_analyzer import SentimentAnalyzer
|
|
73
|
-
|
|
74
|
-
# Initialize with a pre-trained model
|
|
75
|
-
analyzer = SentimentAnalyzer(model_path="path/to/pretrained_model")
|
|
76
|
-
|
|
77
|
-
# Predict sentiment
|
|
78
|
-
result = analyzer.predict("این محصول بسیار عالی است")
|
|
79
|
-
print(result) # Output: 'recommended'
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
### Training Your Own Model
|
|
83
|
-
|
|
84
|
-
``` bash
|
|
85
|
-
from persian_sentiment_analyzer import SentimentAnalyzer
|
|
86
|
-
import pandas as pd
|
|
87
|
-
|
|
88
|
-
# Load your dataset
|
|
89
|
-
data = pd.read_csv("persian_reviews.csv")
|
|
90
|
-
texts = data['text'].tolist()
|
|
91
|
-
labels = data['label'].values # 0: not_recommended, 1: recommended, 2: no_idea
|
|
92
|
-
|
|
93
|
-
# Initialize analyzer
|
|
94
|
-
analyzer = SentimentAnalyzer()
|
|
95
|
-
|
|
96
|
-
# Preprocess and tokenize texts
|
|
97
|
-
tokenized_texts = [analyzer.preprocessor.preprocess_text(text) for text in texts]
|
|
98
|
-
|
|
99
|
-
# Train Word2Vec model
|
|
100
|
-
analyzer.train_word2vec(tokenized_texts, vector_size=100)
|
|
101
|
-
|
|
102
|
-
# Prepare feature vectors
|
|
103
|
-
X = np.array([analyzer.sentence_vector(tokens) for tokens in tokenized_texts])
|
|
104
|
-
|
|
105
|
-
# Train classifier
|
|
106
|
-
analyzer.train_classifier(X, labels)
|
|
107
|
-
|
|
108
|
-
# Save the trained model
|
|
109
|
-
analyzer.save_model("my_persian_model")
|
|
110
|
-
```
|
|
111
|
-
|
|
112
|
-
### Batch Processing
|
|
113
|
-
|
|
114
|
-
```bash
|
|
115
|
-
from persian_sentiment_analyzer import predict_sentiments_for_file
|
|
116
|
-
|
|
117
|
-
# Process a CSV file containing Persian comments
|
|
118
|
-
results_summary = predict_sentiments_for_file(
|
|
119
|
-
analyzer,
|
|
120
|
-
input_file="comments.csv",
|
|
121
|
-
output_file="results.csv",
|
|
122
|
-
summary_file="summary.csv"
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
print(results_summary)
|
|
126
|
-
```
|
|
127
|
-
|
|
128
|
-
## Model Architecture
|
|
129
|
-
1- Text Preprocessing:
|
|
130
|
-
|
|
131
|
-
- Normalization (Hazm)
|
|
132
|
-
|
|
133
|
-
- Tokenization
|
|
134
|
-
|
|
135
|
-
- Stemming
|
|
136
|
-
|
|
137
|
-
- Stopword removal
|
|
138
|
-
|
|
139
|
-
2- Feature Extraction:
|
|
140
|
-
|
|
141
|
-
- Word2Vec embeddings (100 dimensions)
|
|
142
|
-
|
|
143
|
-
- Sentence vectors (average of word vectors)
|
|
144
|
-
|
|
145
|
-
3- Classification:
|
|
146
|
-
|
|
147
|
-
- Logistic Regression with L2 regularization
|
|
148
|
-
|
|
149
|
-
## Performance
|
|
150
|
-
The pre-trained model achieves the following performance on our test set:
|
|
151
|
-
|
|
152
|
-
Metric Value
|
|
153
|
-
Accuracy 85.2%
|
|
154
|
-
Precision 84.7%
|
|
155
|
-
Recall 85.0%
|
|
156
|
-
F1-score 84.8%
|
|
157
|
-
|
|
158
|
-
License
|
|
159
|
-
This project is licensed under the MIT [License](https://github.com/RezaGooner/Sentiment-Survey-Analyzer/blob/main/LICENSE) - see the LICENSE file for details
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
> Github : RezaGooner
|
|
@@ -1,251 +0,0 @@
|
|
|
1
|
-
import pandas as pd
|
|
2
|
-
from hazm import Normalizer, word_tokenize, Stemmer, stopwords_list
|
|
3
|
-
import re
|
|
4
|
-
from tqdm import tqdm
|
|
5
|
-
from gensim.models import Word2Vec
|
|
6
|
-
import numpy as np
|
|
7
|
-
from sklearn.linear_model import LogisticRegression
|
|
8
|
-
from sklearn.model_selection import train_test_split
|
|
9
|
-
import os
|
|
10
|
-
import joblib
|
|
11
|
-
from transformers import pipeline
|
|
12
|
-
import torch
|
|
13
|
-
from torch.nn.functional import softmax
|
|
14
|
-
|
|
15
|
-
class AdvancedPersianSentimentAnalyzer:
|
|
16
|
-
def __init__(self, model_dir='persian_sentiment/model'):
|
|
17
|
-
self.normalizer = Normalizer()
|
|
18
|
-
self.stemmer = Stemmer()
|
|
19
|
-
self.stopwords = set(stopwords_list())
|
|
20
|
-
self.model_dir = model_dir
|
|
21
|
-
self.vectorizer = None
|
|
22
|
-
self.classifier = None
|
|
23
|
-
|
|
24
|
-
# تنظیمات پیشرفته برای مدل احساسات
|
|
25
|
-
self.emotion_analyzer = pipeline(
|
|
26
|
-
"text-classification",
|
|
27
|
-
model="HooshvareLab/bert-fa-base-uncased-emotion",
|
|
28
|
-
tokenizer="HooshvareLab/bert-fa-base-uncased-emotion",
|
|
29
|
-
device=0 if torch.cuda.is_available() else -1,
|
|
30
|
-
return_all_scores=True
|
|
31
|
-
)
|
|
32
|
-
|
|
33
|
-
# پارامترهای بهینه شده
|
|
34
|
-
self.w2v_params = {
|
|
35
|
-
'vector_size': 300,
|
|
36
|
-
'window': 10,
|
|
37
|
-
'min_count': 2,
|
|
38
|
-
'workers': 4,
|
|
39
|
-
'sg': 1 # استفاده از skip-gram
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
# نگاشت برچسبهای احساسات
|
|
43
|
-
self.emotion_labels = {
|
|
44
|
-
'ترس': 'Fear',
|
|
45
|
-
'تعجب': 'Surprise',
|
|
46
|
-
'خشم': 'Anger',
|
|
47
|
-
'تحقیر': 'Contempt',
|
|
48
|
-
'شادی': 'Joy',
|
|
49
|
-
'غم': 'Sadness',
|
|
50
|
-
'انزجار': 'Disgust'
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
os.makedirs(self.model_dir, exist_ok=True)
|
|
54
|
-
|
|
55
|
-
def _preprocess_text(self, text, for_emotion=False):
|
|
56
|
-
"""پیشپردازش متن با قابلیت تنظیم برای تحلیل احساسات"""
|
|
57
|
-
text = self.normalizer.normalize(str(text))
|
|
58
|
-
|
|
59
|
-
if for_emotion:
|
|
60
|
-
# حفظ علائم نگارشی برای تحلیل احساسات
|
|
61
|
-
text = re.sub(r'[۰-۹\d]+', ' ', text)
|
|
62
|
-
else:
|
|
63
|
-
text = re.sub(r'[!()-\[\]{};:\'",؟<>./?@#$%^&*_~۰-۹\d]+', ' ', text)
|
|
64
|
-
|
|
65
|
-
text = re.sub(r'\s+', ' ', text).strip()
|
|
66
|
-
|
|
67
|
-
if not for_emotion:
|
|
68
|
-
tokens = word_tokenize(text)
|
|
69
|
-
return [
|
|
70
|
-
self.stemmer.stem(token)
|
|
71
|
-
for token in tokens
|
|
72
|
-
if token not in self.stopwords and len(token) > 1
|
|
73
|
-
]
|
|
74
|
-
return text
|
|
75
|
-
|
|
76
|
-
def _sentence_vector(self, sentence, model):
|
|
77
|
-
"""تبدیل جمله به بردار با میانگین وزنی"""
|
|
78
|
-
vectors = []
|
|
79
|
-
weights = []
|
|
80
|
-
|
|
81
|
-
for word in sentence:
|
|
82
|
-
try:
|
|
83
|
-
vectors.append(model.wv[word])
|
|
84
|
-
# وزندهی بر اساس فرکانس کلمه
|
|
85
|
-
weights.append(model.wv.get_vecattr(word, "count"))
|
|
86
|
-
except KeyError:
|
|
87
|
-
continue
|
|
88
|
-
|
|
89
|
-
if vectors:
|
|
90
|
-
weights = np.array(weights) / sum(weights)
|
|
91
|
-
return np.average(vectors, axis=0, weights=weights)
|
|
92
|
-
return np.zeros(self.w2v_params['vector_size'])
|
|
93
|
-
|
|
94
|
-
def _emotion_analysis(self, text):
|
|
95
|
-
"""تحلیل احساسات پیشرفته با مدل BERT"""
|
|
96
|
-
try:
|
|
97
|
-
# پیشپردازش متن برای تحلیل احساسات
|
|
98
|
-
processed_text = self._preprocess_text(text, for_emotion=True)
|
|
99
|
-
|
|
100
|
-
# تحلیل احساسات
|
|
101
|
-
results = self.emotion_analyzer(processed_text)[0]
|
|
102
|
-
|
|
103
|
-
# پردازش نتایج
|
|
104
|
-
emotions = {}
|
|
105
|
-
for item in results:
|
|
106
|
-
fa_label = item['label']
|
|
107
|
-
if fa_label in self.emotion_labels:
|
|
108
|
-
en_label = self.emotion_labels[fa_label]
|
|
109
|
-
score = round(float(item['score']), 4)
|
|
110
|
-
emotions[en_label] = score
|
|
111
|
-
|
|
112
|
-
# اگر همه مقادیر صفر بودند، قویترین احساس را برگردان
|
|
113
|
-
if all(v == 0 for v in emotions.values()) and results:
|
|
114
|
-
top_emotion = max(results, key=lambda x: x['score'])
|
|
115
|
-
fa_label = top_emotion['label']
|
|
116
|
-
if fa_label in self.emotion_labels:
|
|
117
|
-
en_label = self.emotion_labels[fa_label]
|
|
118
|
-
emotions[en_label] = round(float(top_emotion['score']), 4)
|
|
119
|
-
|
|
120
|
-
return emotions
|
|
121
|
-
|
|
122
|
-
except Exception as e:
|
|
123
|
-
print(f"خطا در تحلیل احساسات: {str(e)}")
|
|
124
|
-
return {label: 0.0 for label in self.emotion_labels.values()}
|
|
125
|
-
|
|
126
|
-
def train(self, train_csv, test_size=0.2):
|
|
127
|
-
"""آموزش پیشرفته مدل تحلیل توصیه"""
|
|
128
|
-
df = pd.read_csv(train_csv)
|
|
129
|
-
|
|
130
|
-
# تعادل کلاسها
|
|
131
|
-
class_counts = df['recommendation_status'].value_counts()
|
|
132
|
-
max_samples = class_counts.max()
|
|
133
|
-
balanced_df = pd.concat([
|
|
134
|
-
df[df['recommendation_status'] == cls].sample(max_samples, replace=True)
|
|
135
|
-
for cls in class_counts.index
|
|
136
|
-
])
|
|
137
|
-
|
|
138
|
-
balanced_df['tokens'] = balanced_df['body'].progress_apply(
|
|
139
|
-
lambda x: self._preprocess_text(x))
|
|
140
|
-
|
|
141
|
-
# آموزش مدل Word2Vec با پارامترهای بهینه
|
|
142
|
-
self.vectorizer = Word2Vec(
|
|
143
|
-
sentences=balanced_df['tokens'],
|
|
144
|
-
**self.w2v_params
|
|
145
|
-
)
|
|
146
|
-
|
|
147
|
-
# استخراج ویژگیهای پیشرفته
|
|
148
|
-
X = np.array([self._sentence_vector(s, self.vectorizer)
|
|
149
|
-
for s in tqdm(balanced_df['tokens'], desc="Vectorizing")])
|
|
150
|
-
y = balanced_df['recommendation_status'].map({
|
|
151
|
-
"no_idea": 2,
|
|
152
|
-
"recommended": 1,
|
|
153
|
-
"not_recommended": 0
|
|
154
|
-
}).values
|
|
155
|
-
|
|
156
|
-
# تقسیم دادهها با stratify
|
|
157
|
-
X_train, X_test, y_train, y_test = train_test_split(
|
|
158
|
-
X, y, test_size=test_size, stratify=y)
|
|
159
|
-
|
|
160
|
-
# مدل طبقهبندی با تنظیمات بهینه
|
|
161
|
-
self.classifier = LogisticRegression(
|
|
162
|
-
max_iter=2000,
|
|
163
|
-
class_weight='balanced',
|
|
164
|
-
C=0.5,
|
|
165
|
-
solver='saga'
|
|
166
|
-
)
|
|
167
|
-
self.classifier.fit(X_train, y_train)
|
|
168
|
-
|
|
169
|
-
# ذخیره مدل
|
|
170
|
-
self.save_model()
|
|
171
|
-
|
|
172
|
-
# ارزیابی دقیقتر
|
|
173
|
-
train_acc = self.classifier.score(X_train, y_train)
|
|
174
|
-
test_acc = self.classifier.score(X_test, y_test)
|
|
175
|
-
|
|
176
|
-
return {
|
|
177
|
-
'train_accuracy': round(train_acc, 4),
|
|
178
|
-
'test_accuracy': round(test_acc, 4),
|
|
179
|
-
'model_params': self.w2v_params
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
def predict(self, text, mode=0, threshold=0.1):
|
|
183
|
-
"""
|
|
184
|
-
پیشبینی پیشرفته با قابلیت تنظیم آستانه
|
|
185
|
-
|
|
186
|
-
پارامترها:
|
|
187
|
-
text (str): متن ورودی
|
|
188
|
-
mode (int): حالت تحلیل (0=توصیه، 1=احساسات)
|
|
189
|
-
threshold (float): آستانه اطمینان برای تحلیل احساسات
|
|
190
|
-
|
|
191
|
-
بازگشتی:
|
|
192
|
-
dict یا str: نتایج تحلیل
|
|
193
|
-
"""
|
|
194
|
-
if mode == 1:
|
|
195
|
-
emotions = self._emotion_analysis(text)
|
|
196
|
-
# فیلتر کردن احساسات با احتمال پایین
|
|
197
|
-
return {
|
|
198
|
-
k: v for k, v in emotions.items()
|
|
199
|
-
if v >= threshold
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
if not self.classifier or not self.vectorizer:
|
|
203
|
-
raise Exception("Model not trained! Call train() first.")
|
|
204
|
-
|
|
205
|
-
tokens = self._preprocess_text(text)
|
|
206
|
-
vector = self._sentence_vector(tokens, self.vectorizer)
|
|
207
|
-
proba = self.classifier.predict_proba([vector])[0]
|
|
208
|
-
|
|
209
|
-
# تصمیمگیری با در نظر گرفتن احتمال
|
|
210
|
-
if max(proba) < 0.6: # آستانه اطمینان
|
|
211
|
-
return "no_idea"
|
|
212
|
-
|
|
213
|
-
prediction = self.classifier.predict([vector])[0]
|
|
214
|
-
return {
|
|
215
|
-
0: "not_recommended",
|
|
216
|
-
1: "recommended",
|
|
217
|
-
2: "no_idea"
|
|
218
|
-
}[prediction]
|
|
219
|
-
|
|
220
|
-
def save_model(self):
|
|
221
|
-
"""ذخیره مدل با نسخهبندی"""
|
|
222
|
-
version = 1
|
|
223
|
-
while os.path.exists(os.path.join(self.model_dir, f'classifier_v{version}.joblib')):
|
|
224
|
-
version += 1
|
|
225
|
-
|
|
226
|
-
joblib.dump(
|
|
227
|
-
self.classifier,
|
|
228
|
-
os.path.join(self.model_dir, f'classifier_v{version}.joblib')
|
|
229
|
-
)
|
|
230
|
-
self.vectorizer.save(
|
|
231
|
-
os.path.join(self.model_dir, f'word2vec_v{version}.model')
|
|
232
|
-
)
|
|
233
|
-
|
|
234
|
-
def load_model(self, version=None):
|
|
235
|
-
"""بارگذاری مدل با قابلیت انتخاب نسخه"""
|
|
236
|
-
if version is None:
|
|
237
|
-
versions = [
|
|
238
|
-
int(f.split('_v')[-1].split('.')[0])
|
|
239
|
-
for f in os.listdir(self.model_dir)
|
|
240
|
-
if f.startswith('classifier_v')
|
|
241
|
-
]
|
|
242
|
-
version = max(versions) if versions else None
|
|
243
|
-
|
|
244
|
-
if version:
|
|
245
|
-
self.classifier = joblib.load(
|
|
246
|
-
os.path.join(self.model_dir, f'classifier_v{version}.joblib')
|
|
247
|
-
)
|
|
248
|
-
self.vectorizer = Word2Vec.load(
|
|
249
|
-
os.path.join(self.model_dir, f'word2vec_v{version}.model')
|
|
250
|
-
)
|
|
251
|
-
|
|
@@ -1,162 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: PerSent
|
|
3
|
-
Version: 0.1.6
|
|
4
|
-
Summary: Persian Sentiment Analysis Toolkit
|
|
5
|
-
Home-page: https://github.com/RezaGooner/PerSent
|
|
6
|
-
Author: RezaGooner
|
|
7
|
-
Author-email: RezaAsadiProgrammer@Gmail.com
|
|
8
|
-
Keywords: persian sentiment analysis nlp
|
|
9
|
-
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
-
Classifier: Operating System :: OS Independent
|
|
12
|
-
Requires-Python: >=3.8
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
License-File: LICENSE.txt
|
|
15
|
-
Requires-Dist: hazm>=0.7.0
|
|
16
|
-
Requires-Dist: gensim>=4.0.0
|
|
17
|
-
Requires-Dist: scikit-learn>=1.0.0
|
|
18
|
-
Requires-Dist: pandas>=1.3.0
|
|
19
|
-
Requires-Dist: tqdm>=4.62.0
|
|
20
|
-
Requires-Dist: joblib>=1.1.0
|
|
21
|
-
Dynamic: author
|
|
22
|
-
Dynamic: author-email
|
|
23
|
-
Dynamic: classifier
|
|
24
|
-
Dynamic: description
|
|
25
|
-
Dynamic: description-content-type
|
|
26
|
-
Dynamic: home-page
|
|
27
|
-
Dynamic: keywords
|
|
28
|
-
Dynamic: license-file
|
|
29
|
-
Dynamic: requires-dist
|
|
30
|
-
Dynamic: requires-python
|
|
31
|
-
Dynamic: summary
|
|
32
|
-
|
|
33
|
-
# Persian Sentiment Analyzer
|
|
34
|
-
|
|
35
|
-
[](https://www.python.org/)
|
|
36
|
-
[](LICENSE)
|
|
37
|
-
|
|
38
|
-
A Python library for sentiment analysis of Persian (Farsi) text, capable of classifying opinions as "recommended", "not_recommended", or "no_idea".
|
|
39
|
-
|
|
40
|
-
## Features
|
|
41
|
-
|
|
42
|
-
- **Text Preprocessing**: Normalization, tokenization, stemming, and stopword removal for Persian text
|
|
43
|
-
- **Word Embeddings**: Built-in Word2Vec implementation for Persian language
|
|
44
|
-
- **Sentiment Classification**: Logistic Regression classifier trained on Persian sentiment data
|
|
45
|
-
- **Model Persistence**: Save and load trained models for future use
|
|
46
|
-
- **Batch Processing**: Analyze sentiment for multiple texts at once
|
|
47
|
-
|
|
48
|
-
## Installation
|
|
49
|
-
|
|
50
|
-
```bash
|
|
51
|
-
pip install persian-sentiment-analyzer
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
## Dependencies
|
|
55
|
-
- Python 3.6+
|
|
56
|
-
|
|
57
|
-
- hazm
|
|
58
|
-
|
|
59
|
-
- gensim
|
|
60
|
-
|
|
61
|
-
- scikit-learn
|
|
62
|
-
|
|
63
|
-
- numpy
|
|
64
|
-
|
|
65
|
-
- pandas
|
|
66
|
-
|
|
67
|
-
## Usage
|
|
68
|
-
|
|
69
|
-
### Basic Usage
|
|
70
|
-
|
|
71
|
-
``` bash
|
|
72
|
-
from persian_sentiment_analyzer import SentimentAnalyzer
|
|
73
|
-
|
|
74
|
-
# Initialize with a pre-trained model
|
|
75
|
-
analyzer = SentimentAnalyzer(model_path="path/to/pretrained_model")
|
|
76
|
-
|
|
77
|
-
# Predict sentiment
|
|
78
|
-
result = analyzer.predict("این محصول بسیار عالی است")
|
|
79
|
-
print(result) # Output: 'recommended'
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
### Training Your Own Model
|
|
83
|
-
|
|
84
|
-
``` bash
|
|
85
|
-
from persian_sentiment_analyzer import SentimentAnalyzer
|
|
86
|
-
import pandas as pd
|
|
87
|
-
|
|
88
|
-
# Load your dataset
|
|
89
|
-
data = pd.read_csv("persian_reviews.csv")
|
|
90
|
-
texts = data['text'].tolist()
|
|
91
|
-
labels = data['label'].values # 0: not_recommended, 1: recommended, 2: no_idea
|
|
92
|
-
|
|
93
|
-
# Initialize analyzer
|
|
94
|
-
analyzer = SentimentAnalyzer()
|
|
95
|
-
|
|
96
|
-
# Preprocess and tokenize texts
|
|
97
|
-
tokenized_texts = [analyzer.preprocessor.preprocess_text(text) for text in texts]
|
|
98
|
-
|
|
99
|
-
# Train Word2Vec model
|
|
100
|
-
analyzer.train_word2vec(tokenized_texts, vector_size=100)
|
|
101
|
-
|
|
102
|
-
# Prepare feature vectors
|
|
103
|
-
X = np.array([analyzer.sentence_vector(tokens) for tokens in tokenized_texts])
|
|
104
|
-
|
|
105
|
-
# Train classifier
|
|
106
|
-
analyzer.train_classifier(X, labels)
|
|
107
|
-
|
|
108
|
-
# Save the trained model
|
|
109
|
-
analyzer.save_model("my_persian_model")
|
|
110
|
-
```
|
|
111
|
-
|
|
112
|
-
### Batch Processing
|
|
113
|
-
|
|
114
|
-
```bash
|
|
115
|
-
from persian_sentiment_analyzer import predict_sentiments_for_file
|
|
116
|
-
|
|
117
|
-
# Process a CSV file containing Persian comments
|
|
118
|
-
results_summary = predict_sentiments_for_file(
|
|
119
|
-
analyzer,
|
|
120
|
-
input_file="comments.csv",
|
|
121
|
-
output_file="results.csv",
|
|
122
|
-
summary_file="summary.csv"
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
print(results_summary)
|
|
126
|
-
```
|
|
127
|
-
|
|
128
|
-
## Model Architecture
|
|
129
|
-
1- Text Preprocessing:
|
|
130
|
-
|
|
131
|
-
- Normalization (Hazm)
|
|
132
|
-
|
|
133
|
-
- Tokenization
|
|
134
|
-
|
|
135
|
-
- Stemming
|
|
136
|
-
|
|
137
|
-
- Stopword removal
|
|
138
|
-
|
|
139
|
-
2- Feature Extraction:
|
|
140
|
-
|
|
141
|
-
- Word2Vec embeddings (100 dimensions)
|
|
142
|
-
|
|
143
|
-
- Sentence vectors (average of word vectors)
|
|
144
|
-
|
|
145
|
-
3- Classification:
|
|
146
|
-
|
|
147
|
-
- Logistic Regression with L2 regularization
|
|
148
|
-
|
|
149
|
-
## Performance
|
|
150
|
-
The pre-trained model achieves the following performance on our test set:
|
|
151
|
-
|
|
152
|
-
Metric Value
|
|
153
|
-
Accuracy 85.2%
|
|
154
|
-
Precision 84.7%
|
|
155
|
-
Recall 85.0%
|
|
156
|
-
F1-score 84.8%
|
|
157
|
-
|
|
158
|
-
License
|
|
159
|
-
This project is licensed under the MIT [License](https://github.com/RezaGooner/Sentiment-Survey-Analyzer/blob/main/LICENSE) - see the LICENSE file for details
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
> Github : RezaGooner
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
PerSent
|
persent-0.1.6/README.md
DELETED
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
# Persian Sentiment Analyzer
|
|
2
|
-
|
|
3
|
-
[](https://www.python.org/)
|
|
4
|
-
[](LICENSE)
|
|
5
|
-
|
|
6
|
-
A Python library for sentiment analysis of Persian (Farsi) text, capable of classifying opinions as "recommended", "not_recommended", or "no_idea".
|
|
7
|
-
|
|
8
|
-
## Features
|
|
9
|
-
|
|
10
|
-
- **Text Preprocessing**: Normalization, tokenization, stemming, and stopword removal for Persian text
|
|
11
|
-
- **Word Embeddings**: Built-in Word2Vec implementation for Persian language
|
|
12
|
-
- **Sentiment Classification**: Logistic Regression classifier trained on Persian sentiment data
|
|
13
|
-
- **Model Persistence**: Save and load trained models for future use
|
|
14
|
-
- **Batch Processing**: Analyze sentiment for multiple texts at once
|
|
15
|
-
|
|
16
|
-
## Installation
|
|
17
|
-
|
|
18
|
-
```bash
|
|
19
|
-
pip install persian-sentiment-analyzer
|
|
20
|
-
```
|
|
21
|
-
|
|
22
|
-
## Dependencies
|
|
23
|
-
- Python 3.6+
|
|
24
|
-
|
|
25
|
-
- hazm
|
|
26
|
-
|
|
27
|
-
- gensim
|
|
28
|
-
|
|
29
|
-
- scikit-learn
|
|
30
|
-
|
|
31
|
-
- numpy
|
|
32
|
-
|
|
33
|
-
- pandas
|
|
34
|
-
|
|
35
|
-
## Usage
|
|
36
|
-
|
|
37
|
-
### Basic Usage
|
|
38
|
-
|
|
39
|
-
``` bash
|
|
40
|
-
from persian_sentiment_analyzer import SentimentAnalyzer
|
|
41
|
-
|
|
42
|
-
# Initialize with a pre-trained model
|
|
43
|
-
analyzer = SentimentAnalyzer(model_path="path/to/pretrained_model")
|
|
44
|
-
|
|
45
|
-
# Predict sentiment
|
|
46
|
-
result = analyzer.predict("این محصول بسیار عالی است")
|
|
47
|
-
print(result) # Output: 'recommended'
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
### Training Your Own Model
|
|
51
|
-
|
|
52
|
-
``` bash
|
|
53
|
-
from persian_sentiment_analyzer import SentimentAnalyzer
|
|
54
|
-
import pandas as pd
|
|
55
|
-
|
|
56
|
-
# Load your dataset
|
|
57
|
-
data = pd.read_csv("persian_reviews.csv")
|
|
58
|
-
texts = data['text'].tolist()
|
|
59
|
-
labels = data['label'].values # 0: not_recommended, 1: recommended, 2: no_idea
|
|
60
|
-
|
|
61
|
-
# Initialize analyzer
|
|
62
|
-
analyzer = SentimentAnalyzer()
|
|
63
|
-
|
|
64
|
-
# Preprocess and tokenize texts
|
|
65
|
-
tokenized_texts = [analyzer.preprocessor.preprocess_text(text) for text in texts]
|
|
66
|
-
|
|
67
|
-
# Train Word2Vec model
|
|
68
|
-
analyzer.train_word2vec(tokenized_texts, vector_size=100)
|
|
69
|
-
|
|
70
|
-
# Prepare feature vectors
|
|
71
|
-
X = np.array([analyzer.sentence_vector(tokens) for tokens in tokenized_texts])
|
|
72
|
-
|
|
73
|
-
# Train classifier
|
|
74
|
-
analyzer.train_classifier(X, labels)
|
|
75
|
-
|
|
76
|
-
# Save the trained model
|
|
77
|
-
analyzer.save_model("my_persian_model")
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
### Batch Processing
|
|
81
|
-
|
|
82
|
-
```bash
|
|
83
|
-
from persian_sentiment_analyzer import predict_sentiments_for_file
|
|
84
|
-
|
|
85
|
-
# Process a CSV file containing Persian comments
|
|
86
|
-
results_summary = predict_sentiments_for_file(
|
|
87
|
-
analyzer,
|
|
88
|
-
input_file="comments.csv",
|
|
89
|
-
output_file="results.csv",
|
|
90
|
-
summary_file="summary.csv"
|
|
91
|
-
)
|
|
92
|
-
|
|
93
|
-
print(results_summary)
|
|
94
|
-
```
|
|
95
|
-
|
|
96
|
-
## Model Architecture
|
|
97
|
-
1- Text Preprocessing:
|
|
98
|
-
|
|
99
|
-
- Normalization (Hazm)
|
|
100
|
-
|
|
101
|
-
- Tokenization
|
|
102
|
-
|
|
103
|
-
- Stemming
|
|
104
|
-
|
|
105
|
-
- Stopword removal
|
|
106
|
-
|
|
107
|
-
2- Feature Extraction:
|
|
108
|
-
|
|
109
|
-
- Word2Vec embeddings (100 dimensions)
|
|
110
|
-
|
|
111
|
-
- Sentence vectors (average of word vectors)
|
|
112
|
-
|
|
113
|
-
3- Classification:
|
|
114
|
-
|
|
115
|
-
- Logistic Regression with L2 regularization
|
|
116
|
-
|
|
117
|
-
## Performance
|
|
118
|
-
The pre-trained model achieves the following performance on our test set:
|
|
119
|
-
|
|
120
|
-
Metric Value
|
|
121
|
-
Accuracy 85.2%
|
|
122
|
-
Precision 84.7%
|
|
123
|
-
Recall 85.0%
|
|
124
|
-
F1-score 84.8%
|
|
125
|
-
|
|
126
|
-
License
|
|
127
|
-
This project is licensed under the MIT [License](https://github.com/RezaGooner/Sentiment-Survey-Analyzer/blob/main/LICENSE) - see the LICENSE file for details
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
> Github : RezaGooner
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|