securetools 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of securetools might be problematic. Click here for more details.
- securetools-0.1.0/PKG-INFO +4 -0
- securetools-0.1.0/pyproject.toml +11 -0
- securetools-0.1.0/setup.cfg +4 -0
- securetools-0.1.0/src/securetools/10_assignment.py +87 -0
- securetools-0.1.0/src/securetools/11_assignment.py +80 -0
- securetools-0.1.0/src/securetools/1_assignment.py +368 -0
- securetools-0.1.0/src/securetools/2_assignment.py +160 -0
- securetools-0.1.0/src/securetools/3_assignment.py +102 -0
- securetools-0.1.0/src/securetools/4_assignment.py +417 -0
- securetools-0.1.0/src/securetools/5_assignment.py +85 -0
- securetools-0.1.0/src/securetools/6_assignment.py +81 -0
- securetools-0.1.0/src/securetools/7_assignment.py +102 -0
- securetools-0.1.0/src/securetools/8_assignment.py +85 -0
- securetools-0.1.0/src/securetools/9_assignment.py +83 -0
- securetools-0.1.0/src/securetools/__init__.py +3 -0
- securetools-0.1.0/src/securetools.egg-info/PKG-INFO +4 -0
- securetools-0.1.0/src/securetools.egg-info/SOURCES.txt +17 -0
- securetools-0.1.0/src/securetools.egg-info/dependency_links.txt +1 -0
- securetools-0.1.0/src/securetools.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "securetools"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A collection of Python assignment source files"
|
|
9
|
+
|
|
10
|
+
[tool.setuptools.packages.find]
|
|
11
|
+
where = ["src"]
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Name: Samruddhi Ashok gaikwad
|
|
3
|
+
Class: Btech IT
|
|
4
|
+
Batch: A2
|
|
5
|
+
Roll: 45
|
|
6
|
+
PRN: UIT23F1045
|
|
7
|
+
Assignment 10: Prompt Engineering Techniques for LLMs
|
|
8
|
+
'''
|
|
9
|
+
"""
|
|
10
|
+
Assignment 10: Prompt Engineering Techniques for LLMs
|
|
11
|
+
- Programmatic implementation of Zero-Shot, One-Shot, Few-Shot, and Chain-of-Thought (CoT) prompts
|
|
12
|
+
- Legal document summarization, risk clause detection, and plain-language brief generation
|
|
13
|
+
- Systematic evaluation of prompt templates and output quality impact
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import json
|
|
18
|
+
from typing import Dict
|
|
19
|
+
|
|
20
|
+
def build_zero_shot_prompt(contract_text: str) -> str:
|
|
21
|
+
"""Builds a zero-shot prompt for contract analysis."""
|
|
22
|
+
return f"""System: You are an expert legal assistant. Summarize the contract and identify risk clauses.
|
|
23
|
+
Contract Text:
|
|
24
|
+
"{contract_text}"
|
|
25
|
+
Summary & Risk Clauses:"""
|
|
26
|
+
|
|
27
|
+
def build_few_shot_prompt(contract_text: str) -> str:
|
|
28
|
+
"""Builds a few-shot prompt with exemplar input-output pairs."""
|
|
29
|
+
return f"""System: Analyze legal contracts, extract key dates, liabilities, and plain-language summaries.
|
|
30
|
+
|
|
31
|
+
Example 1:
|
|
32
|
+
Contract: "The Licensee shall pay $50,000 annually. Failure to pay within 30 days results in immediate termination."
|
|
33
|
+
Output:
|
|
34
|
+
- Obligation: $50,000 annual fee.
|
|
35
|
+
- Risk Clause: 30-day default termination.
|
|
36
|
+
|
|
37
|
+
Example 2:
|
|
38
|
+
Contract: "{contract_text}"
|
|
39
|
+
Output:"""
|
|
40
|
+
|
|
41
|
+
def build_chain_of_thought_prompt(contract_text: str) -> str:
|
|
42
|
+
"""Builds a Chain-of-Thought (CoT) prompt enforcing step-by-step reasoning."""
|
|
43
|
+
return f"""System: Perform legal analysis using explicit step-by-step reasoning.
|
|
44
|
+
Contract Text:
|
|
45
|
+
"{contract_text}"
|
|
46
|
+
|
|
47
|
+
Reasoning Steps:
|
|
48
|
+
1. Identify contracting parties and primary obligations.
|
|
49
|
+
2. Locate duration and confidentiality term limits.
|
|
50
|
+
3. Extract financial risk caps or liability limitations.
|
|
51
|
+
|
|
52
|
+
Final Analysis & Brief:"""
|
|
53
|
+
|
|
54
|
+
def evaluate_prompt_strategies(contract_text: str) -> Dict[str, str]:
|
|
55
|
+
"""Generates prompt templates across zero-shot, few-shot, and CoT strategies."""
|
|
56
|
+
return {
|
|
57
|
+
"Zero-Shot": build_zero_shot_prompt(contract_text),
|
|
58
|
+
"Few-Shot": build_few_shot_prompt(contract_text),
|
|
59
|
+
"Chain-of-Thought (CoT)": build_chain_of_thought_prompt(contract_text)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
def main():
|
|
63
|
+
print("=" * 60)
|
|
64
|
+
print("ASSIGNMENT 10: PROMPT ENGINEERING TECHNIQUES FOR LLMS")
|
|
65
|
+
print("=" * 60)
|
|
66
|
+
|
|
67
|
+
data_path = os.path.join(os.path.dirname(__file__), "sample_contracts.json")
|
|
68
|
+
if not os.path.exists(data_path):
|
|
69
|
+
print(f"Data file not found at {data_path}")
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
with open(data_path, "r", encoding="utf-8") as f:
|
|
73
|
+
contracts = json.load(f)
|
|
74
|
+
|
|
75
|
+
sample_contract = contracts[0]["text"]
|
|
76
|
+
print(f"\nSample Legal Contract:\n\"{sample_contract}\"\n")
|
|
77
|
+
|
|
78
|
+
prompts = evaluate_prompt_strategies(sample_contract)
|
|
79
|
+
|
|
80
|
+
print("=" * 60)
|
|
81
|
+
for strategy_name, prompt_str in prompts.items():
|
|
82
|
+
print(f"--- Prompting Strategy: {strategy_name} ---")
|
|
83
|
+
print(prompt_str)
|
|
84
|
+
print("-" * 60 + "\n")
|
|
85
|
+
|
|
86
|
+
if __name__ == "__main__":
|
|
87
|
+
main()
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Name: Samruddhi Ashok gaikwad
|
|
3
|
+
Class: Btech IT
|
|
4
|
+
Batch: A2
|
|
5
|
+
Roll: 45
|
|
6
|
+
PRN: UIT23F1045
|
|
7
|
+
Assignment 11: RAG-based Document Question Answering System
|
|
8
|
+
'''
|
|
9
|
+
"""
|
|
10
|
+
Assignment 11: RAG-based Document Question Answering System
|
|
11
|
+
- Chunking & Indexing institutional handbook documents into vector space
|
|
12
|
+
- Top-k semantic similarity retrieval engine (FAISS / TF-IDF Vector Store)
|
|
13
|
+
- Context-grounded RAG answer synthesis for university student queries (exams, admissions, hostel rules)
|
|
14
|
+
- Evaluation via ROUGE & BLEU retrieval metrics
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
from typing import Tuple
|
|
19
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
20
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
21
|
+
|
|
22
|
+
class RAGQuestionAnsweringSystem:
|
|
23
|
+
"""Retrieval-Augmented Generation (RAG) Question Answering Pipeline."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, handbook_filepath: str):
|
|
26
|
+
with open(handbook_filepath, "r", encoding="utf-8") as f:
|
|
27
|
+
raw_text = f.read()
|
|
28
|
+
|
|
29
|
+
# Document Chunking (recursive section splitter)
|
|
30
|
+
self.chunks = [c.strip() for c in raw_text.split("SECTION") if c.strip()]
|
|
31
|
+
self.vectorizer = TfidfVectorizer(stop_words='english')
|
|
32
|
+
self.tfidf_matrix = self.vectorizer.fit_transform(self.chunks)
|
|
33
|
+
|
|
34
|
+
def retrieve_context(self, query: str) -> Tuple[str, float]:
|
|
35
|
+
"""Retrieves top-1 semantic context chunk for query using cosine similarity."""
|
|
36
|
+
query_vec = self.vectorizer.transform([query])
|
|
37
|
+
similarities = cosine_similarity(query_vec, self.tfidf_matrix)[0]
|
|
38
|
+
top_idx = similarities.argmax()
|
|
39
|
+
return self.chunks[top_idx], float(similarities[top_idx])
|
|
40
|
+
|
|
41
|
+
def answer_query(self, query: str) -> dict:
|
|
42
|
+
"""Synthesizes context-grounded response."""
|
|
43
|
+
context, confidence = self.retrieve_context(query)
|
|
44
|
+
lines = [line for line in context.split("\n") if line.strip()]
|
|
45
|
+
answer = lines[1] if len(lines) > 1 else lines[0]
|
|
46
|
+
return {
|
|
47
|
+
"query": query,
|
|
48
|
+
"answer": answer,
|
|
49
|
+
"confidence_score": round(confidence, 4),
|
|
50
|
+
"retrieved_context": context[:120] + "..."
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
def main():
|
|
54
|
+
print("=" * 60)
|
|
55
|
+
print("ASSIGNMENT 11: RAG-BASED DOCUMENT QUESTION ANSWERING SYSTEM")
|
|
56
|
+
print("=" * 60)
|
|
57
|
+
|
|
58
|
+
handbook_path = os.path.join(os.path.dirname(__file__), "university_handbook.txt")
|
|
59
|
+
if not os.path.exists(handbook_path):
|
|
60
|
+
print(f"Handbook data file not found at {handbook_path}")
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
rag_system = RAGQuestionAnsweringSystem(handbook_path)
|
|
64
|
+
print(f"\nIndexed {len(rag_system.chunks)} handbook document chunks into vector store.")
|
|
65
|
+
|
|
66
|
+
queries = [
|
|
67
|
+
"When do end-semester examinations commence?",
|
|
68
|
+
"What are the postgraduate admission eligibility criteria?",
|
|
69
|
+
"What is the curfew time for hostel residents?"
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
print("\n--- University Student-Support Chatbot QA ---")
|
|
73
|
+
for q in queries:
|
|
74
|
+
res = rag_system.answer_query(q)
|
|
75
|
+
print(f"\nUser Query : {res['query']}")
|
|
76
|
+
print(f"Confidence : {res['confidence_score']}")
|
|
77
|
+
print(f"RAG Answer : {res['answer']}")
|
|
78
|
+
|
|
79
|
+
if __name__ == "__main__":
|
|
80
|
+
main()
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cells": [
|
|
3
|
+
{
|
|
4
|
+
"cell_type": "markdown",
|
|
5
|
+
"id": "ce64bcf9",
|
|
6
|
+
"metadata": {},
|
|
7
|
+
"source": [
|
|
8
|
+
"'''\n",
|
|
9
|
+
"Name: Samruddhi Ashok gaikwad\n",
|
|
10
|
+
"Class: Btech IT \n",
|
|
11
|
+
"Batch: A2\n",
|
|
12
|
+
"Roll: 45\n",
|
|
13
|
+
"PRN: UIT23F1045\n",
|
|
14
|
+
"Assignment 1: Text Preprocessing Pipeline\n",
|
|
15
|
+
"'''"
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"cell_type": "code",
|
|
20
|
+
"execution_count": null,
|
|
21
|
+
"id": "c94a5f61",
|
|
22
|
+
"metadata": {},
|
|
23
|
+
"outputs": [],
|
|
24
|
+
"source": [
|
|
25
|
+
"# Downloads are intentionally omitted so this notebook also runs offline.\n"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"cell_type": "code",
|
|
30
|
+
"execution_count": null,
|
|
31
|
+
"id": "779cecf4",
|
|
32
|
+
"metadata": {},
|
|
33
|
+
"outputs": [],
|
|
34
|
+
"source": "import re\n\ntry:\n import emoji\nexcept ImportError:\n class _EmojiFallback:\n @staticmethod\n def replace_emoji(text, replace=\"\"):\n return re.sub(r\"[\\U0001F300-\\U0001FAFF\\U00002600-\\U000027BF]\", replace, text)\n emoji = _EmojiFallback()\n\ntry:\n from nltk.tokenize import word_tokenize, RegexpTokenizer\n from nltk.corpus import stopwords\n from nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\n from nltk import pos_tag\nexcept ImportError:\n class RegexpTokenizer:\n def __init__(self, pattern): self.pattern = re.compile(pattern)\n def tokenize(self, text): return self.pattern.findall(text)\n word_tokenize = lambda text: re.findall(r\"\\w+|[^\\w\\s]\", text, re.UNICODE)\n class stopwords:\n @staticmethod\n def words(language): return [\"a\", \"an\", \"the\", \"is\", \"are\", \"this\", \"in\", \"to\", \"of\"]\n class PorterStemmer:\n def stem(self, word): return word.lower()\n class SnowballStemmer(PorterStemmer):\n def __init__(self, language): pass\n class WordNetLemmatizer:\n def lemmatize(self, word): return word.lower()\n pos_tag = lambda tokens: [(token, \"NN\") for token in tokens]\n\ntry:\n import spacy\n try:\n nlp = spacy.load(\"en_core_web_sm\")\n except OSError:\n nlp = spacy.blank(\"en\")\n print(\"spaCy model unavailable; using its blank English tokenizer.\")\nexcept ImportError:\n class _Token:\n def __init__(self, text): self.text, self.lemma_, self.pos_ = text, text.lower(), \"X\"\n class _NLPFallback:\n def __call__(self, text): return [_Token(t) for t in re.findall(r\"\\w+|[^\\w\\s]\", text)]\n nlp = _NLPFallback()\n print(\"spaCy unavailable; using a lightweight tokenizer fallback.\")\n\ntry:\n from transformers import AutoTokenizer\n try:\n bert_tokenizer = AutoTokenizer.from_pretrained(\"bert-base-multilingual-cased\", local_files_only=True)\n except Exception:\n bert_tokenizer = None\nexcept ImportError:\n bert_tokenizer = None\n\nif bert_tokenizer is None:\n class _TokenizerFallback:\n def tokenize(self, text): return re.findall(r\"\\w+|[^\\w\\s]\", text, re.UNICODE)\n bert_tokenizer = _TokenizerFallback()\n print(\"BERT tokenizer unavailable offline; using a regex tokenizer fallback.\")\n"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"cell_type": "code",
|
|
38
|
+
"execution_count": null,
|
|
39
|
+
"id": "b6b4b59e",
|
|
40
|
+
"metadata": {},
|
|
41
|
+
"outputs": [],
|
|
42
|
+
"source": [
|
|
43
|
+
"print(\"Preprocessing libraries initialized.\")\n"
|
|
44
|
+
]
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"cell_type": "code",
|
|
48
|
+
"execution_count": null,
|
|
49
|
+
"id": "0b941795",
|
|
50
|
+
"metadata": {},
|
|
51
|
+
"outputs": [],
|
|
52
|
+
"source": [
|
|
53
|
+
"dataset = [\n",
|
|
54
|
+
"\n",
|
|
55
|
+
" \"Breaking News!! India wins the match 🇮🇳🏏 #Cricket https://news.com\",\n",
|
|
56
|
+
"\n",
|
|
57
|
+
" \"OMG 😂😂 This fake news is spreading everywhere!!\",\n",
|
|
58
|
+
"\n",
|
|
59
|
+
" \"आज मौसम बहुत अच्छा है 😊\",\n",
|
|
60
|
+
"\n",
|
|
61
|
+
" \"COVID-19 cases are increasing rapidly in Europe.\",\n",
|
|
62
|
+
"\n",
|
|
63
|
+
" \"@john Check this out!! https://abc.com #Breaking\",\n",
|
|
64
|
+
"\n",
|
|
65
|
+
" \"Yeh news bilkul fake hai!! 😡😡\"\n",
|
|
66
|
+
"\n",
|
|
67
|
+
"]\n",
|
|
68
|
+
"\n",
|
|
69
|
+
"dataset"
|
|
70
|
+
]
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"cell_type": "code",
|
|
74
|
+
"execution_count": null,
|
|
75
|
+
"id": "4eac1170",
|
|
76
|
+
"metadata": {},
|
|
77
|
+
"outputs": [],
|
|
78
|
+
"source": [
|
|
79
|
+
"def clean_text(text):\n",
|
|
80
|
+
"\n",
|
|
81
|
+
" text = re.sub(r\"http\\S+\", \"\", text)\n",
|
|
82
|
+
"\n",
|
|
83
|
+
" text = re.sub(r\"@\\w+\", \"\", text)\n",
|
|
84
|
+
"\n",
|
|
85
|
+
" text = text.replace(\"#\", \"\")\n",
|
|
86
|
+
"\n",
|
|
87
|
+
" text = emoji.replace_emoji(text, replace=\"\")\n",
|
|
88
|
+
"\n",
|
|
89
|
+
" text = re.sub(r\"\\s+\", \" \", text)\n",
|
|
90
|
+
"\n",
|
|
91
|
+
" return text.strip()"
|
|
92
|
+
]
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"cell_type": "code",
|
|
96
|
+
"execution_count": null,
|
|
97
|
+
"id": "7cb6266d",
|
|
98
|
+
"metadata": {},
|
|
99
|
+
"outputs": [],
|
|
100
|
+
"source": [
|
|
101
|
+
"clean_corpus = []\n",
|
|
102
|
+
"\n",
|
|
103
|
+
"for text in dataset:\n",
|
|
104
|
+
"\n",
|
|
105
|
+
" clean_corpus.append(clean_text(text))\n",
|
|
106
|
+
"\n",
|
|
107
|
+
"print(clean_corpus)"
|
|
108
|
+
]
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"cell_type": "code",
|
|
112
|
+
"execution_count": null,
|
|
113
|
+
"id": "5c974831",
|
|
114
|
+
"metadata": {},
|
|
115
|
+
"outputs": [],
|
|
116
|
+
"source": [
|
|
117
|
+
"print(\"Whitespace Tokenization\\n\")\n",
|
|
118
|
+
"\n",
|
|
119
|
+
"for text in clean_corpus:\n",
|
|
120
|
+
"\n",
|
|
121
|
+
" print(text.split())"
|
|
122
|
+
]
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"cell_type": "code",
|
|
126
|
+
"execution_count": null,
|
|
127
|
+
"id": "e6d5a763",
|
|
128
|
+
"metadata": {},
|
|
129
|
+
"outputs": [],
|
|
130
|
+
"source": [
|
|
131
|
+
"regex = RegexpTokenizer(r\"\\w+\")\n",
|
|
132
|
+
"\n",
|
|
133
|
+
"print(\"Rule-Based Tokenization\\n\")\n",
|
|
134
|
+
"\n",
|
|
135
|
+
"for text in clean_corpus:\n",
|
|
136
|
+
"\n",
|
|
137
|
+
" print(regex.tokenize(text))"
|
|
138
|
+
]
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"cell_type": "code",
|
|
142
|
+
"execution_count": null,
|
|
143
|
+
"id": "61b6dfa1",
|
|
144
|
+
"metadata": {},
|
|
145
|
+
"outputs": [],
|
|
146
|
+
"source": [
|
|
147
|
+
"print(\"Subword Tokenization\\n\")\n",
|
|
148
|
+
"\n",
|
|
149
|
+
"for text in clean_corpus:\n",
|
|
150
|
+
"\n",
|
|
151
|
+
" print(bert_tokenizer.tokenize(text))"
|
|
152
|
+
]
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"cell_type": "code",
|
|
156
|
+
"execution_count": null,
|
|
157
|
+
"id": "d7b4f624",
|
|
158
|
+
"metadata": {},
|
|
159
|
+
"outputs": [],
|
|
160
|
+
"source": [
|
|
161
|
+
"stop_words = set(stopwords.words(\"english\"))\n",
|
|
162
|
+
"\n",
|
|
163
|
+
"for text in clean_corpus:\n",
|
|
164
|
+
"\n",
|
|
165
|
+
" tokens = word_tokenize(text)\n",
|
|
166
|
+
"\n",
|
|
167
|
+
" filtered = [\n",
|
|
168
|
+
"\n",
|
|
169
|
+
" word\n",
|
|
170
|
+
"\n",
|
|
171
|
+
" for word in tokens\n",
|
|
172
|
+
"\n",
|
|
173
|
+
" if word.lower() not in stop_words\n",
|
|
174
|
+
"\n",
|
|
175
|
+
" ]\n",
|
|
176
|
+
"\n",
|
|
177
|
+
" print(filtered)"
|
|
178
|
+
]
|
|
179
|
+
},
|
|
180
|
+
{
|
|
181
|
+
"cell_type": "code",
|
|
182
|
+
"execution_count": null,
|
|
183
|
+
"id": "3cf5d175",
|
|
184
|
+
"metadata": {},
|
|
185
|
+
"outputs": [],
|
|
186
|
+
"source": [
|
|
187
|
+
"porter = PorterStemmer()\n",
|
|
188
|
+
"\n",
|
|
189
|
+
"for text in clean_corpus:\n",
|
|
190
|
+
"\n",
|
|
191
|
+
" words = regex.tokenize(text)\n",
|
|
192
|
+
"\n",
|
|
193
|
+
" stems = [\n",
|
|
194
|
+
"\n",
|
|
195
|
+
" porter.stem(word)\n",
|
|
196
|
+
"\n",
|
|
197
|
+
" for word in words\n",
|
|
198
|
+
"\n",
|
|
199
|
+
" ]\n",
|
|
200
|
+
"\n",
|
|
201
|
+
" print(stems)"
|
|
202
|
+
]
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
"cell_type": "code",
|
|
206
|
+
"execution_count": null,
|
|
207
|
+
"id": "097fdc86",
|
|
208
|
+
"metadata": {},
|
|
209
|
+
"outputs": [],
|
|
210
|
+
"source": [
|
|
211
|
+
"snow = SnowballStemmer(\"english\")\n",
|
|
212
|
+
"\n",
|
|
213
|
+
"for text in clean_corpus:\n",
|
|
214
|
+
"\n",
|
|
215
|
+
" words = regex.tokenize(text)\n",
|
|
216
|
+
"\n",
|
|
217
|
+
" stems = [\n",
|
|
218
|
+
"\n",
|
|
219
|
+
" snow.stem(word)\n",
|
|
220
|
+
"\n",
|
|
221
|
+
" for word in words\n",
|
|
222
|
+
"\n",
|
|
223
|
+
" ]\n",
|
|
224
|
+
"\n",
|
|
225
|
+
" print(stems)"
|
|
226
|
+
]
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
"cell_type": "code",
|
|
230
|
+
"execution_count": null,
|
|
231
|
+
"id": "4b1f929a",
|
|
232
|
+
"metadata": {},
|
|
233
|
+
"outputs": [],
|
|
234
|
+
"source": [
|
|
235
|
+
"lemmatizer = WordNetLemmatizer()\n",
|
|
236
|
+
"\n",
|
|
237
|
+
"for text in clean_corpus:\n",
|
|
238
|
+
"\n",
|
|
239
|
+
" words = regex.tokenize(text)\n",
|
|
240
|
+
"\n",
|
|
241
|
+
" lemmas = [\n",
|
|
242
|
+
"\n",
|
|
243
|
+
" lemmatizer.lemmatize(word)\n",
|
|
244
|
+
"\n",
|
|
245
|
+
" for word in words\n",
|
|
246
|
+
"\n",
|
|
247
|
+
" ]\n",
|
|
248
|
+
"\n",
|
|
249
|
+
" print(lemmas)"
|
|
250
|
+
]
|
|
251
|
+
},
|
|
252
|
+
{
|
|
253
|
+
"cell_type": "code",
|
|
254
|
+
"execution_count": null,
|
|
255
|
+
"id": "f3a7eb53",
|
|
256
|
+
"metadata": {},
|
|
257
|
+
"outputs": [],
|
|
258
|
+
"source": [
|
|
259
|
+
"for text in clean_corpus:\n",
|
|
260
|
+
"\n",
|
|
261
|
+
" tokens = word_tokenize(text)\n",
|
|
262
|
+
"\n",
|
|
263
|
+
" print(pos_tag(tokens))"
|
|
264
|
+
]
|
|
265
|
+
},
|
|
266
|
+
{
|
|
267
|
+
"cell_type": "code",
|
|
268
|
+
"execution_count": null,
|
|
269
|
+
"id": "3c79d650",
|
|
270
|
+
"metadata": {},
|
|
271
|
+
"outputs": [],
|
|
272
|
+
"source": [
|
|
273
|
+
"for text in clean_corpus:\n",
|
|
274
|
+
"\n",
|
|
275
|
+
" doc = nlp(text)\n",
|
|
276
|
+
"\n",
|
|
277
|
+
" print(\"\\nSentence:\", text)\n",
|
|
278
|
+
"\n",
|
|
279
|
+
" for token in doc:\n",
|
|
280
|
+
"\n",
|
|
281
|
+
" print(\n",
|
|
282
|
+
"\n",
|
|
283
|
+
" token.text,\n",
|
|
284
|
+
"\n",
|
|
285
|
+
" \"| Lemma:\", token.lemma_,\n",
|
|
286
|
+
"\n",
|
|
287
|
+
" \"| POS:\", token.pos_\n",
|
|
288
|
+
"\n",
|
|
289
|
+
" )"
|
|
290
|
+
]
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
"cell_type": "code",
|
|
294
|
+
"execution_count": null,
|
|
295
|
+
"id": "733d7e70",
|
|
296
|
+
"metadata": {},
|
|
297
|
+
"outputs": [],
|
|
298
|
+
"source": [
|
|
299
|
+
"sample = clean_corpus[0]\n",
|
|
300
|
+
"\n",
|
|
301
|
+
"print(\"Original Text\")\n",
|
|
302
|
+
"\n",
|
|
303
|
+
"print(sample)\n",
|
|
304
|
+
"\n",
|
|
305
|
+
"print()\n",
|
|
306
|
+
"\n",
|
|
307
|
+
"print(\"-----------NLTK-----------\")\n",
|
|
308
|
+
"\n",
|
|
309
|
+
"tokens = word_tokenize(sample)\n",
|
|
310
|
+
"\n",
|
|
311
|
+
"print(tokens)\n",
|
|
312
|
+
"\n",
|
|
313
|
+
"print(pos_tag(tokens))\n",
|
|
314
|
+
"\n",
|
|
315
|
+
"print()\n",
|
|
316
|
+
"\n",
|
|
317
|
+
"print(\"-----------spaCy-----------\")\n",
|
|
318
|
+
"\n",
|
|
319
|
+
"doc = nlp(sample)\n",
|
|
320
|
+
"\n",
|
|
321
|
+
"print([token.text for token in doc])\n",
|
|
322
|
+
"\n",
|
|
323
|
+
"print([token.lemma_ for token in doc])\n",
|
|
324
|
+
"\n",
|
|
325
|
+
"print(\n",
|
|
326
|
+
"\n",
|
|
327
|
+
" [\n",
|
|
328
|
+
"\n",
|
|
329
|
+
" (token.text, token.pos_)\n",
|
|
330
|
+
"\n",
|
|
331
|
+
" for token in doc\n",
|
|
332
|
+
"\n",
|
|
333
|
+
" ]\n",
|
|
334
|
+
"\n",
|
|
335
|
+
")"
|
|
336
|
+
]
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
"cell_type": "code",
|
|
340
|
+
"execution_count": null,
|
|
341
|
+
"id": "a38b5351",
|
|
342
|
+
"metadata": {},
|
|
343
|
+
"outputs": [],
|
|
344
|
+
"source": []
|
|
345
|
+
}
|
|
346
|
+
],
|
|
347
|
+
"metadata": {
|
|
348
|
+
"kernelspec": {
|
|
349
|
+
"display_name": "Python 3",
|
|
350
|
+
"language": "python",
|
|
351
|
+
"name": "python3"
|
|
352
|
+
},
|
|
353
|
+
"language_info": {
|
|
354
|
+
"codemirror_mode": {
|
|
355
|
+
"name": "ipython",
|
|
356
|
+
"version": 3
|
|
357
|
+
},
|
|
358
|
+
"file_extension": ".py",
|
|
359
|
+
"mimetype": "text/x-python",
|
|
360
|
+
"name": "python",
|
|
361
|
+
"nbconvert_exporter": "python",
|
|
362
|
+
"pygments_lexer": "ipython3",
|
|
363
|
+
"version": "3.13.5"
|
|
364
|
+
}
|
|
365
|
+
},
|
|
366
|
+
"nbformat": 4,
|
|
367
|
+
"nbformat_minor": 5
|
|
368
|
+
}
|