SinaTools 0.1.11__py2.py3-none-any.whl → 0.1.13__py2.py3-none-any.whl

sinatools/ner/trainers/BertTrainer.py

@@ -0,0 +1,163 @@
+import os
+import logging
+import torch
+import numpy as np
+from sinatools.ner.trainers import BaseTrainer
+from sinatools.ner.metrics import compute_single_label_metrics
+    
+logger = logging.getLogger(__name__)
+
+
+class BertTrainer(BaseTrainer):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def train(self):
+        best_val_loss, test_loss = np.inf, np.inf
+        num_train_batch = len(self.train_dataloader)
+        patience = self.patience
+
+        for epoch_index in range(self.max_epochs):
+            self.current_epoch = epoch_index
+            train_loss = 0
+
+            for batch_index, (_, gold_tags, _, _, logits) in enumerate(self.tag(
+                self.train_dataloader, is_train=True
+            ), 1):
+                self.current_timestep += 1
+                batch_loss = self.loss(logits.view(-1, logits.shape[-1]), gold_tags.view(-1))
+                batch_loss.backward()
+
+                # Avoid exploding gradient by doing gradient clipping
+                torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
+
+                self.optimizer.step()
+                self.scheduler.step()
+                train_loss += batch_loss.item()
+
+                if self.current_timestep % self.log_interval == 0:
+                    logger.info(
+                        "Epoch %d | Batch %d/%d | Timestep %d | LR %.10f | Loss %f",
+                        epoch_index,
+                        batch_index,
+                        num_train_batch,
+                        self.current_timestep,
+                        self.optimizer.param_groups[0]['lr'],
+                        batch_loss.item()
+                    )
+
+            train_loss /= num_train_batch
+
+            logger.info("** Evaluating on validation dataset **")
+            val_preds, segments, valid_len, val_loss = self.eval(self.val_dataloader)
+            val_metrics = compute_single_label_metrics(segments)
+
+            epoch_summary_loss = {
+                "train_loss": train_loss,
+                "val_loss": val_loss
+            }
+            epoch_summary_metrics = {
+                "val_micro_f1": val_metrics.micro_f1,
+                "val_precision": val_metrics.precision,
+                "val_recall": val_metrics.recall
+            }
+
+            logger.info(
+                "Epoch %d | Timestep %d | Train Loss %f | Val Loss %f | F1 %f",
+                epoch_index,
+                self.current_timestep,
+                train_loss,
+                val_loss,
+                val_metrics.micro_f1
+            )
+
+            if val_loss < best_val_loss:
+                patience = self.patience
+                best_val_loss = val_loss
+                logger.info("** Validation improved, evaluating test data **")
+                test_preds, segments, valid_len, test_loss = self.eval(self.test_dataloader)
+                self.segments_to_file(segments, os.path.join(self.output_path, "predictions.txt"))
+                test_metrics = compute_single_label_metrics(segments)
+
+                epoch_summary_loss["test_loss"] = test_loss
+                epoch_summary_metrics["test_micro_f1"] = test_metrics.micro_f1
+                epoch_summary_metrics["test_precision"] = test_metrics.precision
+                epoch_summary_metrics["test_recall"] = test_metrics.recall
+
+                logger.info(
+                    f"Epoch %d | Timestep %d | Test Loss %f | F1 %f",
+                    epoch_index,
+                    self.current_timestep,
+                    test_loss,
+                    test_metrics.micro_f1
+                )
+
+                self.save()
+            else:
+                patience -= 1
+
+            # No improvements, terminating early
+            if patience == 0:
+                logger.info("Early termination triggered")
+                break
+
+            self.summary_writer.add_scalars("Loss", epoch_summary_loss, global_step=self.current_timestep)
+            self.summary_writer.add_scalars("Metrics", epoch_summary_metrics, global_step=self.current_timestep)
+
+    def eval(self, dataloader):
+        golds, preds, segments, valid_lens = list(), list(), list(), list()
+        loss = 0
+
+        for _, gold_tags, tokens, valid_len, logits in self.tag(
+            dataloader, is_train=False
+        ):
+            loss += self.loss(logits.view(-1, logits.shape[-1]), gold_tags.view(-1))
+            preds += torch.argmax(logits, dim=2).detach().cpu().numpy().tolist()
+            segments += tokens
+            valid_lens += list(valid_len)
+
+        loss /= len(dataloader)
+
+        # Update segments, attach predicted tags to each token
+        segments = self.to_segments(segments, preds, valid_lens, dataloader.dataset.vocab)
+
+        return preds, segments, valid_lens, loss.item()
+
+    def infer(self, dataloader):
+        golds, preds, segments, valid_lens = list(), list(), list(), list()
+
+        for _, gold_tags, tokens, valid_len, logits in self.tag(
+            dataloader, is_train=False
+        ):
+            preds += torch.argmax(logits, dim=2).detach().cpu().numpy().tolist()
+            segments += tokens
+            valid_lens += list(valid_len)
+
+        segments = self.to_segments(segments, preds, valid_lens, dataloader.dataset.vocab)
+        return segments
+
+    def to_segments(self, segments, preds, valid_lens, vocab):
+        if vocab is None:
+            vocab = self.vocab
+
+        tagged_segments = list()
+        tokens_stoi = vocab.tokens.get_stoi()
+        tags_itos = vocab.tags[0].get_itos()
+        unk_id = tokens_stoi["UNK"]
+
+        for segment, pred, valid_len in zip(segments, preds, valid_lens):
+            # First, the token at 0th index [CLS] and token at nth index [SEP]
+            # Combine the tokens with their corresponding predictions
+            segment_pred = zip(segment[1:valid_len-1], pred[1:valid_len-1])
+
+            # Ignore the sub-tokens/subwords, which are identified with text being UNK
+            segment_pred = list(filter(lambda t: tokens_stoi[t[0].text] != unk_id, segment_pred))
+
+            # Attach the predicted tags to each token
+            list(map(lambda t: setattr(t[0], 'pred_tag', [{"tag": tags_itos[t[1]]}]), segment_pred))
+
+            # We are only interested in the tagged tokens, we do no longer need raw model predictions
+            tagged_segment = [t for t, _ in segment_pred]
+            tagged_segments.append(tagged_segment)
+
+        return tagged_segments

sinatools/ner/trainers/__init__.py

@@ -0,0 +1,3 @@
+from sinatools.ner.trainers.BaseTrainer import BaseTrainer
+from sinatools.ner.trainers.BertTrainer import BertTrainer
+from sinatools.ner.trainers.BertNestedTrainer import BertNestedTrainer

sinatools/ner/transforms.py

@@ -0,0 +1,119 @@
+import torch
+from transformers import BertTokenizer
+from functools import partial
+import re
+import itertools
+from sinatools.ner import datasets
+
+class BertSeqTransform:
+    def __init__(self, bert_model, vocab, max_seq_len=512):
+        self.tokenizer = BertTokenizer.from_pretrained(bert_model)
+        self.encoder = partial(
+            self.tokenizer.encode,
+            max_length=max_seq_len,
+            truncation=True,
+        )
+        self.max_seq_len = max_seq_len
+        self.vocab = vocab
+
+    def __call__(self, segment):
+        subwords, tags, tokens = list(), list(), list()
+        unk_token = datasets.Token(text="UNK")
+
+        for token in segment:
+            token_subwords = self.encoder(token.text)[1:-1]
+            subwords += token_subwords
+            tags += [self.vocab.tags[0].get_stoi()[token.gold_tag[0]]] + [self.vocab.tags[0].get_stoi()["O"]] * (len(token_subwords) - 1)
+            tokens += [token] + [unk_token] * (len(token_subwords) - 1)
+
+        # Truncate to max_seq_len
+        if len(subwords) > self.max_seq_len - 2:
+            text = " ".join([t.text for t in tokens if t.text != "UNK"])
+          
+            subwords = subwords[:self.max_seq_len - 2]
+            tags = tags[:self.max_seq_len - 2]
+            tokens = tokens[:self.max_seq_len - 2]
+
+        subwords.insert(0, self.tokenizer.cls_token_id)
+        subwords.append(self.tokenizer.sep_token_id)
+
+        tags.insert(0, self.vocab.tags[0].get_stoi()["O"])
+        tags.append(self.vocab.tags[0].get_stoi()["O"])
+
+        tokens.insert(0, unk_token)
+        tokens.append(unk_token)
+
+        return torch.LongTensor(subwords), torch.LongTensor(tags), tokens, len(tokens)
+
+
+class NestedTagsTransform:
+    def __init__(self, bert_model, vocab, max_seq_len=512):
+        self.tokenizer = BertTokenizer.from_pretrained(bert_model)
+        self.encoder = partial(
+            self.tokenizer.encode,
+            max_length=max_seq_len,
+            truncation=True,
+        )
+        self.max_seq_len = max_seq_len
+        self.vocab = vocab
+
+    def __call__(self, segment):
+        tags, tokens, subwords = list(), list(), list()
+        unk_token = datasets.Token(text="UNK")
+
+        # Encode each token and get its subwords and IDs
+        for token in segment:
+            token.subwords = self.encoder(token.text)[1:-1]
+            subwords += token.subwords
+            tokens += [token] + [unk_token] * (len(token.subwords ) - 1)
+
+        # Construct the labels for each tag type
+        # The sequence will have a list of tags for each type
+        # The final tags for a sequence is a matrix NUM_TAG_TYPES x SEQ_LEN
+        # Example:
+        #   [
+        #       [O,     O,     B-PERS, I-PERS, O, O, O]
+        #       [B-ORG, I-ORG, O,      O,      O, O, O]
+        #       [O,     O,     O,      O,      O, O, B-GPE]
+        #   ]
+        for vocab in self.vocab.tags[1:]:
+            vocab_tags = "|".join([t for t in vocab.get_itos() if "-" in t])
+            r = re.compile(vocab_tags)
+
+            # This is really messy
+            # For a given token we find a matching tag_name, BUT we might find
+            # multiple matches (i.e. a token can be labeled B-ORG and I-ORG) in this
+            # case we get only the first tag as we do not have overlapping of same type
+            single_type_tags = [[(list(filter(r.match, token.gold_tag))
+                                or ["O"])[0]] + ["O"] * (len(token.subwords) - 1)
+                                for token in segment]
+            single_type_tags = list(itertools.chain(*single_type_tags))
+            tags.append([vocab.get_stoi()[tag] for tag in single_type_tags])
+
+        # Truncate to max_seq_len
+        if len(subwords) > self.max_seq_len - 2:
+            text = " ".join([t.text for t in tokens if t.text != "UNK"])
+          
+            subwords = subwords[:self.max_seq_len - 2]
+            tags = [t[:self.max_seq_len - 2] for t in tags]
+            tokens = tokens[:self.max_seq_len - 2]
+
+        # Add dummy token at the start end of sequence
+        tokens.insert(0, unk_token)
+        tokens.append(unk_token)
+
+        # Add CLS and SEP at start end of subwords
+        subwords.insert(0, self.tokenizer.cls_token_id)
+        subwords.append(self.tokenizer.sep_token_id)
+        subwords = torch.LongTensor(subwords)
+
+        # Add "O" tags for the first and last subwords
+        tags = torch.Tensor(tags)
+        tags = torch.column_stack((
+            torch.Tensor([vocab.get_stoi()["O"] for vocab in self.vocab.tags[1:]]),
+            tags,
+            torch.Tensor([vocab.get_stoi()["O"] for vocab in self.vocab.tags[1:]]),
+        )).unsqueeze(0)
+
+        mask = torch.ones_like(tags)
+        return subwords, tags, tokens, mask, len(tokens)

sinatools/semantic_relatedness/__init__.py

@@ -0,0 +1,20 @@
+import warnings
+warnings.filterwarnings("ignore")
+from sinatools.DataDownload import downloader
+import os 
+from transformers import BertTokenizer,BertModel
+
+model_file_name = "bert-base-arabertv02_22_May_2021_00h_allglosses_unused01"
+path =downloader.get_appdatadir()
+model_file_path = os.path.join(path, model_file_name)
+
+tokenizer_file_name = "bert-base-arabertv02"
+path =downloader.get_appdatadir()
+tokenizer_file_path = os.path.join(path, tokenizer_file_name)
+
+model = BertModel.from_pretrained('{}'.format(model_file_path),
+                                                      output_hidden_states = True,
+                                                      num_labels=2
+                                                      )
+
+tokenizer = BertTokenizer.from_pretrained('{}'.format(tokenizer_file_path))

sinatools/semantic_relatedness/compute_relatedness.py

@@ -0,0 +1,31 @@
+import torch
+from . import tokenizer 
+from . import model 
+
+#cosine using average embedding 
+def get_similarity_score(sentence1, sentence2):
+
+  # Tokenize and encode sentences
+  inputs1 = tokenizer(sentence1, return_tensors="pt")
+  inputs2 = tokenizer(sentence2, return_tensors="pt")
+
+  # Extract embeddings
+  with torch.no_grad():
+      outputs1 = model(**inputs1)
+      outputs2 = model(**inputs2)
+      
+      embeddings1 = outputs1.last_hidden_state
+      embeddings2 = outputs2.last_hidden_state
+
+  # Mask padding tokens
+  attention_mask1 = inputs1["attention_mask"]
+  attention_mask2 = inputs2["attention_mask"]
+
+  # Average pool across tokens, excluding padding
+  embeddings1_avg = torch.sum(embeddings1 * attention_mask1.unsqueeze(-1), dim=1) / torch.sum(attention_mask1, dim=1, keepdim=True)
+  embeddings2_avg = torch.sum(embeddings2 * attention_mask2.unsqueeze(-1), dim=1) / torch.sum(attention_mask2, dim=1, keepdim=True)
+
+  # Calculate cosine similarity
+  similarity = torch.nn.functional.cosine_similarity(embeddings1_avg, embeddings2_avg)
+
+  return similarity.item()

sinatools/synonyms/__init__.py

@@ -0,0 +1,18 @@
+import pickle
+from sinatools.DataDownload import downloader
+import os 
+
+synonyms_level2_dict = {} 
+level2_dict = 'synonyms_level2.pkl'
+path = downloader.get_appdatadir()
+file_path = os.path.join(path, level2_dict)
+with open(file_path, 'rb') as f:
+    synonyms_level2_dict = pickle.load(f, encoding='utf-8')
+
+
+synonyms_level3_dict = {}    
+#level3_dict = 'synonyms_level3.pkl'
+#path = downloader.get_appdatadir()
+#file_path = os.path.join(path, level3_dict)
+#with open(file_path, 'rb') as f:
+#    synonyms_level3_dict = pickle.load(f, encoding='utf-8')

sinatools/synonyms/synonyms_generator.py

@@ -0,0 +1,192 @@
+from . import synonyms_level2_dict, synonyms_level3_dict
+from copy import deepcopy
+
+def dfs(graph, start, end, level):
+    level = level - 2
+    edge = [(start, [])]
+    while edge:
+        state, path = edge.pop()
+        if path and state == end:
+            yield path
+            continue
+        for next_state in graph[state]:               
+            if next_state not in path:
+               edge.append((next_state, path+[next_state]))
+            
+            if len(path) > level:               
+               break 
+
+
+def find_cycles(level, synset, used_graph):
+    cycles = [] 
+    source_with_unique_candidates = {}
+    for source in synset:
+        source_with_unique_candidates[source] = set()
+
+        for path in dfs(used_graph, source, source, level):  
+            cycle = [source]+path
+            if len(cycle) <= level : 
+               cycles.append(cycle)
+            source_with_unique_candidates[source] =  set(source_with_unique_candidates[source].union(set(cycle)))
+    return cycles, source_with_unique_candidates   
+
+
+
+def get_list_of_unique_synonems(synset,cycles, unique_synonyms, synonems_with_unique_candidates):
+    list_of_unique_synonyms = []
+    for i in range(0, len(unique_synonyms)):
+        synonym = unique_synonyms[i]
+        count = 0
+        syn_count = 0
+        for cycle in cycles:
+            if synonym in cycle:
+               count = count + 1    
+
+        for v in synonems_with_unique_candidates:
+            tmp = list(synonems_with_unique_candidates[v])
+            if synonym in tmp :
+               syn_count = syn_count + 1
+           
+           
+        list_of_unique_synonyms.append([synonym,count, syn_count])        
+    return list_of_unique_synonyms
+
+
+def find_fuzzy_value_for_candidates(level, list_of_unique_synonyms, number_of_cycles, length_of_synset, synset):
+    list_of_synon_with_fuzzy_value = [] 
+
+    if level == 4 : 
+       theta1 = 0.5
+       theta2 = 0.5
+    elif level == 3: 
+       theta1 = 0.4
+       theta2 = 0.6
+    elif level == 2: 
+       theta1 = 0.4
+       theta2 = 0.6   
+    else:
+       theta1 = 0
+       theta2 = 0
+    
+    for unique_syn in list_of_unique_synonyms:
+        if unique_syn[0] not in synset:    
+           equ = ( unique_syn[1] / number_of_cycles ) * theta1 + (unique_syn[2] / length_of_synset ) * theta2     
+           list_of_synon_with_fuzzy_value.append([unique_syn[0], equ])
+    return list_of_synon_with_fuzzy_value
+
+
+def extend_synonyms(synset, level):
+   
+    used_graph = {}
+    if level == 2:
+       used_graph = synonyms_level2_dict
+    elif level == 3:
+       used_graph = synonyms_level3_dict
+    else: 
+       return "Please choose the correct level"   
+    
+    cycles = []
+    nodes = []
+    synonems_with_unique_candidates = {}
+    number_of_cycles = 0
+    final_synset = []
+    if synset != None:
+       synset = synset.split("|")
+    for syn in synset:
+        syn = syn.strip()
+        if syn in list(used_graph.keys()):
+          synonems_with_unique_candidates[syn] = set()
+          final_synset.append(syn)
+          
+          cycles_list = used_graph[syn]
+          number_of_cycles = number_of_cycles + len(cycles_list)
+          for cycle in cycles_list:
+             cycles.append(cycle)
+             for c in cycle:
+                nodes.append(c)
+                synonems_with_unique_candidates[syn] = set(synonems_with_unique_candidates[syn].union(set([c])))                   
+    
+    unique_synonyms = list(set(nodes))
+    list_of_unique_synonyms = get_list_of_unique_synonems(final_synset, cycles, unique_synonyms, synonems_with_unique_candidates)
+    
+    length_of_synset = len(final_synset)
+    
+    list_of_synon_with_fuzzy_value = find_fuzzy_value_for_candidates(level, list_of_unique_synonyms, number_of_cycles, length_of_synset, final_synset)
+
+    list_of_synon_with_fuzzy_value.sort(key=lambda row: (row[1], row[0]), reverse=True)
+
+    return list_of_synon_with_fuzzy_value
+
+def evaluate_synonyms(synset, level):
+
+   used_graph = {}
+   if level == 2:
+      used_graph = synonyms_level2_dict
+   elif level == 3:
+      used_graph = synonyms_level3_dict
+   else: 
+      return "Please choose the correct level"   
+
+   cycles = []
+   synonems_with_unique_candidates = {}
+   number_of_cycles = 0
+   final_synset = []
+
+   if synset != None:
+      synset = synset.split("|")
+
+   for syn in synset:
+       syn = syn.strip()
+       if syn in list(used_graph.keys()):
+         synonems_with_unique_candidates[syn] = set()
+         final_synset.append(syn)
+         
+         cycles_list = used_graph[syn]
+         for cycle in cycles_list:
+            cycles.append(cycle)
+            for c in cycle:
+               synonems_with_unique_candidates[syn] = set(synonems_with_unique_candidates[syn].union(set([c])))                   
+            
+   if len(final_synset) > 1 :
+      fuzzy_result = []
+      for syn in final_synset:
+         included = False
+         tmp_synset = deepcopy(final_synset)
+         tmp_synset.remove(syn)
+   
+         tmp_cycles = deepcopy(cycles)
+         filtered_cycle = [x for x in tmp_cycles if x[0] != syn]
+                  
+         nodes = []
+         for tmp_cycle in filtered_cycle:
+            for c in tmp_cycle:
+               nodes.append(c)
+
+         tmp_synonems_with_unique_candidates = deepcopy(synonems_with_unique_candidates)
+         del tmp_synonems_with_unique_candidates[syn]
+
+         unique_synonyms = list(set(nodes))
+                  
+         tmp_unique_synonyms = deepcopy(unique_synonyms)
+                  
+         number_of_cycles = len(filtered_cycle)
+
+         length_of_synset = len(tmp_synset)
+
+         list_of_unique_synonyms = get_list_of_unique_synonems(tmp_synset, filtered_cycle, tmp_unique_synonyms, tmp_synonems_with_unique_candidates)
+               
+         list_of_synon_with_fuzzy_value = find_fuzzy_value_for_candidates(level, list_of_unique_synonyms, number_of_cycles, length_of_synset, tmp_synset)
+
+         for x in list_of_synon_with_fuzzy_value:
+            if x[0] == syn:
+               fuzzy_result.append(x)
+               included = True
+         
+         if included == False:
+            fuzzy_result.append([syn,0])
+         else:   
+            included = False
+
+      fuzzy_result.sort(key=lambda row: (row[1], row[0]), reverse=True)
+      
+      return fuzzy_result

sinatools/utils/text_dublication_detector.py

@@ -0,0 +1,110 @@
+
+import pandas as pd
+import re
+import math
+from collections import Counter
+from sinatools.utils.parser import arStrip
+from sinatools.utils.parser import remove_punctuation
+
+def validator(sentence, max_tokens=500):
+    tokens = len(sentence.split())
+    if tokens > max_tokens:
+        return f"Invalid: Sentence has {tokens} tokens, which exceeds the maximum allowed ({max_tokens})."
+    else:
+        return "Valid"
+
+
+def removal(csv_file, columnName, finalFileName, deletedFileName, similarityThreshold=0.8):
+    # Read CSV file
+    try:
+        df = pd.read_csv(csv_file)
+    except FileNotFoundError:
+        return "Error: CSV file not found."
+
+    # Check if the specified column exists
+    if columnName not in df.columns:
+        return f"Error: Column '{columnName}' does not exist in the CSV file."
+
+    # Create an empty DataFrame to store the final results
+    finalDf = pd.DataFrame(columns=df.columns)
+
+    # Create temporary DataFrames for deleted sentences
+    deletedSentencesDf = pd.DataFrame(columns=df.columns)
+
+    # Iterate through each row in the DataFrame
+    for index, row in df.iterrows():
+        currentSentence = str(row[columnName])
+
+        # Call the validator function for each sentence
+        #validationResult = validator(currentSentence)
+        validationResult = "Valid"
+
+        if validationResult == "Valid":
+            # Check cosine similarity with all sentences in the final DataFrame
+            isDuplicate = False
+            DublicatedRow = ""
+            for _, finalRow in finalDf.iterrows():
+                finalSentence = str(finalRow[columnName])
+                currentSentence = remove_punctuation(arStrip(currentSentence, diacs = False, smallDiacs = False, shaddah = False,  digit = True, alif = False, specialChars = True))
+                finalSentence = remove_punctuation(arStrip(finalSentence, diacs = False, smallDiacs = False, shaddah = False,  digit = True, alif = False, specialChars = True))
+                if currentSentence != "" and finalSentence != "":
+                   similarity = calculateCosineSimilarity(currentSentence, finalSentence)
+
+                   if similarity >= similarityThreshold:
+                       isDuplicate = True
+                       DublicatedRow = finalSentence
+                       print("DublicatedRow : ", DublicatedRow)
+                       break
+
+            if not isDuplicate:
+                # If not a duplicate, add the sentence to the final DataFrame
+                finalDf = finalDf.append(row, ignore_index=True)
+            else:
+                # If a duplicate, add the sentence to the deleted sentences DataFrame
+                #deletedSentencesDf = deletedSentencesDf.append(row, ignore_index=True)
+                deletedSentencesDf = deletedSentencesDf.append({**row, 'Dublicated': DublicatedRow}, ignore_index=True)
+        else:
+            # If validation fails, return the error message
+            return validationResult
+
+    # Save the final results to CSV files
+    finalDf.to_csv(finalFileName, index=False)
+    deletedSentencesDf.to_csv(deletedFileName, index=False)
+
+
+def calculateCosineSimilarity(sentence1, sentence2):
+    vector1 = textToVector(sentence1)
+    vector2 = textToVector(sentence2)
+    cosine = getCosine(vector1, vector2)
+    
+    return cosine
+
+
+def getCosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+
+    sum1 = sum([vec1[x] ** 2 for x in list(vec1.keys())])
+    sum2 = sum([vec2[x] ** 2 for x in list(vec2.keys())])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+    if not denominator:
+        return 0.0
+    else:
+        return float(numerator) / denominator
+
+
+def textToVector(text):
+    WORD = re.compile(r"\w+")
+    words = WORD.findall(text)
+    return Counter(words)
+
+
+# columnName = "Message"
+# csvFile = "Arabic-Oct7-Feb12.csv"
+# similarityThreshold = 0.8 
+# finalFileName = "Arabic-Oct7-Feb12FINAL.csv" 
+# deletedFileName = "Arabic-Oct7-Feb12DeletedSent.csv"
+
+# result = removal(csvFile, columnName, finalFileName, deletedFileName, similarityThreshold)
+# print(result)

sinatools/wsd/__init__.py

@@ -0,0 +1,11 @@
+from sinatools.wsd import settings 
+import pickle
+from sinatools.DataDownload import downloader
+import os 
+
+settings.glosses_dic = {}
+filename = 'glosses_dic.pickle'
+path =downloader.get_appdatadir()
+file_path = os.path.join(path, filename)
+with open(file_path, 'rb') as f:
+    settings.glosses_dic = pickle.load(f)