pelican-nlp 0.1.0__py3-none-any.whl

pelican_nlp/extraction/distance_from_randomness.py

@@ -0,0 +1,109 @@
+import numpy as np
+import scipy
+from typing import Dict, List, Any
+
+#Type aliases
+DistanceMatrix = np.ndarray
+EmbeddingDict = Dict[str, Dict[str, List[Any]]]
+
+def get_distance_from_randomness(embeddings, config, parallel=False):
+
+    if parallel:
+        print(f'parallel computing not yet set up... '
+              f'continuing without calculating divergence from optimality')
+        return
+
+    else:
+        results_dict = {}
+        result = optimality(
+            embeddings, config['window_size'], config['bootstrap'], config['shuffle_mode']
+        )
+        results_dict[f'section'] = result
+        return results_dict
+
+
+def optimality(embeddings_dict, min_len, bootstrap, shuffle_mode):
+
+    words = list(embeddings_dict.keys())
+    embeddings = list(embeddings_dict.values())
+
+    answer_res = []
+    answer_len = len(words)
+
+    for i in range((answer_len - min_len) + 1):
+
+        window = embeddings[i:i + min_len]
+        dist_matrix = create_semantic_distance_matrix(window)
+
+        # Calculate costs for actual sequence and permutations
+        perm_costs = []
+        for j in range(bootstrap):
+            order = (np.arange(len(window)) if j == 0
+                     else get_shuffled_order(len(window), shuffle_mode, j))
+            cost = calculate_total_distance_covered(dist_matrix, order)
+            perm_costs.append(cost)
+
+            if j == 0:
+                all_pairs_avg = average_similarity(dist_matrix)
+
+        # Normalize costs by number of edges
+        costs_per_edge = np.array(perm_costs) / (min_len - 1)
+        true_cost = costs_per_edge[0]
+
+        # Store results for this window
+        window_results = {
+            "window_index": i,
+            "all_pairs_average": all_pairs_avg,
+            "actual_dist": true_cost,
+            "average_dist": np.mean(costs_per_edge[1:]),
+            "std_dist": np.std(costs_per_edge[1:])
+        }
+        answer_res.append(window_results)
+
+    return answer_res
+
+
+def create_semantic_distance_matrix(embedding_list: List[np.ndarray]) -> DistanceMatrix:
+
+    distances = scipy.spatial.distance.cdist(
+        np.array(embedding_list),
+        np.array(embedding_list),
+        'cosine'
+    )
+    np.fill_diagonal(distances, 0)
+    return distances
+
+def get_shuffled_order(n: int, shuffle_mode: str, seed: int) -> np.ndarray:
+
+    np.random.seed(seed)
+
+    if shuffle_mode == "include0_includeN":
+        order = np.arange(n)
+        np.random.shuffle(order)
+    elif shuffle_mode == "exclude0_includeN":
+        rest = np.arange(1, n)
+        np.random.shuffle(rest)
+        order = np.concatenate(([0], rest))
+    elif shuffle_mode == "exclude0_excludeN":
+        middle = np.arange(1, n - 1)
+        np.random.shuffle(middle)
+        order = np.concatenate(([0], middle, [n - 1]))
+    else:
+        raise ValueError(f"Invalid shuffle mode: {shuffle_mode}")
+
+    return order
+
+def calculate_total_distance_covered(dist_matrix: DistanceMatrix, order: np.ndarray) -> float:
+    distances = dist_matrix[order[:-1], order[1:]]
+    return float(np.sum(distances))
+
+def average_similarity(matrix: DistanceMatrix) -> float:
+
+    n = matrix.shape[0]
+
+    # Only count upper triangle to avoid double counting
+    upper_tri = np.triu(matrix, k=1)
+    total = np.sum(upper_tri)
+    count = (n * (n - 1)) // 2  # Number of pairs
+
+    return float(total / count) if count > 0 else 0.0

pelican_nlp/extraction/extract_embeddings.py

@@ -0,0 +1,57 @@
+from pelican_nlp.extraction.language_model import Model
+from pelican_nlp.preprocessing.text_tokenizer import TextTokenizer
+
+class EmbeddingsExtractor:
+    def __init__(self, embeddings_configurations, project_path):
+        self.embeddings_configurations = embeddings_configurations
+        self.model_name = embeddings_configurations['model_name']  # Embedding model instance (e.g., fastText, RoBERTa)
+        self.model = Model(self.model_name, project_path)
+        self.Tokenizer = TextTokenizer(self.embeddings_configurations['tokenization_method'], self.model_name,
+                                       self.embeddings_configurations['max_length'])
+
+        self.model.load_model()
+        self.model_instance = self.model.model_instance
+
+    def extract_embeddings_from_text(self, text_list):
+
+        doc_entry_list = []
+
+        for text in text_list:
+
+            embeddings = {}
+
+            # Tokenize the input text
+            inputs = self.Tokenizer.tokenize_text(text)
+            print(f'inputs are: {inputs}')
+
+            if self.embeddings_configurations['pytorch_based_model']:
+                #e.g. RoBERTa Model or Llama Model
+                import torch
+                with torch.no_grad():
+                    if 'llama' in self.model_name.lower():
+                        # Handle Llama models which expect input_ids directly
+                        outputs = self.model_instance(input_ids=inputs['input_ids'])
+                    else:
+                        # Handle RoBERTa and other models that accept **inputs
+                        outputs = self.model_instance(**inputs)
+
+                # Get word embeddings (last hidden state)
+                word_embeddings = outputs.last_hidden_state
+
+                # Extract input_ids and convert them back to tokens
+                input_ids = inputs['input_ids'][0].tolist()
+                tokens = self.Tokenizer.tokenizer.convert_ids_to_tokens(input_ids)
+
+                # Now align the tokens and embeddings
+                for token, embedding in zip(tokens, word_embeddings[0]):
+                    embeddings[token]=embedding.tolist()
+
+            else:
+                if self.model_name == 'fastText':
+                    embeddings = []
+                    for token in inputs:
+                        embeddings.append((token, self.model_instance.get_word_vector(token)))
+
+            doc_entry_list.append(embeddings)
+
+        return doc_entry_list, len(inputs)

pelican_nlp/extraction/extract_logits.py

@@ -0,0 +1,102 @@
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+
+class LogitsExtractor:
+    def __init__(self, options, pipeline, project_path):
+
+        self.device = 'cuda' if torch.cuda.is_available()==True else 'cpu'
+        self.options = options
+        self.model_name = self.options['model_name']
+        self.pipeline = pipeline
+        self.PROJECT_PATH = project_path
+
+    def extract_features(self, section, tokenizer, model):
+
+        print(f'section to tokenize: {section}')
+        tokens = tokenizer.tokenize_text(section)
+        print(tokens)
+
+        chunk_size = self.options['chunk_size']
+        overlap_size = self.options['overlap_size']
+
+        input_ids = tokens.to(self.device)
+        chunks = self._split_into_chunks(input_ids, chunk_size, overlap_size)
+
+        per_token_data = []
+
+        total_processed_tokens = 0  # Keep track of total tokens_logits processed to avoid duplicates
+
+        for i, chunk in enumerate(tqdm(chunks, desc="Processing chunks")):
+
+            with torch.no_grad():
+
+                outputs = model(input_ids=chunk)
+                logits = outputs.logits  # Shape: [1, seq_length, vocab_size]
+
+            tokens = tokenizer.convert_ids_to_tokens(chunk.squeeze())
+            num_tokens = len(tokens)
+
+            chunk_data = []
+
+            # Determine the starting index for predictions
+            if i == 0:
+                # For the first chunk, start from index 1 (since the first token has no previous context)
+                start_idx = 1
+            else:
+                # For subsequent chunks, skip tokens_logits that were already processed in the overlap
+                start_idx = overlap_size
+
+            # Loop over the tokens_logits to predict
+            for j in range(start_idx, num_tokens):
+                # Compute per-token metrics
+                per_token_metrics = self._compute_per_token_metrics(logits, chunk, tokens, j, tokenizer)
+                chunk_data.append(per_token_metrics)
+
+            # Append the chunk data to the per_token_data list
+            per_token_data.extend(chunk_data)
+            total_processed_tokens += len(chunk_data)
+
+        return per_token_data
+
+    def _compute_per_token_metrics(self, logits, chunk, tokens, j, tokenizer):
+
+        # The model_instance predicts the token at position j using tokens_logits up to position j-1
+        # Therefore, logits at position j-1 correspond to predictions for token at position j
+        token_logits = logits[:, j - 1, :]  # Shape: [1, vocab_size]
+        token_probs = F.softmax(token_logits, dim=-1)
+        token_logprobs = F.log_softmax(token_logits, dim=-1)
+
+        actual_token_id = chunk[:, j]  # The actual token at position j
+        logprob_actual = token_logprobs[0, actual_token_id].item()
+        max_logprob, max_token_id = torch.max(token_logprobs, dim=-1)
+        max_logprob = max_logprob.item()
+        max_token_id = max_token_id.item()
+        entropy = -(token_probs * token_logprobs).sum().item()
+
+        most_likely_token = tokenizer.convert_ids_to_tokens([max_token_id])[0]
+        token = tokens[j]  # The token at position j
+
+        return {
+            'token': token,
+            'logprob_actual': logprob_actual,
+            'logprob_max': max_logprob,
+            'entropy': entropy,
+            'most_likely_token': most_likely_token
+        }
+
+    def _split_into_chunks(self, input_ids, chunk_size, overlap_size):
+
+        input_ids = input_ids.squeeze()
+        input_length = input_ids.size(0)
+        stride = chunk_size - overlap_size
+        chunks = []
+
+        for i in range(0, input_length, stride):
+            end_index = min(i + chunk_size, input_length)
+            chunk = input_ids[i:end_index]
+            chunks.append(chunk.unsqueeze(0).to(self.device))
+            if end_index == input_length:
+                break
+
+        return chunks

pelican_nlp/extraction/language_model.py

@@ -0,0 +1,71 @@
+import torch
+import psutil
+
+from accelerate import init_empty_weights, infer_auto_device_map, dispatch_model
+from transformers import AutoModelForCausalLM
+
+class Model:
+    def __init__(self, model_name, project_path):
+        self.model_name = model_name
+        self.model_instance = None
+        self.device_map = None
+        self.PROJECT_PATH = project_path
+
+    def load_model(self, empty_weights=False, trust_remote_code=False):
+        """Loads and configures the model"""
+
+        if self.model_name == 'fastText':
+            import fasttext
+            import fasttext.util
+            fasttext.util.download_model('de', if_exists='ignore')
+            self.model_instance = fasttext.load_model('cc.de.300.bin')
+            print('FastText model loaded.')
+        elif self.model_name == 'xlm-roberta-base':
+            from transformers import AutoModel
+            self.model_instance = AutoModel.from_pretrained(
+                self.model_name,
+                trust_remote_code=trust_remote_code,
+                use_safetensors=True
+            )
+            print('RoBERTa model loaded.')
+        elif self.model_name == 'DiscoResearch/Llama3-German-8B-32k':
+            if empty_weights:
+                with init_empty_weights():
+                    self.model_instance = AutoModelForCausalLM.from_pretrained(
+                        self.model_name,
+                        trust_remote_code=trust_remote_code,
+                        use_safetensors=True
+                    )
+            else:
+                self.model_instance = AutoModelForCausalLM.from_pretrained(
+                    self.model_name,
+                    trust_remote_code=trust_remote_code,
+                    use_safetensors=True
+                )
+            print(f'Llama3-German-8B-32k loaded')
+        else:
+            raise ValueError("Invalid model name.")
+
+        if self.model_name == 'xlm-roberta-base' or self.model_name == 'DiscoResearch/Llama3-German-8B-32k':
+            # Additional model setup
+            self.device_map_creation()
+
+            self.model_instance = dispatch_model(self.model_instance, device_map=self.device_map)
+            print('Model dispatched to appropriate devices.')
+
+    def device_map_creation(self):
+        #check if cuda is available
+        if not torch.cuda.is_available():
+            print('Careful: Cuda not available, using CPU. This will be very slow.')
+        else:
+            print(f'{torch.cuda.get_device_name(0)} available.')
+
+        available_VRAM = str(int(torch.cuda.get_device_properties(0).total_memory/(1024 ** 3))-3)+'GB'
+        available_RAM = str(int(psutil.virtual_memory().total/(1024 ** 3))-3)+'GB'
+
+        #create device map and offload directory if it doesn't exist
+        self.device_map = infer_auto_device_map(self.model_instance, max_memory={
+            0: available_VRAM,
+            'cpu': available_RAM,
+            'disk': '200GB'
+        })

pelican_nlp/extraction/semantic_similarity.py

@@ -0,0 +1,60 @@
+import numpy as np
+import scipy
+from scipy.spatial.distance import cdist
+import pandas as pd
+
+def calculate_semantic_similarity(embedding_vectors):
+    # Extract just the vectors from the list of tuples
+    vectors = [vector for _, vector in embedding_vectors]
+    consecutive_similarities = get_consecutive_vector_similarities(vectors)
+    mean_similarity = np.nanmean(consecutive_similarities)
+    return consecutive_similarities, mean_similarity
+
+def get_consecutive_vector_similarities(vectors):
+    return [1 - scipy.spatial.distance.cosine(vectors[i - 1], vectors[i]) for i in range(1, len(vectors))]
+
+def get_cosine_similarity_matrix(embedding_vectors):
+    # Extract just the vectors from the list of tuples
+    vectors = [vector for _, vector in embedding_vectors]
+    similarity_matrix = 1 - cdist(vectors, vectors, 'cosine')
+    np.fill_diagonal(similarity_matrix, np.nan)
+    return similarity_matrix
+
+def get_semantic_similarity_windows(embedding_vectors, window_size):
+    # Extract tokens and vectors from the list of tuples
+    tokens, vectors = zip(*embedding_vectors)
+
+    # Early return if not enough tokens
+    if len(tokens) < 2:
+        return np.nan, np.nan, np.nan, np.nan, np.nan
+
+    # Early return if window size is larger than sequence
+    if window_size > len(tokens):
+        return np.nan, np.nan, np.nan, np.nan, np.nan
+
+    if window_size == 'all':
+        cosine_similarity_matrix = get_cosine_similarity_matrix(embedding_vectors)
+        return calculate_window_statistics(cosine_similarity_matrix)
+
+    # Collect window statistics
+    window_statistics = []
+    for i in range(len(tokens) - window_size + 1):
+        window_vectors = list(zip(tokens[i:i + window_size], vectors[i:i + window_size]))
+        if window_vectors:  # Make sure window is not empty
+            sim_matrix = get_cosine_similarity_matrix(window_vectors)
+            window_statistics.append(calculate_window_statistics(sim_matrix))
+    
+    # Handle case where no valid windows were found
+    if not window_statistics:
+        return np.nan, np.nan, np.nan, np.nan, np.nan
+        
+    # Unzip the statistics
+    window_means, window_stds, window_medians = zip(*window_statistics)
+    
+    return (np.mean(window_means), np.std(window_means), 
+            np.mean(window_stds), np.std(window_stds), 
+            np.mean(window_medians))
+
+def calculate_window_statistics(cosine_similarity_matrix):
+    matrix_values = pd.DataFrame(cosine_similarity_matrix).stack()
+    return matrix_values.mean(), matrix_values.std(), matrix_values.median()