pelican-nlp 0.1.0__py3-none-any.whl

pelican_nlp/__init__.py

@@ -0,0 +1,9 @@
+# Version and metadata
+__version__ = "0.1.0"
+__author__ = "Yves Pauli"
+
+try:
+    from .main import Pelican
+except ImportError as e:
+    print(f"Warning: Could not import Pelican class: {e}")
+

pelican_nlp/core/__init__.py

@@ -0,0 +1,5 @@
+# Import core classes for easier access
+from .corpus import Corpus
+from .document import Document
+from .audio_document import AudioFile
+from .subject import Subject

pelican_nlp/core/audio_document.py

@@ -0,0 +1,20 @@
+import os
+
+class AudioFile:
+    def __init__(self, file_path, name, **kwargs):
+        self.file_path = file_path
+        self.name = name
+        self.file = os.path.join(file_path, name)
+
+        #Initialize optional attributes
+        self.subject_ID = kwargs.get('subject_ID')
+        self.task = kwargs.get('task')
+        self.num_speakers = kwargs.get('num_speakers')
+        self.corpus_name = None
+        self.recording_length = None
+
+        self.opensmile_results = None
+        self.prosogram_features = None
+
+    def __repr__(self):
+        return f"file_name={self.name}"

pelican_nlp/core/corpus.py

@@ -0,0 +1,296 @@
+"""
+This module provides the Corpus class, which aggregates documents where the same processing
+steps applied and results should be aggregated.
+(e.g. all fluency files from task 'animals' or all image-descriptions from the same image)
+
+This class contains the pipelines for homogenous processing and metric extraction of all grouped files.
+"""
+
+from ..preprocessing import TextPreprocessingPipeline
+from ..utils.csv_functions import store_features_to_csv
+from ..extraction.language_model import Model
+from ..preprocessing.speaker_diarization import TextDiarizer
+from ..preprocessing import text_cleaner as textcleaner
+import os
+import pandas as pd
+import re
+
+class Corpus:
+    def __init__(self, corpus_name, documents, configuration_settings):
+        self.name = corpus_name
+        self.documents = documents
+        self.config = configuration_settings
+        self.derivative_dir = self.config['PATH_TO_PROJECT_FOLDER']+'/derivatives'
+        self.pipeline = TextPreprocessingPipeline(self.config)
+        self.task = configuration_settings['task_name']
+        self.results_path = None
+
+    def preprocess_all_documents(self):
+        """Preprocess all documents"""
+        print('Preprocessing all documents...')
+        for document in self.documents:
+            document.detect_sections()
+            document.process_document(self.pipeline)
+
+    def get_all_processed_texts(self):
+        result = {}
+        for subject in self.documents:
+            result[subject.name] = subject.get_processed_texts()
+        return result
+
+    def create_corpus_results_consolidation_csv(self):
+        """Create separate aggregated results CSV files for each metric."""
+        print("Creating aggregated results files per metric...")
+        
+        try:
+            derivatives_path = os.path.dirname(os.path.dirname(self.documents[0].results_path))
+        except (AttributeError, IndexError):
+            print("Error: No valid results path found in documents")
+            return
+        
+        # Create aggregations folder
+        aggregation_path = os.path.join(derivatives_path, 'aggregations')
+        os.makedirs(aggregation_path, exist_ok=True)
+        
+        # Initialize results dictionary with metrics as keys
+        results_by_metric = {}
+        
+        # Walk through all directories in derivatives
+        for root, dirs, files in os.walk(derivatives_path):
+            # Skip the aggregations directory itself
+            if 'aggregations' in root:
+                continue
+                
+            for file in files:
+                if not file.endswith('.csv'):
+                    continue
+                    
+                file_path = os.path.join(root, file)
+                try:
+                    df = pd.read_csv(file_path)
+                    subject_key = os.path.basename(file).split('_')[0]
+                    
+                    # Determine metric type from file path
+                    if 'semantic-similarity-window' in file:
+                        metric = 'semantic-similarity'
+                    elif 'distance-from-randomness' in file:
+                        metric = 'distance-from-randomness'
+                    else:
+                        continue
+                    
+                    # Initialize metric dict if not exists
+                    if metric not in results_by_metric:
+                        results_by_metric[metric] = {}
+                    
+                    # Initialize subject dict if not exists
+                    if subject_key not in results_by_metric[metric]:
+                        results_by_metric[metric][subject_key] = {}
+                    
+                    # Process based on metric type
+                    if metric == 'semantic-similarity':
+                        window_size = re.search(r'window-(\d+)', file).group(1)
+                        for _, row in df.iterrows():
+                            if 'Metric' in df.columns and 'Similarity_Score' in df.columns:
+                                metric_name = f"window_{window_size}_{row['Metric']}"
+                                results_by_metric[metric][subject_key][metric_name] = row['Similarity_Score']
+
+                except Exception as e:
+                    print(f"Error processing {file_path}: {e}")
+                    continue
+        
+        # Save separate aggregated results for each metric
+        for metric, metric_results in results_by_metric.items():
+            if metric_results:
+                output_file = os.path.join(aggregation_path, f'{self.name}_{metric}_aggregated_results.csv')
+                pd.DataFrame(metric_results).T.to_csv(output_file)
+                print(f"Aggregated results for {metric} saved to: {output_file}")
+            
+        if not results_by_metric:
+            print("No results to aggregate")
+
+    def extract_logits(self):
+        from pelican_nlp.extraction.extract_logits import LogitsExtractor
+        from pelican_nlp.preprocessing.text_tokenizer import TextTokenizer
+        logits_options = self.config['options_logits']
+        project_path = self.config['PATH_TO_PROJECT_FOLDER']
+
+        print('logits extraction in progress')
+        model_name = logits_options['model_name']
+        logitsExtractor = LogitsExtractor(logits_options,
+                                          self.pipeline,
+                                          project_path)
+        model = Model(model_name, project_path)
+        model.load_model()
+        model_instance = model.model_instance
+        tokenizer = TextTokenizer(logits_options['tokenization_method'], model_name=logits_options['model_name'])
+        for i in range(len(self.documents)):
+
+            for key, section in self.documents[i].cleaned_sections.items():
+
+                if self.config['discourse'] == True:
+                    section = TextDiarizer.parse_speaker(section, self.config['subject_speakertag'],
+                                                         logits_options['keep_speakertags'])
+                    #print(f'parsed section is {section}')
+                else:
+                    section = [section]
+
+                print(f'Extracting Logits for section {key}')
+
+                for part in section:
+                    print(part)
+                    logits = logitsExtractor.extract_features(part, tokenizer, model_instance)
+                    print(logits)
+                    self.documents[i].logits.append(logits)
+
+                    #'logits' list of dictionaries; keys token, logprob_actual, logprob_max, entropy, most_likely_token
+                    store_features_to_csv(logits,
+                                          self.derivative_dir,
+                                          self.documents[i],
+                                          metric='logits')
+
+    def extract_embeddings(self):
+        from pelican_nlp.extraction.extract_embeddings import EmbeddingsExtractor
+
+        embedding_options = self.config['options_embeddings']
+        print('Embeddings extraction in progress...')
+        embeddingsExtractor = EmbeddingsExtractor(embedding_options, self.config['PATH_TO_PROJECT_FOLDER'])
+        for i in range(len(self.documents)):
+            for key, section in self.documents[i].cleaned_sections.items():
+                print(f'Processing section {key}')
+                
+                if self.config['discourse']:
+                    section = TextDiarizer.parse_speaker(section, self.config['subject_speakertag'], embedding_options['keep_speakertags'])
+                else:
+                    section = [section]
+
+                embeddings, token_count = embeddingsExtractor.extract_embeddings_from_text(section)
+                self.documents[i].embeddings.append(embeddings)
+
+                if self.task == 'fluency':
+                    self.documents[i].fluency_word_count = token_count
+                
+                for utterance in embeddings:
+
+                    if self.config['options_embeddings']['semantic-similarity']:
+                        from pelican_nlp.extraction.semantic_similarity import calculate_semantic_similarity, \
+                            get_semantic_similarity_windows
+                        consecutive_similarities, mean_similarity = calculate_semantic_similarity(utterance)
+                        print(f'Mean semantic similarity: {mean_similarity:.4f}')
+
+                        for window_size in self.config['options_semantic-similarity']['window_sizes']:
+                            window_stats = get_semantic_similarity_windows(utterance, window_size)
+                            if isinstance(window_stats, tuple) and len(window_stats) == 5:
+                                window_data = {
+                                    'mean_of_window_means': window_stats[0],
+                                    'std_of_window_means': window_stats[1],
+                                    'mean_of_window_stds': window_stats[2],
+                                    'std_of_window_stds': window_stats[3],
+                                    'mean_of_window_medians': window_stats[4]
+                                }
+                                print(f'Window {window_size} stats - mean: {window_stats[0]:.4f}, std: {window_stats[1]:.4f}, median: {window_stats[4]:.4f}')
+                            else:
+                                window_data = {
+                                    'mean': window_stats[0] if isinstance(window_stats, tuple) else window_stats,
+                                    'std': window_stats[1] if isinstance(window_stats, tuple) and len(window_stats) > 1 else None
+                                }
+                            
+                            store_features_to_csv(window_data,
+                                                  self.derivative_dir,
+                                                  self.documents[i],
+                                                  metric=f'semantic-similarity-window-{window_size}')
+
+                    if self.config['options_embeddings']['distance-from-randomness']:
+                        from pelican_nlp.extraction.distance_from_randomness import get_distance_from_randomness
+                        divergence = get_distance_from_randomness(utterance, self.config["options_dis_from_randomness"])
+                        print(f'Divergence from optimality metrics: {divergence}')
+                        store_features_to_csv(divergence,
+                                              self.derivative_dir,
+                                              self.documents[i],
+                                              metric='distance-from-randomness')
+
+                    # Process tokens
+                    if embedding_options['clean_embedding_tokens']:
+                        cleaned_embeddings = []
+                        if isinstance(utterance, dict):
+                            # Handle dictionary case (PyTorch models)
+                            for token, embedding in utterance.items():
+                                if 'xlm-roberta-base' in self.config['options_embeddings']['model_name'].lower():
+                                    cleaned_token = textcleaner.clean_subword_token_RoBERTa(token)
+                                else:
+                                    cleaned_token = textcleaner.clean_token_generic(token)
+                                if cleaned_token is not None:
+                                    cleaned_embeddings.append((cleaned_token, embedding))
+                        else:
+                            # Handle list of tuples case (fastText)
+                            for token, embedding in utterance:
+                                cleaned_token = textcleaner.clean_token_generic(token)
+                                if cleaned_token is not None:
+                                    cleaned_embeddings.append((cleaned_token, embedding))
+                    else:
+                        cleaned_embeddings = utterance if isinstance(utterance, list) else [(k, v) for k, v in utterance.items()]
+
+                    store_features_to_csv(cleaned_embeddings,
+                                          self.derivative_dir,
+                                          self.documents[i],
+                                          metric='embeddings')
+        return
+
+    def extract_opensmile_features(self):
+        from pelican_nlp.extraction.acoustic_feature_extraction import AudioFeatureExtraction
+        for i in range(len(self.documents)):
+            results, recording_length = AudioFeatureExtraction.opensmile_extraction(self.documents[i].file, self.config['opensmile_configurations'])
+            self.documents[i].recording_length = recording_length  # Store the recording length
+            results['subject_ID'] = self.documents[i].subject_ID  # Set the subject ID
+            print('results obtained')
+            store_features_to_csv(results,
+                                self.derivative_dir,
+                                self.documents[i],
+                                metric='opensmile-features')
+
+    def extract_prosogram(self):
+        from pelican_nlp.extraction.acoustic_feature_extraction import AudioFeatureExtraction
+        for i in range(len(self.documents)):
+            results = AudioFeatureExtraction.extract_prosogram_profile(self.documents[i].file)
+            print('prosogram obtained')
+
+    def create_document_information_csv(self):
+        """Create CSV file with summarized document parameters based on config specifications."""
+        print("Creating document information summary...")
+        
+        try:
+            derivatives_path = os.path.dirname(os.path.dirname(self.documents[0].results_path))
+        except (AttributeError, IndexError):
+            print("Error: No valid results path found in documents")
+            return
+        
+        # Create document_information folder inside aggregations
+        doc_info_path = os.path.join(derivatives_path, 'aggregations', 'document_information')
+        os.makedirs(doc_info_path, exist_ok=True)
+        
+        # Define output file path
+        output_file = os.path.join(doc_info_path, f'{self.name}_document-information.csv')
+        
+        # Get parameters to include from config
+        parameters_to_include = self.config.get('document_information_output', {}).get('parameters', [])
+        
+        if not parameters_to_include:
+            print("Warning: No parameters specified in config for document information output")
+            return
+        
+        # Get document information based on specified parameters
+        document_info = []
+        for doc in self.documents:
+            # Get all attributes using vars()
+            attrs = vars(doc)
+            # Filter based on specified parameters
+            info = {
+                param: attrs.get(param) 
+                for param in parameters_to_include 
+                if param in attrs
+            }
+            document_info.append(info)
+        
+        # Convert to DataFrame and save to CSV
+        df = pd.DataFrame(document_info)
+        df.to_csv(output_file, index=False)
+        print(f"Document information saved to: {output_file}")

pelican_nlp/core/document.py

@@ -0,0 +1,226 @@
+"""
+This module provides the Document class, each instance representing one file within a corpus.
+The Document class stores all document specific information.
+"""
+
+import os
+import re
+from pelican_nlp.preprocessing import TextImporter
+from collections import defaultdict, OrderedDict
+
+class Document:
+
+    def __init__(self, file_path, name, **kwargs):
+        """Initialize Document object.
+        
+        Args:
+            file_path: Path to document file
+            name: Document name
+            **kwargs: Optional document attributes
+        """
+        self.file_path = file_path
+        self.name = name
+        self.file = os.path.join(file_path, name)
+        
+        # Initialize optional attributes
+        self.subject_ID = kwargs.get('subject_ID')
+        self.task = kwargs.get('task')
+        self.num_speakers = kwargs.get('num_speakers')
+        self.has_sections = kwargs.get('has_sections', False)
+        self.has_section_titles = kwargs.get('has_section_titles')
+        self.section_identifier = kwargs.get('section_identifier')
+        self.number_of_sections = kwargs.get('number_of_sections')
+        self.lines = kwargs.get('lines', [])
+        self.new_parameter = kwargs.get('new_parameter')
+        self.another_metric = kwargs.get('another_metric')
+        
+        # Derived attributes
+        self.has_segments = self.task == "discourse"
+        self.segments = [] if self.has_segments else ["default"] * len(self.lines)
+        self.sections = None
+        
+        # Initialize processing attributes
+        self._init_processing_attributes()
+        self._init_document_metrics()
+
+    def _init_processing_attributes(self):
+        """Initialize attributes related to text processing."""
+        self.extension = None
+        self.session = None
+        self.corpus_name = None
+        self.sections = {}
+        self.section_metrics = {}
+        
+        # Load raw text
+        self.importer = TextImporter(self.file_path)
+        self.raw_text = self.importer.load_text(self.file)
+        
+        # Text processing state
+        self.fluency_word_count = None
+        self.fluency_duplicate_count = None
+        self.cleaned_sections = {}
+        self.tokens_logits = []
+        self.tokens_embeddings = []
+        self.normalized_tokens = None
+        self.processed_text = None
+        self.logits = []
+        self.embeddings = []
+        self.acoustic_features = None
+
+    def _init_document_metrics(self):
+        """Initialize document metrics."""
+        self.length_in_lines = len(self.lines)
+        self.length_in_words = sum(line.length_in_words for line in self.lines)
+        self.fluency = None
+        self.number_of_duplicates = None
+        self.number_of_hyphenated_words = None
+
+    def __repr__(self):
+        return f"file_name={self.name}"
+
+    def add_line(self, line):
+        self.lines.append(line)
+        self.length_in_lines = len(self.lines)
+        self.length_in_words += line.length_in_words
+        if not self.has_segments:
+            self.segments.append("default")
+
+    def compile_texts_and_tags(self):
+        self.words, self.word_tags, self.word_segments = [], [], []
+        for line, segment in zip(self.lines, self.segments):
+            line_words = line.text.split()
+            tag = "i" if line.speaker.lower() == "investigator" else "s"
+
+            self.word_segments.extend([segment] * len(line_words))
+            self.words.extend(line_words)
+            self.word_tags.extend([tag] * len(line_words))
+
+    def segment_task(self, protocol, cutoff=1):
+        if not self.has_segments:
+            return self.segments
+
+        patterns = {
+            section: re.compile("|".join(f"(?:\\b{re.escape(term)}\\b)" for term in terms), re.IGNORECASE)
+            for section, terms in protocol.items()
+        }
+
+        match_scores = defaultdict(list)
+        for section, pattern in patterns.items():
+            for line_index, line in enumerate(self.lines):
+                if pattern.search(line.text):
+                    match_scores[section].append(line_index)
+
+        section_order = sorted(protocol.keys(), key=lambda x: int(x))
+        section_starts = OrderedDict()
+        last_index_used = -1
+
+        for section in section_order:
+            line_indices = match_scores[section]
+            valid_starts = [idx for idx in line_indices if idx > last_index_used and len(line_indices) >= cutoff]
+            if valid_starts:
+                start_line = min(valid_starts)
+                section_starts[section] = start_line
+                last_index_used = start_line
+
+        segment_names = ["1"] * len(self.lines)
+        current_section = None
+        for i in range(len(self.lines)):
+            if i in section_starts.values():
+                current_section = [sec for sec, start in section_starts.items() if start == i][0]
+            segment_names[i] = current_section if current_section else "default"
+
+        self.segments = segment_names
+        self.sections = self._create_sections(segment_names)
+        return segment_names
+
+    def _create_sections(self, segment_names):
+        sections = defaultdict(list)
+        for line, segment in zip(self.lines, segment_names):
+            sections[segment].append(line)
+        return sections
+
+    def detect_sections(self):
+        print(f'detecting sections...')
+        if not self.raw_text:
+            raise ValueError("Raw text must be loaded before detecting sections.")
+
+        lines = self.raw_text.splitlines()
+        if not self.has_sections:
+            if self.has_section_titles and lines:
+                title, content = (lines[0].strip(), "\n".join(lines[1:]).strip()) if lines else ("untitled section", "")
+            else:
+                title, content = "untitled section", "\n".join(lines).strip()
+            self.sections = {title: content}
+            print(self.sections)
+            return
+
+        sections = {}
+        current_title, current_content = None, []
+        section_titles = []
+
+        for line in lines:
+            if line.startswith(self.section_identifier):
+                if current_title:
+                    sections[current_title] = "\n".join(current_content).strip()
+
+                current_title = line.strip()
+                section_titles.append(current_title)
+                current_content = []
+            else:
+                if current_title:
+                    current_content.append(line)
+
+        if current_title:
+            sections[current_title] = "\n".join(current_content).strip()
+
+        self.sections = sections
+
+        if self.number_of_sections is not None and len(self.sections) != self.number_of_sections:
+            raise ValueError("Incorrect number of sections detected.")
+
+    def process_document(self, pipeline):
+        print(f"Processing document: {self.name}")
+        pipeline.process_document(self)
+
+    def clean_text(self, cleaner):
+        if not self.sections:
+            raise ValueError("Text must be divided into sections before cleaning.")
+
+        self.cleaned_sections = self.sections.copy()
+        for title, content in self.sections.items():
+            if self.fluency:
+                self.cleaned_sections[title] = (
+                    cleaner.clean_fluency_transcripts(self, content)
+                )
+            else:
+                self.cleaned_sections[title] = (
+                    cleaner.clean(self, content)
+                )
+
+    def tokenize_text(self, tokenizer, purpose):
+        if not self.cleaned_sections:
+            raise ValueError("Text must be cleaned before tokenizing.")
+
+        for _, content in self.cleaned_sections.items():
+            tokens = tokenizer.tokenize(content)
+            if purpose == "logits":
+                self.tokens_logits.append(tokens)
+            elif purpose == "embeddings":
+                self.tokens_embeddings.append(tokens)
+
+    def normalize_text(self, normalizer):
+        if not self.tokens_logits:
+            raise ValueError("Text must be tokenized before normalization.")
+
+        self.normalized_tokens = normalizer.normalize(self.tokens_logits)
+
+    def get_processed_text(self):
+        return " ".join(self.normalized_tokens) if self.normalized_tokens else self.tokens_logits
+
+    def get_document_metadata(self):
+        return {
+            "file_path": self.file_path,
+            "task": self.task,
+            "num_speakers": self.num_speakers,
+            "has_sections": self.has_sections,
+        }

pelican_nlp/core/subject.py

@@ -0,0 +1,30 @@
+"""
+This module provides the Subject class, each instance representing one subject.
+The Subject class stores all subject specific information and a list of corresponding documents.
+"""
+
+class Subject:
+    def __init__(self, subjectID, description=None):
+
+        self.subjectID = subjectID
+        self.gender = None
+        self.age = None
+        self.name = None
+        self.description = description  # Description of the subject
+        self.documents = []  # List of TextDocument instances
+        self.numberOfSessions = None
+
+    def __repr__(self):
+        return f"Subject(ID={self.subjectID})"
+
+    def add_document(self, document):
+        self.documents.append(document)
+        document.subject = self
+
+    def process_subject(self, importer, cleaner, tokenizer, normalizer):
+        print(f'Subject {self.subjectID} is being processed')
+        for document in self.documents:
+            continue
+
+    def get_subject_info(self):
+        return f"Subject: {self.name}\nDescription: {self.description}\nNumber of files: {len(self.documents)}"

pelican_nlp/extraction/__init__.py

@@ -0,0 +1,2 @@
+# Import extraction related classes and functions
+from .language_model import Model

pelican_nlp/extraction/acoustic_feature_extraction.py

@@ -0,0 +1,71 @@
+import audiofile
+import pandas as pd
+
+class AudioFeatureExtraction:
+
+    @staticmethod
+    def opensmile_extraction(file, opensmile_configurations):
+        print(f'opensmile extraction in progress...')
+        import opensmile
+
+        print(f'audio file is: {file}')
+
+        signal, sampling_rate = audiofile.read(
+            file,
+            always_2d=True,
+            duration=opensmile_configurations['duration'],
+            offset=opensmile_configurations['offset']
+        )
+
+        # extract eGeMAPSv02 feature set
+        smile = opensmile.Smile(
+            feature_set=opensmile.FeatureSet.eGeMAPSv02,
+            feature_level=opensmile.FeatureLevel.Functionals
+        )
+
+        output = smile.process_signal(
+            signal,
+            sampling_rate
+        )
+
+        # Create result dictionary with only the values we want
+        result = {}
+        result['subject_ID'] = None  # This will be set by the calling function
+        for feature in smile.feature_names:
+            # Extract just the numerical value from the output
+            result[feature] = float(output[feature].values[0])
+
+        # Get recording length from the index
+        recording_length = output.index[0][1].total_seconds()  # This gets the end time from the MultiIndex
+
+        return result, recording_length
+
+    @staticmethod
+    def extract_prosogram_profile(file):
+        """
+        Extract prosodic features using prosogram through parselmouth.
+
+        Returns:
+            profile (DataFrame): Prosogram analysis results
+        """
+        import parselmouth
+        from pelican.praat import PROSOGRAM_SCRIPT
+        try:
+            sound = parselmouth.Sound(file)
+            # Common Prosogram parameters
+            result = parselmouth.praat.run_file(
+                PROSOGRAM_SCRIPT,
+                arguments=[sound, "save=yes", "draw=no"],
+                capture_output=True
+            )
+            
+            # Convert result into a DataFrame with the same format
+            profile = pd.read_csv(pd.compat.StringIO(result), sep="\t")
+            
+            return profile
+
+        except Exception as e:
+            print(f"Error processing {file}")
+            print(f"Full error message: {str(e)}")
+            raise  # This will show the full stack trace
+            return None