pelican-nlp 0.1.0__py3-none-any.whl

pelican_nlp/utils/setup_functions.py

@@ -0,0 +1,93 @@
+import os
+from pelican_nlp.core.subject import Subject
+import shutil
+import yaml
+import sys
+
+def subject_instantiator(config):
+    project_folder = config['PATH_TO_PROJECT_FOLDER']
+    path_to_subjects = os.path.join(project_folder, 'subjects')
+    print('Instantiating Subjects...')
+    subjects = [Subject(subject) for subject in os.listdir(path_to_subjects)]
+
+    # Identifying all subject files
+    for subject in subjects:
+        if config['multiple_sessions']:
+            paths = _get_subject_sessions(subject, project_folder)
+        else:
+            paths = [os.path.join(path_to_subjects, subject.subjectID)]
+
+        for path in paths:
+            file_path = os.path.join(path, config['task_name'])
+            subject.documents.extend(_instantiate_documents(file_path, subject.subjectID, config))
+        print(f'all identified subject documents for subject {subject.subjectID}: {subject.documents}')
+        for document in subject.documents:
+            parts = document.file_path.split(os.sep)
+            
+            # Adjust path components based on whether session exists
+            if config.get('multiple_sessions', False):
+                subject_ID, session, task = parts[-4], parts[-3], parts[-2]
+                document.results_path = os.path.join(project_folder, 'derivatives', subject_ID, session, task)
+            else:
+                subject_ID, task = parts[-3], parts[-2]
+                document.results_path = os.path.join(project_folder, 'derivatives', subject_ID, task)
+
+    return subjects
+
+def _get_subject_sessions(subject, project_path):
+    session_dir = os.path.join(os.path.join(project_path, 'subjects'), subject.subjectID)
+    session_paths = [
+        os.path.join(session_dir, session)
+        for session in os.listdir(session_dir)
+        if os.path.isdir(os.path.join(session_dir, session))
+    ]
+    subject.numberOfSessions = len(session_paths)
+    return session_paths
+
+def _instantiate_documents(filepath, subject, config):
+
+    if config['input_file']=='text':
+        from pelican_nlp.core.document import Document
+        return [
+            Document(
+                filepath,
+                file_name,
+                subject_ID = subject,
+                task=config['task_name'],
+                fluency=config['fluency_task'],
+                has_sections=config['has_multiple_sections'],
+                section_identifier=config['section_identification'],
+                number_of_sections=config['number_of_sections'],
+                num_speakers=config['number_of_speakers'],
+                has_section_titles=config['has_section_titles']
+            )
+            for file_name in os.listdir(filepath)
+        ]
+
+    elif config['input_file']=='audio':
+        from pelican_nlp.core.audio_document import AudioFile
+        return [
+            AudioFile(
+                filepath,
+                file_name,
+                subject_ID=subject,
+                task=config['task_name'],
+                fluency=config['fluency_task'],
+                num_speakers=config['number_of_speakers'],
+            )
+            for file_name in os.listdir(filepath)
+        ]
+
+def remove_previous_derivative_dir(output_directory):
+    if os.path.isdir(output_directory):
+        shutil.rmtree(output_directory)
+
+def ignore_files(directory, files):
+    return [f for f in files if os.path.isfile(os.path.join(directory, f))]
+
+def load_config(config_path):
+    try:
+        with open(config_path, 'r') as stream:
+            return yaml.safe_load(stream)
+    except yaml.YAMLError as exc:
+        sys.exit(f"Error loading configuration: {exc}")

pelican_nlp-0.1.0.dist-info/METADATA

@@ -0,0 +1,146 @@
+Metadata-Version: 2.4
+Name: pelican_nlp
+Version: 0.1.0
+Summary: Preprocessing and Extraction of Linguistic Information for Computational Analysis
+Author-email: Yves Pauli <yves.pauli@gmail.com>
+License-Expression: CC-BY-NC-4.0
+Project-URL: Homepage, https://github.com/ypauli/PELICAN_nlp
+Project-URL: Repository, https://github.com/ypauli/PELICAN_nlp
+Project-URL: Documentation, https://github.com/ypauli/PELICAN_nlp#readme
+Project-URL: Bug Tracker, https://github.com/ypauli/PELICAN_nlp/issues
+Keywords: nlp,linguistics,preprocessing,language-processing,text-analysis
+Classifier: Development Status :: 1 - Planning
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Natural Language :: English
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: numpy==2.0.1
+Requires-Dist: pandas==2.2.3
+Requires-Dist: PyYAML>=6.0.2
+Requires-Dist: torch==2.5.1
+Requires-Dist: spacy==3.8.2
+Requires-Dist: transformers==4.49.0
+Requires-Dist: docx2txt>=0.9
+Requires-Dist: striprtf>=0.0.28
+Requires-Dist: chardet>=4.0.0
+Requires-Dist: scikit_learn>=1.6.1
+Requires-Dist: scipy==1.15.2
+Requires-Dist: fasttext-wheel==0.9.2
+Requires-Dist: matplotlib>=3.10.0
+Requires-Dist: seaborn>=0.13.2
+Requires-Dist: accelerate==1.4.0
+Requires-Dist: editdistance>=0.8.1
+Requires-Dist: psutil>=6.1.0
+Requires-Dist: tqdm==4.67.1
+Requires-Dist: pytest>=8.3.4
+Requires-Dist: statsmodels>=0.14.4
+Requires-Dist: datasets==3.3.2
+Requires-Dist: huggingface_hub==0.29.2
+Dynamic: license-file
+
+====================================
+PELICAN_nlp
+====================================
+
+PELICAN_nlp stands for "Preprocessing and Extraction of Linguistic Information for Computational Analysis - Natural Language Processing". This package enables the creation of standardized and reproducible language processing pipelines, extracting linguistic features from various tasks like discourse, fluency, and image descriptions.
+
+.. image:: https://img.shields.io/pypi/v/package-name.svg
+    :target: https://pypi.org/project/package-name/
+    :alt: PyPI version
+
+.. image:: https://img.shields.io/github/license/username/package-name.svg
+    :target: https://github.com/ypauli/PELICAN-nlp/blob/main/LICENSE
+    :alt: License
+
+.. image:: https://img.shields.io/pypi/pyversions/package-name.svg
+    :target: https://pypi.org/project/package-name/
+    :alt: Supported Python Versions
+
+Installation
+============
+
+Install the package using pip:
+
+.. code-block:: bash
+
+    pip install pelican_nlp
+
+For the latest development version:
+
+.. code-block:: bash
+
+    pip install git+https://github.com/ypauli/PELICAN_nlp.git
+
+Usage
+=====
+
+To use the pelican_nlp package:
+
+.. code-block:: python
+
+    from pelican_nlp.main import Pelican
+
+    configuration_file = "/path/to/your/config/file"
+    pelican = Pelican(configuration_file)
+    pelican.run()
+
+For reliable operation, data must be stored in the *Language Processing Data Structure (LPDS)* format, inspired by brain imaging data structure conventions.
+
+Text and audio files should follow this naming convention:
+
+subjectID_sessionID_task_task-supplement_corpus.extension
+
+- subjectID: ID of subject (e.g., sub-01), mandatory
+- sessionID: ID of session (e.g., ses-01), if available
+- task: task used for file creation, mandatory
+- task-supplement: additional information regarding the task, if available
+- corpus: (e.g., healthy-control / patient) specify files belonging to the same group, mandatory
+- extension: file extension (e.g., txt / pdf / docx / rtf), mandatory
+
+Example filenames:
+- sub-01_ses-01_interview_schizophrenia.rtf
+- sub-03_ses-02_fluency_semantic_animals.docx
+
+To optimize performance, close other programs and limit GPU usage during language processing.
+
+Features
+========
+
+- **Feature 1: Cleaning text files**
+    - Handles whitespaces, timestamps, punctuation, special characters, and case-sensitivity.
+
+- **Feature 2: Linguistic Feature Extraction**
+    - Extracts semantic embeddings, logits, distance from optimality, and semantic similarity.
+
+Examples
+========
+
+Here's a detailed usage example:
+
+.. code-block:: python
+
+    from package_name import SomeClass
+
+    configuration_file = "config_fluency.yml"
+    pelican.run(configuration_file)
+
+*Link to config_fluency.yml*
+
+Sample folder for data collection of the semantic fluency task:
+*Link to sample_folder*
+
+Contributing
+============
+
+Contributions are welcome! Please check out the `contributing guide <https://github.com/ypauli/PELICAN/blob/main/CONTRIBUTING.md>`_.
+
+License
+=======
+
+This project is licensed under Attribution-NonCommercial 4.0 International. See the `LICENSE <https://github.com/ypauli/PELICAN/blob/main/LICENSE>`_ file for details.

pelican_nlp-0.1.0.dist-info/RECORD

@@ -0,0 +1,39 @@
+pelican_nlp/__init__.py,sha256=yLyG5Amt7nuHQMHz0tuGUVsGBtgVmXT0LMeRo3an-OU,192
+pelican_nlp/main.py,sha256=xKUqqA3sh9kbk07lKA_poILIU1c8oIeaSsVqPOPY5Tk,7596
+pelican_nlp/core/__init__.py,sha256=whJc5dWsGsKn2IAw-D4BvCvUKW1sVtWYE1WJIuUr5uI,165
+pelican_nlp/core/audio_document.py,sha256=hhSJNgeqSYa6_uws2ho66agHhAdHuKN3EIEdIsIcXKg,586
+pelican_nlp/core/corpus.py,sha256=6pDRmeO0XoHylhjLE4Fi5Tc3HCMQJ-Xk0YRzEfz5Z1Y,15168
+pelican_nlp/core/document.py,sha256=j2HP5FX6cfmXHo7OWVFCX6cMsDyqsOmNlnGNNNfCm2c,8467
+pelican_nlp/core/subject.py,sha256=-pi3jDzb2zLiG8JNAi9i-9Jd-VtsPxDO4ShQci2QSMg,1059
+pelican_nlp/extraction/__init__.py,sha256=hfqFiaKpQBS6cwRm9Yd7MpOcV60_xJmwuQ2Kegary5k,84
+pelican_nlp/extraction/acoustic_feature_extraction.py,sha256=6Csrr6uotarhuAzxYlGFAil9K4PLUqa9vWw607peRoA,2319
+pelican_nlp/extraction/distance_from_randomness.py,sha256=yikZ3GK2dqpzuNFPVsjuUK0lo6kHOIoIhKPaVrGXRMQ,3365
+pelican_nlp/extraction/extract_embeddings.py,sha256=e5bcNlskd7f-JkWtfd7YutGV5bqcURKrAkETRyTx93Q,2457
+pelican_nlp/extraction/extract_logits.py,sha256=Lc7Es86T8mlSvLMhiDHpFdCc0kCZ9fNr3-VFnOyeybs,3869
+pelican_nlp/extraction/language_model.py,sha256=4tHJZIRCEeHVTwEf2jmOtu-zDGkdXiDjKmlpuxDuLiw,2929
+pelican_nlp/extraction/semantic_similarity.py,sha256=QhY5CAOAorxEo3UBWPlMegFvbySF0KH6j4j3m2I3_NY,2552
+pelican_nlp/extraction/test_documents/test_features.csv,sha256=LR_3m4vIm-YWKw5gI5ziswhS-NF9VhKv14c2udLxtJU,488482
+pelican_nlp/extraction/test_documents/wallace_1.15_3.txt,sha256=ShXxOHUZzGPNUqIcOn6-OYkarzNtTC22V05a_Xpvtlw,3731
+pelican_nlp/extraction/test_documents/wallace_1.1_3.txt,sha256=gs5REE10myK3Nm9JBOV8hjqKcMRkrl7BasuK7HSBe5M,3695
+pelican_nlp/extraction/test_documents/wallace_1_4.txt,sha256=95Z7gS92KERCocrbOAFbJntf5QoE-6p0GL67XQEffqI,3963
+pelican_nlp/metrics_statistics/embeddings_metrics_statistics.py,sha256=svXXyLEA62mLa0KUfSiOSFFMjYk17K7BJbxUoLf0l9w,1468
+pelican_nlp/preprocessing/LPDS.py,sha256=4UWkMMSrdU-nWVi8eKiWQSGD7f7lemB42aI0fFn6ZLU,4097
+pelican_nlp/preprocessing/__init__.py,sha256=ZYgOUlKPXmltYez3urPZmsAWRWSEqZ3_l_gN2aqd15s,293
+pelican_nlp/preprocessing/pipeline.py,sha256=t2zJAvZRO12MdAKQgm8XZxfZND7_8gFtzHF9Rq2L2aE,1796
+pelican_nlp/preprocessing/speaker_diarization.py,sha256=N6dZCa2AHHGw__g9e-ZUyZM_In0-nzFOkZ44cBnoKLk,1122
+pelican_nlp/preprocessing/text_cleaner.py,sha256=QKqxwoRR8dnuBYiY-PXK1kB7744TVUcUMJb7dbKvXGk,7512
+pelican_nlp/preprocessing/text_importer.py,sha256=FtSyJjFXDxVle7Jpyw6EqCLDbLTCRxqVQi9ymWWtPB4,1356
+pelican_nlp/preprocessing/text_normalizer.py,sha256=huo5VFqJ0p2jq-ud1047XvMu1qNeaiuG879SF3zkJoM,894
+pelican_nlp/preprocessing/text_tokenizer.py,sha256=h875bXr0YuMrLh4HtQUvpHmASScddtkQXGaF9mm7uwU,1642
+pelican_nlp/sample_configuration_files/config_discourse.yml,sha256=xVHIUpSORV6iR0nEvuess6rfiAvuGEkqmaMWD_6kyFE,3618
+pelican_nlp/sample_configuration_files/config_fluency.yml,sha256=oQ6Y2BhRLExEMpS3VRH2pFrGHi788L66aSYUm05nV_A,3038
+pelican_nlp/sample_configuration_files/config_general.yml,sha256=UuGnZUa-SVmioE9NmXWOMKuv3uG5mNjIuXgA6-Y0JS0,3440
+pelican_nlp/utils/__init__.py,sha256=q1tGdOOj5UPRC2mGhoMUh8p4cbFCkkbD21bQaOVvFao,189
+pelican_nlp/utils/csv_functions.py,sha256=hsG73gm3Up9sAerp6gIxuNHaeP1vJj6HSh7ggVm1SSo,7272
+pelican_nlp/utils/sample_usage.py,sha256=W__OVMjWND-ZtxxRhfGJDHwbVpGlB-anXDxyA5P4cME,353
+pelican_nlp/utils/setup_functions.py,sha256=s0QcarswU8qeFBcEQNIYC1ooaD-xwRiTJn--yPEId8E,3612
+pelican_nlp-0.1.0.dist-info/licenses/LICENSE,sha256=m3jshBZIXKiBX6qhmhtJcLTVJ1N6BEkQGIflneXvpYg,19336
+pelican_nlp-0.1.0.dist-info/METADATA,sha256=kIWgpFUOeQC1c-DYvSPoN82OXBgV7TJtPLUGLNC5KDs,4947
+pelican_nlp-0.1.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+pelican_nlp-0.1.0.dist-info/top_level.txt,sha256=F0qlyqy5FCd3sTS_npUYPeLKN9_BZq6wD4qo9pI0xbg,12
+pelican_nlp-0.1.0.dist-info/RECORD,,

pelican_nlp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (78.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+