pelican-nlp 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

pelican_nlp/_version.py

@@ -1 +1 @@
-__version__ = "0.2.5"
+__version__ = "0.2.7"

pelican_nlp/cli.py

@@ -7,6 +7,10 @@ def main():
         print("No .yml configuration file found in the current directory.")
         return
 
+    if len(config_files) > 1:
+        print("More than one configuration file found - remove unneeded files from project directory")
+        return
+
     config_file = config_files[0]  # You could also add logic to choose or validate
     print(f"Using configuration file: {config_file}")
 

pelican_nlp/extraction/language_model.py

@@ -1,5 +1,6 @@
 import torch
 import psutil
+import os
 
 from accelerate import init_empty_weights, infer_auto_device_map, dispatch_model
 from transformers import AutoModelForCausalLM
@@ -17,8 +18,31 @@ class Model:
         if self.model_name == 'fastText':
             import fasttext
             import fasttext.util
-            fasttext.util.download_model('de', if_exists='ignore')
-            self.model_instance = fasttext.load_model('cc.de.300.bin')
+            
+            # Create a model directory if it doesn't exist
+            model_dir = os.path.join(os.path.expanduser('~'), '.fasttext')
+            os.makedirs(model_dir, exist_ok=True)
+            
+            # Set the model path using proper OS path joining
+            model_path = os.path.join(model_dir, 'cc.de.300.bin')
+            
+            # Download only if model doesn't exist
+            if not os.path.exists(model_path):
+                try:
+                    fasttext.util.download_model('de', if_exists='ignore')
+                except OSError:
+                    # Direct download fallback for Windows
+                    import urllib.request
+                    url = 'https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.de.300.bin.gz'
+                    urllib.request.urlretrieve(url, model_path + '.gz')
+                    # Decompress the file
+                    import gzip
+                    with gzip.open(model_path + '.gz', 'rb') as f_in:
+                        with open(model_path, 'wb') as f_out:
+                            f_out.write(f_in.read())
+                    os.remove(model_path + '.gz')
+            
+            self.model_instance = fasttext.load_model(model_path)
             print('FastText model loaded.')
         elif self.model_name == 'xlm-roberta-base':
             from transformers import AutoModel

pelican_nlp/main.py

@@ -24,6 +24,8 @@ from pelican_nlp.core import Corpus
 from pelican_nlp.utils.setup_functions import subject_instantiator, load_config, remove_previous_derivative_dir
 from pelican_nlp.preprocessing import LPDS
 
+project_path = '/home/yvespauli/PycharmProjects/PyPI_testing_fluency/config_fluency.yml'
+
 class Pelican:
 
     """Main class for the Pelican project handling document processing and metric extraction."""
@@ -31,7 +33,7 @@ class Pelican:
     def __init__(self, config_path: str = None, dev_mode: bool = True) -> None:
         self.dev_mode = dev_mode
         
-        # If no config path is provided, use the default config from package
+        # If no config path is provided, use the default config from package; used for dev-mode
         if config_path is None:
             package_dir = Path(__file__).parent
             default_config = package_dir / 'configuration_files' / 'config_fluency.yml'
@@ -205,4 +207,4 @@ class Pelican:
 
 
 if __name__ == '__main__':
-    Pelican().run()
+    Pelican(project_path).run()

pelican_nlp/sample_configuration_files/config_general.yml

@@ -5,7 +5,7 @@
 # -------------
 input_file: "text"  # Options: 'text' or 'audio'
 language: "german"
-recompute_everything: true  # If false, reuses previously computed results
+recompute_everything: true #If false will give warning if output folder already exists
 
 # Task Configuration
 # -----------------
@@ -20,7 +20,7 @@ corpus_names:  # List of task corpora
 multiple_sessions: false
 number_of_subjects: null  # If null, auto-detected
 number_of_speakers: 1
-subject_speakertag: null  # Speaker tag for subject (e.g., "B")
+subject_speakertag: null  # Speaker tag for subject (e.g., "B"), only for discourse
 
 # Document Structure
 # ----------------

{pelican_nlp-0.2.5.dist-info → pelican_nlp-0.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pelican_nlp
-Version: 0.2.5
+Version: 0.2.7
 Summary: Preprocessing and Extraction of Linguistic Information for Computational Analysis
 Author-email: Yves Pauli <yves.pauli@gmail.com>
 License-Expression: CC-BY-NC-4.0
@@ -65,6 +65,18 @@ pelican_nlp stands for "Preprocessing and Extraction of Linguistic Information f
 Installation
 ============
 
+Create conda environment
+
+.. code-block:: bash
+
+    conda create -n pelican-nlp -c defaults python=3.10
+
+Activate environment
+
+.. code-block:: bash
+
+    conda activate pelican-nlp
+
 Install the package using pip:
 
 .. code-block:: bash
@@ -80,12 +92,12 @@ For the latest development version:
 Usage
 =====
 
-To use the pelican_nlp package:
+To run pelican_nlp you need a configuration.yml file in your project directory, which specifies the configurations used for your project.
+Sample configuration files can be found on the pelican_nlp github repository: https://github.com/ypauli/pelican_nlp/tree/main/sample_configuration_files
 
-Adapt your configuration file to your needs.
-ALWAYS change the specified project folder location.
+Adapt your configuration file to your needs and save your personal configuration.yml file to your main project directory.
 
-Save configuration file to main project directory.
+Running pelican_nlp with your configurations can be done directly from the command line interface or via Python script.
 
 Run from command line:
 
@@ -93,13 +105,16 @@ Navigate to main project directory in command line and enter the following comma
 
 .. code-block:: bash
 
+    conda activate pelican-nlp
     pelican-run
 
 
 Run with python script:
 
 Create python file with IDE of your choice (e.g. Visual Studio Code, Pycharm, etc.) and copy the following code into the file:
+Make sure to use the previously created conda environment 'pelican-nlp' for your project.
 
+Run the following Python code:
 .. code-block:: python
 
     from pelican_nlp.main import Pelican
@@ -142,8 +157,7 @@ Features
 Examples
 ========
 
-You can find example setups in the [`examples/`](https://github.com/ypauli/pelican_nlp/tree/main/examples) folder.
-ALWAYS change the path to the project folder specified in the configuration file to your specific project location.
+You can find example setups on the github repository in the `examples <https://github.com/ypauli/pelican_nlp/tree/main/examples>`_ folder:
 
 Contributing
 ============

{pelican_nlp-0.2.5.dist-info → pelican_nlp-0.2.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
 pelican_nlp/__init__.py,sha256=TD5xjKeXXAH6nUWG-6igbClgovi5r8RIEqI_ix1QeYo,204
-pelican_nlp/_version.py,sha256=_VIP0UQ_JqCj1G-6kKLIYhaPRNuXStwJz-CIhCgKrHg,21
-pelican_nlp/cli.py,sha256=uXouL67mTjBynFMWpBTaGvGtEBWDnbS1BI7aRdxV-0M,439
-pelican_nlp/main.py,sha256=iTaV0xuftFijsn1QLk_6iZ2AQDs-r6Z9l0sACOXFwGo,7539
+pelican_nlp/_version.py,sha256=LIho7asb0pp1iNbJvXEhRMluyGN4gB4RHIIbAKpROsc,21
+pelican_nlp/cli.py,sha256=mPz-ASIMUme69G6YGVpTnHr5VfM3XA4h29WFd7DXpa4,588
+pelican_nlp/main.py,sha256=HX2Rbl4j7RXaMXlGCtggBBqcg3gRh-ey1PdLsQcDX30,7660
 pelican_nlp/Nils_backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pelican_nlp/Nils_backup/extract_acoustic_features.py,sha256=eSP8lXxbZ15YE1HqxGtma9uWOcSN-fI-ig-NwQ9eOA8,10771
 pelican_nlp/Nils_backup/speaker_diarization_Nils.py,sha256=3RIhjKihu4Z1rruMt9KESFE2lqesfzIpRr7rLummUEo,10219
@@ -46,7 +46,7 @@ pelican_nlp/extraction/acoustic_feature_extraction.py,sha256=6Csrr6uotarhuAzxYlG
 pelican_nlp/extraction/distance_from_randomness.py,sha256=yikZ3GK2dqpzuNFPVsjuUK0lo6kHOIoIhKPaVrGXRMQ,3365
 pelican_nlp/extraction/extract_embeddings.py,sha256=e5bcNlskd7f-JkWtfd7YutGV5bqcURKrAkETRyTx93Q,2457
 pelican_nlp/extraction/extract_logits.py,sha256=Lc7Es86T8mlSvLMhiDHpFdCc0kCZ9fNr3-VFnOyeybs,3869
-pelican_nlp/extraction/language_model.py,sha256=4tHJZIRCEeHVTwEf2jmOtu-zDGkdXiDjKmlpuxDuLiw,2929
+pelican_nlp/extraction/language_model.py,sha256=npew_4ziTCNE87pjN8LL0eTPujlewVr8pMT7BsmzEjo,4038
 pelican_nlp/extraction/semantic_similarity.py,sha256=QhY5CAOAorxEo3UBWPlMegFvbySF0KH6j4j3m2I3_NY,2552
 pelican_nlp/extraction/test_documents/test_features.csv,sha256=LR_3m4vIm-YWKw5gI5ziswhS-NF9VhKv14c2udLxtJU,488482
 pelican_nlp/extraction/test_documents/wallace_1.15_3.txt,sha256=ShXxOHUZzGPNUqIcOn6-OYkarzNtTC22V05a_Xpvtlw,3731
@@ -64,14 +64,14 @@ pelican_nlp/preprocessing/text_normalizer.py,sha256=huo5VFqJ0p2jq-ud1047XvMu1qNe
 pelican_nlp/preprocessing/text_tokenizer.py,sha256=h875bXr0YuMrLh4HtQUvpHmASScddtkQXGaF9mm7uwU,1642
 pelican_nlp/sample_configuration_files/config_discourse.yml,sha256=OaTCoMwhDjrOIBpw1nKXWIoSWRUUFNjGQdgQZHVrJn0,3570
 pelican_nlp/sample_configuration_files/config_fluency.yml,sha256=JYpq90K4AF5TslzESJK6Nidw6-D1IiqD_6cdmlCd5-w,2990
-pelican_nlp/sample_configuration_files/config_general.yml,sha256=T-rIYROERuG3PDKlUUlfNUeh35q7icW25Mg6Otzc1U4,3392
+pelican_nlp/sample_configuration_files/config_general.yml,sha256=-GAVATlqXuQq4ANSW0JauwIGhr7ET_oZiBiM7I40AkA,3424
 pelican_nlp/utils/__init__.py,sha256=q1tGdOOj5UPRC2mGhoMUh8p4cbFCkkbD21bQaOVvFao,189
 pelican_nlp/utils/csv_functions.py,sha256=hsG73gm3Up9sAerp6gIxuNHaeP1vJj6HSh7ggVm1SSo,7272
 pelican_nlp/utils/sample_usage.py,sha256=W__OVMjWND-ZtxxRhfGJDHwbVpGlB-anXDxyA5P4cME,353
 pelican_nlp/utils/setup_functions.py,sha256=t4WG5qd5iYpNNBGklje_8ukwmJp_C9RMLLi7veDgNeA,3574
-pelican_nlp-0.2.5.dist-info/licenses/LICENSE,sha256=m3jshBZIXKiBX6qhmhtJcLTVJ1N6BEkQGIflneXvpYg,19336
-pelican_nlp-0.2.5.dist-info/METADATA,sha256=G0-X8_BeAE27zXEjBDYF3ri0i6-Ft1oxkTsL5gdinnw,5562
-pelican_nlp-0.2.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-pelican_nlp-0.2.5.dist-info/entry_points.txt,sha256=znlG0paAfju9P10UM3rm5HcCHoj4tarTllNpeaqH_gc,53
-pelican_nlp-0.2.5.dist-info/top_level.txt,sha256=F0qlyqy5FCd3sTS_npUYPeLKN9_BZq6wD4qo9pI0xbg,12
-pelican_nlp-0.2.5.dist-info/RECORD,,
+pelican_nlp-0.2.7.dist-info/licenses/LICENSE,sha256=m3jshBZIXKiBX6qhmhtJcLTVJ1N6BEkQGIflneXvpYg,19336
+pelican_nlp-0.2.7.dist-info/METADATA,sha256=YyZBYza89dtKbvLLHXkxOEZ1BODloXBjh-zZSODLfVI,6155
+pelican_nlp-0.2.7.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+pelican_nlp-0.2.7.dist-info/entry_points.txt,sha256=znlG0paAfju9P10UM3rm5HcCHoj4tarTllNpeaqH_gc,53
+pelican_nlp-0.2.7.dist-info/top_level.txt,sha256=F0qlyqy5FCd3sTS_npUYPeLKN9_BZq6wD4qo9pI0xbg,12
+pelican_nlp-0.2.7.dist-info/RECORD,,