assert-llm-tools 0.0.3__tar.gz

assert_llm_tools-0.0.3/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.1
+Name: assert_llm_tools
+Version: 0.0.3
+Summary: A library for generating test assertions using Large Language Models
+Author-email: Charlie Douglas <cdouglas@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yourusername/assert-llm
+Project-URL: Bug Tracker, https://github.com/yourusername/assert-llm/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: anthropic>=0.3.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=0.19.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.0.0; extra == "dev"
+Requires-Dist: flake8>=4.0.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+
+# ASSERT TOOLS LLM
+
+This repository contains tools for evaluating the quality of summaries generated by LLMs.
+
+## Installation
+
+```bash
+pip install assert-tools-llm
+```
+
+## Usage
+
+```python
+# test_assert.py
+from assert_tools_llm.core import evaluate_summary
+
+# Example text from an article
+full_text = """
+Artificial intelligence is rapidly transforming the world economy. Companies 
+are investing billions in AI research and development, leading to breakthroughs 
+in automation, data analysis, and decision-making processes. While this 
+technology offers immense benefits, it also raises concerns about job 
+displacement and ethical considerations.
+"""
+
+# Example summary
+summary = """
+AI is transforming the economy through major investments, bringing advances in 
+automation and analytics while raising job and ethical concerns.
+"""
+
+# Get evaluation metrics
+metrics = evaluate_summary(full_text, summary)
+
+# Print results
+print("\nOriginal Text:")
+print(full_text)
+print("\nSummary:")
+print(summary)
+print("\nEvaluation Metrics:")
+for metric, score in metrics.items():
+    print(f"{metric}: {score:.4f}")
+
+```
+
+## LICENSE
+
+MIT License
+
+Copyright (c) 2024
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+## Acknowledgements
+
+- [ROUGE](https://github.com/google-research/google-research/tree/master/rouge)
+- [NLTK](https://www.nltk.org/)

assert_llm_tools-0.0.3/README.md

@@ -0,0 +1,73 @@
+# ASSERT TOOLS LLM
+
+This repository contains tools for evaluating the quality of summaries generated by LLMs.
+
+## Installation
+
+```bash
+pip install assert-tools-llm
+```
+
+## Usage
+
+```python
+# test_assert.py
+from assert_tools_llm.core import evaluate_summary
+
+# Example text from an article
+full_text = """
+Artificial intelligence is rapidly transforming the world economy. Companies 
+are investing billions in AI research and development, leading to breakthroughs 
+in automation, data analysis, and decision-making processes. While this 
+technology offers immense benefits, it also raises concerns about job 
+displacement and ethical considerations.
+"""
+
+# Example summary
+summary = """
+AI is transforming the economy through major investments, bringing advances in 
+automation and analytics while raising job and ethical concerns.
+"""
+
+# Get evaluation metrics
+metrics = evaluate_summary(full_text, summary)
+
+# Print results
+print("\nOriginal Text:")
+print(full_text)
+print("\nSummary:")
+print(summary)
+print("\nEvaluation Metrics:")
+for metric, score in metrics.items():
+    print(f"{metric}: {score:.4f}")
+
+```
+
+## LICENSE
+
+MIT License
+
+Copyright (c) 2024
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+## Acknowledgements
+
+- [ROUGE](https://github.com/google-research/google-research/tree/master/rouge)
+- [NLTK](https://www.nltk.org/)

assert_llm_tools-0.0.3/assert_llm_tools/__init__.py

@@ -0,0 +1,37 @@
+import nltk
+import os
+from pathlib import Path
+
+
+def initialize_nltk():
+    """
+    Initialize required NLTK data, downloading if necessary.
+    Creates a data directory in the user's home directory if needed.
+    """
+    try:
+        # Set NLTK data path to user's home directory
+        nltk_data_dir = str(Path.home() / ".assert_tools" / "nltk_data")
+        os.makedirs(nltk_data_dir, exist_ok=True)
+        nltk.data.path.append(nltk_data_dir)
+
+        # List of required NLTK resources
+        required_resources = ["punkt", "punkt_tab"]
+
+        # Download all required resources
+        for resource in required_resources:
+            try:
+                nltk.data.find(f"tokenizers/{resource}")
+            except LookupError:
+                print(f"Downloading {resource}...")
+                nltk.download(resource, download_dir=nltk_data_dir, quiet=False)
+
+        return True
+    except Exception as e:
+        print(f"Warning: Failed to initialize NLTK data: {str(e)}")
+        return False
+
+
+# Initialize when the package is imported
+print("Initializing NLTK resources...")
+initialize_nltk()
+print("NLTK initialization complete.")

assert_llm_tools-0.0.3/assert_llm_tools/core.py

@@ -0,0 +1,26 @@
+from typing import Dict, Union, List
+from .metrics.rouge import calculate_rouge
+from .metrics.bleu import calculate_bleu
+
+
+def evaluate_summary(full_text: str, summary: str) -> Dict[str, float]:
+    """
+    Evaluate a summary against the original text using ROUGE and BLEU scores.
+
+    Args:
+        full_text (str): The original full text
+        summary (str): The summary to evaluate
+
+    Returns:
+        Dict[str, float]: Dictionary containing various evaluation metrics
+    """
+    # Calculate ROUGE scores
+    rouge_scores = calculate_rouge(full_text, summary)
+
+    # Calculate BLEU score
+    bleu_score = calculate_bleu(full_text, summary)
+
+    # Combine all metrics
+    metrics = {**rouge_scores, "bleu": bleu_score}
+
+    return metrics

assert_llm_tools-0.0.3/assert_llm_tools/metrics/bleu.py

@@ -0,0 +1,16 @@
+from nltk.translate.bleu_score import sentence_bleu
+from nltk.tokenize import word_tokenize
+from .. import initialize_nltk
+
+
+def calculate_bleu(reference: str, candidate: str) -> float:
+    """
+    Calculate BLEU score for a candidate summary against a reference text.
+    """
+    # Ensure NLTK is initialized before calculating scores
+    initialize_nltk()
+
+    reference_tokens = [word_tokenize(reference)]
+    candidate_tokens = word_tokenize(candidate)
+
+    return sentence_bleu(reference_tokens, candidate_tokens)

assert_llm_tools-0.0.3/assert_llm_tools/metrics/rouge.py

@@ -0,0 +1,15 @@
+from rouge_score import rouge_scorer
+
+
+def calculate_rouge(reference: str, candidate: str) -> dict[str, float]:
+    """
+    Calculate ROUGE scores for a candidate summary against a reference text.
+    """
+    scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True)
+    scores = scorer.score(reference, candidate)
+
+    return {
+        "rouge1": scores["rouge1"].fmeasure,
+        "rouge2": scores["rouge2"].fmeasure,
+        "rougeL": scores["rougeL"].fmeasure,
+    }

assert_llm_tools-0.0.3/assert_llm_tools/utils.py

@@ -0,0 +1,9 @@
+def preprocess_text(text: str) -> str:
+    """
+    Preprocess text by cleaning and normalizing it.
+    """
+    # Remove extra whitespace
+    text = " ".join(text.split())
+    # Convert to lowercase
+    text = text.lower()
+    return text

assert_llm_tools-0.0.3/assert_llm_tools.egg-info/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.1
+Name: assert_llm_tools
+Version: 0.0.3
+Summary: A library for generating test assertions using Large Language Models
+Author-email: Charlie Douglas <cdouglas@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yourusername/assert-llm
+Project-URL: Bug Tracker, https://github.com/yourusername/assert-llm/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: anthropic>=0.3.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=0.19.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.0.0; extra == "dev"
+Requires-Dist: flake8>=4.0.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+
+# ASSERT TOOLS LLM
+
+This repository contains tools for evaluating the quality of summaries generated by LLMs.
+
+## Installation
+
+```bash
+pip install assert-tools-llm
+```
+
+## Usage
+
+```python
+# test_assert.py
+from assert_tools_llm.core import evaluate_summary
+
+# Example text from an article
+full_text = """
+Artificial intelligence is rapidly transforming the world economy. Companies 
+are investing billions in AI research and development, leading to breakthroughs 
+in automation, data analysis, and decision-making processes. While this 
+technology offers immense benefits, it also raises concerns about job 
+displacement and ethical considerations.
+"""
+
+# Example summary
+summary = """
+AI is transforming the economy through major investments, bringing advances in 
+automation and analytics while raising job and ethical concerns.
+"""
+
+# Get evaluation metrics
+metrics = evaluate_summary(full_text, summary)
+
+# Print results
+print("\nOriginal Text:")
+print(full_text)
+print("\nSummary:")
+print(summary)
+print("\nEvaluation Metrics:")
+for metric, score in metrics.items():
+    print(f"{metric}: {score:.4f}")
+
+```
+
+## LICENSE
+
+MIT License
+
+Copyright (c) 2024
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+## Acknowledgements
+
+- [ROUGE](https://github.com/google-research/google-research/tree/master/rouge)
+- [NLTK](https://www.nltk.org/)

assert_llm_tools-0.0.3/assert_llm_tools.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+assert_llm_tools/__init__.py
+assert_llm_tools/core.py
+assert_llm_tools/utils.py
+assert_llm_tools.egg-info/PKG-INFO
+assert_llm_tools.egg-info/SOURCES.txt
+assert_llm_tools.egg-info/dependency_links.txt
+assert_llm_tools.egg-info/requires.txt
+assert_llm_tools.egg-info/top_level.txt
+assert_llm_tools/metrics/bleu.py
+assert_llm_tools/metrics/rouge.py

assert_llm_tools-0.0.3/assert_llm_tools.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

assert_llm_tools-0.0.3/assert_llm_tools.egg-info/requires.txt

@@ -0,0 +1,11 @@
+anthropic>=0.3.0
+openai>=1.0.0
+python-dotenv>=0.19.0
+
+[dev]
+pytest>=7.0
+black>=23.0.0
+isort>=5.0.0
+flake8>=4.0.0
+build>=0.10.0
+twine>=4.0.0

assert_llm_tools-0.0.3/assert_llm_tools.egg-info/top_level.txt

@@ -0,0 +1 @@
+assert_llm_tools

assert_llm_tools-0.0.3/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "assert_llm_tools"
+version = "0.0.3"
+description = "A library for generating test assertions using Large Language Models"
+authors = [
+    {name = "Charlie Douglas", email = "cdouglas@gmail.com"},
+]
+readme = "README.md"
+license = {text = "MIT"}
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "anthropic>=0.3.0",
+    "openai>=1.0.0",
+    "python-dotenv>=0.19.0",
+]
+requires-python = ">=3.8"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "black>=23.0.0",
+    "isort>=5.0.0",
+    "flake8>=4.0.0",
+    "build>=0.10.0",
+    "twine>=4.0.0",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/yourusername/assert-llm"
+"Bug Tracker" = "https://github.com/yourusername/assert-llm/issues"
+
+[tool.setuptools]
+packages = [
+    "assert_llm_tools",
+    "assert_llm_tools.metrics"
+]

assert_llm_tools-0.0.3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+