fragmentshot 0.1.0__tar.gz

fragmentshot-0.1.0/LICENSE

@@ -0,0 +1,25 @@
+Creative Commons Attribution-ShareAlike 4.0 International Public License (CC BY-SA 4.0)
+
+By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License").
+
+You are free to:
+  - Share — copy and redistribute the material in any medium or format
+  - Adapt — remix, transform, and build upon the material
+     for any purpose, even commercially.
+
+Under the following terms:
+  - Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made.
+    You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+
+  - ShareAlike — If you remix, transform, or build upon the material, you must distribute your contributions under
+    the same license as the original.
+
+No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.
+
+Notices:
+  - You do not have to comply with the license for elements of the material in the public domain or where your use is permitted by an applicable exception or limitation.
+  - No warranties are given. The license may not give you all of the permissions necessary for your intended use.
+    For example, other rights such as publicity, privacy, or moral rights may limit how you use the material.
+
+For the full license text, please visit:
+https://creativecommons.org/licenses/by-sa/4.0/legalcode

fragmentshot-0.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: fragmentshot
+Version: 0.1.0
+Summary: Prompt generation for Fragment-Shot and Pivoted Fragment-Shot translation using LLMs
+Author-email: Samuel Frontull <samuel.frontull@uibk.ac.at>
+License: MIT
+Project-URL: Homepage, https://github.com/schtailmuel/fragmentshot
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# FragmentShot Retriever
+
+A Python package for retrieving exemplary translations for text based on fragments from parallel corpora.  
+
+## Features
+
+- Fragment extraction from source and target texts
+- Configurable maximum fragment size
+- Option to enable or disable fragment overlaps
+- Easy integration for retrieval workflows
+
+## Installation
+
+You can install this package locally:
+
+```bash
+pip install .
+```
+
+Or clone the repo and install in editable mode:
+
+```bash
+git clone https://github.com/schtailmuel/fragmentshot.git
+cd fragmentshot
+pip install -e .
+```
+
+## Usage 
+
+```python 
+from fragmentshot.retriever import FragmentShotsRetriever
+
+src_texts = [
+    "This is a sample source sentence.",
+    "Another example source sentence."
+]
+
+tgt_texts = [
+    "Dies ist ein Beispiel im Zieltext.",
+    "Noch ein Beispiel."
+]
+
+result = FragmentShotsRetriever(src_texts, tgt_texts, max_fragment_size=5, overlaps=False)
+
+# Access extracted shots
+print(result)
+```
+
+## Testing 
+
+Run unit tests with:
+
+```bash
+python -m unittest discover tests
+```
+
+## License 
+
+This project is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
+
+Created by Samuel Frontull

fragmentshot-0.1.0/README.md

@@ -0,0 +1,61 @@
+# FragmentShot Retriever
+
+A Python package for retrieving exemplary translations for text based on fragments from parallel corpora.  
+
+## Features
+
+- Fragment extraction from source and target texts
+- Configurable maximum fragment size
+- Option to enable or disable fragment overlaps
+- Easy integration for retrieval workflows
+
+## Installation
+
+You can install this package locally:
+
+```bash
+pip install .
+```
+
+Or clone the repo and install in editable mode:
+
+```bash
+git clone https://github.com/schtailmuel/fragmentshot.git
+cd fragmentshot
+pip install -e .
+```
+
+## Usage 
+
+```python 
+from fragmentshot.retriever import FragmentShotsRetriever
+
+src_texts = [
+    "This is a sample source sentence.",
+    "Another example source sentence."
+]
+
+tgt_texts = [
+    "Dies ist ein Beispiel im Zieltext.",
+    "Noch ein Beispiel."
+]
+
+result = FragmentShotsRetriever(src_texts, tgt_texts, max_fragment_size=5, overlaps=False)
+
+# Access extracted shots
+print(result)
+```
+
+## Testing 
+
+Run unit tests with:
+
+```bash
+python -m unittest discover tests
+```
+
+## License 
+
+This project is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
+
+Created by Samuel Frontull

fragmentshot-0.1.0/fragmentshot/__init__.py

@@ -0,0 +1,9 @@
+# fragmentshot/__init__.py
+
+__version__ = "0.1.0"
+
+from .retriever import FragmentShotsRetriever
+
+__all__ = [
+    "get_fragment_shots"
+]

fragmentshot-0.1.0/fragmentshot/retriever.py

@@ -0,0 +1,110 @@
+import re
+import random
+
+class FragmentShotsRetriever:
+
+    def __init__(self, src_texts, tgt_texts, max_fragment_size=7, overlaps=False):
+
+        if len(src_texts) != len(tgt_texts):
+            raise ValueError(
+                "Source and target files must have the same number of lines."
+            )
+
+        self.src_texts = src_texts
+        self.tgt_texts = tgt_texts
+        self.overlaps = overlaps
+        self.max_fragment_size = max_fragment_size
+
+        self.corpus_fragments_str = {}
+        self.corpus_fragments_idx = {}
+        self._init_corpus_fragments()
+
+    def _init_corpus_fragments(self):
+
+        src_tok = [self._remove_punctuation(s).split() for s in self.src_texts]
+
+        for size in range(1, self.max_fragment_size + 1):
+
+            self.corpus_fragments_str[size] = []
+            self.corpus_fragments_idx[size] = []
+
+            for i, src_sent in enumerate(src_tok):
+                res = self._create_fragments(src_sent, size)
+                for tok in res:
+                    self.corpus_fragments_str[size].append(tok)
+                    self.corpus_fragments_idx[size].append(i)
+
+    def _remove_punctuation(self, text):
+        text = re.sub(r"[ ]+", " ", text)
+        text = re.sub(r"[.,!?:;]", "", text)
+        return text
+
+    def _create_fragments(self, xs, n):
+        
+        fragments = []
+        
+        for i in range(len(xs) - n + 1):
+            fragments.append(xs[i : i + n])
+        
+        return fragments
+
+    def get_fragment_shots(self, text, num_shots=6):
+        """
+        Retrieve fragments based on a text.
+        """
+        shots = []
+
+        text_tokenized = self._remove_punctuation(text).split()
+        start_size = min(len(text_tokenized), self.max_fragment_size)
+
+        for size in range(start_size, 0, -1):
+
+            src_fragments = self._create_fragments(text_tokenized, size)
+            wi_marked = []
+
+            for f_idx, fragment in enumerate(src_fragments):
+                
+                if ("#" in fragment) or (f_idx in wi_marked and not self.overlaps):
+                    continue
+                
+                fragment_lower = [x.lower() for x in fragment]
+
+                match_idxs = [
+                    _idx
+                    for _idx, x in enumerate(self.corpus_fragments_str[size])
+                    if x == fragment or x == fragment_lower
+                ]
+
+                if match_idxs:
+                    
+                    sent_ids = [self.corpus_fragments_idx[size][i] for i in match_idxs]
+                    random.shuffle(sent_ids)
+
+                    examples = []
+
+                    for sent_id in sent_ids:
+                        
+                        examples.append({
+                            "src_text": self.src_texts[sent_id], 
+                            "tgt_text": self.tgt_texts[sent_id]
+                        })
+
+                        if len(examples) >= num_shots:
+                            break
+
+                    shots.append(
+                        {
+                            "fragment": " ".join(fragment),
+                            "examples": examples
+                        }
+                    )
+
+                    for j in range(f_idx, f_idx + size):
+                        text_tokenized[j] = "#"
+                        wi_marked.append(j)
+
+        return {
+            "shots": shots,
+            "num_words": len(text_tokenized),
+            "unknown": [x for x in text_tokenized if x != "#"]
+        }

fragmentshot-0.1.0/fragmentshot.egg-info/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: fragmentshot
+Version: 0.1.0
+Summary: Prompt generation for Fragment-Shot and Pivoted Fragment-Shot translation using LLMs
+Author-email: Samuel Frontull <samuel.frontull@uibk.ac.at>
+License: MIT
+Project-URL: Homepage, https://github.com/schtailmuel/fragmentshot
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# FragmentShot Retriever
+
+A Python package for retrieving exemplary translations for text based on fragments from parallel corpora.  
+
+## Features
+
+- Fragment extraction from source and target texts
+- Configurable maximum fragment size
+- Option to enable or disable fragment overlaps
+- Easy integration for retrieval workflows
+
+## Installation
+
+You can install this package locally:
+
+```bash
+pip install .
+```
+
+Or clone the repo and install in editable mode:
+
+```bash
+git clone https://github.com/schtailmuel/fragmentshot.git
+cd fragmentshot
+pip install -e .
+```
+
+## Usage 
+
+```python 
+from fragmentshot.retriever import FragmentShotsRetriever
+
+src_texts = [
+    "This is a sample source sentence.",
+    "Another example source sentence."
+]
+
+tgt_texts = [
+    "Dies ist ein Beispiel im Zieltext.",
+    "Noch ein Beispiel."
+]
+
+result = FragmentShotsRetriever(src_texts, tgt_texts, max_fragment_size=5, overlaps=False)
+
+# Access extracted shots
+print(result)
+```
+
+## Testing 
+
+Run unit tests with:
+
+```bash
+python -m unittest discover tests
+```
+
+## License 
+
+This project is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
+
+Created by Samuel Frontull

fragmentshot-0.1.0/fragmentshot.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+LICENSE
+README.md
+pyproject.toml
+fragmentshot/__init__.py
+fragmentshot/retriever.py
+fragmentshot.egg-info/PKG-INFO
+fragmentshot.egg-info/SOURCES.txt
+fragmentshot.egg-info/dependency_links.txt
+fragmentshot.egg-info/top_level.txt
+tests/test_retriever.py

fragmentshot-0.1.0/fragmentshot.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fragmentshot-0.1.0/fragmentshot.egg-info/top_level.txt

@@ -0,0 +1 @@
+fragmentshot

fragmentshot-0.1.0/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fragmentshot"
+version = "0.1.0"
+description = "Prompt generation for Fragment-Shot and Pivoted Fragment-Shot translation using LLMs"
+authors = [{ name="Samuel Frontull", email="samuel.frontull@uibk.ac.at" }]
+license = { text = "MIT" }
+readme = "README.md"
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/schtailmuel/fragmentshot"

fragmentshot-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fragmentshot-0.1.0/tests/test_retriever.py

@@ -0,0 +1,43 @@
+import unittest
+from fragmentshot.retriever import FragmentShotsRetriever
+
+class TestFragmentShotsRetriever(unittest.TestCase):
+
+    @classmethod
+    def setUp(self):
+        self.src_texts = [
+            "The cat sleeps on the couch",
+            "A dog runs in the garden"
+        ]
+        self.tgt_texts = [
+            "Il gatto dorme sul divano",
+            "Un cane corre nel giardino"
+        ]
+
+    def test_constructor_with_valid_input(self):
+        retriever = FragmentShotsRetriever(self.src_texts, self.tgt_texts)
+        self.assertEqual(len(retriever.src_texts), 2)
+        self.assertEqual(len(retriever.tgt_texts), 2)
+        self.assertFalse(retriever.overlaps)
+        self.assertEqual(retriever.max_fragment_size, 7)
+
+    def test_constructor_raises_error_on_mismatched_input(self):
+        with self.assertRaises(ValueError):
+            FragmentShotsRetriever(self.src_texts, self.tgt_texts + ["extra"])
+
+    def test_fragment_initialization(self):
+        retriever = FragmentShotsRetriever(self.src_texts, self.tgt_texts, max_fragment_size=3)
+        
+        self.assertIn(3, retriever.corpus_fragments_str)
+        self.assertIn(3, retriever.corpus_fragments_idx)
+        self.assertTrue(len(retriever.corpus_fragments_str[3]) > 0)
+        self.assertEqual(len(retriever.corpus_fragments_str[3]), len(retriever.corpus_fragments_idx[3]))
+
+    def test_retrieve(self):
+        input_sentence = "my dogs sleeps on the floow"
+        retriever = FragmentShotsRetriever(self.src_texts, self.tgt_texts)
+        results = retriever.get_fragment_shots(input_sentence)
+        self.assertEqual(len(results['shots']), 1)
+
+if __name__ == "__main__":
+    unittest.main()