sembr 0.2.2__tar.gz → 0.2.3__tar.gz

{sembr-0.2.2/sembr.egg-info → sembr-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sembr
-Version: 0.2.2
+Version: 0.2.3
 Summary: A semantic linebreaker powered by transformers
 Author: admk
 License-Expression: MIT
@@ -14,14 +14,15 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: accelerate
-Requires-Dist: transformers
-Requires-Dist: torch
+Requires-Dist: fastmcp
+Requires-Dist: flask
+Requires-Dist: mcp[cli]
 Requires-Dist: numpy
-Requires-Dist: tqdm
+Requires-Dist: pydantic
 Requires-Dist: requests
-Requires-Dist: flask
-Requires-Dist: mcp[cli]>=1.2.0
-Requires-Dist: fastmcp>=2.10.6
+Requires-Dist: torch
+Requires-Dist: tqdm
+Requires-Dist: transformers
 Dynamic: license-file
 
 # Semantic Line Breaker (SemBr)

{sembr-0.2.2 → sembr-0.2.3}/pyproject.toml

@@ -15,14 +15,15 @@ license-files = ["LICEN[CS]E*"]
 requires-python = ">=3.10"
 dependencies = [
     "accelerate",
-    "transformers",
-    "torch",
+    "fastmcp",
+    "flask",
+    "mcp[cli]",
     "numpy",
-    "tqdm",
+    "pydantic",
     "requests",
-    "flask",
-    "mcp[cli]>=1.2.0",
-    "fastmcp>=2.10.6",
+    "torch",
+    "tqdm",
+    "transformers",
 ]
 classifiers = [
     "Programming Language :: Python :: 3",

{sembr-0.2.2 → sembr-0.2.3}/sembr/__init__.py

@@ -1,5 +1,5 @@
 __toolname__ = __name__
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 __author__ = "admk"
 __license__ = "MIT"
 __url__ = f"https://github.com/admk/{__name__}"

{sembr-0.2.2 → sembr-0.2.3}/sembr/inference.py

@@ -108,7 +108,7 @@ def inference(
     if text.strip() == '':
         return []
     collator = DataCollatorForTokenClassification(tokenizer, padding='longest')
-    results = processor(text, split=isinstance(text, str))
+    results = processor.parse_text(text, split=isinstance(text, str))
     results = processor.tokenize_with_modes(tokenizer, results)
     logits, counts = _tiled_inference(
         model, collator, results, batch_size, overlap_divisor)

{sembr-0.2.2 → sembr-0.2.3}/sembr/process.py

@@ -143,6 +143,20 @@ class SemBrProcessor(object):
             'base_indent': base_indent,
         }
 
+    def parse_text(self, text, split=True):
+        text = text.replace('\t', ' ' * self.spaces)
+        if split:
+            text = re.split(r'\n(?:\s*\n)+', text)
+        elif isinstance(text, str):
+            raise ValueError(
+                'Text must be a list of strings if split=True.')
+        paras = []
+        for p in text:
+            if not p.strip():
+                continue
+            paras.append(self._process_paragraph(p))
+        return paras
+
     def _tokenize_with_modes(
         self, tokenizer, text, line_modes, line_mode_offsets, line_indents
     ):
@@ -201,19 +215,6 @@ class SemBrProcessor(object):
             new_results.append(tokenized)
         return new_results
 
-    def __call__(self, text, split=True):
-        if split:
-            text = re.split(r'\n(?:\s*\n)+', text)
-        elif isinstance(text, str):
-            raise ValueError(
-                'Text must be a list of strings if split=True.')
-        paras = []
-        for p in text:
-            if not p.strip():
-                continue
-            paras.append(self._process_paragraph(p))
-        return paras
-
     def _replace_newlines(self, words, modes, indents):
         new_words, new_modes, new_indents = [], [], []
         next_mode = None
@@ -288,11 +289,10 @@ class SemBrProcessor(object):
 
 
 if __name__ == '__main__':
-    # test = open('./data/test/mair.tex', 'r').read()
-    test = open('./data/example.tex', 'r').read()
+    test = open('./data/raw/example.tex', 'r').read()
     processor = SemBrProcessor()
     tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
-    results = processor(test)
+    results = processor.parse_text(test)
     results = processor.tokenize_with_modes(tokenizer, results)
     print('--- Processed ---')
     print(processor.generate(results))

{sembr-0.2.2 → sembr-0.2.3/sembr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sembr
-Version: 0.2.2
+Version: 0.2.3
 Summary: A semantic linebreaker powered by transformers
 Author: admk
 License-Expression: MIT
@@ -14,14 +14,15 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: accelerate
-Requires-Dist: transformers
-Requires-Dist: torch
+Requires-Dist: fastmcp
+Requires-Dist: flask
+Requires-Dist: mcp[cli]
 Requires-Dist: numpy
-Requires-Dist: tqdm
+Requires-Dist: pydantic
 Requires-Dist: requests
-Requires-Dist: flask
-Requires-Dist: mcp[cli]>=1.2.0
-Requires-Dist: fastmcp>=2.10.6
+Requires-Dist: torch
+Requires-Dist: tqdm
+Requires-Dist: transformers
 Dynamic: license-file
 
 # Semantic Line Breaker (SemBr)

{sembr-0.2.2 → sembr-0.2.3}/sembr.egg-info/requires.txt

@@ -1,9 +1,10 @@
 accelerate
-transformers
-torch
+fastmcp
+flask
+mcp[cli]
 numpy
-tqdm
+pydantic
 requests
-flask
-mcp[cli]>=1.2.0
-fastmcp>=2.10.6
+torch
+tqdm
+transformers