chartoken-vp 1.0.0__tar.gz

chartoken_vp-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025-2026 F000NK / Voluntas Progressus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chartoken_vp-1.0.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: chartoken-vp
+Version: 1.0.0
+Summary: Character-level vocabulary and morphological feature encoding
+Author: F000NK, Voluntas Progressus
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.14
+License-File: LICENSE
+Requires-Dist: torch>=2.0.0
+Dynamic: license-file

chartoken_vp-1.0.0/README.md

@@ -0,0 +1,37 @@
+# chartoken
+
+Character-level vocabulary and morphological feature encoding for NLP pipelines.
+
+## Installation
+
+```bash
+pip install chartoken
+```
+
+Requires Python >= 3.14 and PyTorch >= 2.0.
+
+## Features
+
+- **CharVocab** — character-level tokenizer with NFKC normalization, SOS/EOS/PAD tokens
+- **FeatureVocab** — morphological feature encoder (UniMorph tag set) with padding masks
+- Encode text to padded tensors or plain ID lists
+- Serialize/deserialize vocabularies for checkpoint compatibility
+
+## Quick Start
+
+```python
+from chartoken import CharVocab, FeatureVocab
+
+# Build vocab from texts
+vocab = CharVocab.from_texts(["hello", "world"])
+ids = vocab.encode("hello", max_len=32)
+print(vocab.decode(ids.tolist()))  # "hello"
+
+# Feature vocab
+feat_vocab = FeatureVocab.from_tags([["V", "IND", "PRS"], ["N", "SG"]])
+feat_ids, feat_mask = feat_vocab.encode(["V", "IND"], max_features=12)
+```
+
+## License
+
+See LICENSE file in the repository root.

chartoken_vp-1.0.0/chartoken/__init__.py

@@ -0,0 +1,4 @@
+from chartoken.vocab import CharVocab, PAD, SOS, EOS, normalize_text, SPECIAL_TOKENS
+from chartoken.feature_vocab import FeatureVocab, FEATURE_PAD
+
+__version__ = "1.0.0"

chartoken_vp-1.0.0/chartoken/feature_vocab.py

@@ -0,0 +1,58 @@
+"""Morphological feature vocabulary (UniMorph tag set)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+
+FEATURE_PAD = 0
+FEATURE_SPECIAL = ["<pad>"]
+
+
+@dataclass
+class FeatureVocab:
+    stoi: dict[str, int]
+    itos: list[str]
+
+    @property
+    def size(self) -> int:
+        return len(self.itos)
+
+    @classmethod
+    def from_tags(cls, all_tags: list[list[str]]) -> FeatureVocab:
+        unique_tags = sorted({tag for tags in all_tags for tag in tags})
+        itos = FEATURE_SPECIAL + unique_tags
+        stoi = {tag: idx for idx, tag in enumerate(itos)}
+        return cls(stoi=stoi, itos=itos)
+
+    def encode(
+        self, tags: list[str], *, max_features: int,
+    ) -> tuple[list[int], list[float]]:
+        """Encode *tags* into padded ``(feature_ids, feature_mask)`` lists of length *max_features*."""
+        ids = [self.stoi[tag] for tag in tags if tag in self.stoi]
+        ids = ids[:max_features]
+        mask = [1.0] * len(ids)
+        padding_length = max_features - len(ids)
+        if padding_length > 0:
+            ids.extend([FEATURE_PAD] * padding_length)
+            mask.extend([0.0] * padding_length)
+        return ids, mask
+
+    def encode_tensor(
+        self, tags: list[str], *, max_features: int, device: torch.device | None = None,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        ids, mask = self.encode(tags, max_features=max_features)
+        ids_tensor = torch.tensor(ids, dtype=torch.long)
+        mask_tensor = torch.tensor(mask, dtype=torch.float)
+        if device is not None:
+            ids_tensor = ids_tensor.to(device)
+            mask_tensor = mask_tensor.to(device)
+        return ids_tensor, mask_tensor
+
+    def to_dict(self) -> dict:
+        return {"stoi": self.stoi, "itos": self.itos}
+
+    @classmethod
+    def from_dict(cls, data: dict) -> FeatureVocab:
+        return cls(stoi=data["stoi"], itos=data["itos"])

chartoken_vp-1.0.0/chartoken/vocab.py

@@ -0,0 +1,74 @@
+"""Character-level vocabulary with NFKC normalization."""
+
+from __future__ import annotations
+
+import unicodedata
+from dataclasses import dataclass
+
+import torch
+
+PAD = 0
+SOS = 1
+EOS = 2
+SPECIAL_TOKENS = ["<pad>", "<sos>", "<eos>"]
+
+
+def normalize_text(text: str) -> str:
+    return unicodedata.normalize("NFKC", text)
+
+
+@dataclass
+class CharVocab:
+    stoi: dict[str, int]
+    itos: list[str]
+
+    @property
+    def size(self) -> int:
+        return len(self.itos)
+
+    @classmethod
+    def from_texts(cls, texts: list[str]) -> CharVocab:
+        chars = sorted({char for text in texts for char in normalize_text(text)})
+        itos = SPECIAL_TOKENS + [char for char in chars if char not in SPECIAL_TOKENS]
+        stoi = {char: idx for idx, char in enumerate(itos)}
+        return cls(stoi=stoi, itos=itos)
+
+    def encode(
+        self, text: str, *, max_len: int, device: torch.device | None = None,
+    ) -> torch.Tensor:
+        """Encode *text* into a padded long tensor ``[SOS, ...chars, EOS, PAD...]``."""
+        text = normalize_text(text)
+        token_ids = [SOS] + [self.stoi[char] for char in text if char in self.stoi] + [EOS]
+        token_ids = token_ids[:max_len]
+        padding_length = max_len - len(token_ids)
+        if padding_length > 0:
+            token_ids.extend([PAD] * padding_length)
+        tensor = torch.tensor(token_ids, dtype=torch.long)
+        return tensor if device is None else tensor.to(device)
+
+    def encode_ids(self, text: str, max_len: int) -> list[int]:
+        text = normalize_text(text)
+        token_ids = [SOS] + [self.stoi[char] for char in text if char in self.stoi] + [EOS]
+        token_ids = token_ids[:max_len]
+        padding_length = max_len - len(token_ids)
+        if padding_length > 0:
+            token_ids.extend([PAD] * padding_length)
+        return token_ids
+
+    def decode(self, token_ids: list[int]) -> str:
+        chars: list[str] = []
+        for token_id in token_ids:
+            if token_id in (PAD, SOS):
+                continue
+            if token_id == EOS:
+                break
+            if 0 <= token_id < len(self.itos):
+                chars.append(self.itos[token_id])
+        return "".join(chars)
+
+    def to_dict(self) -> dict:
+        return {"stoi": self.stoi, "itos": self.itos}
+
+    @classmethod
+    def from_dict(cls, data: dict) -> CharVocab:
+        return cls(stoi=data["stoi"], itos=data["itos"])

chartoken_vp-1.0.0/chartoken_vp.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: chartoken-vp
+Version: 1.0.0
+Summary: Character-level vocabulary and morphological feature encoding
+Author: F000NK, Voluntas Progressus
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.14
+License-File: LICENSE
+Requires-Dist: torch>=2.0.0
+Dynamic: license-file

chartoken_vp-1.0.0/chartoken_vp.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+LICENSE
+README.md
+pyproject.toml
+chartoken/__init__.py
+chartoken/feature_vocab.py
+chartoken/vocab.py
+chartoken_vp.egg-info/PKG-INFO
+chartoken_vp.egg-info/SOURCES.txt
+chartoken_vp.egg-info/dependency_links.txt
+chartoken_vp.egg-info/requires.txt
+chartoken_vp.egg-info/top_level.txt

chartoken_vp-1.0.0/chartoken_vp.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

chartoken_vp-1.0.0/chartoken_vp.egg-info/requires.txt

@@ -0,0 +1 @@
+torch>=2.0.0

chartoken_vp-1.0.0/chartoken_vp.egg-info/top_level.txt

@@ -0,0 +1 @@
+chartoken

chartoken_vp-1.0.0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=69.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chartoken-vp"
+version = "1.0.0"
+authors = [
+    {name = "F000NK"},
+    {name = "Voluntas Progressus"},
+]
+description = "Character-level vocabulary and morphological feature encoding"
+license = "MIT"
+requires-python = ">=3.14"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.14",
+    "Operating System :: OS Independent",
+    "Topic :: Text Processing :: Linguistic",
+]
+dependencies = ["torch>=2.0.0"]
+
+[tool.setuptools.packages.find]
+include = ["chartoken*"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py314"

chartoken_vp-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+