khmerns 0.0.3__tar.gz → 0.0.4__tar.gz

{khmerns-0.0.3 → khmerns-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: khmerns
-Version: 0.0.3
+Version: 0.0.4
 Summary: Khmer Neural Segmenter
 Keywords: khmer,nlp,segmentation,tokenization,neural-network
 Author-Email: Seanghay Yath <seanghay.dev@gmail.com>
@@ -35,11 +35,15 @@ pip install khmerns
 ## Usage
 
 ```python
-from khmerns import tokenize
+from khmerns import tokenize, normalize
 
 # Returns a list of words
 words = tokenize("សួស្តីបងប្អូន")
-# ['សួស្តី', 'បង', 'ប្អូន']
+# => ['សួស្តី', 'បង', 'ប្អូន']
+
+# normalize and reorder Khmer characters
+words = tokenize(normalize("សួស្តីបងប្អូន"))
+# => ['សួស្តី', 'បង', 'ប្អូន']
 ```
 
 You can also use the class-based API if you prefer:
@@ -48,8 +52,10 @@ You can also use the class-based API if you prefer:
 from khmerns import KhmerSegmenter
 
 segmenter = KhmerSegmenter()
+
 words = segmenter.tokenize("សួស្តីបងប្អូន")
 # or
+
 words = segmenter("សួស្តីបងប្អូន")
 ```
 

{khmerns-0.0.3 → khmerns-0.0.4}/README.md

@@ -13,11 +13,15 @@ pip install khmerns
 ## Usage
 
 ```python
-from khmerns import tokenize
+from khmerns import tokenize, normalize
 
 # Returns a list of words
 words = tokenize("សួស្តីបងប្អូន")
-# ['សួស្តី', 'បង', 'ប្អូន']
+# => ['សួស្តី', 'បង', 'ប្អូន']
+
+# normalize and reorder Khmer characters
+words = tokenize(normalize("សួស្តីបងប្អូន"))
+# => ['សួស្តី', 'បង', 'ប្អូន']
 ```
 
 You can also use the class-based API if you prefer:
@@ -26,8 +30,10 @@ You can also use the class-based API if you prefer:
 from khmerns import KhmerSegmenter
 
 segmenter = KhmerSegmenter()
+
 words = segmenter.tokenize("សួស្តីបងប្អូន")
 # or
+
 words = segmenter("សួស្តីបងប្អូន")
 ```
 

{khmerns-0.0.3 → khmerns-0.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "khmerns"
-version = "0.0.3"
+version = "0.0.4"
 license = "MIT"
 license-files = ["LICENSE"]
 description = "Khmer Neural Segmenter"
@@ -31,6 +31,7 @@ Issues = "https://github.com/seanghay/khmer-neural-segmenter/issues"
 [tool.scikit-build]
 minimum-version = "build-system.requires"
 wheel.packages = ["src/khmerns"]
+sdist.exclude = ["training", "img", "test", ".github"]
 
 [tool.cibuildwheel]
 build-frontend = "build[uv]"

khmerns-0.0.4/src/khmerns/__init__.py

@@ -0,0 +1,4 @@
+from ._core import KhmerSegmenter, tokenize, __version__
+from .khnormal import normalize
+
+__all__ = ["KhmerSegmenter", "tokenize", "__version__", "normalize"]

{khmerns-0.0.3 → khmerns-0.0.4}/src/khmerns/__init__.pyi

@@ -14,3 +14,7 @@ class KhmerSegmenter:
 def tokenize(text: str) -> List[str]:
   """Segment Khmer text and return a list of words."""
   ...
+
+def normalize(text: str) -> str:
+  """Normalize and reorder Khmer character"""
+  ...

{khmerns-0.0.3/training → khmerns-0.0.4/src/khmerns}/khnormal.py

@@ -2,7 +2,8 @@
 # Copyright (c) 2021-2024, SIL Global.
 # Licensed under MIT license: https://opensource.org/licenses/MIT
 
-import enum, re, regex
+import enum
+import re
 
 
 class Cats(enum.Enum):
@@ -137,7 +138,7 @@ def lunar(m, base):
   return chr(v + base)
 
 
-def khnormal(txt, lang="km"):
+def normalize(txt, lang="km"):
   """Returns khmer normalised string, without fixing or marking errors"""
   # Mark final coengs in Middle Khmer
   if lang == "xhm":

khmerns-0.0.3/.github/workflows/wheels.yml

@@ -1,93 +0,0 @@
-name: Wheels
-
-on:
-  workflow_dispatch:
-  pull_request:
-  push:
-    branches:
-      - main
-  release:
-    types:
-      - published
-
-env:
-  FORCE_COLOR: 3
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build_sdist:
-    name: Build SDist
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          submodules: true
-
-      - name: Build SDist
-        run: pipx run build --sdist
-
-      - name: Check metadata
-        run: pipx run twine check dist/*
-
-      - uses: actions/upload-artifact@v5
-        with:
-          name: cibw-sdist
-          path: dist/*.tar.gz
-
-  build_wheels:
-    name: Wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, macos-15-intel, windows-latest, ubuntu-24.04-arm]
-    env:
-      MACOSX_DEPLOYMENT_TARGET: "11.0"
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          submodules: true
-
-      - uses: astral-sh/setup-uv@v7
-
-      - uses: pypa/cibuildwheel@v3.3
-
-      - name: Verify clean directory
-        run: git diff --exit-code
-        shell: bash
-
-      - uses: actions/upload-artifact@v5
-        with:
-          name: cibw-wheels-${{ matrix.os }}
-          path: wheelhouse/*.whl
-
-  upload_all:
-    name: Upload if release
-    needs: [build_wheels, build_sdist]
-    runs-on: ubuntu-latest
-    if: github.event_name == 'release' && github.event.action == 'published'
-    environment: pypi
-    permissions:
-      id-token: write
-      attestations: write
-
-    steps:
-      - uses: actions/setup-python@v6
-        with:
-          python-version: "3.x"
-
-      - uses: actions/download-artifact@v6
-        with:
-          pattern: cibw-*
-          merge-multiple: true
-          path: dist
-
-      - name: Generate artifact attestation for sdist and wheels
-        uses: actions/attest-build-provenance@v3
-        with:
-          subject-path: "dist/*"
-
-      - uses: pypa/gh-action-pypi-publish@release/v1

khmerns-0.0.3/src/khmerns/__init__.py

@@ -1,3 +0,0 @@
-from ._core import KhmerSegmenter, tokenize, __version__
-
-__all__ = ["KhmerSegmenter", "tokenize", "__version__"]

khmerns-0.0.3/test/example.py

@@ -1,9 +0,0 @@
-from khmerns import tokenize
-
-print(
-  tokenize(
-    "តុលាការបារាំងបើកការស៊ើបអង្កេតលើក្រុមហ៊ុនបណ្តាញសង្គម X (Twitter) ក្នុងសំណុំរឿងឧក្រិដ្ឋកម្មអនឡាញ"
-  )
-)
-
-# ['តុលាការ', 'បារាំង', 'បើក', 'ការ', 'ស៊ើប', 'អង្កេត', 'លើ', 'ក្រុមហ៊ុន', 'បណ្តាញ', 'សង្គម', ' ', 'X', ' ', '(', 'T', 'w', 'i', 't', 't', 'e', 'r', ')', ' ', 'ក្នុង', 'សំណុំ', 'រឿង', 'ឧក្រិដ្ឋ', 'កម្ម', 'អនឡាញ']

khmerns-0.0.3/training/convert_to_gguf.py

@@ -1,124 +0,0 @@
-#!/usr/bin/env python3
-"""Convert PyTorch Khmer segmenter model to GGUF format."""
-
-import argparse
-import sys
-from pathlib import Path
-
-import numpy as np
-import torch
-
-# Add parent directory for imports
-sys.path.insert(0, str(Path(__file__).parent.parent))
-from model import Segmenter
-from tokenizer import Tokenizer
-
-try:
-  from gguf import GGUFWriter
-except ImportError:
-  print("Error: gguf package not installed. Run: pip install gguf")
-  sys.exit(1)
-
-
-def convert_to_gguf(model_path: str, output_path: str):
-  """Convert PyTorch model to GGUF format."""
-
-  # Load tokenizer and model
-  print(f"Loading model from {model_path}...")
-  tokenizer = Tokenizer()
-  model = Segmenter(
-    vocab_size=len(tokenizer),
-    embedding_dim=256,
-    hidden_dim=256,
-    num_labels=3,
-  )
-  model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=True))
-  model.eval()
-
-  # Create GGUF writer
-  print(f"Creating GGUF file: {output_path}")
-  writer = GGUFWriter(output_path, "khmer-segmenter")
-
-  # Write metadata
-  writer.add_uint32("khmer.vocab_size", len(tokenizer))
-  writer.add_uint32("khmer.embedding_dim", 256)
-  writer.add_uint32("khmer.hidden_dim", 256)
-  writer.add_uint32("khmer.num_labels", 3)
-
-  # Write tensors
-  print("Writing tensors...")
-
-  # Embedding: [vocab_size, embedding_dim]
-  embed_weight = model.embedding.weight.detach().numpy().astype(np.float32)
-  writer.add_tensor("embedding.weight", embed_weight)
-  print(f"  embedding.weight: {embed_weight.shape}")
-
-  # GRU forward weights
-  # PyTorch GRU weight_ih_l0: [3*hidden, input]
-  # PyTorch GRU weight_hh_l0: [3*hidden, hidden]
-  gru = model.gru
-  writer.add_tensor("gru.weight_ih_l0", gru.weight_ih_l0.detach().numpy().astype(np.float32))
-  writer.add_tensor("gru.weight_hh_l0", gru.weight_hh_l0.detach().numpy().astype(np.float32))
-  writer.add_tensor("gru.bias_ih_l0", gru.bias_ih_l0.detach().numpy().astype(np.float32))
-  writer.add_tensor("gru.bias_hh_l0", gru.bias_hh_l0.detach().numpy().astype(np.float32))
-  print(f"  gru.weight_ih_l0: {gru.weight_ih_l0.shape}")
-  print(f"  gru.weight_hh_l0: {gru.weight_hh_l0.shape}")
-
-  # GRU backward (reverse) weights
-  writer.add_tensor("gru.weight_ih_l0_reverse", gru.weight_ih_l0_reverse.detach().numpy().astype(np.float32))
-  writer.add_tensor("gru.weight_hh_l0_reverse", gru.weight_hh_l0_reverse.detach().numpy().astype(np.float32))
-  writer.add_tensor("gru.bias_ih_l0_reverse", gru.bias_ih_l0_reverse.detach().numpy().astype(np.float32))
-  writer.add_tensor("gru.bias_hh_l0_reverse", gru.bias_hh_l0_reverse.detach().numpy().astype(np.float32))
-  print(f"  gru.weight_ih_l0_reverse: {gru.weight_ih_l0_reverse.shape}")
-
-  # Linear layer: [num_labels, 2*hidden]
-  fc_weight = model.fc.weight.detach().numpy().astype(np.float32)
-  fc_bias = model.fc.bias.detach().numpy().astype(np.float32)
-  writer.add_tensor("fc.weight", fc_weight)
-  writer.add_tensor("fc.bias", fc_bias)
-  print(f"  fc.weight: {fc_weight.shape}")
-  print(f"  fc.bias: {fc_bias.shape}")
-
-  # CRF parameters
-  crf = model.crf
-  writer.add_tensor("crf.start_transitions", crf.start_transitions.detach().numpy().astype(np.float32))
-  writer.add_tensor("crf.end_transitions", crf.end_transitions.detach().numpy().astype(np.float32))
-  writer.add_tensor("crf.transitions", crf.transitions.detach().numpy().astype(np.float32))
-  print(f"  crf.start_transitions: {crf.start_transitions.shape}")
-  print(f"  crf.end_transitions: {crf.end_transitions.shape}")
-  print(f"  crf.transitions: {crf.transitions.shape}")
-
-  # Finalize
-  writer.write_header_to_file()
-  writer.write_kv_data_to_file()
-  writer.write_tensors_to_file()
-  writer.close()
-
-  print(f"\nGGUF model saved to: {output_path}")
-
-  # Print file size
-  size_mb = Path(output_path).stat().st_size / (1024 * 1024)
-  print(f"File size: {size_mb:.2f} MB")
-
-
-def main():
-  parser = argparse.ArgumentParser(
-    description="Convert PyTorch Khmer segmenter to GGUF format"
-  )
-  parser.add_argument(
-    "model_path",
-    type=str,
-    help="Path to PyTorch model file (best_model.pt)",
-  )
-  parser.add_argument(
-    "output_path",
-    type=str,
-    help="Output GGUF file path",
-  )
-  args = parser.parse_args()
-
-  convert_to_gguf(args.model_path, args.output_path)
-
-
-if __name__ == "__main__":
-  main()

khmerns-0.0.3/training/data.py

@@ -1,78 +0,0 @@
-import random
-import torch
-import re
-from torch.utils.data import DataLoader
-from tokenizer import Tokenizer
-from torch.utils.data import Dataset
-
-
-def yield_chunks(data, n, s):
-  for i in range(0, len(data), s):
-    yield data[i : i + n]
-
-
-re_khmer = re.compile(r"[\u1780-\u17ff]+")
-
-
-class TextDataset(Dataset):
-  def __init__(self, tokenizer: Tokenizer, split="train", train_ratio=0.95, seed=42):
-    super().__init__()
-    self.tokenizer = tokenizer
-    with open("data/train.txt") as infile:
-      lines = [line.rstrip("\n") for line in infile]
-
-    rng = random.Random(seed)
-    all_items = [c for c in yield_chunks(lines, 128, rng.randint(1, 128))]
-    split_idx = int(len(all_items) * train_ratio)
-
-    if split == "train":
-      self.items = all_items[:split_idx]
-    else:
-      self.items = all_items[split_idx:]
-
-  def __len__(self):
-    return len(self.items)
-
-  def __getitem__(self, i):
-    inputs = []
-    tags = []
-    for w in self.items[i]:
-      is_khmer = re_khmer.search(w)
-      token_ids = self.tokenizer.encode(w)
-      for idx, token_id in enumerate(token_ids):
-        inputs.append(token_id)
-        if is_khmer:
-          if idx == 0:
-            tags.append(1)
-          else:
-            tags.append(2)
-        else:
-          tags.append(0)
-
-    inputs = [self.tokenizer.bos_id] + inputs + [self.tokenizer.eos_id]
-    tags = [0] + tags + [0]
-
-    return torch.LongTensor(inputs), torch.LongTensor(tags)
-
-
-def collate_fn(batch):
-  inputs, tags = zip(*batch)
-  lengths = [len(x) for x in inputs]
-  max_len = max(lengths)
-
-  padded_inputs = torch.zeros(len(batch), max_len, dtype=torch.long)
-  padded_tags = torch.zeros(len(batch), max_len, dtype=torch.long)
-  mask = torch.zeros(len(batch), max_len, dtype=torch.bool)
-
-  for i, (inp, tag) in enumerate(zip(inputs, tags)):
-    padded_inputs[i, : lengths[i]] = inp
-    padded_tags[i, : lengths[i]] = tag
-    mask[i, : lengths[i]] = True
-
-  return padded_inputs, padded_tags, mask
-
-
-if __name__ == "__main__":
-  dataset = TextDataset(tokenizer=Tokenizer())
-  inputs, targets = dataset[1]
-  # print(inputs, targets)

khmerns-0.0.3/training/export_onnx.py

@@ -1,169 +0,0 @@
-import torch
-import torch.nn as nn
-import numpy as np
-from model import Segmenter
-from tokenizer import Tokenizer
-
-
-class SegmenterEmissions(nn.Module):
-  """Wrapper that outputs emissions only (for ONNX export)."""
-
-  def __init__(self, segmenter):
-    super().__init__()
-    self.embedding = segmenter.embedding
-    self.gru = segmenter.gru
-    self.fc = segmenter.fc
-
-  def forward(self, x):
-    embedded = self.embedding(x)
-    gru_out, _ = self.gru(embedded)
-    emissions = self.fc(gru_out)
-    return emissions
-
-
-def export_to_onnx(
-  model_path="best_model.pt",
-  onnx_path="segmenter.onnx",
-  crf_path="crf_params.npz",
-):
-  tokenizer = Tokenizer()
-  model = Segmenter(
-    vocab_size=len(tokenizer),
-    embedding_dim=256,
-    hidden_dim=256,
-    num_labels=3,
-  )
-  model.load_state_dict(torch.load(model_path, map_location="cpu"))
-  model.eval()
-
-  # Extract CRF parameters
-  crf = model.crf
-  print(crf.start_transitions)
-  print(crf.end_transitions)
-  print(crf.transitions)
-
-  np.savez(
-    crf_path,
-    start_transitions=crf.start_transitions.detach().numpy(),
-    end_transitions=crf.end_transitions.detach().numpy(),
-    transitions=crf.transitions.detach().numpy(),
-  )
-  print(f"Saved CRF parameters to {crf_path}")
-
-  # Create emissions-only model
-  emissions_model = SegmenterEmissions(model)
-  emissions_model.eval()
-
-  # Create dummy input for tracing
-  dummy_input = torch.randint(0, len(tokenizer), (1, 32), dtype=torch.long)
-
-  # Export to ONNX (use legacy export to avoid dynamo issues)
-  torch.onnx.export(
-    emissions_model,
-    dummy_input,
-    onnx_path,
-    input_names=["input_ids"],
-    output_names=["emissions"],
-    dynamic_axes={
-      "input_ids": {0: "batch_size", 1: "sequence_length"},
-      "emissions": {0: "batch_size", 1: "sequence_length"},
-    },
-    opset_version=14,
-    dynamo=False,
-  )
-  print(f"Exported ONNX model to {onnx_path}")
-
-
-def viterbi_decode(emissions, start_transitions, end_transitions, transitions):
-  """Viterbi decoding for CRF inference."""
-  seq_length, _ = emissions.shape
-
-  # Initialize
-  score = start_transitions + emissions[0]
-  history = []
-
-  # Forward pass
-  for i in range(1, seq_length):
-    broadcast_score = score.reshape(-1, 1)
-    broadcast_emissions = emissions[i].reshape(1, -1)
-    next_score = broadcast_score + transitions + broadcast_emissions
-    indices = next_score.argmax(axis=0)
-    score = next_score.max(axis=0)
-    history.append(indices)
-
-  # Add end transitions
-  score += end_transitions
-
-  # Backtrack
-  best_tags = [int(score.argmax())]
-  for hist in reversed(history):
-    best_tags.append(int(hist[best_tags[-1]]))
-  best_tags.reverse()
-
-  return best_tags
-
-
-def segment_onnx(text, session, tokenizer, crf_params):
-  """Segment text using ONNX Runtime."""
-  token_ids = tokenizer.encode(text)
-  inputs = [tokenizer.bos_id] + token_ids + [tokenizer.eos_id]
-  input_array = np.array([inputs], dtype=np.int64)
-
-  # Run inference
-  emissions = session.run(None, {"input_ids": input_array})[0][0]
-
-  # Viterbi decode
-  predictions = viterbi_decode(
-    emissions,
-    crf_params["start_transitions"],
-    crf_params["end_transitions"],
-    crf_params["transitions"],
-  )
-
-  # Remove BOS/EOS predictions
-  predictions = predictions[1:-1]
-
-  # Segment based on B-WORD (1) tags
-  words = []
-  current_word = []
-
-  for char, tag in zip(text, predictions):
-    if tag == 1:  # B-WORD
-      if current_word:
-        words.append("".join(current_word))
-      current_word = [char]
-    elif tag == 2:  # I-WORD
-      current_word.append(char)
-    else:  # 0 (non-Khmer)
-      if current_word:
-        words.append("".join(current_word))
-        current_word = []
-      words.append(char)
-
-  if current_word:
-    words.append("".join(current_word))
-
-  return words
-
-
-if __name__ == "__main__":
-  # Export model
-  export_to_onnx()
-
-  # Test ONNX inference
-  import onnxruntime as ort
-
-  tokenizer = Tokenizer()
-  session = ort.InferenceSession("segmenter.onnx")
-  crf_params = np.load("crf_params.npz")
-
-  text = "គិតចាប់ពី ខែធ្នូ ឆ្នាំ២០២៤ មកដល់ថ្ងៃទី១១".replace("\u200b", "")
-  words = segment_onnx(text, session, tokenizer, crf_params)
-  print(f"ONNX result: {'|'.join(words)}")
-
-  # Compare with PyTorch
-  from infer import load_model, segment
-
-  model, tokenizer = load_model()
-  words_pt = segment(text, model, tokenizer)
-  print(f"PyTorch result: {'|'.join(words_pt)}")

khmerns-0.0.3/training/generate.py

@@ -1,52 +0,0 @@
-import os
-import regex as re
-from khmersegment import Segmenter
-from nltk.tokenize import TweetTokenizer
-from khnormal import khnormal
-
-tknzr = TweetTokenizer(reduce_len=True, strip_handles=False)
-
-segmenter = Segmenter("-m assets/km-5tag-seg-model")
-re_pre_segment = re.compile(
-  r"([\u1780-\u17dd]+)|([\u17e0-\u17e90-9]+)|([^\u1780-\u17ff]+)"
-)
-
-
-def segment(text: str):
-  for m in re_pre_segment.finditer(text):
-    if m[2]:
-      yield m[2]
-      continue
-    if m[1]:
-      for segment in segmenter(m[1], deep=True):
-        yield segment
-      continue
-
-    if len(m[0].strip()) == 0:
-      yield m[0]
-      continue
-
-    tokens = tknzr.tokenize(m[0])
-    if len(tokens) == 0:
-      yield m[0]
-      continue
-    yield from tokens
-
-
-if __name__ == "__main__":
-  text_path = "/Users/seanghay/Projects/github/khmer-text-crawler/train.txt"
-
-  os.makedirs("data", exist_ok=True)
-  c = 0
-  with open("data/train.txt", "w") as outfile:
-    with open(text_path) as infile:
-      for line in infile:
-        line = line.rstrip("\n")
-        #print(line)
-        line = khnormal(line)
-        for s in segment(line):
-          c += 1
-          outfile.write(s + "\n")
-          print(c)
-        if c > 10_000_000:
-          break

khmerns-0.0.3/training/infer.py

@@ -1,63 +0,0 @@
-from khmercut import tokenize
-import torch
-from model import Segmenter
-from tokenizer import Tokenizer
-
-
-def load_model(path, device="cpu"):
-  tokenizer = Tokenizer()
-  model = Segmenter(
-    vocab_size=len(tokenizer),
-    embedding_dim=256,
-    hidden_dim=256,
-    num_labels=3,
-  )
-  model.load_state_dict(torch.load(path, map_location=device))
-  model.to(device)
-  model.eval()
-  return model, tokenizer
-
-
-def segment(text, model, tokenizer, device="cpu"):
-  token_ids = tokenizer.encode(text)
-  inputs = [tokenizer.bos_id] + token_ids + [tokenizer.eos_id]
-  inputs = torch.LongTensor(inputs).unsqueeze(0).to(device)
-
-  with torch.no_grad():
-    predictions = model(inputs)[0]
-
-  # Remove BOS/EOS predictions
-  predictions = predictions[1:-1]
-
-  # Segment based on B-WORD (1) tags
-  words = []
-  current_word = []
-
-  for char, tag in zip(text, predictions):
-    if tag == 1:  # B-WORD
-      if current_word:
-        words.append("".join(current_word))
-      current_word = [char]
-    elif tag == 2:  # I-WORD
-      current_word.append(char)
-    else:  # 0 (non-Khmer)
-      if current_word:
-        words.append("".join(current_word))
-        current_word = []
-      words.append(char)
-
-  if current_word:
-    words.append("".join(current_word))
-
-  return words
-
-
-if __name__ == "__main__":
-  device = "cpu"
-  model, tokenizer = load_model("best_model.pt", device=device)
-  text = "ប្រជាជនទីបេរស់នៅ​ក្រៅស្រុក ទូទាំងពិភពលោក បានចាប់ផ្តើមនីតិវិធីបោះឆ្នោត ដើម្បីជ្រើសរើសថ្នាក់ដឹកនាំរដ្ឋាភិបាលភៀស​ខ្លួន ដែលមានទីតាំងស្ថិតនៅ​ទីក្រុង Dharamsala ភាគខាងជើងប្រទេសឥណ្ឌា។ ជាជំហាន​ដំបូង ថ្ងៃទី​១កុម្ភៈ ប្រជាជនទីបេត្រូវបោះឆ្នោត តែងតាំង​​បេក្ខជន​ជាមុនសិន ហើយជំហានបន្ទាប់ នៅថ្ងៃទី​២៦មេសា គឺត្រូវសម្រេចជ្រើសរើសក្នុងចំណោមបេក្ខជន​ឈរឈ្មោះទាំងអស់។​ លទ្ធផលជាស្ថាពរចុងក្រោយ នឹងត្រូវប្រកាស​នៅថ្ងៃ​ទី​១៣ខែឧសភា​។".replace(
-    "\u200b", ""
-  )
-
-  words = segment(text, model, tokenizer, device=device)
-  print("|".join(words))

khmerns-0.0.3/training/model.py

@@ -1,41 +0,0 @@
-import torch
-import torch.nn as nn
-from torchcrf import CRF
-
-
-class Segmenter(nn.Module):
-  def __init__(self, vocab_size, embedding_dim, hidden_dim, num_labels):
-    super(Segmenter, self).__init__()
-    self.embedding = nn.Embedding(vocab_size, embedding_dim)
-    self.gru = nn.GRU(
-      embedding_dim,
-      hidden_dim,
-      bidirectional=True,
-      batch_first=True,
-    )
-
-    self.fc = nn.Linear(hidden_dim * 2, num_labels)
-    self.crf = CRF(num_labels, batch_first=True)
-
-  def forward(self, x, tags=None, mask=None):
-    embedded = self.embedding(x)
-    gru_out, _ = self.gru(embedded)
-    emissions = self.fc(gru_out)
-    if tags is not None:
-      log_likelihood = self.crf(emissions, tags, mask=mask, reduction="mean")
-      return -log_likelihood
-    else:
-      return self.crf.decode(emissions, mask=mask)
-
-
-if __name__ == "__main__":
-  model = Segmenter(vocab_size=200, embedding_dim=256, hidden_dim=256, num_labels=5)
-  input_data = torch.randint(0, 200, (4, 10)).long()
-  target_tags = torch.randint(0, 5, (4, 10)).long()
-
-  loss = model(input_data, tags=target_tags)
-  loss.backward()
-
-  with torch.no_grad():
-    best_paths = model(input_data)
-    print(f"Predicted Tag Sequence: {best_paths}")

khmerns-0.0.3/training/requirements.txt

@@ -1,7 +0,0 @@
-numpy
-torch
-tqdm
-pytorch-crf
-khmersegment
-regex
-gguf

khmerns-0.0.3/training/tokenizer.py

@@ -1,202 +0,0 @@
-class Tokenizer:
-  def __init__(self):
-    self.vocab = [
-      " ",
-      "!",
-      "#",
-      "$",
-      "%",
-      "&",
-      "(",
-      ")",
-      "+",
-      ",",
-      "-",
-      ".",
-      "/",
-      "0",
-      "1",
-      "2",
-      "3",
-      "4",
-      "5",
-      "6",
-      "7",
-      "8",
-      "9",
-      ":",
-      ";",
-      "=",
-      "?",
-      "@",
-      "A",
-      "B",
-      "C",
-      "D",
-      "E",
-      "F",
-      "G",
-      "H",
-      "I",
-      "J",
-      "K",
-      "L",
-      "M",
-      "N",
-      "O",
-      "P",
-      "Q",
-      "R",
-      "S",
-      "T",
-      "U",
-      "V",
-      "W",
-      "X",
-      "Y",
-      "Z",
-      "_",
-      "a",
-      "b",
-      "c",
-      "d",
-      "e",
-      "f",
-      "g",
-      "h",
-      "i",
-      "j",
-      "k",
-      "l",
-      "m",
-      "n",
-      "o",
-      "p",
-      "q",
-      "r",
-      "s",
-      "t",
-      "u",
-      "v",
-      "w",
-      "x",
-      "y",
-      "z",
-      "«",
-      "°",
-      "»",
-      "á",
-      "é",
-      "ë",
-      "ó",
-      "ö",
-      "ü",
-      "ក",
-      "ខ",
-      "គ",
-      "ឃ",
-      "ង",
-      "ច",
-      "ឆ",
-      "ជ",
-      "ឈ",
-      "ញ",
-      "ដ",
-      "ឋ",
-      "ឌ",
-      "ឍ",
-      "ណ",
-      "ត",
-      "ថ",
-      "ទ",
-      "ធ",
-      "ន",
-      "ប",
-      "ផ",
-      "ព",
-      "ភ",
-      "ម",
-      "យ",
-      "រ",
-      "ល",
-      "វ",
-      "ស",
-      "ហ",
-      "ឡ",
-      "អ",
-      "ឤ",
-      "ឥ",
-      "ឦ",
-      "ឧ",
-      "ឪ",
-      "ឫ",
-      "ឬ",
-      "ឭ",
-      "ឮ",
-      "ឯ",
-      "ឱ",
-      "ឲ",
-      "ា",
-      "ិ",
-      "ី",
-      "ឹ",
-      "ឺ",
-      "ុ",
-      "ូ",
-      "ួ",
-      "ើ",
-      "ឿ",
-      "ៀ",
-      "េ",
-      "ែ",
-      "ៃ",
-      "ោ",
-      "ៅ",
-      "ំ",
-      "ះ",
-      "ៈ",
-      "៉",
-      "៊",
-      "់",
-      "៌",
-      "៍",
-      "៏",
-      "័",
-      "្",
-      "។",
-      "៕",
-      "៖",
-      "ៗ",
-      "៘",
-      "៛",
-      "០",
-      "១",
-      "២",
-      "៣",
-      "៤",
-      "៥",
-      "៦",
-      "៧",
-      "៨",
-      "៩",
-    ]
-
-    self.pad_id = 0
-    self.bos_id = 1
-    self.eos_id = 2
-    self.unk_id = 3
-
-  def __len__(self):
-    return len(self.vocab) + 4
-
-  def decode(self, ids) -> str:
-    return "".join([self.vocab[i - 4] for i in ids if i - 4 >= 0])
-
-  def encode(self, text: str):
-    return [(self.vocab.index(c) + 4) if c in self.vocab else self.unk_id for c in text]
-
-
-if __name__ == "__main__":
-  t = Tokenizer()
-  ids = t.encode("មិនដឹង")
-  print(t.decode(ids), len(t))

khmerns-0.0.3/training/train.py

@@ -1,112 +0,0 @@
-import torch
-from torch.utils.data import DataLoader
-from torch.optim import AdamW
-from torch.optim.lr_scheduler import ReduceLROnPlateau
-from model import Segmenter
-from data import TextDataset, collate_fn
-from tokenizer import Tokenizer
-from tqdm import tqdm
-
-
-def train():
-  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-  print(f"Using device: {device}")
-
-  tokenizer = Tokenizer()
-
-  train_dataset = TextDataset(tokenizer=tokenizer, split="train")
-  eval_dataset = TextDataset(tokenizer=tokenizer, split="eval")
-
-  train_loader = DataLoader(
-    train_dataset,
-    batch_size=256,
-    shuffle=True,
-    collate_fn=collate_fn,
-  )
-
-  eval_loader = DataLoader(
-    eval_dataset,
-    batch_size=256,
-    shuffle=False,
-    collate_fn=collate_fn,
-  )
-
-  print(f"Train samples: {len(train_dataset)}, Eval samples: {len(eval_dataset)}")
-
-  model = Segmenter(
-    vocab_size=len(tokenizer),
-    embedding_dim=256,
-    hidden_dim=256,
-    num_labels=3,
-  )
-
-  model.to(device)
-
-  optimizer = AdamW(model.parameters(), lr=1e-5)
-  scheduler = ReduceLROnPlateau(optimizer, mode="min", factor=0.5, patience=1)
-  num_epochs = 20
-  best_eval_loss = float("inf")
-
-  for epoch in range(num_epochs):
-    # Training
-    model.train()
-    total_loss = 0.0
-
-    for batch_idx, (inputs, tags, mask) in enumerate(tqdm(train_loader, desc="Train")):
-      inputs = inputs.to(device)
-      tags = tags.to(device)
-      mask = mask.to(device)
-
-      optimizer.zero_grad()
-      loss = model(inputs, tags=tags, mask=mask)
-      loss.backward()
-      optimizer.step()
-
-      total_loss += loss.item()
-
-    avg_train_loss = total_loss / len(train_loader)
-
-    # Evaluation
-    model.eval()
-    eval_loss = 0.0
-    correct = 0
-    total = 0
-
-    with torch.no_grad():
-      for inputs, tags, mask in tqdm(eval_loader, desc="Eval"):
-        inputs = inputs.to(device)
-        tags = tags.to(device)
-        mask = mask.to(device)
-
-        loss = model(inputs, tags=tags, mask=mask)
-        eval_loss += loss.item()
-
-        predictions = model(inputs, mask=mask)
-        for pred, target, m in zip(predictions, tags, mask):
-          for p, t, valid in zip(pred, target, m):
-            if valid:
-              total += 1
-              if p == t.item():
-                correct += 1
-
-    avg_eval_loss = eval_loss / len(eval_loader)
-    accuracy = correct / total if total > 0 else 0
-
-    current_lr = optimizer.param_groups[0]["lr"]
-    print(
-      f"Epoch [{epoch + 1}/{num_epochs}] Train Loss: {avg_train_loss:.4f}, Eval Loss: {avg_eval_loss:.4f}, Accuracy: {accuracy:.4f}, LR: {current_lr:.6f}"
-    )
-
-    scheduler.step(avg_eval_loss)
-
-    if avg_eval_loss < best_eval_loss:
-      best_eval_loss = avg_eval_loss
-      torch.save(model.state_dict(), "best_model.pt")
-      print(f"Best model saved with eval loss: {best_eval_loss:.4f}")
-
-  torch.save(model.state_dict(), "model.pt")
-  print("Final model saved to model.pt")
-
-
-if __name__ == "__main__":
-  train()