khmerns 0.0.3__cp314-cp314-win32.whl

include/gguf.h

@@ -0,0 +1,202 @@
+// This file contains functionality related to "GGUF" files, the binary file format used by ggml.
+// GGUF files have the following structure:
+//
+// 1. File magic "GGUF" (4 bytes).
+// 2. File version (uint32_t).
+// 3. Number of ggml tensors in file (int64_t).
+// 4. Number of key-value-pairs in file (int64_t).
+// 5. For each KV pair:
+//   1. The key (string).
+//   2. The value type (gguf_type).
+//   3a. If the value type is GGUF_TYPE_ARRAY:
+//     1. The type of the array (gguf_type).
+//     2. The number of elements in the array (uint64_t).
+//     3. The binary representation of each element in the array.
+//   3b. Otherwise:
+//     1. The binary representation of the value.
+// 6. For each ggml tensor:
+//   1. The tensor name (string).
+//   2. The number of dimensions of the tensor (uint32_t).
+//   3. For each dimension:
+//     1. The size of the tensor in the dimension (int64_t).
+//   4. The tensor data type (ggml_type).
+//   5. The tensor data offset in the tensor data binary blob (uint64_t).
+// 7. The tensor data binary blob (optional, aligned).
+//
+// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator.
+// All enums are stored as int32_t.
+// All bool values are stored as int8_t.
+// If the special key "general.alignment" (uint32_t) is defined it is used for alignment,
+//   otherwise GGUF_DEFAULT_ALIGNMENT is used.
+//
+// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de)
+
+#pragma once
+
+#include "ggml.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#define GGUF_MAGIC   "GGUF"
+#define GGUF_VERSION 3
+
+#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment"
+
+#define GGUF_DEFAULT_ALIGNMENT 32
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+    // types that can be stored as GGUF KV data
+    enum gguf_type {
+        GGUF_TYPE_UINT8   = 0,
+        GGUF_TYPE_INT8    = 1,
+        GGUF_TYPE_UINT16  = 2,
+        GGUF_TYPE_INT16   = 3,
+        GGUF_TYPE_UINT32  = 4,
+        GGUF_TYPE_INT32   = 5,
+        GGUF_TYPE_FLOAT32 = 6,
+        GGUF_TYPE_BOOL    = 7,
+        GGUF_TYPE_STRING  = 8,
+        GGUF_TYPE_ARRAY   = 9,
+        GGUF_TYPE_UINT64  = 10,
+        GGUF_TYPE_INT64   = 11,
+        GGUF_TYPE_FLOAT64 = 12,
+        GGUF_TYPE_COUNT,       // marks the end of the enum
+    };
+
+    struct gguf_context;
+
+    struct gguf_init_params {
+        bool no_alloc;
+
+        // if not NULL, create a ggml_context and allocate the tensor data in it
+        struct ggml_context ** ctx;
+    };
+
+    GGML_API struct gguf_context * gguf_init_empty(void);
+    GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
+    //GGML_API struct gguf_context * gguf_init_from_buffer(..);
+
+    GGML_API void gguf_free(struct gguf_context * ctx);
+
+    GGML_API const char * gguf_type_name(enum gguf_type type);
+
+    GGML_API uint32_t gguf_get_version    (const struct gguf_context * ctx);
+    GGML_API size_t   gguf_get_alignment  (const struct gguf_context * ctx);
+    GGML_API size_t   gguf_get_data_offset(const struct gguf_context * ctx);
+
+    GGML_API int64_t      gguf_get_n_kv(const struct gguf_context * ctx);
+    GGML_API int64_t      gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found
+    GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id);
+
+    GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id);
+
+    // will abort if the wrong type is used for the key
+    GGML_API uint8_t      gguf_get_val_u8  (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API int8_t       gguf_get_val_i8  (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API uint16_t     gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API int16_t      gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API uint32_t     gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API int32_t      gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API float        gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API uint64_t     gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API int64_t      gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API double       gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API bool         gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id);
+    GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id);
+    GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id);
+    GGML_API size_t       gguf_get_arr_n   (const struct gguf_context * ctx, int64_t key_id);
+
+    // get raw pointer to the first element of the array with the given key_id
+    // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference)
+    GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id);
+
+    // get ith C string from array with given key_id
+    GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i);
+
+    GGML_API int64_t        gguf_get_n_tensors    (const struct gguf_context * ctx);
+    GGML_API int64_t        gguf_find_tensor      (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found
+    GGML_API size_t         gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);
+    GGML_API const char *   gguf_get_tensor_name  (const struct gguf_context * ctx, int64_t tensor_id);
+    GGML_API enum ggml_type gguf_get_tensor_type  (const struct gguf_context * ctx, int64_t tensor_id);
+    GGML_API size_t         gguf_get_tensor_size  (const struct gguf_context * ctx, int64_t tensor_id);
+
+    // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist)
+    GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key);
+
+    // overrides an existing KV pair or adds a new one, the new KV pair is always at the back
+    GGML_API void gguf_set_val_u8  (struct gguf_context * ctx, const char * key, uint8_t      val);
+    GGML_API void gguf_set_val_i8  (struct gguf_context * ctx, const char * key, int8_t       val);
+    GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t     val);
+    GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t      val);
+    GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t     val);
+    GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t      val);
+    GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float        val);
+    GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t     val);
+    GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t      val);
+    GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double       val);
+    GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool         val);
+    GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
+
+    // creates a new array with n elements of the given type and copies the corresponding number of bytes from data
+    GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n);
+
+    // creates a new array with n strings and copies the corresponding strings from data
+    GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n);
+
+    // set or add KV pairs from another context
+    GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src);
+
+    // add tensor to GGUF context, tensor name must be unique
+    GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
+
+    // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated
+    //   in such a way that the tensor data remains as one contiguous block (except for padding)
+    GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
+
+    // assumes that at least gguf_get_tensor_size bytes can be read from data
+    GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data);
+
+    // writing gguf files can be done in 3 ways:
+    //
+    // - write the entire gguf_context to a binary file in a single pass:
+    //
+    //   gguf_write_to_file(ctx, fname, /*only_meta =*/ false);
+    //
+    // - write only the meta data to a file, then re-open the file and append the tensor data:
+    //
+    //   gguf_write_to_file(ctx, fname, /*only_meta =*/ true);
+    //   FILE * f = fopen(fname, "ab");
+    //   fwrite(f, ...); // write tensor data
+    //   fclose(f);
+    //
+    // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
+    //
+    //   FILE * f = fopen(fname, "wb");
+    //   const size_t size_meta = gguf_get_meta_size(ctx);
+    //   fseek(f, size_meta, SEEK_SET);
+    //   fwrite(f, ...); // write tensor data
+    //   void * data = malloc(size_meta);
+    //   gguf_get_meta_data(ctx, data);
+    //   rewind(f);
+    //   fwrite(data, 1, data, f);
+    //   free(data);
+    //   fclose(f);
+    //
+
+    // write the entire context to a binary file
+    GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
+
+    // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
+    GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
+
+    // writes the meta data to pointer "data"
+    GGML_API void   gguf_get_meta_data(const struct gguf_context * ctx, void * data);
+
+#ifdef  __cplusplus
+}
+#endif

khmerns/__init__.py

@@ -0,0 +1,3 @@
+from ._core import KhmerSegmenter, tokenize, __version__
+
+__all__ = ["KhmerSegmenter", "tokenize", "__version__"]

khmerns/__init__.pyi

@@ -0,0 +1,16 @@
+from typing import List
+
+__version__: str
+
+class KhmerSegmenter:
+  def __init__(self) -> None: ...
+  def tokenize(self, text: str) -> List[str]:
+    """Segment Khmer text and return a list of words."""
+    ...
+  def __call__(self, text: str) -> List[str]:
+    """Segment Khmer text and return a list of words."""
+    ...
+
+def tokenize(text: str) -> List[str]:
+  """Segment Khmer text and return a list of words."""
+  ...

khmerns-0.0.3.dist-info/METADATA

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: khmerns
+Version: 0.0.3
+Summary: Khmer Neural Segmenter
+Keywords: khmer,nlp,segmentation,tokenization,neural-network
+Author-Email: Seanghay Yath <seanghay.dev@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Project-URL: Homepage, https://github.com/seanghay/khmer-neural-segmenter
+Project-URL: Repository, https://github.com/seanghay/khmer-neural-segmenter
+Project-URL: Issues, https://github.com/seanghay/khmer-neural-segmenter/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# Khmer Neural Segmenter
+
+A fast Khmer word segmentation library.
+
+<img src="img/graph.png" alt="" width=500>
+
+## Installation
+
+```
+pip install khmerns
+```
+
+## Usage
+
+```python
+from khmerns import tokenize
+
+# Returns a list of words
+words = tokenize("សួស្តីបងប្អូន")
+# ['សួស្តី', 'បង', 'ប្អូន']
+```
+
+You can also use the class-based API if you prefer:
+
+```python
+from khmerns import KhmerSegmenter
+
+segmenter = KhmerSegmenter()
+words = segmenter.tokenize("សួស្តីបងប្អូន")
+# or
+words = segmenter("សួស្តីបងប្អូន")
+```
+
+## Training
+
+The training pipeline lives in the `training/` directory. It trains a BiGRU + CRF model on character-level BIO tags, then converts the result to GGUF for the C++ inference backend.
+
+### Data format
+
+Training data is a plain text file at `training/data/train.txt`. One word per line. Words that appear on consecutive lines are treated as part of the same sentence. The model learns word boundaries from this.
+
+Example `training/data/train.txt`:
+
+```
+សួស្តី
+បង
+ប្អូន
+ខ្ញុំ
+ទៅ
+ផ្សារ
+```
+
+Non-Khmer tokens (spaces, punctuation, numbers, Latin text) are tagged as `NON-KHMER`. Khmer tokens get `B-WORD` on the first character and `I-WORD` on the rest.
+
+### Steps
+
+```bash
+cd training
+pip install -r requirements.txt
+```
+
+**1. Prepare training data**
+
+Place your segmented text in `data/train.txt` (one word per line). If you have raw unsegmented Khmer text, you can use the generation script to pre-segment it:
+
+```bash
+python generate.py
+```
+
+This requires `khmersegment` and a source text file. Edit the path in `generate.py` to point to your raw text.
+
+**2. Train**
+
+```bash
+python train.py
+```
+
+Trains for 20 epochs with AdamW (lr=1e-5) and ReduceLROnPlateau. Saves `best_model.pt` (best eval loss) and `model.pt` (final). Uses CUDA if available.
+
+**3. Convert to GGUF**
+
+```bash
+python convert_to_gguf.py best_model.pt model.gguf
+```
+
+This produces a GGUF file (~3.3MB) containing all model weights.
+
+**4. Embed in the C++ binary**
+
+To use the new model in the library, convert the GGUF file to a C header and replace `src/model_data.h`, then rebuild:
+
+```bash
+xxd -i model.gguf > ../src/model_data.h
+pip install -e ..
+```
+
+## License
+
+MIT

khmerns-0.0.3.dist-info/RECORD

@@ -0,0 +1,29 @@
+include/ggml-alloc.h,sha256=q5pVcoR6mOwx7lf-IC84IWvgHaPmPjYYPX2LnzR1ag8,3709
+include/ggml-backend.h,sha256=ATnc9x8vL0X51r3r5XFpz0hBRGAtv6uHHL4Gta0NaRQ,21426
+include/ggml-blas.h,sha256=XGbWYj60VJBBaYGwpd8u6REiwJLQx5wg4SE5-fM4M4E,611
+include/ggml-cann.h,sha256=dc2nq-Z7-_9UZbJln2t1o6maduJqgCe1pCJPQ1ziMOg,4676
+include/ggml-cpp.h,sha256=7tG7ys4Uqvre6aJ7t0CgofIDzmWilcv1M8fn3vD6yd4,1691
+include/ggml-cpu.h,sha256=e4riopzIzfZq0rK7V8PEchNsSPc2wr_F678kBUJPOqk,7674
+include/ggml-cuda.h,sha256=qqFiJffdn4w6MVhOOw6g0Uo54GFnmZsjBWYZLSvNxZ0,1632
+include/ggml-metal.h,sha256=PPz5g2PKsENZLJEE-JuSugbHlChrzomGSHQ7RX4VXms,2202
+include/ggml-opt.h,sha256=qTpDaSWZqnIi8JKcMKEZNhXmFPJuCclY6zMy8B56nw0,14295
+include/ggml-rpc.h,sha256=jfGW1PTGF_rq85edNxWfm_LB4KtXJ_DIZn6n4TqlrBo,1086
+include/ggml-sycl.h,sha256=WJW-Sk6poGXcMmXuwPCcmpJG3fJy5uyx4bB12oNHrKo,1812
+include/ggml-virtgpu.h,sha256=L3KdW5jwhgIHee2FgiAOucBH09Tgz_GzPB_pV1mYreE,266
+include/ggml-vulkan.h,sha256=C29iUCGwR5B9JLYLPKRB6Kl9RCNfDTKr6yZlESw1QY4,981
+include/ggml-webgpu.h,sha256=YoxXN2KYJOwzJmIKXESA3olVy_gsNSvu2ORuAlFPs5I,347
+include/ggml-zendnn.h,sha256=rlX7HjkqaAory2NoxpQSVZgPjEnZHZ6dKjQAOLK_8lE,520
+include/ggml.h,sha256=8AuUtoOyhyNaf65atAKT1NBO-FrquzEXbYB-2VYjC6I,106093
+include/gguf.h,sha256=dGmizdUtMG2dBzbEiu0ebZfF2Abz5smJuCXuEpMFToc,10426
+khmerns/__init__.py,sha256=56VwZ3fseBKsGGMhVtE1EI1Tqi4uhigZ7OdRKf6MAKE,117
+khmerns/__init__.pyi,sha256=5Ywjkugs1j6YWPS6bkFFTl077E9mb9-fSXPgu-kQhbw,437
+khmerns/_core.cp314-win32.pyd,sha256=yuCZ1Xjmkv6CyPwLXKNxUIiJuNUsRlsHA_8NyDg6Okk,3675648
+lib/cmake/ggml/ggml-config.cmake,sha256=OPnz2F8SEuCgmqv8vJc3xlADN6NkgootaXOIKZ9uQ_Y,12170
+lib/cmake/ggml/ggml-version.cmake,sha256=cmpuq1NZlHRY3WdA7y81233b6KomdObooai5LL4-Dyc,2827
+lib/ggml-base.lib,sha256=OmtZI_juR08w9OXQcBR1ZuTOk-eSRo-2ds0nYLvXr8Y,1217308
+lib/ggml-cpu.lib,sha256=-djxu2q6TZUH8HBwq-v6oxrfjY7f5W4rYaUgosUEKlU,1130366
+lib/ggml.lib,sha256=ktfyHQWCPnA0cBlPMbnY4M3IeIJmbQr6hzrliPV2gdw,190804
+khmerns-0.0.3.dist-info/METADATA,sha256=NI0Lq5nTQGzEEI5prGscnoTSKg2WFwkoL1QbDhCEAaI,3263
+khmerns-0.0.3.dist-info/WHEEL,sha256=xwLkizf5NInhysT1HPChN8-LXYl1drA7qIdg1OQtfC8,102
+khmerns-0.0.3.dist-info/licenses/LICENSE,sha256=NJbBwbQTQpJwxvCeUDhqpUe3HKE3geRtx5iqIlQ5q0c,1089
+khmerns-0.0.3.dist-info/RECORD,,

khmerns-0.0.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: scikit-build-core 0.11.6
+Root-Is-Purelib: false
+Tag: cp314-cp314-win32
+

khmerns-0.0.3.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Seanghay Yath
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.