tokana 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 mnaoizy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,164 @@
+# tokana
+
+Modern Japanese morphological analyzer for TypeScript. Works in Node.js and the browser.
+
+- Zero runtime dependencies
+- Full TypeScript support with typed tokens per dictionary format
+- ESM and CommonJS
+- Supports IPAdic, UniDic, and NEologd dictionaries
+- Built-in CLI for compiling MeCab-compatible dictionary sources
+- Browser support via `fetch` + `DecompressionStream`
+
+## Install
+
+```bash
+npm install tokana
+```
+
+## Dictionary Setup
+
+tokana requires compiled dictionary files. Use the CLI to compile from MeCab-compatible sources (e.g. [mecab-ipadic](https://taku910.github.io/mecab/)):
+
+```bash
+# Download and extract MeCab IPAdic source, then compile:
+npx tokana build ./mecab-ipadic-2.7.0-20070801 ./dict
+```
+
+This generates `.dat.gz` files in the output directory.
+
+## Usage
+
+### Node.js
+
+```typescript
+import { createTokenizer } from "tokana";
+
+const tokenizer = await createTokenizer({
+  format: "ipadic",
+  dicPath: "./dict",
+});
+
+const tokens = tokenizer.tokenize("東京都に住んでいる");
+
+for (const token of tokens) {
+  console.log(token.surface, token.pos, token.baseForm, token.reading);
+}
+// 東京  名詞  東京  トウキョウ
+// 都    名詞  都    ト
+// に    助詞  に    ニ
+// 住ん  動詞  住む  スン
+// で    助詞  で    デ
+// いる  動詞  いる  イル
+```
+
+### Browser
+
+```typescript
+import { createTokenizer } from "tokana";
+
+// dicPath is a URL path — serve .dat.gz files from your static assets
+const tokenizer = await createTokenizer({
+  format: "ipadic",
+  dicPath: "/dict",
+});
+
+const tokens = tokenizer.tokenize("形態素解析");
+```
+
+## Dictionary Formats
+
+| Format | Description | Features |
+|--------|-------------|----------|
+| `ipadic` | MeCab IPAdic (default) | 9 fields: pos, posDetail1-3, conjugationType/Form, baseForm, reading, pronunciation |
+| `unidic` | UniDic | 17 fields: pos1-4, cType/cForm, lemma, orth, pron, goshu, etc. |
+| `neologd` | mecab-ipadic-NEologd | Same as ipadic, with added neologisms |
+
+```typescript
+// UniDic example
+const tokenizer = await createTokenizer({
+  format: "unidic",
+  dicPath: "./unidic-dict",
+});
+```
+
+## API
+
+### `createTokenizer(options): Promise<Tokenizer<T>>`
+
+Creates a tokenizer instance. Loads and parses dictionary files asynchronously.
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `dicPath` | `string` | (required) | Path to compiled dictionary directory |
+| `format` | `"ipadic" \| "unidic" \| "neologd"` | `"ipadic"` | Dictionary format |
+
+### `Tokenizer<T>.tokenize(text: string): T[]`
+
+Tokenizes input text into an array of typed token objects.
+
+### Token Types
+
+**`IpadicToken`** (ipadic / neologd):
+
+```typescript
+interface IpadicToken {
+  surface: string;         // Surface form
+  pos: string;             // Part-of-speech
+  posDetail1: string;      // POS subcategory 1
+  posDetail2: string;      // POS subcategory 2
+  posDetail3: string;      // POS subcategory 3
+  conjugationType: string; // Conjugation type
+  conjugationForm: string; // Conjugation form
+  baseForm: string;        // Base form (lemma)
+  reading: string;         // Reading
+  pronunciation: string;   // Pronunciation
+  // + wordId, type, offset, length, cost
+}
+```
+
+**`UnidicToken`** (unidic):
+
+```typescript
+interface UnidicToken {
+  surface: string;
+  pos1: string;     pos2: string;     pos3: string;    pos4: string;
+  cType: string;    cForm: string;
+  lForm: string;    lemma: string;
+  orth: string;     orthBase: string;
+  pron: string;     pronBase: string;
+  goshu: string;
+  iType: string;    iForm: string;
+  fType: string;    fForm: string;
+  // + wordId, type, offset, length, cost
+}
+```
+
+## CLI
+
+```
+tokana build <source-dir> <output-dir>   Compile MeCab dictionary source
+tokana info <dict-dir>                   Show dictionary info
+```
+
+### Compile a dictionary
+
+```bash
+npx tokana build ./mecab-ipadic-2.7.0-20070801 ./dict
+```
+
+Reads `matrix.def`, `char.def`, `unk.def`, and `*.csv` from the source directory and outputs compiled `.dat.gz` files.
+
+## Demo
+
+A Vite + React demo app is included in [`demo/`](./demo):
+
+```bash
+cd demo
+npm install
+npm run setup-dict   # downloads and compiles mecab-ipadic
+npm run dev          # http://localhost:5173
+```
+
+## License
+
+[MIT](./LICENSE)

package/dist/cli.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+var args = process.argv.slice(2);
+var command = args[0];
+async function main() {
+  switch (command) {
+    case "build":
+    case "compile": {
+      const { compile } = await import("./compile-VQJF62SJ.js");
+      await compile(args.slice(1));
+      break;
+    }
+    case "info": {
+      const { info } = await import("./info-OVE32SWZ.js");
+      await info(args.slice(1));
+      break;
+    }
+    default:
+      console.log(`tokana - Modern Japanese Morphological Analyzer
+
+Usage:
+  tokana build <source-dir> <output-dir>  Compile MeCab dictionary source
+  tokana info <dict-dir>                  Show dictionary info
+
+Options:
+  --format <ipadic|unidic|neologd>  Dictionary format (default: ipadic)
+`);
+      if (command && command !== "help" && command !== "--help") {
+        console.error(`Unknown command: ${command}`);
+        process.exit(1);
+      }
+  }
+}
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["/**\n * CLI entry point for tokana dictionary compiler.\n */\n\nconst args = process.argv.slice(2);\nconst command = args[0];\n\nasync function main() {\n  switch (command) {\n    case \"build\":\n    case \"compile\": {\n      const { compile } = await import(\"./cli/commands/compile.js\");\n      await compile(args.slice(1));\n      break;\n    }\n    case \"info\": {\n      const { info } = await import(\"./cli/commands/info.js\");\n      await info(args.slice(1));\n      break;\n    }\n    default:\n      console.log(`tokana - Modern Japanese Morphological Analyzer\n\nUsage:\n  tokana build <source-dir> <output-dir>  Compile MeCab dictionary source\n  tokana info <dict-dir>                  Show dictionary info\n\nOptions:\n  --format <ipadic|unidic|neologd>  Dictionary format (default: ipadic)\n`);\n      if (command && command !== \"help\" && command !== \"--help\") {\n        console.error(`Unknown command: ${command}`);\n        process.exit(1);\n      }\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n"],"mappings":";;;AAIA,IAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AACjC,IAAM,UAAU,KAAK,CAAC;AAEtB,eAAe,OAAO;AACpB,UAAQ,SAAS;AAAA,IACf,KAAK;AAAA,IACL,KAAK,WAAW;AACd,YAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,uBAA2B;AAC5D,YAAM,QAAQ,KAAK,MAAM,CAAC,CAAC;AAC3B;AAAA,IACF;AAAA,IACA,KAAK,QAAQ;AACX,YAAM,EAAE,KAAK,IAAI,MAAM,OAAO,oBAAwB;AACtD,YAAM,KAAK,KAAK,MAAM,CAAC,CAAC;AACxB;AAAA,IACF;AAAA,IACA;AACE,cAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAQjB;AACK,UAAI,WAAW,YAAY,UAAU,YAAY,UAAU;AACzD,gBAAQ,MAAM,oBAAoB,OAAO,EAAE;AAC3C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,EACJ;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}