shevchenko 2.1.3 → 3.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/npm-publish.yml +36 -0
- package/README.md +8 -7
- package/dist/cjs/anthroponym-declension/anthroponym-inflector.js +39 -0
- package/dist/cjs/anthroponym-declension/anthroponym-inflector.js.map +1 -0
- package/dist/cjs/anthroponym-declension/family-name-inflector.js +69 -0
- package/dist/cjs/anthroponym-declension/family-name-inflector.js.map +1 -0
- package/dist/cjs/anthroponym-declension/given-name-inflector.js +40 -0
- package/dist/cjs/anthroponym-declension/given-name-inflector.js.map +1 -0
- package/dist/cjs/anthroponym-declension/name-inflector.js +30 -0
- package/dist/cjs/anthroponym-declension/name-inflector.js.map +1 -0
- package/dist/cjs/anthroponym-declension/patronymic-name-inflector.js +40 -0
- package/dist/cjs/anthroponym-declension/patronymic-name-inflector.js.map +1 -0
- package/dist/cjs/bootstrap.js +29 -0
- package/dist/cjs/bootstrap.js.map +1 -0
- package/dist/cjs/gender-detection/detect-gender.js +55 -0
- package/dist/cjs/gender-detection/detect-gender.js.map +1 -0
- package/dist/cjs/gender-detection/given-names-genders.json.js +1158 -0
- package/dist/cjs/gender-detection/given-names-genders.json.js.map +1 -0
- package/dist/cjs/index.js +36 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/input-validation.js +83 -0
- package/dist/cjs/input-validation.js.map +1 -0
- package/dist/cjs/language/alphabet.js +54 -0
- package/dist/cjs/language/alphabet.js.map +1 -0
- package/dist/cjs/language/grammatical-case.js +27 -0
- package/dist/cjs/language/grammatical-case.js.map +1 -0
- package/dist/cjs/language/grammatical-gender.js +22 -0
- package/dist/cjs/language/grammatical-gender.js.map +1 -0
- package/dist/cjs/language/letter-case.js +49 -0
- package/dist/cjs/language/letter-case.js.map +1 -0
- package/dist/cjs/language/linguistics.js +35 -0
- package/dist/cjs/language/linguistics.js.map +1 -0
- package/dist/cjs/language/word-class.js +22 -0
- package/dist/cjs/language/word-class.js.map +1 -0
- package/dist/cjs/shevchenko.js +67 -154555
- package/dist/cjs/shevchenko.js.map +1 -1
- package/dist/cjs/word-class-recognition/artifacts/model.bundle.json.js +199 -0
- package/dist/cjs/word-class-recognition/artifacts/model.bundle.json.js.map +1 -0
- package/dist/cjs/word-class-recognition/cache/incorrect-predictions.json.js +1459 -0
- package/dist/cjs/word-class-recognition/cache/incorrect-predictions.json.js.map +1 -0
- package/dist/cjs/word-class-recognition/model-bundle-loader.js +74 -0
- package/dist/cjs/word-class-recognition/model-bundle-loader.js.map +1 -0
- package/dist/cjs/word-class-recognition/model-config.js +20 -0
- package/dist/cjs/word-class-recognition/model-config.js.map +1 -0
- package/dist/cjs/word-class-recognition/word-class-recognizer.js +76 -0
- package/dist/cjs/word-class-recognition/word-class-recognizer.js.map +1 -0
- package/dist/cjs/word-class-recognition/word-class-transformer.js +52 -0
- package/dist/cjs/word-class-recognition/word-class-transformer.js.map +1 -0
- package/dist/cjs/word-class-recognition/word-transformer.js +37 -0
- package/dist/cjs/word-class-recognition/word-transformer.js.map +1 -0
- package/dist/cjs/word-declension/append-command-runner.js +28 -0
- package/dist/cjs/word-declension/append-command-runner.js.map +1 -0
- package/dist/cjs/word-declension/command-runner-factory.js +38 -0
- package/dist/cjs/word-declension/command-runner-factory.js.map +1 -0
- package/dist/cjs/word-declension/declension-rule-inflector.js +65 -0
- package/dist/cjs/word-declension/declension-rule-inflector.js.map +1 -0
- package/dist/cjs/word-declension/declension-rule-matchers.js +43 -0
- package/dist/cjs/word-declension/declension-rule-matchers.js.map +1 -0
- package/dist/cjs/word-declension/declension-rules.json.js +7405 -0
- package/dist/cjs/word-declension/declension-rules.json.js.map +1 -0
- package/dist/cjs/word-declension/declension-types.js +24 -0
- package/dist/cjs/word-declension/declension-types.js.map +1 -0
- package/dist/cjs/word-declension/index.js +28 -0
- package/dist/cjs/word-declension/index.js.map +1 -0
- package/dist/cjs/word-declension/replace-command-runner.js +28 -0
- package/dist/cjs/word-declension/replace-command-runner.js.map +1 -0
- package/dist/es/anthroponym-declension/anthroponym-inflector.d.ts +16 -0
- package/dist/es/anthroponym-declension/anthroponym-inflector.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/anthroponym-inflector.js +35 -0
- package/dist/es/anthroponym-declension/anthroponym-inflector.js.map +1 -0
- package/dist/es/anthroponym-declension/anthroponym.d.ts +6 -0
- package/dist/es/anthroponym-declension/anthroponym.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/family-name-inflector.d.ts +14 -0
- package/dist/es/anthroponym-declension/family-name-inflector.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/family-name-inflector.js +65 -0
- package/dist/es/anthroponym-declension/family-name-inflector.js.map +1 -0
- package/dist/es/anthroponym-declension/given-name-inflector.d.ts +12 -0
- package/dist/es/anthroponym-declension/given-name-inflector.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/given-name-inflector.js +36 -0
- package/dist/es/anthroponym-declension/given-name-inflector.js.map +1 -0
- package/dist/es/anthroponym-declension/index.d.ts +6 -0
- package/dist/es/anthroponym-declension/index.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/name-inflector.d.ts +12 -0
- package/dist/es/anthroponym-declension/name-inflector.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/name-inflector.js +26 -0
- package/dist/es/anthroponym-declension/name-inflector.js.map +1 -0
- package/dist/es/anthroponym-declension/patronymic-name-inflector.d.ts +12 -0
- package/dist/es/anthroponym-declension/patronymic-name-inflector.d.ts.map +1 -0
- package/dist/es/anthroponym-declension/patronymic-name-inflector.js +36 -0
- package/dist/es/anthroponym-declension/patronymic-name-inflector.js.map +1 -0
- package/dist/es/bootstrap.d.ts +4 -0
- package/dist/{types → es}/bootstrap.d.ts.map +1 -1
- package/dist/es/bootstrap.js +25 -0
- package/dist/es/bootstrap.js.map +1 -0
- package/dist/es/contracts.d.ts +21 -0
- package/dist/es/contracts.d.ts.map +1 -0
- package/dist/es/gender-detection/detect-gender.d.ts +11 -0
- package/dist/es/gender-detection/detect-gender.d.ts.map +1 -0
- package/dist/es/gender-detection/detect-gender.js +51 -0
- package/dist/es/gender-detection/detect-gender.js.map +1 -0
- package/dist/es/gender-detection/detect-gender.test.d.ts +2 -0
- package/dist/es/gender-detection/detect-gender.test.d.ts.map +1 -0
- package/dist/es/gender-detection/given-names-genders.json.js +1156 -0
- package/dist/es/gender-detection/given-names-genders.json.js.map +1 -0
- package/dist/es/gender-detection/index.d.ts +2 -0
- package/dist/es/gender-detection/index.d.ts.map +1 -0
- package/dist/es/index.d.ts +2 -0
- package/dist/es/index.d.ts.map +1 -0
- package/dist/es/index.js +13 -0
- package/dist/es/index.js.map +1 -0
- package/dist/es/input-validation.d.ts +14 -0
- package/dist/es/input-validation.d.ts.map +1 -0
- package/dist/es/input-validation.js +78 -0
- package/dist/es/input-validation.js.map +1 -0
- package/dist/es/input-validation.test.d.ts +2 -0
- package/dist/es/input-validation.test.d.ts.map +1 -0
- package/dist/es/language/alphabet.d.ts +48 -0
- package/dist/es/language/alphabet.d.ts.map +1 -0
- package/dist/es/language/alphabet.js +52 -0
- package/dist/es/language/alphabet.js.map +1 -0
- package/dist/es/language/grammatical-case.d.ts +13 -0
- package/dist/es/language/grammatical-case.d.ts.map +1 -0
- package/dist/es/language/grammatical-case.js +25 -0
- package/dist/es/language/grammatical-case.js.map +1 -0
- package/dist/es/language/grammatical-gender.d.ts +8 -0
- package/dist/es/language/grammatical-gender.d.ts.map +1 -0
- package/dist/es/language/grammatical-gender.js +20 -0
- package/dist/es/language/grammatical-gender.js.map +1 -0
- package/dist/es/language/index.d.ts +7 -0
- package/dist/es/language/index.d.ts.map +1 -0
- package/dist/es/language/letter-case.d.ts +6 -0
- package/dist/es/language/letter-case.d.ts.map +1 -0
- package/dist/es/language/letter-case.js +45 -0
- package/dist/es/language/letter-case.js.map +1 -0
- package/dist/es/language/letter-case.test.d.ts +2 -0
- package/dist/es/language/letter-case.test.d.ts.map +1 -0
- package/dist/es/language/linguistics.d.ts +7 -0
- package/dist/es/language/linguistics.d.ts.map +1 -0
- package/dist/es/language/linguistics.js +30 -0
- package/dist/es/language/linguistics.js.map +1 -0
- package/dist/es/language/linguistics.test.d.ts +2 -0
- package/dist/es/language/linguistics.test.d.ts.map +1 -0
- package/dist/es/language/word-class.d.ts +8 -0
- package/dist/es/language/word-class.d.ts.map +1 -0
- package/dist/es/language/word-class.js +20 -0
- package/dist/es/language/word-class.js.map +1 -0
- package/dist/es/shevchenko.d.ts +54 -0
- package/dist/es/shevchenko.d.ts.map +1 -0
- package/dist/es/shevchenko.js +60 -154480
- package/dist/es/shevchenko.js.map +1 -1
- package/dist/es/word-class-recognition/artifacts/model.bundle.json.js +190 -0
- package/dist/es/word-class-recognition/artifacts/model.bundle.json.js.map +1 -0
- package/dist/es/word-class-recognition/cache/incorrect-predictions.json.js +1457 -0
- package/dist/es/word-class-recognition/cache/incorrect-predictions.json.js.map +1 -0
- package/dist/es/word-class-recognition/data.utils.d.ts +16 -0
- package/dist/es/word-class-recognition/data.utils.d.ts.map +1 -0
- package/dist/es/word-class-recognition/index.d.ts +3 -0
- package/dist/es/word-class-recognition/index.d.ts.map +1 -0
- package/dist/es/word-class-recognition/model-bundle-loader.d.ts +15 -0
- package/dist/es/word-class-recognition/model-bundle-loader.d.ts.map +1 -0
- package/dist/es/word-class-recognition/model-bundle-loader.js +69 -0
- package/dist/es/word-class-recognition/model-bundle-loader.js.map +1 -0
- package/dist/es/word-class-recognition/model-bundle-saver.d.ts +12 -0
- package/dist/es/word-class-recognition/model-bundle-saver.d.ts.map +1 -0
- package/dist/es/word-class-recognition/model-config.d.ts +5 -0
- package/dist/es/word-class-recognition/model-config.d.ts.map +1 -0
- package/dist/es/word-class-recognition/model-config.js +16 -0
- package/dist/es/word-class-recognition/model-config.js.map +1 -0
- package/dist/es/word-class-recognition/scripts/cache-incorrect-predictions.d.ts +2 -0
- package/dist/es/word-class-recognition/scripts/cache-incorrect-predictions.d.ts.map +1 -0
- package/dist/es/word-class-recognition/scripts/train-model.d.ts +7 -0
- package/dist/es/word-class-recognition/scripts/train-model.d.ts.map +1 -0
- package/dist/es/word-class-recognition/word-class-recognizer.d.ts +18 -0
- package/dist/es/word-class-recognition/word-class-recognizer.d.ts.map +1 -0
- package/dist/es/word-class-recognition/word-class-recognizer.js +52 -0
- package/dist/es/word-class-recognition/word-class-recognizer.js.map +1 -0
- package/dist/es/word-class-recognition/word-class-transformer.d.ts +6 -0
- package/dist/es/word-class-recognition/word-class-transformer.d.ts.map +1 -0
- package/dist/es/word-class-recognition/word-class-transformer.js +48 -0
- package/dist/es/word-class-recognition/word-class-transformer.js.map +1 -0
- package/dist/es/word-class-recognition/word-class-transformer.test.d.ts +2 -0
- package/dist/es/word-class-recognition/word-class-transformer.test.d.ts.map +1 -0
- package/dist/es/word-class-recognition/word-transformer.d.ts +7 -0
- package/dist/es/word-class-recognition/word-transformer.d.ts.map +1 -0
- package/dist/es/word-class-recognition/word-transformer.js +33 -0
- package/dist/es/word-class-recognition/word-transformer.js.map +1 -0
- package/dist/es/word-class-recognition/word-transformer.test.d.ts +2 -0
- package/dist/es/word-class-recognition/word-transformer.test.d.ts.map +1 -0
- package/dist/es/word-declension/append-command-runner.d.ts +12 -0
- package/dist/es/word-declension/append-command-runner.d.ts.map +1 -0
- package/dist/es/word-declension/append-command-runner.js +24 -0
- package/dist/es/word-declension/append-command-runner.js.map +1 -0
- package/dist/es/word-declension/command-runner-factory.d.ts +9 -0
- package/dist/es/word-declension/command-runner-factory.d.ts.map +1 -0
- package/dist/es/word-declension/command-runner-factory.js +34 -0
- package/dist/es/word-declension/command-runner-factory.js.map +1 -0
- package/dist/es/word-declension/declension-rule-inflector.d.ts +16 -0
- package/dist/es/word-declension/declension-rule-inflector.d.ts.map +1 -0
- package/dist/es/word-declension/declension-rule-inflector.js +60 -0
- package/dist/es/word-declension/declension-rule-inflector.js.map +1 -0
- package/dist/es/word-declension/declension-rule-matchers.d.ts +19 -0
- package/dist/es/word-declension/declension-rule-matchers.d.ts.map +1 -0
- package/dist/es/word-declension/declension-rule-matchers.js +36 -0
- package/dist/es/word-declension/declension-rule-matchers.js.map +1 -0
- package/dist/es/word-declension/declension-rules.json.js +7403 -0
- package/dist/es/word-declension/declension-rules.json.js.map +1 -0
- package/dist/es/word-declension/declension-types.d.ts +36 -0
- package/dist/es/word-declension/declension-types.d.ts.map +1 -0
- package/dist/es/word-declension/declension-types.js +22 -0
- package/dist/es/word-declension/declension-types.js.map +1 -0
- package/dist/es/word-declension/index.d.ts +7 -0
- package/dist/es/word-declension/index.d.ts.map +1 -0
- package/dist/es/word-declension/index.js +20 -0
- package/dist/es/word-declension/index.js.map +1 -0
- package/dist/es/word-declension/inflection-command-runner.d.ts +4 -0
- package/dist/es/word-declension/inflection-command-runner.d.ts.map +1 -0
- package/dist/es/word-declension/replace-command-runner.d.ts +12 -0
- package/dist/es/word-declension/replace-command-runner.d.ts.map +1 -0
- package/dist/es/word-declension/replace-command-runner.js +24 -0
- package/dist/es/word-declension/replace-command-runner.js.map +1 -0
- package/dist/es/word-declension/scripts/load-declension-rules.d.ts +2 -0
- package/dist/es/word-declension/scripts/load-declension-rules.d.ts.map +1 -0
- package/dist/umd/shevchenko.min.js +4680 -68
- package/dist/umd/shevchenko.min.js.map +1 -1
- package/package.json +38 -37
- package/src/anthroponym-declension/README.md +3 -0
- package/src/anthroponym-declension/anthroponym-inflector.ts +58 -0
- package/src/anthroponym-declension/anthroponym.ts +5 -0
- package/src/anthroponym-declension/family-name-inflector.ts +75 -0
- package/src/anthroponym-declension/given-name-inflector.ts +38 -0
- package/src/anthroponym-declension/index.ts +5 -0
- package/src/anthroponym-declension/name-inflector.ts +38 -0
- package/src/anthroponym-declension/patronymic-name-inflector.ts +38 -0
- package/src/bootstrap.ts +20 -47
- package/src/contracts.ts +22 -0
- package/src/gender-detection/README.md +5 -0
- package/src/gender-detection/detect-gender.ts +43 -0
- package/src/gender-detection/given-names-genders.json +1144 -0
- package/src/gender-detection/index.ts +1 -0
- package/src/index.ts +1 -0
- package/src/input-validation.ts +80 -0
- package/src/language/README.md +3 -0
- package/src/language/alphabet.ts +49 -0
- package/src/language/grammatical-case.ts +12 -0
- package/src/language/grammatical-gender.ts +7 -0
- package/src/language/index.ts +6 -0
- package/src/language/letter-case.ts +35 -0
- package/src/language/linguistics.ts +20 -0
- package/src/language/word-class.ts +7 -0
- package/src/shevchenko.ts +65 -33
- package/src/word-class-recognition/README.md +5 -0
- package/src/word-class-recognition/artifacts/model.bundle.json +7 -0
- package/src/word-class-recognition/artifacts/model.json +1 -0
- package/src/word-class-recognition/artifacts/weights.bin +0 -0
- package/src/word-class-recognition/cache/incorrect-predictions.json +1445 -0
- package/src/word-class-recognition/data.utils.ts +57 -0
- package/src/word-class-recognition/index.ts +2 -0
- package/src/word-class-recognition/model-bundle-loader.ts +73 -0
- package/src/word-class-recognition/model-bundle-saver.ts +105 -0
- package/src/word-class-recognition/model-config.ts +4 -0
- package/src/word-class-recognition/scripts/cache-incorrect-predictions.ts +46 -0
- package/src/word-class-recognition/scripts/train-model.ts +95 -0
- package/src/word-class-recognition/word-class-recognizer.ts +54 -0
- package/src/word-class-recognition/word-class-transformer.ts +38 -0
- package/src/word-class-recognition/word-transformer.ts +23 -0
- package/src/word-declension/README.md +3 -0
- package/src/word-declension/append-command-runner.ts +18 -0
- package/src/word-declension/command-runner-factory.ts +23 -0
- package/src/{anthroponym-inflection/rule-inflector.ts → word-declension/declension-rule-inflector.ts} +17 -7
- package/src/word-declension/declension-rule-matchers.ts +30 -0
- package/src/{resources/inflector/rules.json → word-declension/declension-rules.json} +1119 -1119
- package/src/word-declension/declension-types.ts +41 -0
- package/src/word-declension/index.ts +9 -0
- package/src/word-declension/inflection-command-runner.ts +3 -0
- package/src/word-declension/replace-command-runner.ts +18 -0
- package/src/word-declension/scripts/load-declension-rules.ts +14 -0
- package/dist/types/anthroponym-inflection/anthroponym-inflector.d.ts +0 -15
- package/dist/types/anthroponym-inflection/anthroponym-inflector.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/append-command-runner.d.ts +0 -12
- package/dist/types/anthroponym-inflection/append-command-runner.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/command-runner-factory.d.ts +0 -9
- package/dist/types/anthroponym-inflection/command-runner-factory.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/first-name-inflector.d.ts +0 -12
- package/dist/types/anthroponym-inflection/first-name-inflector.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/index.d.ts +0 -6
- package/dist/types/anthroponym-inflection/index.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/inflector-command-runner.d.ts +0 -4
- package/dist/types/anthroponym-inflection/inflector-command-runner.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/inflector.types.d.ts +0 -37
- package/dist/types/anthroponym-inflection/inflector.types.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/last-name-inflector.d.ts +0 -14
- package/dist/types/anthroponym-inflection/last-name-inflector.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/middle-name-inflector.d.ts +0 -12
- package/dist/types/anthroponym-inflection/middle-name-inflector.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/name-inflector.d.ts +0 -12
- package/dist/types/anthroponym-inflection/name-inflector.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/replace-command-runner.d.ts +0 -12
- package/dist/types/anthroponym-inflection/replace-command-runner.d.ts.map +0 -1
- package/dist/types/anthroponym-inflection/rule-inflector.d.ts +0 -12
- package/dist/types/anthroponym-inflection/rule-inflector.d.ts.map +0 -1
- package/dist/types/bootstrap.d.ts +0 -4
- package/dist/types/core/anthroponym.interface.d.ts +0 -8
- package/dist/types/core/anthroponym.interface.d.ts.map +0 -1
- package/dist/types/core/gender.enum.d.ts +0 -5
- package/dist/types/core/gender.enum.d.ts.map +0 -1
- package/dist/types/core/grammatical-case.enum.d.ts +0 -10
- package/dist/types/core/grammatical-case.enum.d.ts.map +0 -1
- package/dist/types/core/index.d.ts +0 -5
- package/dist/types/core/index.d.ts.map +0 -1
- package/dist/types/core/validator.d.ts +0 -8
- package/dist/types/core/validator.d.ts.map +0 -1
- package/dist/types/internal.d.ts +0 -2
- package/dist/types/internal.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/index.d.ts +0 -6
- package/dist/types/part-of-speech-recognition/index.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/neural-network-config.d.ts +0 -6
- package/dist/types/part-of-speech-recognition/neural-network-config.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/neural-network-training-data.d.ts +0 -5
- package/dist/types/part-of-speech-recognition/neural-network-training-data.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/neural-network.d.ts +0 -28
- package/dist/types/part-of-speech-recognition/neural-network.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/part-of-speech-decoder.d.ts +0 -9
- package/dist/types/part-of-speech-recognition/part-of-speech-decoder.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/part-of-speech-encoder.d.ts +0 -10
- package/dist/types/part-of-speech-recognition/part-of-speech-encoder.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/part-of-speech-recognition-rule.d.ts +0 -19
- package/dist/types/part-of-speech-recognition/part-of-speech-recognition-rule.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/part-of-speech-recognizer.d.ts +0 -14
- package/dist/types/part-of-speech-recognition/part-of-speech-recognizer.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/part-of-speech.enum.d.ts +0 -5
- package/dist/types/part-of-speech-recognition/part-of-speech.enum.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/word-encoder.d.ts +0 -9
- package/dist/types/part-of-speech-recognition/word-encoder.d.ts.map +0 -1
- package/dist/types/part-of-speech-recognition/word-transformer.d.ts +0 -7
- package/dist/types/part-of-speech-recognition/word-transformer.d.ts.map +0 -1
- package/dist/types/shevchenko.d.ts +0 -32
- package/dist/types/shevchenko.d.ts.map +0 -1
- package/dist/types/utils/lang.utils.d.ts +0 -7
- package/dist/types/utils/lang.utils.d.ts.map +0 -1
- package/dist/types/utils/regexp.utils.d.ts +0 -5
- package/dist/types/utils/regexp.utils.d.ts.map +0 -1
- package/dist/types/utils/string.utils.d.ts +0 -14
- package/dist/types/utils/string.utils.d.ts.map +0 -1
- package/nuxt/.eslintrc.json +0 -27
- package/nuxt/.prettierrc.json +0 -6
- package/nuxt/README.md +0 -27
- package/nuxt/app.config.ts +0 -3
- package/nuxt/app.vue +0 -5
- package/nuxt/assets/img/shevchenko_pixelized_304x304.jpg +0 -0
- package/nuxt/assets/img/shevchenko_pixelized_304x304.png +0 -0
- package/nuxt/assets/img/shevchenko_pixelized_608x608.jpg +0 -0
- package/nuxt/assets/style/main.scss +0 -3
- package/nuxt/components/about-modal.vue +0 -58
- package/nuxt/components/api-spec.vue +0 -25
- package/nuxt/components/contact-me-button.vue +0 -11
- package/nuxt/components/contact-me-modal.vue +0 -63
- package/nuxt/components/copy-button.vue +0 -38
- package/nuxt/components/declension-demo.vue +0 -38
- package/nuxt/components/declension-form.vue +0 -158
- package/nuxt/components/declension-preview.vue +0 -170
- package/nuxt/components/declension-results.vue +0 -135
- package/nuxt/components/error-404.vue +0 -15
- package/nuxt/components/github-issue-count.vue +0 -18
- package/nuxt/components/how-it-works.vue +0 -51
- package/nuxt/components/installation-instructions.vue +0 -33
- package/nuxt/components/library-docs.vue +0 -27
- package/nuxt/components/page-copyright.vue +0 -15
- package/nuxt/components/page-footer.vue +0 -32
- package/nuxt/components/page-header.vue +0 -4
- package/nuxt/components/preview-banner.vue +0 -32
- package/nuxt/components/share-links.vue +0 -108
- package/nuxt/components/top-nav-bar.vue +0 -165
- package/nuxt/components/ukrainian-flag.vue +0 -37
- package/nuxt/components/usage-example.vue +0 -32
- package/nuxt/composables/declension.ts +0 -124
- package/nuxt/composables/page-i18n.ts +0 -11
- package/nuxt/composables/page-meta.ts +0 -5
- package/nuxt/composables/route-utils.ts +0 -17
- package/nuxt/config.ts +0 -20
- package/nuxt/data/female-first-names.json +0 -556
- package/nuxt/data/male-first-names.json +0 -603
- package/nuxt/error.vue +0 -25
- package/nuxt/layouts/default.vue +0 -5
- package/nuxt/locales/en-US.json +0 -63
- package/nuxt/locales/uk-UA.json +0 -65
- package/nuxt/nuxt.config.ts +0 -99
- package/nuxt/package-lock.json +0 -19524
- package/nuxt/package.json +0 -37
- package/nuxt/pages/en-US/index.vue +0 -41
- package/nuxt/pages/index.vue +0 -68
- package/nuxt/plugins/i18n.ts +0 -26
- package/nuxt/public/.nojekyll +0 -0
- package/nuxt/public/CNAME +0 -1
- package/nuxt/public/android-icon-144x144.png +0 -0
- package/nuxt/public/android-icon-192x192.png +0 -0
- package/nuxt/public/android-icon-36x36.png +0 -0
- package/nuxt/public/android-icon-48x48.png +0 -0
- package/nuxt/public/android-icon-72x72.png +0 -0
- package/nuxt/public/android-icon-96x96.png +0 -0
- package/nuxt/public/apple-icon-114x114.png +0 -0
- package/nuxt/public/apple-icon-120x120.png +0 -0
- package/nuxt/public/apple-icon-144x144.png +0 -0
- package/nuxt/public/apple-icon-152x152.png +0 -0
- package/nuxt/public/apple-icon-180x180.png +0 -0
- package/nuxt/public/apple-icon-57x57.png +0 -0
- package/nuxt/public/apple-icon-60x60.png +0 -0
- package/nuxt/public/apple-icon-72x72.png +0 -0
- package/nuxt/public/apple-icon-76x76.png +0 -0
- package/nuxt/public/apple-icon-precomposed.png +0 -0
- package/nuxt/public/apple-icon.png +0 -0
- package/nuxt/public/browserconfig.xml +0 -2
- package/nuxt/public/favicon-16x16.png +0 -0
- package/nuxt/public/favicon-32x32.png +0 -0
- package/nuxt/public/favicon-96x96.png +0 -0
- package/nuxt/public/favicon.ico +0 -0
- package/nuxt/public/manifest.json +0 -41
- package/nuxt/public/ms-icon-144x144.png +0 -0
- package/nuxt/public/ms-icon-150x150.png +0 -0
- package/nuxt/public/ms-icon-310x310.png +0 -0
- package/nuxt/public/ms-icon-70x70.png +0 -0
- package/nuxt/public/preview-608x608.jpg +0 -0
- package/nuxt/scripts/postgenerate.js +0 -7
- package/nuxt/scripts/publish.js +0 -7
- package/nuxt/server/plugins/google-tag-manager.js +0 -37
- package/nuxt/tsconfig.json +0 -11
- package/src/anthroponym-inflection/anthroponym-inflector.ts +0 -38
- package/src/anthroponym-inflection/append-command-runner.ts +0 -18
- package/src/anthroponym-inflection/command-runner-factory.ts +0 -23
- package/src/anthroponym-inflection/first-name-inflector.ts +0 -41
- package/src/anthroponym-inflection/index.ts +0 -5
- package/src/anthroponym-inflection/inflector-command-runner.ts +0 -3
- package/src/anthroponym-inflection/inflector.types.ts +0 -42
- package/src/anthroponym-inflection/last-name-inflector.ts +0 -54
- package/src/anthroponym-inflection/middle-name-inflector.ts +0 -29
- package/src/anthroponym-inflection/name-inflector.ts +0 -19
- package/src/anthroponym-inflection/replace-command-runner.ts +0 -18
- package/src/core/anthroponym.interface.ts +0 -8
- package/src/core/gender.enum.ts +0 -4
- package/src/core/grammatical-case.enum.ts +0 -9
- package/src/core/index.ts +0 -4
- package/src/core/validator.ts +0 -39
- package/src/internal.ts +0 -1
- package/src/part-of-speech-recognition/index.ts +0 -5
- package/src/part-of-speech-recognition/neural-network-config.ts +0 -5
- package/src/part-of-speech-recognition/neural-network-training-data.ts +0 -5
- package/src/part-of-speech-recognition/neural-network.ts +0 -66
- package/src/part-of-speech-recognition/part-of-speech-decoder.ts +0 -26
- package/src/part-of-speech-recognition/part-of-speech-encoder.ts +0 -20
- package/src/part-of-speech-recognition/part-of-speech-recognition-rule.ts +0 -33
- package/src/part-of-speech-recognition/part-of-speech-recognizer.ts +0 -26
- package/src/part-of-speech-recognition/part-of-speech.enum.ts +0 -4
- package/src/part-of-speech-recognition/word-encoder.ts +0 -23
- package/src/part-of-speech-recognition/word-transformer.ts +0 -8
- package/src/resources/neural-networks/kosmii/README.md +0 -5
- package/src/resources/neural-networks/kosmii/cache.json +0 -8
- package/src/resources/neural-networks/kosmii/samples.json +0 -9435
- package/src/resources/neural-networks/kosmii/structure.json +0 -47898
- package/src/resources/neural-networks/pelykh/README.md +0 -5
- package/src/resources/neural-networks/pelykh/cache.json +0 -1
- package/src/resources/neural-networks/pelykh/samples.json +0 -221
- package/src/resources/neural-networks/pelykh/structure.json +0 -47898
- package/src/resources/neural-networks/pohorielova/README.md +0 -5
- package/src/resources/neural-networks/pohorielova/cache.json +0 -542
- package/src/resources/neural-networks/pohorielova/samples.json +0 -32147
- package/src/resources/neural-networks/pohorielova/structure.json +0 -47898
- package/src/utils/lang.utils.ts +0 -13
- package/src/utils/regexp.utils.ts +0 -11
- package/src/utils/string.utils.ts +0 -49
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An interface representing a split of a dataset into training and validation sets.
|
|
3
|
+
*/
|
|
4
|
+
export interface SplitData<T> {
|
|
5
|
+
trainingData: T[];
|
|
6
|
+
validationData: T[];
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Splits an array of objects into training and validation datasets, grouped by a specific feature and shuffled randomly within each group.
|
|
11
|
+
*/
|
|
12
|
+
export function splitData<T>(data: T[], features: (keyof T)[], splitRatio: number): SplitData<T> {
|
|
13
|
+
if (splitRatio < 0 || splitRatio > 1) {
|
|
14
|
+
throw new TypeError('"splitRatio" must be between 0 and 1.');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Group the data by feature value.
|
|
18
|
+
const featureGroups = new Map<string, T[]>();
|
|
19
|
+
for (const dataItem of data) {
|
|
20
|
+
const featureGroup = features.map((splitKey) => dataItem[splitKey]).join('::');
|
|
21
|
+
if (!featureGroups.has(featureGroup)) {
|
|
22
|
+
featureGroups.set(featureGroup, []);
|
|
23
|
+
}
|
|
24
|
+
featureGroups.get(featureGroup)?.push(dataItem);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Shuffle each feature group data.
|
|
28
|
+
for (const featureData of featureGroups.values()) {
|
|
29
|
+
shuffleData(featureData);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Split each feature group into training and validation sets.
|
|
33
|
+
const trainingData: T[] = [];
|
|
34
|
+
const validationData: T[] = [];
|
|
35
|
+
|
|
36
|
+
for (const featureData of featureGroups.values()) {
|
|
37
|
+
const trainingDataSize = Math.round(splitRatio * featureData.length);
|
|
38
|
+
trainingData.push(...featureData.slice(0, trainingDataSize));
|
|
39
|
+
validationData.push(...featureData.slice(trainingDataSize));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Shuffle the training and validation sets again to ensure randomness.
|
|
43
|
+
shuffleData(trainingData);
|
|
44
|
+
shuffleData(validationData);
|
|
45
|
+
|
|
46
|
+
return { trainingData, validationData };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Shuffles the given array in place using the Fisher-Yates shuffle algorithm.
|
|
51
|
+
*/
|
|
52
|
+
export function shuffleData<T>(data: T[]): void {
|
|
53
|
+
for (let i = data.length - 1; i > 0; i--) {
|
|
54
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
55
|
+
[data[i], data[j]] = [data[j], data[i]];
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import * as tf from '@tensorflow/tfjs';
|
|
2
|
+
import modelBundle from './artifacts/model.bundle.json';
|
|
3
|
+
|
|
4
|
+
type ModelMetadata = Omit<tf.io.ModelJSON, 'modelTopology' | 'weightsManifest'>;
|
|
5
|
+
|
|
6
|
+
// Use Buffer on Node.js instead of Blob/atob/btoa
|
|
7
|
+
const useNodeBuffer =
|
|
8
|
+
typeof Buffer !== 'undefined' &&
|
|
9
|
+
(typeof Blob === 'undefined' || typeof atob === 'undefined' || typeof btoa === 'undefined');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Decodes a base64 string as an ArrayBuffer.
|
|
13
|
+
*/
|
|
14
|
+
export function base64StringToArrayBuffer(str: string): ArrayBuffer {
|
|
15
|
+
if (useNodeBuffer) {
|
|
16
|
+
const buf = Buffer.from(str, 'base64');
|
|
17
|
+
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const s = atob(str);
|
|
21
|
+
const buffer = new Uint8Array(s.length);
|
|
22
|
+
for (let i = 0; i < s.length; ++i) {
|
|
23
|
+
buffer.set([s.charCodeAt(i)], i);
|
|
24
|
+
}
|
|
25
|
+
return buffer.buffer;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @see {@link https://github.com/tensorflow/tfjs/blob/57e0f212f7fc3643ec1f70130725c7d3c716ef7a/tfjs-core/src/io/local_storage.ts}
|
|
30
|
+
*/
|
|
31
|
+
// eslint-disable-next-line import/namespace
|
|
32
|
+
export class ModelBundleLoader implements tf.io.IOHandler {
|
|
33
|
+
/**
|
|
34
|
+
* Saves the model artifacts from the artifacts directory in JSON format.
|
|
35
|
+
*/
|
|
36
|
+
async load(): Promise<tf.io.ModelArtifacts> {
|
|
37
|
+
if (modelBundle.modelInfo.modelTopologyType !== 'JSON') {
|
|
38
|
+
throw new Error(
|
|
39
|
+
`${this.constructor.name} does not support loading non-JSON model topology yet.`,
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const out: tf.io.ModelArtifacts = {};
|
|
44
|
+
|
|
45
|
+
out.modelTopology = modelBundle.modelTopology;
|
|
46
|
+
out.weightSpecs = modelBundle.weightSpecs as tf.io.WeightsManifestEntry[];
|
|
47
|
+
out.weightData = base64StringToArrayBuffer(modelBundle.weightData);
|
|
48
|
+
|
|
49
|
+
if (modelBundle.modelMetadata != null) {
|
|
50
|
+
const metadata = modelBundle.modelMetadata as ModelMetadata;
|
|
51
|
+
out.format = metadata.format;
|
|
52
|
+
out.generatedBy = metadata.generatedBy;
|
|
53
|
+
out.convertedBy = metadata.convertedBy;
|
|
54
|
+
if (metadata.signature != null) {
|
|
55
|
+
out.signature = metadata.signature;
|
|
56
|
+
}
|
|
57
|
+
if (metadata.userDefinedMetadata != null) {
|
|
58
|
+
out.userDefinedMetadata = metadata.userDefinedMetadata;
|
|
59
|
+
}
|
|
60
|
+
if (metadata.modelInitializer != null) {
|
|
61
|
+
out.modelInitializer = metadata.modelInitializer;
|
|
62
|
+
}
|
|
63
|
+
if (metadata.initializerSignature != null) {
|
|
64
|
+
out.initializerSignature = metadata.initializerSignature;
|
|
65
|
+
}
|
|
66
|
+
if (metadata.trainingConfig != null) {
|
|
67
|
+
out.trainingConfig = metadata.trainingConfig;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { writeFile } from 'fs/promises';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import * as tf from '@tensorflow/tfjs';
|
|
4
|
+
|
|
5
|
+
const MODEL_BUNDLE_FILEPATH = path.join(__dirname, 'artifacts/model.bundle.json');
|
|
6
|
+
|
|
7
|
+
type ModelMetadata = Omit<tf.io.ModelJSON, 'modelTopology' | 'weightsManifest'>;
|
|
8
|
+
|
|
9
|
+
interface ModelBundle {
|
|
10
|
+
modelInfo: tf.io.ModelArtifactsInfo;
|
|
11
|
+
modelTopology: Record<string, unknown>;
|
|
12
|
+
weightSpecs: tf.io.WeightsManifestEntry[];
|
|
13
|
+
weightData: string;
|
|
14
|
+
modelMetadata: ModelMetadata;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Use Buffer on Node.js instead of Blob/atob/btoa
|
|
18
|
+
const useNodeBuffer =
|
|
19
|
+
typeof Buffer !== 'undefined' &&
|
|
20
|
+
(typeof Blob === 'undefined' || typeof atob === 'undefined' || typeof btoa === 'undefined');
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Encodes an ArrayBuffer as a base64 encoded string.
|
|
24
|
+
*/
|
|
25
|
+
export function arrayBufferToBase64String(buffer: ArrayBuffer): string {
|
|
26
|
+
if (useNodeBuffer) {
|
|
27
|
+
return Buffer.from(buffer).toString('base64');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const buf = new Uint8Array(buffer);
|
|
31
|
+
let s = '';
|
|
32
|
+
for (let i = 0, l = buf.length; i < l; i++) {
|
|
33
|
+
s += String.fromCharCode(buf[i]);
|
|
34
|
+
}
|
|
35
|
+
return btoa(s);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// eslint-disable-next-line import/namespace
|
|
39
|
+
export class ModelBundleSaver implements tf.io.IOHandler {
|
|
40
|
+
/**
|
|
41
|
+
* Saves the model artifacts to the artifacts directory in JSON format.
|
|
42
|
+
*/
|
|
43
|
+
async save(modelArtifacts: tf.io.ModelArtifacts): Promise<tf.io.SaveResult> {
|
|
44
|
+
if (modelArtifacts.modelTopology instanceof ArrayBuffer) {
|
|
45
|
+
throw new Error(
|
|
46
|
+
`${this.constructor.name} does not support saving non-JSON model topology yet.`,
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const modelBundle: Partial<ModelBundle> = {};
|
|
51
|
+
|
|
52
|
+
// Save model info.
|
|
53
|
+
const modelArtifactsInfo: tf.io.ModelArtifactsInfo =
|
|
54
|
+
tf.io.getModelArtifactsInfoForJSON(modelArtifacts);
|
|
55
|
+
modelBundle.modelInfo = modelArtifactsInfo;
|
|
56
|
+
|
|
57
|
+
// Save topology.
|
|
58
|
+
if (modelArtifacts.modelTopology == null) {
|
|
59
|
+
throw new Error(`The topology is missing.`);
|
|
60
|
+
}
|
|
61
|
+
modelBundle.modelTopology = modelArtifacts.modelTopology;
|
|
62
|
+
|
|
63
|
+
// Save weight specs.
|
|
64
|
+
if (modelArtifacts.weightSpecs == null) {
|
|
65
|
+
throw new Error(`The weight specs are missing.`);
|
|
66
|
+
}
|
|
67
|
+
modelBundle.weightSpecs = modelArtifacts.weightSpecs;
|
|
68
|
+
|
|
69
|
+
// Save weight data.
|
|
70
|
+
if (modelArtifacts.weightData == null) {
|
|
71
|
+
throw new Error(`The binary weight values are missing.`);
|
|
72
|
+
}
|
|
73
|
+
modelBundle.weightData = arrayBufferToBase64String(modelArtifacts.weightData);
|
|
74
|
+
|
|
75
|
+
// Load meta-data fields.
|
|
76
|
+
// Note that JSON.stringify doesn't write out keys that have undefined
|
|
77
|
+
// values, so for some keys, we set undefined instead of a null-ish value.
|
|
78
|
+
const metadata: ModelMetadata = {
|
|
79
|
+
format: modelArtifacts.format,
|
|
80
|
+
generatedBy: modelArtifacts.generatedBy,
|
|
81
|
+
convertedBy: modelArtifacts.convertedBy,
|
|
82
|
+
signature: modelArtifacts.signature != null ? modelArtifacts.signature : undefined,
|
|
83
|
+
userDefinedMetadata:
|
|
84
|
+
modelArtifacts.userDefinedMetadata != null ? modelArtifacts.userDefinedMetadata : undefined,
|
|
85
|
+
modelInitializer:
|
|
86
|
+
modelArtifacts.modelInitializer != null ? modelArtifacts.modelInitializer : undefined,
|
|
87
|
+
initializerSignature:
|
|
88
|
+
modelArtifacts.initializerSignature != null
|
|
89
|
+
? modelArtifacts.initializerSignature
|
|
90
|
+
: undefined,
|
|
91
|
+
trainingConfig:
|
|
92
|
+
modelArtifacts.trainingConfig != null ? modelArtifacts.trainingConfig : undefined,
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
modelBundle.modelMetadata = metadata;
|
|
96
|
+
|
|
97
|
+
await writeFile(
|
|
98
|
+
path.join(MODEL_BUNDLE_FILEPATH),
|
|
99
|
+
JSON.stringify(modelBundle, null, 2),
|
|
100
|
+
'utf-8',
|
|
101
|
+
);
|
|
102
|
+
|
|
103
|
+
return { modelArtifactsInfo };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { createReadStream } from 'fs';
|
|
2
|
+
import { writeFile } from 'fs/promises';
|
|
3
|
+
import { join as joinPath } from 'path';
|
|
4
|
+
import * as tf from '@tensorflow/tfjs-node';
|
|
5
|
+
import { parse as createCsvParser } from 'csv';
|
|
6
|
+
import { ModelBundleLoader } from '../model-bundle-loader';
|
|
7
|
+
import { MODEL_INPUT_SIZE } from '../model-config';
|
|
8
|
+
import { WordClassTransformer } from '../word-class-transformer';
|
|
9
|
+
import { WordTransformer } from '../word-transformer';
|
|
10
|
+
|
|
11
|
+
const TRAINING_DATASET_FILEPATH = joinPath(__dirname, '../datasets/training.csv');
|
|
12
|
+
const INCORRECT_PREDICTIONS_FILEPATH = joinPath(__dirname, '../cache/incorrect-predictions.json');
|
|
13
|
+
|
|
14
|
+
type IncorrectPredictions = Record<string, string>;
|
|
15
|
+
|
|
16
|
+
async function retrieveIncorrectPredictions(): Promise<IncorrectPredictions> {
|
|
17
|
+
const dataParser = createCsvParser({ columns: true });
|
|
18
|
+
createReadStream(TRAINING_DATASET_FILEPATH).pipe(dataParser);
|
|
19
|
+
|
|
20
|
+
const model = await tf.loadLayersModel(new ModelBundleLoader());
|
|
21
|
+
const wordTransformer = new WordTransformer(MODEL_INPUT_SIZE);
|
|
22
|
+
const wordClassTransformer = new WordClassTransformer();
|
|
23
|
+
|
|
24
|
+
const incorrectPredictions: IncorrectPredictions = {};
|
|
25
|
+
for await (const dataRow of dataParser) {
|
|
26
|
+
const input = wordTransformer.encode(dataRow['Family Name']);
|
|
27
|
+
const output = await (model.predict(tf.tensor2d([input])) as tf.Tensor).data();
|
|
28
|
+
const prediction = wordClassTransformer.decode(output);
|
|
29
|
+
if (prediction !== dataRow['Word Class']) {
|
|
30
|
+
incorrectPredictions[dataRow['Family Name']] = dataRow['Word Class'];
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return incorrectPredictions;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function saveIncorrectPredictions(cache: IncorrectPredictions): Promise<void> {
|
|
38
|
+
await writeFile(INCORRECT_PREDICTIONS_FILEPATH, JSON.stringify(cache, null, 2), 'utf-8');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function main(): Promise<void> {
|
|
42
|
+
const incorrectPredictions = await retrieveIncorrectPredictions();
|
|
43
|
+
await saveIncorrectPredictions(incorrectPredictions);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
void main();
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { createReadStream } from 'fs';
|
|
2
|
+
import { join as joinPath } from 'path';
|
|
3
|
+
import * as tf from '@tensorflow/tfjs-node';
|
|
4
|
+
import { parse as createCsvParser } from 'csv';
|
|
5
|
+
import { ALPHABET_SIZE, WordClass } from '../../language';
|
|
6
|
+
import { SplitData, splitData } from '../data.utils';
|
|
7
|
+
import { ModelBundleSaver } from '../model-bundle-saver';
|
|
8
|
+
import { MODEL_INPUT_SIZE } from '../model-config';
|
|
9
|
+
import { WordClassTransformer } from '../word-class-transformer';
|
|
10
|
+
import { WordTransformer } from '../word-transformer';
|
|
11
|
+
|
|
12
|
+
const TRAINING_DATASET_FILEPATH = joinPath(__dirname, '../datasets/training.csv');
|
|
13
|
+
const MODEL_ARTIFACTS_SOURCE = 'file://' + joinPath(__dirname, '../artifacts');
|
|
14
|
+
|
|
15
|
+
export interface DataItem {
|
|
16
|
+
familyName: string;
|
|
17
|
+
wordClass: WordClass;
|
|
18
|
+
wordEnding: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function loadData(): Promise<SplitData<DataItem>> {
|
|
22
|
+
const data: DataItem[] = [];
|
|
23
|
+
const dataParser = createCsvParser({ columns: true });
|
|
24
|
+
createReadStream(TRAINING_DATASET_FILEPATH).pipe(dataParser);
|
|
25
|
+
|
|
26
|
+
for await (const dataRow of dataParser) {
|
|
27
|
+
data.push({
|
|
28
|
+
familyName: dataRow['Family Name'],
|
|
29
|
+
wordClass: dataRow['Word Class'] as WordClass,
|
|
30
|
+
wordEnding: dataRow['Family Name'].slice(-4),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return splitData(data, ['wordClass', 'wordEnding'], 0.8);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const wordTransformer = new WordTransformer(MODEL_INPUT_SIZE);
|
|
38
|
+
const wordClassTransformer = new WordClassTransformer();
|
|
39
|
+
function transformDataEntry(dataItem: DataItem) {
|
|
40
|
+
return {
|
|
41
|
+
xs: wordTransformer.encode(dataItem.familyName),
|
|
42
|
+
ys: wordClassTransformer.encode(dataItem.wordClass),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function trainModel({
|
|
47
|
+
trainingData,
|
|
48
|
+
validationData,
|
|
49
|
+
}: SplitData<DataItem>): Promise<tf.LayersModel> {
|
|
50
|
+
const model = tf.sequential({
|
|
51
|
+
layers: [
|
|
52
|
+
tf.layers.embedding({ inputDim: ALPHABET_SIZE + 1, outputDim: 16 }),
|
|
53
|
+
tf.layers.simpleRNN({ units: 16, activation: 'relu' }),
|
|
54
|
+
tf.layers.dense({ units: 1, activation: 'sigmoid' }),
|
|
55
|
+
],
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
model.compile({
|
|
59
|
+
optimizer: tf.train.adam(0.001),
|
|
60
|
+
loss: 'binaryCrossentropy',
|
|
61
|
+
metrics: ['accuracy'],
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
model.summary();
|
|
65
|
+
|
|
66
|
+
const trainingDataset = tf.data
|
|
67
|
+
.array(trainingData.map(transformDataEntry))
|
|
68
|
+
.shuffle(5_000)
|
|
69
|
+
.batch(500);
|
|
70
|
+
|
|
71
|
+
const validationDataset = tf.data
|
|
72
|
+
.array(validationData.map(transformDataEntry))
|
|
73
|
+
.shuffle(5_000)
|
|
74
|
+
.batch(500);
|
|
75
|
+
|
|
76
|
+
await model.fitDataset(trainingDataset, {
|
|
77
|
+
epochs: 100,
|
|
78
|
+
validationData: validationDataset,
|
|
79
|
+
callbacks: tf.callbacks.earlyStopping({ patience: 3 }),
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
return model;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function saveModel(model: tf.LayersModel): Promise<void> {
|
|
86
|
+
await Promise.all([model.save(MODEL_ARTIFACTS_SOURCE), model.save(new ModelBundleSaver())]);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async function main(): Promise<void> {
|
|
90
|
+
const splitData = await loadData();
|
|
91
|
+
const model = await trainModel(splitData);
|
|
92
|
+
await saveModel(model);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
void main();
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import * as tf from '@tensorflow/tfjs';
|
|
2
|
+
import { WordClass } from '../language';
|
|
3
|
+
import incorrectPredictionsCache from './cache/incorrect-predictions.json';
|
|
4
|
+
import { MODEL_INPUT_SIZE } from './model-config';
|
|
5
|
+
import { WordClassTransformer } from './word-class-transformer';
|
|
6
|
+
import { WordTransformer } from './word-transformer';
|
|
7
|
+
|
|
8
|
+
// Disable Node.js environment warning message in production code.
|
|
9
|
+
// See https://github.com/tensorflow/tfjs/issues/5349
|
|
10
|
+
tf.env().set('IS_NODE', false);
|
|
11
|
+
|
|
12
|
+
type IncorrectPredictionWord = keyof typeof incorrectPredictionsCache;
|
|
13
|
+
|
|
14
|
+
export class WordClassRecognizer {
|
|
15
|
+
private readonly modelLoader: tf.io.IOHandler;
|
|
16
|
+
private modelPromise: Promise<tf.LayersModel> | null;
|
|
17
|
+
private readonly wordTransformer: WordTransformer;
|
|
18
|
+
private readonly wordClassTransformer: WordClassTransformer;
|
|
19
|
+
|
|
20
|
+
constructor(modelLoader: tf.io.IOHandler) {
|
|
21
|
+
this.modelLoader = modelLoader;
|
|
22
|
+
this.modelPromise = null;
|
|
23
|
+
this.wordTransformer = new WordTransformer(MODEL_INPUT_SIZE);
|
|
24
|
+
this.wordClassTransformer = new WordClassTransformer();
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Predicts a word class of the given word.
|
|
29
|
+
*/
|
|
30
|
+
async recognize(word: string): Promise<WordClass> {
|
|
31
|
+
const cachedWordClass =
|
|
32
|
+
incorrectPredictionsCache[word.toLowerCase() as IncorrectPredictionWord];
|
|
33
|
+
if (cachedWordClass) {
|
|
34
|
+
return cachedWordClass as WordClass;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const model = await this.loadModel();
|
|
38
|
+
const input = this.wordTransformer.encode(word);
|
|
39
|
+
const output = await (model.predict(tf.tensor2d([input])) as tf.Tensor).data();
|
|
40
|
+
const wordClass = this.wordClassTransformer.decode(output);
|
|
41
|
+
|
|
42
|
+
return wordClass;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Loads the model from the storage. If called multiple times, resolves the same model instance.
|
|
47
|
+
*/
|
|
48
|
+
private async loadModel(): Promise<tf.LayersModel> {
|
|
49
|
+
if (this.modelPromise == null) {
|
|
50
|
+
this.modelPromise = tf.loadLayersModel(this.modelLoader);
|
|
51
|
+
}
|
|
52
|
+
return this.modelPromise;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { WordClass } from '../language';
|
|
2
|
+
|
|
3
|
+
export class WordClassTransformer {
|
|
4
|
+
encode(wordClass: WordClass): Uint8Array {
|
|
5
|
+
const values = new Uint8Array(1);
|
|
6
|
+
|
|
7
|
+
switch (wordClass) {
|
|
8
|
+
case WordClass.NOUN:
|
|
9
|
+
values[0] = 1;
|
|
10
|
+
break;
|
|
11
|
+
case WordClass.ADJECTIVE:
|
|
12
|
+
values[0] = 0;
|
|
13
|
+
break;
|
|
14
|
+
default:
|
|
15
|
+
throw new TypeError(`Invalid word class: "${wordClass}".`);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
return values;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
decode(values: Uint8Array | Int32Array | Float32Array): WordClass {
|
|
22
|
+
if (values.length < 1 || values.length > 1) {
|
|
23
|
+
throw new RangeError('Invalid vector length.');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const value = values[0];
|
|
27
|
+
|
|
28
|
+
if (value < 0 || value > 1) {
|
|
29
|
+
throw new RangeError('Invalid vector value.');
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if (value >= 0.5) {
|
|
33
|
+
return WordClass.NOUN;
|
|
34
|
+
} else {
|
|
35
|
+
return WordClass.ADJECTIVE;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { AlphabetEncoding, Letter } from '../language';
|
|
2
|
+
|
|
3
|
+
export class WordTransformer {
|
|
4
|
+
private readonly vectorSize: number;
|
|
5
|
+
private readonly unknownCharcode: number;
|
|
6
|
+
|
|
7
|
+
constructor(vectorSize: number, unknownCharcode = 0) {
|
|
8
|
+
this.vectorSize = vectorSize;
|
|
9
|
+
this.unknownCharcode = unknownCharcode;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
encode(word: string): Uint8Array {
|
|
13
|
+
const values = new Uint8Array(this.vectorSize);
|
|
14
|
+
|
|
15
|
+
const letters = word.slice(-this.vectorSize).toLowerCase().padStart(this.vectorSize, '-');
|
|
16
|
+
for (let index = 0; index < letters.length; index += 1) {
|
|
17
|
+
const letter = letters[index] as Letter;
|
|
18
|
+
values[index] = AlphabetEncoding[letter] ?? this.unknownCharcode;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
return values;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { InflectionCommand } from './declension-types';
|
|
2
|
+
import { InflectionCommandRunner } from './inflection-command-runner';
|
|
3
|
+
|
|
4
|
+
export class AppendCommandRunner implements InflectionCommandRunner {
|
|
5
|
+
private readonly command: InflectionCommand;
|
|
6
|
+
|
|
7
|
+
constructor(command: InflectionCommand) {
|
|
8
|
+
this.command = command;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Appends the command value to the given value.
|
|
13
|
+
* Returns a new value.
|
|
14
|
+
*/
|
|
15
|
+
exec(value: string): string {
|
|
16
|
+
return value + this.command.value;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { AppendCommandRunner } from './append-command-runner';
|
|
2
|
+
import { InflectionCommand, InflectionCommandAction } from './declension-types';
|
|
3
|
+
import { InflectionCommandRunner } from './inflection-command-runner';
|
|
4
|
+
import { ReplaceCommandRunner } from './replace-command-runner';
|
|
5
|
+
|
|
6
|
+
export class CommandRunnerFactory {
|
|
7
|
+
/**
|
|
8
|
+
* Creates a new command runner for the given command.
|
|
9
|
+
*/
|
|
10
|
+
make(command: InflectionCommand): InflectionCommandRunner {
|
|
11
|
+
switch (command.action) {
|
|
12
|
+
case InflectionCommandAction.APPEND: {
|
|
13
|
+
return new AppendCommandRunner(command);
|
|
14
|
+
}
|
|
15
|
+
case InflectionCommandAction.REPLACE: {
|
|
16
|
+
return new ReplaceCommandRunner(command);
|
|
17
|
+
}
|
|
18
|
+
default: {
|
|
19
|
+
throw new TypeError(`Invalid command action: ${command.action}.`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -1,14 +1,24 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { copyLetterCase } from '../utils/string.utils';
|
|
3
|
-
import { GrammaticalCase } from '../core';
|
|
4
|
-
import { InflectorRule } from './inflector.types';
|
|
1
|
+
import { copyLetterCase, GrammaticalCase } from '../language';
|
|
5
2
|
import { CommandRunnerFactory } from './command-runner-factory';
|
|
3
|
+
import { DeclensionRule } from './declension-types';
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Counts a number of groups in a given regular expression.
|
|
7
|
+
*/
|
|
8
|
+
export function countGroups(src: RegExp | string): number {
|
|
9
|
+
const pattern = new RegExp(`${src.toString()}|`);
|
|
10
|
+
const matches = pattern.exec('');
|
|
11
|
+
if (matches == null) {
|
|
12
|
+
return 0;
|
|
13
|
+
}
|
|
14
|
+
return matches.length - 1;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class DeclensionRuleInflector {
|
|
18
|
+
private readonly rule: DeclensionRule;
|
|
9
19
|
private readonly commandRunnerFactory: CommandRunnerFactory;
|
|
10
20
|
|
|
11
|
-
constructor(rule:
|
|
21
|
+
constructor(rule: DeclensionRule) {
|
|
12
22
|
this.rule = rule;
|
|
13
23
|
this.commandRunnerFactory = new CommandRunnerFactory();
|
|
14
24
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { GrammaticalGender } from '../language';
|
|
2
|
+
import { DeclensionRule } from './declension-types';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Determines if a given declension rule applies to a specified gender.
|
|
6
|
+
*/
|
|
7
|
+
export function isGenderApplicable(rule: DeclensionRule, gender: GrammaticalGender): boolean {
|
|
8
|
+
return rule.gender.includes(gender);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Determines if a given declension rule applies to a specified application.
|
|
13
|
+
*/
|
|
14
|
+
export function isApplicable(rule: DeclensionRule, application: string): boolean {
|
|
15
|
+
return rule.application.length === 0 || rule.application.includes(application);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Determines if a given declension rule strictly applies to a specified application.
|
|
20
|
+
*/
|
|
21
|
+
export function isStriclyApplicable(rule: DeclensionRule, application: string): boolean {
|
|
22
|
+
return rule.application.includes(application);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Determines if a given declension rule applies to a specified word.
|
|
27
|
+
*/
|
|
28
|
+
export function isWordApplicable(rule: DeclensionRule, word: string): boolean {
|
|
29
|
+
return new RegExp(rule.pattern.find, 'gi').test(word);
|
|
30
|
+
}
|