react-native-executorch 0.10.5 → 0.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/android/build.gradle.kts +4 -3
- package/ios/RnExecutorch.mm +15 -17
- package/lib/module/fetcher/fetcher.js +35 -122
- package/lib/module/fetcher/fetcher.js.map +1 -1
- package/lib/module/fetcher/telemetry.js +1 -1
- package/lib/module/index.js +12 -13
- package/lib/module/index.js.map +1 -1
- package/lib/module/models.js +4 -2
- package/lib/module/models.js.map +1 -1
- package/lib/typescript/src/fetcher/fetcher.d.ts +3 -7
- package/lib/typescript/src/fetcher/fetcher.d.ts.map +1 -1
- package/lib/typescript/src/models.d.ts.map +1 -1
- package/package.json +9 -7
- package/react-native-executorch.podspec +6 -2
- package/src/fetcher/fetcher.ts +38 -131
- package/src/fetcher/telemetry.ts +1 -1
- package/src/models.ts +6 -1
- package/third-party/common/phonemis/CMakeLists.txt +70 -14
- package/third-party/common/phonemis/README.md +183 -33
- package/third-party/common/phonemis/THIRD_PARTY_NOTICES.md +57 -0
- package/third-party/common/phonemis/knowledge/code_style.md +642 -0
- package/third-party/common/phonemis/licenses/Unicode-3.0.txt +39 -0
- package/third-party/common/phonemis/licenses/misaki-Apache-2.0.txt +201 -0
- package/third-party/common/phonemis/licenses/nlohmann-json-MIT.txt +21 -0
- package/third-party/common/phonemis/licenses/xsimd-BSD-3-Clause.txt +29 -0
- package/third-party/common/phonemis/src/phonemis/base/config.h +6 -0
- package/third-party/common/phonemis/src/phonemis/base/ipipeline.cpp +75 -0
- package/third-party/common/phonemis/src/phonemis/base/ipipeline.h +21 -0
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/constants.h +2 -5
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/lexicon_phonemizer.cpp +3 -4
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/neural_phonemizer.cpp +17 -8
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/neural_phonemizer.h +4 -0
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/tokenizer.cpp +21 -6
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/tokenizer.h +13 -4
- package/third-party/common/phonemis/src/phonemis/base/pipeline.cpp +8 -0
- package/third-party/common/phonemis/src/phonemis/base/pipeline.h +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.cpp +37 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.h +45 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/constants.h +10 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/context.cpp +105 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/context.h +49 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/document.cpp +173 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/document.h +67 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/text.cpp +75 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/text.h +60 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/types.h +18 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/config.h +53 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layer.h +9 -7
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/constants.h +20 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/layer.cpp +89 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/layer.h +35 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/types.h +16 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/kind.h +40 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/layer.h +382 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/notation.h +30 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/numerals.cpp +51 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/numerals.h +31 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern.h +30 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern_set.cpp +17 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern_set.h +31 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.cpp +811 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.h +285 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.cpp +64 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.h +23 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/types.h +94 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sanitizer/layer.cpp +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/{sanitizer_layer.h → layers/sanitizer/layer.h} +8 -8
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.cpp +65 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.h +29 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/layer.cpp +170 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/layer.h +40 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/types.h +25 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/trim/layer.cpp +47 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/trim/layer.h +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/preprocessor.cpp +60 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/preprocessor.h +45 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/processor.cpp +6 -7
- package/third-party/common/phonemis/src/phonemis/base/processor/processor.h +14 -10
- package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.cpp +89 -52
- package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.h +16 -9
- package/third-party/common/phonemis/src/phonemis/base/tokenizer/tokenizer.cpp +13 -12
- package/third-party/common/phonemis/src/phonemis/lang/de/constants.h +183 -19
- package/third-party/common/phonemis/src/phonemis/lang/de/num2word.cpp +355 -309
- package/third-party/common/phonemis/src/phonemis/lang/de/num2word.h +42 -17
- package/third-party/common/phonemis/src/phonemis/lang/de/patterns.cpp +56 -0
- package/third-party/common/phonemis/src/phonemis/lang/de/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/de/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/en/constants.h +181 -30
- package/third-party/common/phonemis/src/phonemis/lang/en/hmm_tagger.cpp +5 -7
- package/third-party/common/phonemis/src/phonemis/lang/en/num2word.cpp +232 -135
- package/third-party/common/phonemis/src/phonemis/lang/en/num2word.h +41 -16
- package/third-party/common/phonemis/src/phonemis/lang/en/pipeline.h +3 -10
- package/third-party/common/phonemis/src/phonemis/lang/es/constants.h +182 -60
- package/third-party/common/phonemis/src/phonemis/lang/es/num2word.cpp +247 -212
- package/third-party/common/phonemis/src/phonemis/lang/es/num2word.h +38 -18
- package/third-party/common/phonemis/src/phonemis/lang/es/patterns.cpp +50 -0
- package/third-party/common/phonemis/src/phonemis/lang/es/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/es/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/fr/constants.h +152 -26
- package/third-party/common/phonemis/src/phonemis/lang/fr/num2word.cpp +252 -179
- package/third-party/common/phonemis/src/phonemis/lang/fr/num2word.h +38 -15
- package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.cpp +83 -0
- package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/fr/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/hi/constants.h +62 -10
- package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.cpp +74 -123
- package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.h +27 -16
- package/third-party/common/phonemis/src/phonemis/lang/hi/pipeline.h +5 -3
- package/third-party/common/phonemis/src/phonemis/lang/it/constants.h +122 -42
- package/third-party/common/phonemis/src/phonemis/lang/it/num2word.cpp +200 -207
- package/third-party/common/phonemis/src/phonemis/lang/it/num2word.h +30 -18
- package/third-party/common/phonemis/src/phonemis/lang/it/patterns.cpp +25 -0
- package/third-party/common/phonemis/src/phonemis/lang/it/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/it/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/pl/constants.h +346 -77
- package/third-party/common/phonemis/src/phonemis/lang/pl/num2word.cpp +631 -246
- package/third-party/common/phonemis/src/phonemis/lang/pl/num2word.h +47 -15
- package/third-party/common/phonemis/src/phonemis/lang/pl/pipeline.h +6 -6
- package/third-party/common/phonemis/src/phonemis/lang/pt/constants.h +201 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/num2word.cpp +365 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/num2word.h +53 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/pipeline.h +4 -2
- package/third-party/common/phonemis/src/phonemis/main.cpp +42 -11
- package/third-party/common/phonemis/src/phonemis/protophone/meta.h +24 -0
- package/third-party/common/phonemis/src/phonemis/protophone/numeric.h +26 -0
- package/third-party/common/phonemis/src/phonemis/protophone/protophone.cpp +525 -0
- package/third-party/common/phonemis/src/phonemis/protophone/protophone.h +80 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor.h +76 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor_utils.h +89 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor_view.h +125 -0
- package/third-party/common/phonemis/src/phonemis/protophone/types.h +43 -0
- package/third-party/common/phonemis/src/phonemis/utils/strings.h +20 -14
- package/third-party/common/phonemis/src/phonemis/utils/unicode.h +15 -1
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_arithmetic.hpp +267 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_bit.hpp +232 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_cast.hpp +43 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_complex.hpp +112 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_details.hpp +386 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_logical.hpp +240 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_math.hpp +2601 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_memory.hpp +1006 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_rounding.hpp +72 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_swizzle.hpp +260 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_trigo.hpp +996 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/utils/shifts.hpp +82 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx.hpp +2111 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx2.hpp +1591 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx2_128.hpp +170 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512bw.hpp +724 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512cd.hpp +28 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512dq.hpp +270 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512er.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512f.hpp +2919 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512ifma.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512pf.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vbmi.hpp +76 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vbmi2.hpp +130 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vnni_avx512bw.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vnni_avx512vbmi2.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx_128.hpp +164 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avxvnni.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_common.hpp +26 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_common_fwd.hpp +127 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_constants.hpp +393 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_emulated.hpp +787 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_avx.hpp +93 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_avx2.hpp +46 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_sse.hpp +97 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma4.hpp +92 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_i8mm_neon64.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_isa.hpp +148 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_neon.hpp +3580 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_neon64.hpp +1614 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_rvv.hpp +1515 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_scalar.hpp +1210 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse2.hpp +2333 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse3.hpp +66 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse4_1.hpp +510 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse4_2.hpp +44 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_ssse3.hpp +186 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sve.hpp +1186 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_vsx.hpp +947 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_vxe.hpp +787 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_wasm.hpp +1819 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_arch.hpp +242 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_config.hpp +633 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_arm.hpp +132 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_ppc.hpp +54 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_riscv.hpp +95 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_s390x.hpp +56 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_x86.hpp +1121 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpuid.hpp +144 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_getauxval.hpp +159 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_macros.hpp +78 -0
- package/third-party/common/phonemis/src/third-party/xsimd/math/xsimd_rem_pio2.hpp +707 -0
- package/third-party/common/phonemis/src/third-party/xsimd/memory/xsimd_aligned_allocator.hpp +349 -0
- package/third-party/common/phonemis/src/third-party/xsimd/memory/xsimd_alignment.hpp +102 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_all_registers.hpp +57 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_api.hpp +3106 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx2_register.hpp +57 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512bw_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512cd_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512dq_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512er_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512f_register.hpp +77 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512ifma_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512pf_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vbmi2_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vbmi_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vnni_avx512bw_register.hpp +54 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vnni_avx512vbmi2_register.hpp +53 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx_register.hpp +79 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avxvnni_register.hpp +44 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch.hpp +1666 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch_constant.hpp +572 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch_fwd.hpp +41 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_common_arch.hpp +49 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_emulated_register.hpp +80 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_avx2_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_avx_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_sse_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma4_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_i8mm_neon64_register.hpp +55 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_neon64_register.hpp +55 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_neon_register.hpp +130 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_register.hpp +96 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_rvv_register.hpp +491 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse2_register.hpp +59 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse3_register.hpp +49 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse4_1_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse4_2_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_ssse3_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sve_register.hpp +216 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_traits.hpp +409 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_utils.hpp +462 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_vsx_register.hpp +84 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_vxe_register.hpp +86 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_wasm_register.hpp +59 -0
- package/third-party/common/phonemis/src/third-party/xsimd/utils/bits.hpp +122 -0
- package/third-party/common/phonemis/src/third-party/xsimd/utils/xsimd_type_traits.hpp +145 -0
- package/third-party/common/phonemis/src/third-party/xsimd/xsimd.hpp +43 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/constants.h +0 -11
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/config.h +0 -16
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/layer.cpp +0 -271
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/layer.h +0 -45
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/types.h +0 -29
- package/third-party/common/phonemis/src/phonemis/base/processor/sanitizer_layer.cpp +0 -22
- package/third-party/common/phonemis/src/phonemis/base/processor/trim_layer.cpp +0 -40
- package/third-party/common/phonemis/src/phonemis/base/processor/trim_layer.h +0 -22
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
|
+
#include <phonemis/base/processor/layers/num2word/types.h>
|
|
4
|
+
#include <phonemis/base/processor/common/conditions.h>
|
|
5
|
+
#include <phonemis/base/processor/layers/num2word/notation.h>
|
|
6
|
+
#include <phonemis/base/processor/layers/term2word/types.h>
|
|
3
7
|
#include <phonemis/base/tokenizer/types.h>
|
|
4
8
|
|
|
5
9
|
#include <array>
|
|
6
10
|
#include <cstdint>
|
|
7
11
|
#include <map>
|
|
8
12
|
#include <string>
|
|
13
|
+
#include <string_view>
|
|
9
14
|
#include <unordered_map>
|
|
10
15
|
#include <unordered_set>
|
|
11
16
|
#include <vector>
|
|
@@ -22,86 +27,350 @@ namespace sanitizer {
|
|
|
22
27
|
|
|
23
28
|
// --- Number to Word Normalization ---
|
|
24
29
|
namespace num2word {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
{
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
{
|
|
72
|
-
{
|
|
73
|
-
{
|
|
74
|
-
{
|
|
75
|
-
{
|
|
76
|
-
{
|
|
77
|
-
{
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
{
|
|
85
|
-
{
|
|
86
|
-
{
|
|
87
|
-
{
|
|
88
|
-
{
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
U"
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
U"
|
|
30
|
+
using ::phonemis::processor::num2word::Case;
|
|
31
|
+
using ::phonemis::processor::num2word::DateFormat;
|
|
32
|
+
using ::phonemis::processor::num2word::Joiners;
|
|
33
|
+
using ::phonemis::processor::num2word::Keys;
|
|
34
|
+
using ::phonemis::processor::num2word::keys_of;
|
|
35
|
+
using ::phonemis::processor::num2word::Notation;
|
|
36
|
+
using ::phonemis::processor::num2word::PhoneFormat;
|
|
37
|
+
using ::phonemis::processor::num2word::SuffixFormat;
|
|
38
|
+
|
|
39
|
+
// A noun's forms after a number: singular, then plural, each in the order of Case.
|
|
40
|
+
struct Noun {
|
|
41
|
+
std::array<std::u32string, 6> singular;
|
|
42
|
+
std::array<std::u32string, 6> plural;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// A noun from its twelve forms, separated by spaces, or by "|" when they are made of several words.
|
|
46
|
+
inline Noun noun(std::u32string_view forms) {
|
|
47
|
+
char32_t separator = forms.find(U'|') != std::u32string_view::npos ? U'|' : U' ';
|
|
48
|
+
Noun result;
|
|
49
|
+
for (size_t i = 0, start = 0; i < 12; ++i) {
|
|
50
|
+
size_t end = std::min(forms.find(separator, start), forms.size());
|
|
51
|
+
(i < 6 ? result.singular[i] : result.plural[i - 6]) = forms.substr(start, end - start);
|
|
52
|
+
start = end + 1;
|
|
53
|
+
}
|
|
54
|
+
return result;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// A regular masculine noun from its stem and its locative: metr, metrze.
|
|
58
|
+
inline Noun masculine(std::u32string_view stem, std::u32string_view locative) {
|
|
59
|
+
std::u32string s(stem);
|
|
60
|
+
return {{s, s + U"a", s + U"owi", s, s + U"em", std::u32string(locative)},
|
|
61
|
+
{s + U"y", s + U"ów", s + U"om", s + U"y", s + U"ami", s + U"ach"}};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Cardinals: nominative/accusative, genitive/locative, dative, instrumental.
|
|
65
|
+
inline const std::map<uint64_t, std::array<std::u32string_view, 4>> kCardinals = {
|
|
66
|
+
{0, {U"zero", U"zera", U"zeru", U"zerem"}},
|
|
67
|
+
{2, {U"dwa", U"dwóch", U"dwóm", U"dwoma"}},
|
|
68
|
+
{3, {U"trzy", U"trzech", U"trzem", U"trzema"}},
|
|
69
|
+
{4, {U"cztery", U"czterech", U"czterem", U"czterema"}},
|
|
70
|
+
{5, {U"pięć", U"pięciu", U"pięciu", U"pięcioma"}},
|
|
71
|
+
{6, {U"sześć", U"sześciu", U"sześciu", U"sześcioma"}},
|
|
72
|
+
{7, {U"siedem", U"siedmiu", U"siedmiu", U"siedmioma"}},
|
|
73
|
+
{8, {U"osiem", U"ośmiu", U"ośmiu", U"ośmioma"}},
|
|
74
|
+
{9, {U"dziewięć", U"dziewięciu", U"dziewięciu", U"dziewięcioma"}},
|
|
75
|
+
{10, {U"dziesięć", U"dziesięciu", U"dziesięciu", U"dziesięcioma"}},
|
|
76
|
+
{11, {U"jedenaście", U"jedenastu", U"jedenastu", U"jedenastoma"}},
|
|
77
|
+
{12, {U"dwanaście", U"dwunastu", U"dwunastu", U"dwunastoma"}},
|
|
78
|
+
{13, {U"trzynaście", U"trzynastu", U"trzynastu", U"trzynastoma"}},
|
|
79
|
+
{14, {U"czternaście", U"czternastu", U"czternastu", U"czternastoma"}},
|
|
80
|
+
{15, {U"piętnaście", U"piętnastu", U"piętnastu", U"piętnastoma"}},
|
|
81
|
+
{16, {U"szesnaście", U"szesnastu", U"szesnastu", U"szesnastoma"}},
|
|
82
|
+
{17, {U"siedemnaście", U"siedemnastu", U"siedemnastu", U"siedemnastoma"}},
|
|
83
|
+
{18, {U"osiemnaście", U"osiemnastu", U"osiemnastu", U"osiemnastoma"}},
|
|
84
|
+
{19, {U"dziewiętnaście", U"dziewiętnastu", U"dziewiętnastu", U"dziewiętnastoma"}},
|
|
85
|
+
{20, {U"dwadzieścia", U"dwudziestu", U"dwudziestu", U"dwudziestoma"}},
|
|
86
|
+
{30, {U"trzydzieści", U"trzydziestu", U"trzydziestu", U"trzydziestoma"}},
|
|
87
|
+
{40, {U"czterdzieści", U"czterdziestu", U"czterdziestu", U"czterdziestoma"}},
|
|
88
|
+
{50, {U"pięćdziesiąt", U"pięćdziesięciu", U"pięćdziesięciu", U"pięćdziesięcioma"}},
|
|
89
|
+
{60, {U"sześćdziesiąt", U"sześćdziesięciu", U"sześćdziesięciu", U"sześćdziesięcioma"}},
|
|
90
|
+
{70, {U"siedemdziesiąt", U"siedemdziesięciu", U"siedemdziesięciu", U"siedemdziesięcioma"}},
|
|
91
|
+
{80, {U"osiemdziesiąt", U"osiemdziesięciu", U"osiemdziesięciu", U"osiemdziesięcioma"}},
|
|
92
|
+
{90, {U"dziewięćdziesiąt", U"dziewięćdziesięciu", U"dziewięćdziesięciu", U"dziewięćdziesięcioma"}},
|
|
93
|
+
{100, {U"sto", U"stu", U"stu", U"stoma"}},
|
|
94
|
+
{200, {U"dwieście", U"dwustu", U"dwustu", U"dwustoma"}},
|
|
95
|
+
{300, {U"trzysta", U"trzystu", U"trzystu", U"trzystoma"}},
|
|
96
|
+
{400, {U"czterysta", U"czterystu", U"czterystu", U"czterystoma"}},
|
|
97
|
+
{500, {U"pięćset", U"pięciuset", U"pięciuset", U"pięciuset"}},
|
|
98
|
+
{600, {U"sześćset", U"sześciuset", U"sześciuset", U"sześciuset"}},
|
|
99
|
+
{700, {U"siedemset", U"siedmiuset", U"siedmiuset", U"siedmiuset"}},
|
|
100
|
+
{800, {U"osiemset", U"ośmiuset", U"ośmiuset", U"ośmiuset"}},
|
|
101
|
+
{900, {U"dziewięćset", U"dziewięciuset", U"dziewięciuset", U"dziewięciuset"}}
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
// "1" by gender (masculine, feminine, neuter), each in the order of Case.
|
|
105
|
+
inline const std::array<std::array<std::u32string_view, 6>, 3> kOne = {{
|
|
106
|
+
{U"jeden", U"jednego", U"jednemu", U"jeden", U"jednym", U"jednym"},
|
|
107
|
+
{U"jedna", U"jednej", U"jednej", U"jedną", U"jedną", U"jednej"},
|
|
108
|
+
{U"jedno", U"jednego", U"jednemu", U"jedno", U"jednym", U"jednym"}
|
|
109
|
+
}};
|
|
110
|
+
|
|
111
|
+
// Stems of ordinals, which take adjective endings: pierwsz-y, drug-iego.
|
|
112
|
+
inline const std::map<uint64_t, std::u32string_view> kOrdinalStems = {
|
|
113
|
+
{1, U"pierwsz"}, {2, U"drug"}, {3, U"trzec"}, {4, U"czwart"}, {5, U"piąt"},
|
|
114
|
+
{6, U"szóst"}, {7, U"siódm"}, {8, U"ósm"}, {9, U"dziewiąt"}, {10, U"dziesiąt"},
|
|
115
|
+
{11, U"jedenast"}, {12, U"dwunast"}, {13, U"trzynast"}, {14, U"czternast"}, {15, U"piętnast"},
|
|
116
|
+
{16, U"szesnast"}, {17, U"siedemnast"}, {18, U"osiemnast"}, {19, U"dziewiętnast"}, {20, U"dwudziest"},
|
|
117
|
+
{30, U"trzydziest"}, {40, U"czterdziest"}, {50, U"pięćdziesiąt"}, {60, U"sześćdziesiąt"},
|
|
118
|
+
{70, U"siedemdziesiąt"}, {80, U"osiemdziesiąt"}, {90, U"dziewięćdziesiąt"},
|
|
119
|
+
{100, U"setn"}, {200, U"dwusetn"}, {300, U"trzechsetn"}, {400, U"czterechsetn"}, {500, U"pięćsetn"},
|
|
120
|
+
{600, U"sześćsetn"}, {700, U"siedemsetn"}, {800, U"osiemsetn"}, {900, U"dziewięćsetn"}
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
// Adjective endings by gender (masculine, feminine, neuter, plural), each in the order of Case.
|
|
124
|
+
inline const std::array<std::array<std::u32string_view, 6>, 4> kAdjectiveEndings = {{
|
|
125
|
+
{U"y", U"ego", U"emu", U"y", U"ym", U"ym"},
|
|
126
|
+
{U"a", U"ej", U"ej", U"ą", U"ą", U"ej"},
|
|
127
|
+
{U"e", U"ego", U"emu", U"e", U"ym", U"ym"},
|
|
128
|
+
{U"e", U"ych", U"ym", U"e", U"ymi", U"ych"}
|
|
129
|
+
}};
|
|
130
|
+
|
|
131
|
+
// Forms of numbers inside compounds: 45-metrowe -> czterdziestopięciometrowe.
|
|
132
|
+
inline const std::map<uint64_t, std::u32string_view> kCombining = {
|
|
133
|
+
{1, U"jedno"}, {2, U"dwu"}, {3, U"trzy"}, {4, U"cztero"}, {5, U"pięcio"},
|
|
134
|
+
{6, U"sześcio"}, {7, U"siedmio"}, {8, U"ośmio"}, {9, U"dziewięcio"}, {10, U"dziesięcio"},
|
|
135
|
+
{11, U"jedenasto"}, {12, U"dwunasto"}, {13, U"trzynasto"}, {14, U"czternasto"}, {15, U"piętnasto"},
|
|
136
|
+
{16, U"szesnasto"}, {17, U"siedemnasto"}, {18, U"osiemnasto"}, {19, U"dziewiętnasto"}, {20, U"dwudziesto"},
|
|
137
|
+
{30, U"trzydziesto"}, {40, U"czterdziesto"}, {50, U"pięćdziesięcio"}, {60, U"sześćdziesięcio"},
|
|
138
|
+
{70, U"siedemdziesięcio"}, {80, U"osiemdziesięcio"}, {90, U"dziewięćdziesięcio"},
|
|
139
|
+
{100, U"stu"}, {200, U"dwustu"}, {300, U"trzystu"}, {400, U"czterystu"}, {500, U"pięćset"},
|
|
140
|
+
{600, U"sześćset"}, {700, U"siedemset"}, {800, U"osiemset"}, {900, U"dziewięćset"}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
// Hyphenated endings which make a number an ordinal (lat 70-tych), with the adjective ending they stand for.
|
|
144
|
+
inline const std::unordered_map<std::u32string, std::u32string_view> kOrdinalSuffixes = {
|
|
145
|
+
{U"tych", U"ych"}, {U"ych", U"ych"}, {U"ty", U"y"}, {U"y", U"y"}, {U"szy", U"y"}, {U"gi", U"y"},
|
|
146
|
+
{U"ci", U"y"}, {U"go", U"ego"}, {U"ego", U"ego"}, {U"tego", U"ego"}, {U"ej", U"ej"}, {U"tej", U"ej"},
|
|
147
|
+
{U"ym", U"ym"}, {U"tym", U"ym"}, {U"ta", U"a"}, {U"a", U"a"}, {U"te", U"e"}, {U"e", U"e"},
|
|
148
|
+
{U"mu", U"emu"}, {U"temu", U"emu"}
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
// Nouns counted by numbers
|
|
152
|
+
inline const Noun kThousand = noun(U"tysiąc tysiąca tysiącowi tysiąc tysiącem tysiącu tysiące tysięcy tysiącom tysiące tysiącami tysiącach");
|
|
153
|
+
inline const Noun kMillion = masculine(U"milion", U"milionie");
|
|
154
|
+
inline const Noun kBillion = masculine(U"miliard", U"miliardzie");
|
|
155
|
+
inline const Noun kZloty = noun(U"złoty złotego złotemu złoty złotym złotym złote złotych złotym złote złotymi złotych");
|
|
156
|
+
inline const Noun kGrosz = noun(U"grosz grosza groszowi grosz groszem groszu grosze groszy groszom grosze groszami groszach");
|
|
157
|
+
inline const Noun kPercent = noun(U"procent procent procentowi procent procentem procencie procent procent procentom procent procentami procentach");
|
|
158
|
+
inline const Noun kEuro = noun(U"euro euro euro euro euro euro euro euro euro euro euro euro");
|
|
159
|
+
inline const Noun kDollar = masculine(U"dolar", U"dolarze");
|
|
160
|
+
inline constexpr std::u32string_view kDegree = U"stopień stopnia stopniowi stopień stopniem stopniu stopnie stopni stopniom stopnie stopniami stopniach";
|
|
161
|
+
|
|
162
|
+
inline const std::unordered_map<std::u32string, Noun> kCurrencies = {
|
|
163
|
+
{U"zł", kZloty}, {U"PLN", kZloty}, {U"€", kEuro}, {U"EUR", kEuro}, {U"$", kDollar}, {U"USD", kDollar},
|
|
164
|
+
{U"£", masculine(U"funt", U"funcie")}, {U"gr", kGrosz}
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
inline const std::unordered_map<std::u32string, Noun> kScales = {
|
|
168
|
+
{U"tys.", kThousand}, {U"mln", kMillion}, {U"mld", kBillion}
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
inline const std::unordered_map<std::u32string, Noun> kUnits = {
|
|
172
|
+
{U"km", masculine(U"kilometr", U"kilometrze")}, {U"m", masculine(U"metr", U"metrze")},
|
|
173
|
+
{U"cm", masculine(U"centymetr", U"centymetrze")}, {U"mm", masculine(U"milimetr", U"milimetrze")},
|
|
174
|
+
{U"nm", masculine(U"nanometr", U"nanometrze")}, {U"kg", masculine(U"kilogram", U"kilogramie")},
|
|
175
|
+
{U"g", masculine(U"gram", U"gramie")}, {U"mg", masculine(U"miligram", U"miligramie")},
|
|
176
|
+
{U"l", masculine(U"litr", U"litrze")}, {U"ml", masculine(U"mililitr", U"mililitrze")},
|
|
177
|
+
{U"ha", masculine(U"hektar", U"hektarze")}, {U"W", masculine(U"wat", U"wacie")},
|
|
178
|
+
{U"kW", masculine(U"kilowat", U"kilowacie")}, {U"MW", masculine(U"megawat", U"megawacie")},
|
|
179
|
+
{U"V", masculine(U"wolt", U"wolcie")}, {U"GB", masculine(U"gigabajt", U"gigabajcie")},
|
|
180
|
+
{U"MB", masculine(U"megabajt", U"megabajcie")}, {U"TB", masculine(U"terabajt", U"terabajcie")},
|
|
181
|
+
{U"kWh", noun(U"kilowatogodzina kilowatogodziny kilowatogodzinie kilowatogodzinę kilowatogodziną kilowatogodzinie kilowatogodziny kilowatogodzin kilowatogodzinom kilowatogodziny kilowatogodzinami kilowatogodzinach")},
|
|
182
|
+
{U"Hz", noun(U"herc herca hercowi herc hercem hercu herce herców hercom herce hercami hercach")},
|
|
183
|
+
{U"°C", noun(U"stopień Celsjusza|stopnia Celsjusza|stopniowi Celsjusza|stopień Celsjusza|stopniem Celsjusza|stopniu Celsjusza|stopnie Celsjusza|stopni Celsjusza|stopniom Celsjusza|stopnie Celsjusza|stopniami Celsjusza|stopniach Celsjusza")},
|
|
184
|
+
{U"°", noun(kDegree)},
|
|
185
|
+
{U"m²", noun(U"metr kwadratowy|metra kwadratowego|metrowi kwadratowemu|metr kwadratowy|metrem kwadratowym|metrze kwadratowym|metry kwadratowe|metrów kwadratowych|metrom kwadratowym|metry kwadratowe|metrami kwadratowymi|metrach kwadratowych")},
|
|
186
|
+
{U"t", noun(U"tona tony tonie tonę toną tonie tony ton tonom tony tonami tonach")},
|
|
187
|
+
{U"h", noun(U"godzina godziny godzinie godzinę godziną godzinie godziny godzin godzinom godziny godzinami godzinach")},
|
|
188
|
+
{U"min", noun(U"minuta minuty minucie minutę minutą minucie minuty minut minutom minuty minutami minutach")},
|
|
189
|
+
{U"s", noun(U"sekunda sekundy sekundzie sekundę sekundą sekundzie sekundy sekund sekundom sekundy sekundami sekundach")}
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
// Months in the genitive, as dates read them: 27 marca
|
|
193
|
+
inline const std::array<std::u32string_view, 12> kMonthsGenitive = {
|
|
194
|
+
U"stycznia", U"lutego", U"marca", U"kwietnia", U"maja", U"czerwca",
|
|
195
|
+
U"lipca", U"sierpnia", U"września", U"października", U"listopada", U"grudnia"
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
inline const std::unordered_set<std::u32string> kMonths = {
|
|
199
|
+
U"stycznia", U"lutego", U"marca", U"kwietnia", U"maja", U"czerwca", U"lipca", U"sierpnia", U"września",
|
|
200
|
+
U"października", U"listopada", U"grudnia", U"styczeń", U"luty", U"marzec", U"kwiecień", U"maj", U"czerwiec",
|
|
201
|
+
U"lipiec", U"sierpień", U"wrzesień", U"październik", U"listopad", U"grudzień"
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
// The case a preposition gives the number after it: od pięciu, z trzema
|
|
205
|
+
inline const std::unordered_map<std::u32string, Case> kPrepositionCases = {
|
|
206
|
+
{U"od", Case::GENITIVE}, {U"do", Case::GENITIVE}, {U"bez", Case::GENITIVE}, {U"dla", Case::GENITIVE},
|
|
207
|
+
{U"u", Case::GENITIVE}, {U"około", Case::GENITIVE}, {U"koło", Case::GENITIVE}, {U"obok", Case::GENITIVE},
|
|
208
|
+
{U"podczas", Case::GENITIVE}, {U"według", Case::GENITIVE}, {U"wśród", Case::GENITIVE},
|
|
209
|
+
{U"spośród", Case::GENITIVE}, {U"sprzed", Case::GENITIVE}, {U"spod", Case::GENITIVE}, {U"zza", Case::GENITIVE},
|
|
210
|
+
{U"znad", Case::GENITIVE}, {U"oprócz", Case::GENITIVE}, {U"prócz", Case::GENITIVE}, {U"zamiast", Case::GENITIVE},
|
|
211
|
+
{U"wokół", Case::GENITIVE}, {U"dookoła", Case::GENITIVE}, {U"naprzeciwko", Case::GENITIVE},
|
|
212
|
+
{U"blisko", Case::GENITIVE}, {U"niedaleko", Case::GENITIVE}, {U"powyżej", Case::GENITIVE},
|
|
213
|
+
{U"poniżej", Case::GENITIVE},
|
|
214
|
+
{U"dzięki", Case::DATIVE}, {U"przeciw", Case::DATIVE}, {U"przeciwko", Case::DATIVE}, {U"ku", Case::DATIVE},
|
|
215
|
+
{U"wbrew", Case::DATIVE},
|
|
216
|
+
{U"przez", Case::ACCUSATIVE}, {U"ponad", Case::ACCUSATIVE}, {U"na", Case::ACCUSATIVE}, {U"o", Case::ACCUSATIVE},
|
|
217
|
+
{U"za", Case::ACCUSATIVE}, {U"po", Case::ACCUSATIVE}, {U"w", Case::ACCUSATIVE}, {U"we", Case::ACCUSATIVE},
|
|
218
|
+
{U"z", Case::INSTRUMENTAL}, {U"ze", Case::INSTRUMENTAL}, {U"nad", Case::INSTRUMENTAL},
|
|
219
|
+
{U"pod", Case::INSTRUMENTAL}, {U"przed", Case::INSTRUMENTAL}, {U"między", Case::INSTRUMENTAL},
|
|
220
|
+
{U"pomiędzy", Case::INSTRUMENTAL},
|
|
221
|
+
{U"przy", Case::LOCATIVE}
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
inline const std::unordered_set<std::u32string> kLocativePrepositions = {U"w", U"we", U"o", U"po", U"przy", U"na"};
|
|
225
|
+
|
|
226
|
+
// Prepositional phrases which govern the genitive, with the preposition they need: w ciągu dwóch tygodni
|
|
227
|
+
inline const std::unordered_map<std::u32string, std::u32string> kComplexPrepositions = {
|
|
228
|
+
{U"ciągu", U"w"}, {U"trakcie", U"w"}, {U"czasie", U"w"}, {U"wyniku", U"w"}, {U"przypadku", U"w"},
|
|
229
|
+
{U"zakresie", U"w"}, {U"ramach", U"w"}, {U"celu", U"w"}, {U"postaci", U"w"}, {U"powodu", U"z"}
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
// Stems of amount nouns: a number after them is their genitive attribute (od kwoty trzystu tysięcy)
|
|
233
|
+
inline const std::vector<std::u32string> kAmountNouns = {
|
|
234
|
+
U"rzęd", U"wysokoś", U"gruboś", U"średnic", U"długoś", U"szerokoś", U"głębokoś", U"wiek", U"kwot", U"kwoc",
|
|
235
|
+
U"poziom", U"stan", U"strat", U"strac", U"wartoś", U"liczb", U"cen", U"koszt", U"opłat", U"kar", U"moc",
|
|
236
|
+
U"wag", U"wadz", U"pojemnoś", U"powierzchn", U"prędkoś", U"temperatur", U"objętoś", U"odległoś", U"armi",
|
|
237
|
+
U"grup", U"okres", U"limit", U"próg", U"prog", U"budżet", U"deficyt", U"nadwyżk", U"dochod", U"dochód",
|
|
238
|
+
U"przychod", U"przychód", U"wydatk", U"wzrost", U"spad", U"zysk", U"dotacj", U"wynagrodzen", U"pensj"
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
inline const std::unordered_set<std::u32string> kNounEndings = {
|
|
242
|
+
U"", U"a", U"ą", U"ę", U"y", U"i", U"u", U"e", U"ie", U"em", U"iem", U"om", U"ów", U"ami", U"ach", U"ć", U"ci",
|
|
243
|
+
U"cią", U"ze", U"ia", U"ię", U"ii", U"ek", U"ku"
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
// Stems of verbs whose object is in the genitive: potrzebowali stu czterdziestu gramów
|
|
247
|
+
inline const std::vector<std::u32string> kGenitiveVerbs = {
|
|
248
|
+
U"potrzeb", U"używ", U"użyj", U"użyć", U"wymag", U"szuk", U"poszuk", U"unik", U"brak", U"zabrak", U"życz",
|
|
249
|
+
U"dotycz", U"uczy", U"nauczy", U"słucha", U"pilnu"
|
|
102
250
|
};
|
|
251
|
+
|
|
252
|
+
// Words skipped when looking for the verb governing a number: brakuje nam trzech osób
|
|
253
|
+
inline const std::unordered_set<std::u32string> kClitics = {
|
|
254
|
+
U"mi", U"ci", U"mu", U"jej", U"nam", U"wam", U"im", U"się", U"sobie", U"już", U"jeszcze", U"teraz", U"tu",
|
|
255
|
+
U"tam", U"aż", U"nawet"
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
inline const std::unordered_set<std::u32string> kCopulas = {
|
|
259
|
+
U"jest", U"są", U"był", U"była", U"było", U"były", U"będzie", U"będą", U"to"
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
// Prepositions before a bare year (od 2020 do 2022), with the case of the year ordinal after them
|
|
263
|
+
inline const std::unordered_map<std::u32string, Case> kYearPrepositions = {
|
|
264
|
+
{U"od", Case::GENITIVE}, {U"do", Case::GENITIVE}, {U"z", Case::GENITIVE}, {U"ze", Case::GENITIVE},
|
|
265
|
+
{U"sprzed", Case::GENITIVE}, {U"około", Case::GENITIVE}, {U"przed", Case::INSTRUMENTAL},
|
|
266
|
+
{U"po", Case::LOCATIVE}, {U"na", Case::ACCUSATIVE}
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
inline const std::unordered_set<std::u32string> kCountNouns = {
|
|
270
|
+
U"osób", U"lat", U"dni", U"godzin", U"minut", U"sekund", U"razy", U"sztuk", U"ludzi", U"dzieci", U"tygodni",
|
|
271
|
+
U"miesięcy"
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
inline const std::unordered_set<std::u32string> kCurrencyWords = {U"zł", U"złoty", U"złote", U"złotych"};
|
|
275
|
+
|
|
276
|
+
// Stems of nouns whose numbers are feminine: dwie osoby, dwie godziny
|
|
277
|
+
inline const std::vector<std::u32string> kFemininePlurals = {
|
|
278
|
+
U"osob", U"osób", U"godzin", U"minut", U"sekund", U"złotówk", U"kobiet", U"stron", U"sztuk", U"rzecz", U"częś",
|
|
279
|
+
U"noc", U"dob", U"klas", U"wersj", U"sesj", U"edycj", U"zapał", U"tablic", U"butelk", U"szklank", U"łyżk"
|
|
280
|
+
};
|
|
281
|
+
|
|
282
|
+
// Words before a time with a dot, which make it a time: o 18.30
|
|
283
|
+
inline const std::unordered_set<std::u32string> kTimeWords = {
|
|
284
|
+
U"o", U"godzinie", U"godz", U"od", U"do", U"około", U"koło", U"po", U"przed", U"na", U"między", U"za"
|
|
285
|
+
};
|
|
286
|
+
|
|
287
|
+
// Stems of headings: Punkt 3., w punkcie 5.; the ones of kOrdinalHeadings read as ordinals even without a dot
|
|
288
|
+
inline const std::vector<std::u32string> kHeadings = {
|
|
289
|
+
U"podpunkt", U"punkt", U"punkc", U"rozdział", U"paragraf", U"artykuł", U"pkt"
|
|
290
|
+
};
|
|
291
|
+
inline const std::vector<std::u32string> kOrdinalHeadings = {U"podpunkt", U"punkt", U"punkc", U"paragraf", U"pkt"};
|
|
292
|
+
|
|
293
|
+
// Words before the ordinal of a decade, with its case: z lat 70., w latach 80.
|
|
294
|
+
inline const std::unordered_map<std::u32string, Case> kDecadeWords = {
|
|
295
|
+
{U"lata", Case::NOMINATIVE}, {U"lat", Case::GENITIVE}, {U"latach", Case::LOCATIVE},
|
|
296
|
+
{U"latami", Case::INSTRUMENTAL}, {U"roczników", Case::GENITIVE}, {U"rocznikach", Case::LOCATIVE}
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
inline const std::unordered_set<std::u32string> kConjunctions = {U"a", U"i", U"lub", U"albo", U"czy", U"oraz"};
|
|
300
|
+
|
|
301
|
+
// --- Notation: how numbers are written ---
|
|
302
|
+
inline const Notation kNotation = {
|
|
303
|
+
.group_separators = U" \u00a0\u202f.", .decimal_separators = U",.", .time_separators = U":.",
|
|
304
|
+
.currencies = keys_of(kCurrencies), .scales = keys_of(kScales), .units = keys_of(kUnits)
|
|
305
|
+
};
|
|
306
|
+
|
|
307
|
+
// Words between numbers: 3.8.10, 10.0.0.51, 10-15, 1920x1080, 2:1, -5, 2 + 2
|
|
308
|
+
inline const Joiners kJoiners = {
|
|
309
|
+
.point = U"kropka", .dot = U"kropka", .range = U"-", .by = U"na", .score = U" do ", .minus = U"minus", .plus = U"plus"
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
// 15.08.2020 r.: the year has four digits, "r." may follow it
|
|
313
|
+
inline const DateFormat kDateFormat = {.separators = U".", .boundaries = U".,", .short_year = false, .suffixes = {U"r."}};
|
|
314
|
+
|
|
315
|
+
// 600 123 456, 22 123 45 67 (after a word of kPhoneWords), 600-123-456, 68-324-22-55
|
|
316
|
+
inline const std::vector<PhoneFormat> kPhoneFormats = {
|
|
317
|
+
{.groups = {{3, 3}, {3, 3}, {3, 3}}, .separators = U" ", .boundaries = U"", .closing = U""},
|
|
318
|
+
{.groups = {{2, 2}, {3, 3}, {2, 2}, {2, 2}}, .separators = U" ", .boundaries = U"", .closing = U""},
|
|
319
|
+
{.groups = {{3, 3}, {3, 3}, {3, 3}}, .separators = U"-", .boundaries = U".,-", .closing = U"-"},
|
|
320
|
+
{.groups = {{2, 2}, {3, 3}, {2, 2}, {2, 2}}, .separators = U"-", .boundaries = U".,-", .closing = U"-"}
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
// Words before a phone number written with spaces, which else reads as a count: numer 600 123 456
|
|
324
|
+
inline const std::unordered_set<std::u32string> kPhoneWords = {
|
|
325
|
+
U"numer", U"numerem", U"numeru", U"tel", U"telefon", U"telefonu", U"telefonem", U"infolinia", U"faks", U"fax"
|
|
326
|
+
};
|
|
327
|
+
|
|
328
|
+
// 2020 r.: a year with "roku" after it
|
|
329
|
+
inline const Keys kYearSuffixes = {U"r."};
|
|
330
|
+
inline const SuffixFormat kYearFormat = {.space = true, .sentence_dot = true};
|
|
103
331
|
} // namespace num2word
|
|
104
332
|
|
|
333
|
+
// --- Symbols to Word Normalization ---
|
|
334
|
+
namespace sym2word {
|
|
335
|
+
using ::phonemis::processor::Readings;
|
|
336
|
+
|
|
337
|
+
inline const Readings kSymbols = {
|
|
338
|
+
{U"&", {U"and"}}, {U"+", {U"plus"}}, {U"=", {U"równa się"}}, {U"%", {U"procent"}}, {U"°", {U"stopni"}},
|
|
339
|
+
{U"#", {U""}}, {U"/", {U""}}, {U"²", {U"do kwadratu"}}, {U"³", {U"do sześcianu"}},
|
|
340
|
+
{U"π", {U"pi"}}, {U"×", {U"razy"}}, {U"€", {U"euro"}}, {U"$", {U"dolarów"}}, {U"@", {U"małpa"}}
|
|
341
|
+
};
|
|
342
|
+
} // namespace sym2word
|
|
343
|
+
|
|
344
|
+
// --- Terms to Word Normalization ---
|
|
345
|
+
namespace term2word {
|
|
346
|
+
using ::phonemis::processor::before_number;
|
|
347
|
+
using ::phonemis::processor::term2word::Terms;
|
|
348
|
+
|
|
349
|
+
inline const Terms kTerms = {
|
|
350
|
+
.abbreviations = {
|
|
351
|
+
// Before a number only: nr 5
|
|
352
|
+
{U"nr", {{U"numer", before_number}}},
|
|
353
|
+
{U"Nr", {{U"numer", before_number}}},
|
|
354
|
+
{U"nr.", {{U"numer", before_number}}},
|
|
355
|
+
{U"ust.", {{U"ustęp", before_number}}},
|
|
356
|
+
{U"pkt", {{U"punkt", before_number}}},
|
|
357
|
+
{U"art.", {{U"artykuł", before_number}}},
|
|
358
|
+
{U"str.", {{U"strona", before_number}}},
|
|
359
|
+
{U"poz.", {{U"pozycja", before_number}}},
|
|
360
|
+
{U"ok.", {{U"około", before_number}}},
|
|
361
|
+
// Anywhere
|
|
362
|
+
{U"np.", {U"na przykład"}},
|
|
363
|
+
{U"itd.", {U"i tak dalej"}},
|
|
364
|
+
{U"itp.", {U"i tym podobne"}},
|
|
365
|
+
{U"tzn.", {U"to znaczy"}},
|
|
366
|
+
{U"tj.", {U"to jest"}},
|
|
367
|
+
{U"m.in.", {U"między innymi"}},
|
|
368
|
+
{U"wg", {U"według"}},
|
|
369
|
+
},
|
|
370
|
+
.web = {.dot = U"kropka", .at = U"małpa", .slash = U"ukośnik"}
|
|
371
|
+
};
|
|
372
|
+
} // namespace term2word
|
|
373
|
+
|
|
105
374
|
// --- Tokenization Rules ---
|
|
106
375
|
namespace tokenizer {
|
|
107
376
|
using ::phonemis::tokenizer::split::Rule;
|