react-native-executorch 0.10.6 → 0.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/module/fetcher/telemetry.js +1 -1
- package/package.json +1 -1
- package/react-native-executorch.podspec +2 -0
- package/src/fetcher/telemetry.ts +1 -1
- package/third-party/common/phonemis/CMakeLists.txt +70 -14
- package/third-party/common/phonemis/README.md +183 -33
- package/third-party/common/phonemis/THIRD_PARTY_NOTICES.md +57 -0
- package/third-party/common/phonemis/knowledge/code_style.md +642 -0
- package/third-party/common/phonemis/licenses/Unicode-3.0.txt +39 -0
- package/third-party/common/phonemis/licenses/misaki-Apache-2.0.txt +201 -0
- package/third-party/common/phonemis/licenses/nlohmann-json-MIT.txt +21 -0
- package/third-party/common/phonemis/licenses/xsimd-BSD-3-Clause.txt +29 -0
- package/third-party/common/phonemis/src/phonemis/base/config.h +6 -0
- package/third-party/common/phonemis/src/phonemis/base/ipipeline.cpp +75 -0
- package/third-party/common/phonemis/src/phonemis/base/ipipeline.h +21 -0
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/constants.h +2 -5
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/lexicon_phonemizer.cpp +3 -4
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/neural_phonemizer.cpp +17 -8
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/neural_phonemizer.h +4 -0
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/tokenizer.cpp +21 -6
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/tokenizer.h +13 -4
- package/third-party/common/phonemis/src/phonemis/base/pipeline.cpp +8 -0
- package/third-party/common/phonemis/src/phonemis/base/pipeline.h +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.cpp +37 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.h +45 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/constants.h +10 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/context.cpp +105 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/context.h +49 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/document.cpp +173 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/document.h +67 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/text.cpp +75 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/text.h +60 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/types.h +18 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/config.h +53 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layer.h +9 -7
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/constants.h +20 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/layer.cpp +89 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/layer.h +35 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/types.h +16 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/kind.h +40 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/layer.h +382 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/notation.h +30 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/numerals.cpp +51 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/numerals.h +31 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern.h +30 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern_set.cpp +17 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern_set.h +31 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.cpp +811 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.h +285 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.cpp +64 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.h +23 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/types.h +94 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sanitizer/layer.cpp +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/{sanitizer_layer.h → layers/sanitizer/layer.h} +8 -8
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.cpp +65 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.h +29 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/layer.cpp +170 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/layer.h +40 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/types.h +25 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/trim/layer.cpp +47 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/trim/layer.h +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/preprocessor.cpp +60 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/preprocessor.h +45 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/processor.cpp +6 -7
- package/third-party/common/phonemis/src/phonemis/base/processor/processor.h +14 -10
- package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.cpp +89 -52
- package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.h +16 -9
- package/third-party/common/phonemis/src/phonemis/base/tokenizer/tokenizer.cpp +13 -12
- package/third-party/common/phonemis/src/phonemis/lang/de/constants.h +183 -19
- package/third-party/common/phonemis/src/phonemis/lang/de/num2word.cpp +355 -309
- package/third-party/common/phonemis/src/phonemis/lang/de/num2word.h +42 -17
- package/third-party/common/phonemis/src/phonemis/lang/de/patterns.cpp +56 -0
- package/third-party/common/phonemis/src/phonemis/lang/de/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/de/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/en/constants.h +181 -30
- package/third-party/common/phonemis/src/phonemis/lang/en/hmm_tagger.cpp +5 -7
- package/third-party/common/phonemis/src/phonemis/lang/en/num2word.cpp +232 -135
- package/third-party/common/phonemis/src/phonemis/lang/en/num2word.h +41 -16
- package/third-party/common/phonemis/src/phonemis/lang/en/pipeline.h +3 -10
- package/third-party/common/phonemis/src/phonemis/lang/es/constants.h +182 -60
- package/third-party/common/phonemis/src/phonemis/lang/es/num2word.cpp +247 -212
- package/third-party/common/phonemis/src/phonemis/lang/es/num2word.h +38 -18
- package/third-party/common/phonemis/src/phonemis/lang/es/patterns.cpp +50 -0
- package/third-party/common/phonemis/src/phonemis/lang/es/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/es/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/fr/constants.h +152 -26
- package/third-party/common/phonemis/src/phonemis/lang/fr/num2word.cpp +252 -179
- package/third-party/common/phonemis/src/phonemis/lang/fr/num2word.h +38 -15
- package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.cpp +83 -0
- package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/fr/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/hi/constants.h +62 -10
- package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.cpp +74 -123
- package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.h +27 -16
- package/third-party/common/phonemis/src/phonemis/lang/hi/pipeline.h +5 -3
- package/third-party/common/phonemis/src/phonemis/lang/it/constants.h +122 -42
- package/third-party/common/phonemis/src/phonemis/lang/it/num2word.cpp +200 -207
- package/third-party/common/phonemis/src/phonemis/lang/it/num2word.h +30 -18
- package/third-party/common/phonemis/src/phonemis/lang/it/patterns.cpp +25 -0
- package/third-party/common/phonemis/src/phonemis/lang/it/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/it/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/pl/constants.h +346 -77
- package/third-party/common/phonemis/src/phonemis/lang/pl/num2word.cpp +631 -246
- package/third-party/common/phonemis/src/phonemis/lang/pl/num2word.h +47 -15
- package/third-party/common/phonemis/src/phonemis/lang/pl/pipeline.h +6 -6
- package/third-party/common/phonemis/src/phonemis/lang/pt/constants.h +201 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/num2word.cpp +365 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/num2word.h +53 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/pipeline.h +4 -2
- package/third-party/common/phonemis/src/phonemis/main.cpp +42 -11
- package/third-party/common/phonemis/src/phonemis/protophone/meta.h +24 -0
- package/third-party/common/phonemis/src/phonemis/protophone/numeric.h +26 -0
- package/third-party/common/phonemis/src/phonemis/protophone/protophone.cpp +525 -0
- package/third-party/common/phonemis/src/phonemis/protophone/protophone.h +80 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor.h +76 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor_utils.h +89 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor_view.h +125 -0
- package/third-party/common/phonemis/src/phonemis/protophone/types.h +43 -0
- package/third-party/common/phonemis/src/phonemis/utils/strings.h +20 -14
- package/third-party/common/phonemis/src/phonemis/utils/unicode.h +15 -1
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_arithmetic.hpp +267 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_bit.hpp +232 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_cast.hpp +43 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_complex.hpp +112 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_details.hpp +386 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_logical.hpp +240 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_math.hpp +2601 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_memory.hpp +1006 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_rounding.hpp +72 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_swizzle.hpp +260 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_trigo.hpp +996 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/utils/shifts.hpp +82 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx.hpp +2111 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx2.hpp +1591 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx2_128.hpp +170 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512bw.hpp +724 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512cd.hpp +28 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512dq.hpp +270 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512er.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512f.hpp +2919 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512ifma.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512pf.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vbmi.hpp +76 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vbmi2.hpp +130 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vnni_avx512bw.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vnni_avx512vbmi2.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx_128.hpp +164 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avxvnni.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_common.hpp +26 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_common_fwd.hpp +127 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_constants.hpp +393 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_emulated.hpp +787 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_avx.hpp +93 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_avx2.hpp +46 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_sse.hpp +97 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma4.hpp +92 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_i8mm_neon64.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_isa.hpp +148 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_neon.hpp +3580 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_neon64.hpp +1614 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_rvv.hpp +1515 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_scalar.hpp +1210 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse2.hpp +2333 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse3.hpp +66 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse4_1.hpp +510 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse4_2.hpp +44 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_ssse3.hpp +186 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sve.hpp +1186 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_vsx.hpp +947 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_vxe.hpp +787 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_wasm.hpp +1819 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_arch.hpp +242 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_config.hpp +633 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_arm.hpp +132 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_ppc.hpp +54 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_riscv.hpp +95 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_s390x.hpp +56 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_x86.hpp +1121 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpuid.hpp +144 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_getauxval.hpp +159 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_macros.hpp +78 -0
- package/third-party/common/phonemis/src/third-party/xsimd/math/xsimd_rem_pio2.hpp +707 -0
- package/third-party/common/phonemis/src/third-party/xsimd/memory/xsimd_aligned_allocator.hpp +349 -0
- package/third-party/common/phonemis/src/third-party/xsimd/memory/xsimd_alignment.hpp +102 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_all_registers.hpp +57 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_api.hpp +3106 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx2_register.hpp +57 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512bw_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512cd_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512dq_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512er_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512f_register.hpp +77 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512ifma_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512pf_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vbmi2_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vbmi_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vnni_avx512bw_register.hpp +54 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vnni_avx512vbmi2_register.hpp +53 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx_register.hpp +79 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avxvnni_register.hpp +44 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch.hpp +1666 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch_constant.hpp +572 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch_fwd.hpp +41 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_common_arch.hpp +49 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_emulated_register.hpp +80 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_avx2_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_avx_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_sse_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma4_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_i8mm_neon64_register.hpp +55 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_neon64_register.hpp +55 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_neon_register.hpp +130 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_register.hpp +96 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_rvv_register.hpp +491 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse2_register.hpp +59 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse3_register.hpp +49 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse4_1_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse4_2_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_ssse3_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sve_register.hpp +216 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_traits.hpp +409 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_utils.hpp +462 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_vsx_register.hpp +84 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_vxe_register.hpp +86 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_wasm_register.hpp +59 -0
- package/third-party/common/phonemis/src/third-party/xsimd/utils/bits.hpp +122 -0
- package/third-party/common/phonemis/src/third-party/xsimd/utils/xsimd_type_traits.hpp +145 -0
- package/third-party/common/phonemis/src/third-party/xsimd/xsimd.hpp +43 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/constants.h +0 -11
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/config.h +0 -16
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/layer.cpp +0 -271
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/layer.h +0 -45
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/types.h +0 -29
- package/third-party/common/phonemis/src/phonemis/base/processor/sanitizer_layer.cpp +0 -22
- package/third-party/common/phonemis/src/phonemis/base/processor/trim_layer.cpp +0 -40
- package/third-party/common/phonemis/src/phonemis/base/processor/trim_layer.h +0 -22
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include "common/conditions.h"
|
|
4
|
+
|
|
5
|
+
namespace phonemis::processor {
|
|
6
|
+
|
|
7
|
+
// Configuration of preprocessing: which layers run, and words added to a language's own tables.
|
|
8
|
+
struct Config {
|
|
9
|
+
/**
|
|
10
|
+
* Write equivalent characters in one form (NormalizeLayer): quotes as ", apostrophes as ', dashes as - or " - ".
|
|
11
|
+
* The other layers expect it: without it, they miss "’" in 90’s or "–" in 10–15.
|
|
12
|
+
*/
|
|
13
|
+
bool normalize = true;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Normalize whitespaces (TrimLayer): no whitespace at the edges or before punctuation, a single one between words.
|
|
17
|
+
* The tokenizer expects a trimmed text, so turn it off only for an already normalized input.
|
|
18
|
+
*/
|
|
19
|
+
bool trim = true;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Read abbreviations and web addresses as words (Term2WordLayer): e.g. -> for example.
|
|
23
|
+
*/
|
|
24
|
+
bool terms = true;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Read numbers as words (Num2WordLayer): 25 -> twenty-five, $5 -> five dollars.
|
|
28
|
+
* Without it, "$5" reads "dollars 5" if symbols stay on.
|
|
29
|
+
*/
|
|
30
|
+
bool numbers = true;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Read symbols as words (Sym2WordLayer): & -> and.
|
|
34
|
+
*/
|
|
35
|
+
bool symbols = true;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Replace or remove the characters a language does not expect (SanitizerLayer), if it has such rules.
|
|
39
|
+
*/
|
|
40
|
+
bool sanitize = true;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Abbreviations added to the language's own, replacing those with the same key: {U"approx.", {U"approximately"}}.
|
|
44
|
+
*/
|
|
45
|
+
Readings abbreviations;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Symbols added to the language's own, replacing those with the same key: {U"™", {U"trademark"}}.
|
|
49
|
+
*/
|
|
50
|
+
Readings symbol_words;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
} // namespace phonemis::processor
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
|
-
#include
|
|
4
|
-
#include <string_view>
|
|
3
|
+
#include "common/document.h"
|
|
5
4
|
|
|
6
5
|
namespace phonemis::processor {
|
|
7
6
|
|
|
7
|
+
/**
|
|
8
|
+
* An interface (contract) for all preprocessing steps.
|
|
9
|
+
*/
|
|
8
10
|
class Layer {
|
|
9
11
|
public:
|
|
10
|
-
|
|
12
|
+
virtual ~Layer() = default;
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
+
// A single preprocessing step: reads the document and queues its edits.
|
|
15
|
+
virtual void apply(Document& document) const = 0;
|
|
14
16
|
};
|
|
15
|
-
|
|
16
|
-
} // namespace phonemis::processor
|
|
17
|
+
|
|
18
|
+
} // namespace phonemis::processor
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include "types.h"
|
|
4
|
+
|
|
5
|
+
#include <string_view>
|
|
6
|
+
|
|
7
|
+
namespace phonemis::processor::normalize::constants {
|
|
8
|
+
|
|
9
|
+
// The character classes of Echo (echo/preprocessing/characters.py), the default of every language
|
|
10
|
+
inline const Characters kCharacters = {
|
|
11
|
+
{U"…", U"..."}, // Ellipsis
|
|
12
|
+
{U"«»„“”‟˝", U"\""}, // Quotes
|
|
13
|
+
{U"‘’‚‛´`ʼ", U"'"}, // Apostrophes
|
|
14
|
+
{U"\u200b\u200c\u200d\ufeff", U""} // Invisible: zero-width space, non-joiner, joiner, byte order mark
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
// Dashes besides "-": em, en, figure dash, horizontal bar
|
|
18
|
+
inline constexpr std::u32string_view kDashes = U"—–‒―";
|
|
19
|
+
|
|
20
|
+
} // namespace phonemis::processor::normalize::constants
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#include "layer.h"
|
|
2
|
+
#include <phonemis/utils/strings.h>
|
|
3
|
+
#include <phonemis/utils/unicode.h>
|
|
4
|
+
|
|
5
|
+
namespace phonemis::processor::normalize {
|
|
6
|
+
|
|
7
|
+
using namespace utils;
|
|
8
|
+
|
|
9
|
+
namespace {
|
|
10
|
+
// Whether a character is a dash or a hyphen
|
|
11
|
+
bool is_dash(char32_t c) {
|
|
12
|
+
return c == U'-' || strings::contains(constants::kDashes, c);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// The end of a dash starting at a position: a dash character or two hyphens and more (--); the position if none
|
|
16
|
+
size_t dash_end(std::u32string_view text, size_t position) {
|
|
17
|
+
if (position < text.size() && strings::contains(constants::kDashes, text[position])) {
|
|
18
|
+
return position + 1;
|
|
19
|
+
}
|
|
20
|
+
size_t end = position;
|
|
21
|
+
while (end < text.size() && text[end] == U'-') {
|
|
22
|
+
end++;
|
|
23
|
+
}
|
|
24
|
+
return end - position >= 2 ? end : position;
|
|
25
|
+
}
|
|
26
|
+
} // namespace
|
|
27
|
+
|
|
28
|
+
NormalizeLayer::NormalizeLayer(const Characters& characters) {
|
|
29
|
+
for (const auto& [members, canonical] : characters) {
|
|
30
|
+
for (char32_t c : members) {
|
|
31
|
+
canonical_[c] = canonical;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
void NormalizeLayer::apply(Document& document) const {
|
|
37
|
+
std::u32string_view text = document.text();
|
|
38
|
+
|
|
39
|
+
for (size_t i = 0; i < text.size();) {
|
|
40
|
+
if (auto dash = read_dash(text, i)) {
|
|
41
|
+
document.replace(dash->span, std::move(dash->text));
|
|
42
|
+
i = dash->span.end;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (auto it = canonical_.find(text[i]); it != canonical_.end()) {
|
|
46
|
+
document.replace({i, i + 1}, it->second);
|
|
47
|
+
}
|
|
48
|
+
i++;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
std::optional<Document::Edit> NormalizeLayer::read_dash(std::u32string_view text, size_t position) const {
|
|
53
|
+
size_t end = dash_end(text, position);
|
|
54
|
+
if (end == position) {
|
|
55
|
+
return std::nullopt;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// 1. Inside a word or a number: Szczurek–Żelazko, 1990–2000, a--b
|
|
59
|
+
if (position > 0 && unicode::isword(text[position - 1]) && end < text.size() && unicode::isword(text[end])) {
|
|
60
|
+
return Document::Edit{.span = {position, end}, .text = U"-"};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// 2. Elsewhere, the whole run of dashes with the spaces around it: "a – b", "a – — b", "a ---- b"
|
|
64
|
+
size_t begin = position;
|
|
65
|
+
while (begin > 0 && unicode::isspace(text[begin - 1])) {
|
|
66
|
+
begin--;
|
|
67
|
+
}
|
|
68
|
+
for (size_t next = end; next < text.size();) {
|
|
69
|
+
while (next < text.size() && unicode::isspace(text[next])) {
|
|
70
|
+
next++;
|
|
71
|
+
}
|
|
72
|
+
if (next == text.size() || !is_dash(text[next])) {
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
while (next < text.size() && is_dash(text[next])) {
|
|
76
|
+
next++;
|
|
77
|
+
}
|
|
78
|
+
end = next;
|
|
79
|
+
}
|
|
80
|
+
while (end < text.size() && unicode::isspace(text[end])) {
|
|
81
|
+
end++;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Between numbers it stays a range: 2020 – 2022 -> 2020-2022
|
|
85
|
+
bool range = begin > 0 && unicode::isdigit(text[begin - 1]) && end < text.size() && unicode::isdigit(text[end]);
|
|
86
|
+
return Document::Edit{.span = {begin, end}, .text = range ? U"-" : U" - "};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
} // namespace phonemis::processor::normalize
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include "../../layer.h"
|
|
4
|
+
#include "constants.h"
|
|
5
|
+
#include "types.h"
|
|
6
|
+
|
|
7
|
+
#include <optional>
|
|
8
|
+
#include <string>
|
|
9
|
+
#include <string_view>
|
|
10
|
+
#include <unordered_map>
|
|
11
|
+
|
|
12
|
+
namespace phonemis::processor::normalize {
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Writes equivalent characters in one canonical form, before anything reads the text:
|
|
16
|
+
* quotes as ", apostrophes as ', an ellipsis as ..., dashes as - or " - ", and no invisible characters.
|
|
17
|
+
*/
|
|
18
|
+
class NormalizeLayer : public Layer {
|
|
19
|
+
public:
|
|
20
|
+
explicit NormalizeLayer(const Characters& characters = constants::kCharacters);
|
|
21
|
+
|
|
22
|
+
void apply(Document& document) const override;
|
|
23
|
+
|
|
24
|
+
private:
|
|
25
|
+
/**
|
|
26
|
+
* A dash at a position: "-" inside a word or a number (1990–2000), and " - " for a run of dashes with
|
|
27
|
+
* the spaces around it elsewhere (a – b), except between numbers, which stay a range (2020 – 2022 -> 2020-2022).
|
|
28
|
+
* @returns its edit, or std::nullopt if no dash starts there ("-" alone is a hyphen).
|
|
29
|
+
*/
|
|
30
|
+
std::optional<Document::Edit> read_dash(std::u32string_view text, size_t position) const;
|
|
31
|
+
|
|
32
|
+
std::unordered_map<char32_t, std::u32string> canonical_; // Of each character of the classes: ’ -> '
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
} // namespace phonemis::processor::normalize
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <string>
|
|
4
|
+
#include <vector>
|
|
5
|
+
|
|
6
|
+
namespace phonemis::processor::normalize {
|
|
7
|
+
|
|
8
|
+
// A class of characters written in one canonical form: all quotes as ".
|
|
9
|
+
struct CharacterClass {
|
|
10
|
+
std::u32string characters; // « » „ “ ” ‟ ˝
|
|
11
|
+
std::u32string canonical; // "; empty to remove them
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
using Characters = std::vector<CharacterClass>;
|
|
15
|
+
|
|
16
|
+
} // namespace phonemis::processor::normalize
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
namespace phonemis::processor::num2word {
|
|
4
|
+
|
|
5
|
+
// Types of numeric expressions, in priority order: at a given position of a text,
|
|
6
|
+
// patterns of earlier kinds are tried first. Languages add patterns, but never kinds.
|
|
7
|
+
enum class Kind {
|
|
8
|
+
// Whole expressions whose parts are numbers
|
|
9
|
+
DATE = 0, // 12.05.2024, 2024-05-12, 1. 5.
|
|
10
|
+
INTERNATIONAL_PHONE, // +48 600 123 456, +1 (555) 123-4567: before a language's own formats of its groups
|
|
11
|
+
PHONE, // 555-123-4567, 06 12 34 56 78
|
|
12
|
+
VERSION, // 3.8.10, 10.0.0.51
|
|
13
|
+
|
|
14
|
+
// Symbols between numbers (only the symbol is matched)
|
|
15
|
+
DIFFERENCE, // the " - " of 5 - 3
|
|
16
|
+
SUM, // the "+" of 2 + 2 and 90+6
|
|
17
|
+
RANGE, // the "-" of 10-15 and 2020-2022
|
|
18
|
+
DIMENSION, // the "x" of 1920x1080 and 3 m x 4 m
|
|
19
|
+
FRACTION, // 3/4
|
|
20
|
+
SIGN, // the minus of -5
|
|
21
|
+
TIME, // 18:30, 7pm, 18 Uhr, 21h30
|
|
22
|
+
SCORE, // the ":" of 2:1
|
|
23
|
+
|
|
24
|
+
// Numbers with what they count
|
|
25
|
+
MONEY, // $5, 12,99 zł, 10-15 €, 3 mln zł
|
|
26
|
+
SCALE, // 3 mln, 500k, 1.5 million
|
|
27
|
+
PERCENT, // 20%, 10-15%, 8%ige
|
|
28
|
+
UNIT, // 5 kg, 10-15 km
|
|
29
|
+
|
|
30
|
+
// Numbers marked by a suffix or joined to letters
|
|
31
|
+
ORDINAL, // 21st, 3., 1er, 1º
|
|
32
|
+
DECADE, // 1990s, 90er, '80
|
|
33
|
+
COMPOUND, // 21-year-old, 45-metrowe
|
|
34
|
+
CODE, // MP3, A380, COVID-19
|
|
35
|
+
|
|
36
|
+
// Any other number
|
|
37
|
+
NUMBER, // 42, 1 000,5, .5, 2020 r.
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
} // namespace phonemis::processor::num2word
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include "../../common/context.h"
|
|
4
|
+
#include "../../layer.h"
|
|
5
|
+
#include "notation.h"
|
|
6
|
+
#include "numerals.h"
|
|
7
|
+
#include "pattern_set.h"
|
|
8
|
+
#include "patterns.h"
|
|
9
|
+
#include "primitives.h"
|
|
10
|
+
#include <phonemis/utils/unicode.h>
|
|
11
|
+
|
|
12
|
+
#include <memory>
|
|
13
|
+
#include <optional>
|
|
14
|
+
#include <string>
|
|
15
|
+
#include <vector>
|
|
16
|
+
|
|
17
|
+
namespace phonemis::processor::num2word {
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A skeleton of numbers verbalization, specialized by every language.
|
|
21
|
+
*
|
|
22
|
+
* Each pattern, in priority order (see kind.h), rewrites the text in turn: it finds its expressions (scan),
|
|
23
|
+
* decides how to read each of them from its context (resolve) and says it (verbalize).
|
|
24
|
+
* A language provides the patterns specific to it, the resolution and the readings.
|
|
25
|
+
*
|
|
26
|
+
* @tparam Reading a language's decision on how to read a match: the features its words agree with.
|
|
27
|
+
*/
|
|
28
|
+
template <typename Reading>
|
|
29
|
+
class Num2WordLayer : public Layer {
|
|
30
|
+
public:
|
|
31
|
+
void apply(Document& document) const override;
|
|
32
|
+
|
|
33
|
+
protected:
|
|
34
|
+
Num2WordLayer(const Notation& notation, Joiners joiners);
|
|
35
|
+
|
|
36
|
+
// Adds a language's own pattern; it goes before the generic patterns of its kind.
|
|
37
|
+
void add_pattern(std::unique_ptr<Pattern> pattern);
|
|
38
|
+
|
|
39
|
+
// ---------------------
|
|
40
|
+
// ----- Resolution -----
|
|
41
|
+
// ---------------------
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Whether a match really is what its pattern found: a time can turn out to be a decimal (1.13).
|
|
45
|
+
* A rejected match is left to the patterns of later kinds.
|
|
46
|
+
*/
|
|
47
|
+
virtual bool accept(const Match& match, const Context& context) const { return true; }
|
|
48
|
+
|
|
49
|
+
// Decides how to read a match from its context.
|
|
50
|
+
virtual Reading resolve(const Match& match, const Context& context) const = 0;
|
|
51
|
+
|
|
52
|
+
// ---------------------------------
|
|
53
|
+
// ----- Readings of a language -----
|
|
54
|
+
// ---------------------------------
|
|
55
|
+
|
|
56
|
+
virtual std::u32string read_number(const Number& number, const Reading& reading) const = 0;
|
|
57
|
+
virtual std::u32string read_code_digits(std::u32string_view digits) const = 0; // The 380 of A380
|
|
58
|
+
|
|
59
|
+
virtual std::u32string read_date(const Match& match, const Reading& reading) const;
|
|
60
|
+
virtual std::u32string read_phone(const Match& match, const Reading& reading) const;
|
|
61
|
+
virtual std::u32string read_fraction(const Match& match, const Reading& reading) const;
|
|
62
|
+
virtual std::u32string read_time(const Match& match, const Reading& reading) const;
|
|
63
|
+
virtual std::u32string read_ordinal(const Match& match, const Reading& reading) const;
|
|
64
|
+
virtual std::u32string read_decade(const Match& match, const Reading& reading) const;
|
|
65
|
+
virtual std::u32string read_compound(const Match& match, const Reading& reading) const;
|
|
66
|
+
virtual std::u32string read_plain(const Match& match, const Reading& reading) const; // A NUMBER match
|
|
67
|
+
|
|
68
|
+
// Amounts: each reads the last number of a match, a range (10-15 km) gets its start prepended
|
|
69
|
+
virtual std::u32string read_money(const Number& number, const Match& match, const Reading& reading) const;
|
|
70
|
+
virtual std::u32string read_scale(const Number& number, const Match& match, const Reading& reading) const;
|
|
71
|
+
virtual std::u32string read_percent(const Number& number, const Match& match, const Reading& reading) const;
|
|
72
|
+
virtual std::u32string read_unit(const Number& number, const Match& match, const Reading& reading) const;
|
|
73
|
+
|
|
74
|
+
// Symbols between numbers (DIFFERENCE, SUM, RANGE, DIMENSION, SIGN, SCORE), from the joiners by default
|
|
75
|
+
virtual std::u32string read_joiner(const Match& match, const Reading& reading) const;
|
|
76
|
+
virtual std::u32string read_range(const Number& from, const Number& to, const Reading& reading) const;
|
|
77
|
+
|
|
78
|
+
// The plural of a reading, for codes like F-16s; none by default.
|
|
79
|
+
virtual std::optional<std::u32string> plural(std::u32string_view words) const { return std::nullopt; }
|
|
80
|
+
|
|
81
|
+
Joiners joiners_;
|
|
82
|
+
|
|
83
|
+
private:
|
|
84
|
+
using AmountReading = std::u32string (Num2WordLayer::*)(const Number&, const Match&, const Reading&) const;
|
|
85
|
+
|
|
86
|
+
std::u32string verbalize(const Match& match, const Reading& reading) const;
|
|
87
|
+
|
|
88
|
+
// Shared readings
|
|
89
|
+
std::u32string read_each(const Match& match, const Reading& reading) const; // Its numbers one by one
|
|
90
|
+
std::u32string read_version(const Match& match, const Reading& reading) const;
|
|
91
|
+
std::u32string read_international_phone(const Match& match, const Reading& reading) const;
|
|
92
|
+
std::u32string read_code(const Match& match, const Reading& reading) const;
|
|
93
|
+
std::u32string read_amount(const Match& match, const Reading& reading, AmountReading read) const;
|
|
94
|
+
|
|
95
|
+
PatternSet patterns_;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
// ------------------------
|
|
99
|
+
// ----- Implementation -----
|
|
100
|
+
// ------------------------
|
|
101
|
+
|
|
102
|
+
template <typename Reading>
|
|
103
|
+
Num2WordLayer<Reading>::Num2WordLayer(const Notation& notation, Joiners joiners) : joiners_(std::move(joiners)) {
|
|
104
|
+
for (auto& pattern : generic_patterns(notation)) {
|
|
105
|
+
patterns_.add(std::move(pattern), Origin::GENERIC);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
template <typename Reading>
|
|
110
|
+
void Num2WordLayer<Reading>::add_pattern(std::unique_ptr<Pattern> pattern) {
|
|
111
|
+
patterns_.add(std::move(pattern), Origin::LANGUAGE);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
template <typename Reading>
|
|
115
|
+
void Num2WordLayer<Reading>::apply(Document& document) const {
|
|
116
|
+
// A draft rewritten by each pattern in turn: later patterns see the words of earlier ones
|
|
117
|
+
Document draft(document.text());
|
|
118
|
+
|
|
119
|
+
for (const auto& [pattern, origin] : patterns_) {
|
|
120
|
+
std::u32string_view text = draft.text();
|
|
121
|
+
|
|
122
|
+
for (size_t position = 0; position < text.size();) {
|
|
123
|
+
// 1. Scan
|
|
124
|
+
auto match = pattern->match(text, position);
|
|
125
|
+
if (!match) {
|
|
126
|
+
position++;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
match->text = text.substr(match->span.begin, match->span.end - match->span.begin);
|
|
130
|
+
|
|
131
|
+
// 2. Resolve and verbalize
|
|
132
|
+
Context context(text, match->span);
|
|
133
|
+
if (accept(*match, context)) {
|
|
134
|
+
std::u32string words = verbalize(*match, resolve(*match, context));
|
|
135
|
+
if (words != match->text) {
|
|
136
|
+
draft.replace(match->span, std::move(words));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
position = match->span.end;
|
|
140
|
+
}
|
|
141
|
+
draft.commit();
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Each change of the draft is an edit of the document
|
|
145
|
+
for (auto& [span, words] : draft.changes()) {
|
|
146
|
+
document.replace(span, std::move(words));
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
template <typename Reading>
|
|
151
|
+
std::u32string Num2WordLayer<Reading>::verbalize(const Match& match, const Reading& reading) const {
|
|
152
|
+
switch (match.kind) {
|
|
153
|
+
case Kind::DATE:
|
|
154
|
+
return read_date(match, reading);
|
|
155
|
+
case Kind::INTERNATIONAL_PHONE:
|
|
156
|
+
return read_international_phone(match, reading);
|
|
157
|
+
case Kind::PHONE:
|
|
158
|
+
return read_phone(match, reading);
|
|
159
|
+
case Kind::VERSION:
|
|
160
|
+
return read_version(match, reading);
|
|
161
|
+
case Kind::DIFFERENCE:
|
|
162
|
+
case Kind::SUM:
|
|
163
|
+
case Kind::RANGE:
|
|
164
|
+
case Kind::DIMENSION:
|
|
165
|
+
case Kind::SIGN:
|
|
166
|
+
case Kind::SCORE:
|
|
167
|
+
return read_joiner(match, reading);
|
|
168
|
+
case Kind::FRACTION:
|
|
169
|
+
return read_fraction(match, reading);
|
|
170
|
+
case Kind::TIME:
|
|
171
|
+
return read_time(match, reading);
|
|
172
|
+
case Kind::MONEY:
|
|
173
|
+
return read_amount(match, reading, &Num2WordLayer::read_money);
|
|
174
|
+
case Kind::SCALE:
|
|
175
|
+
return read_amount(match, reading, &Num2WordLayer::read_scale);
|
|
176
|
+
case Kind::PERCENT:
|
|
177
|
+
return read_amount(match, reading, &Num2WordLayer::read_percent);
|
|
178
|
+
case Kind::UNIT:
|
|
179
|
+
return read_amount(match, reading, &Num2WordLayer::read_unit);
|
|
180
|
+
case Kind::ORDINAL:
|
|
181
|
+
return read_ordinal(match, reading);
|
|
182
|
+
case Kind::DECADE:
|
|
183
|
+
return read_decade(match, reading);
|
|
184
|
+
case Kind::COMPOUND:
|
|
185
|
+
return read_compound(match, reading);
|
|
186
|
+
case Kind::CODE:
|
|
187
|
+
return read_code(match, reading);
|
|
188
|
+
case Kind::NUMBER:
|
|
189
|
+
return read_plain(match, reading);
|
|
190
|
+
}
|
|
191
|
+
return std::u32string(match.text);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Default readings - all numbers read one by one
|
|
195
|
+
|
|
196
|
+
template <typename Reading>
|
|
197
|
+
std::u32string Num2WordLayer<Reading>::read_date(const Match& match, const Reading& reading) const {
|
|
198
|
+
return read_each(match, reading);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
template <typename Reading>
|
|
202
|
+
std::u32string Num2WordLayer<Reading>::read_phone(const Match& match, const Reading& reading) const {
|
|
203
|
+
std::vector<std::u32string> words;
|
|
204
|
+
for (const auto& number : match.numbers) {
|
|
205
|
+
words.push_back(read_code_digits(number.integer));
|
|
206
|
+
}
|
|
207
|
+
return join(words);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
template <typename Reading>
|
|
211
|
+
std::u32string Num2WordLayer<Reading>::read_fraction(const Match& match, const Reading& reading) const {
|
|
212
|
+
return read_each(match, reading);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
template <typename Reading>
|
|
216
|
+
std::u32string Num2WordLayer<Reading>::read_time(const Match& match, const Reading& reading) const {
|
|
217
|
+
return read_each(match, reading);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
template <typename Reading>
|
|
221
|
+
std::u32string Num2WordLayer<Reading>::read_ordinal(const Match& match, const Reading& reading) const {
|
|
222
|
+
return read_number(match.numbers[0], reading);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
template <typename Reading>
|
|
226
|
+
std::u32string Num2WordLayer<Reading>::read_decade(const Match& match, const Reading& reading) const {
|
|
227
|
+
return read_number(match.numbers[0], reading);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
template <typename Reading>
|
|
231
|
+
std::u32string Num2WordLayer<Reading>::read_compound(const Match& match, const Reading& reading) const {
|
|
232
|
+
return read_number(match.numbers[0], reading) + U"-" + std::u32string(match.suffix);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
template <typename Reading>
|
|
236
|
+
std::u32string Num2WordLayer<Reading>::read_plain(const Match& match, const Reading& reading) const {
|
|
237
|
+
return read_number(match.numbers[0], reading);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
template <typename Reading>
|
|
241
|
+
std::u32string Num2WordLayer<Reading>::read_money(const Number& number, const Match& match,
|
|
242
|
+
const Reading& reading) const {
|
|
243
|
+
return read_number(number, reading) + U" " + std::u32string(match.unit);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
template <typename Reading>
|
|
247
|
+
std::u32string Num2WordLayer<Reading>::read_scale(const Number& number, const Match& match,
|
|
248
|
+
const Reading& reading) const {
|
|
249
|
+
return read_number(number, reading) + U" " + std::u32string(match.scale);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
template <typename Reading>
|
|
253
|
+
std::u32string Num2WordLayer<Reading>::read_percent(const Number& number, const Match& match,
|
|
254
|
+
const Reading& reading) const {
|
|
255
|
+
return read_number(number, reading) + U" %";
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
template <typename Reading>
|
|
259
|
+
std::u32string Num2WordLayer<Reading>::read_unit(const Number& number, const Match& match,
|
|
260
|
+
const Reading& reading) const {
|
|
261
|
+
return read_number(number, reading) + U" " + std::u32string(match.unit);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
template <typename Reading>
|
|
265
|
+
std::u32string Num2WordLayer<Reading>::read_joiner(const Match& match, const Reading& reading) const {
|
|
266
|
+
switch (match.kind) {
|
|
267
|
+
case Kind::DIFFERENCE:
|
|
268
|
+
return U" " + joiners_.minus + U" ";
|
|
269
|
+
case Kind::SUM:
|
|
270
|
+
return U" " + joiners_.plus + U" ";
|
|
271
|
+
case Kind::RANGE:
|
|
272
|
+
return read_range(match.numbers[0], match.numbers[1], reading);
|
|
273
|
+
case Kind::DIMENSION:
|
|
274
|
+
return U" " + joiners_.by + U" ";
|
|
275
|
+
case Kind::SIGN:
|
|
276
|
+
return joiners_.minus + U" ";
|
|
277
|
+
default:
|
|
278
|
+
return joiners_.score;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
template <typename Reading>
|
|
283
|
+
std::u32string Num2WordLayer<Reading>::read_range(const Number& from, const Number& to, const Reading& reading) const {
|
|
284
|
+
return joiners_.range;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Shared readings
|
|
288
|
+
|
|
289
|
+
template <typename Reading>
|
|
290
|
+
std::u32string Num2WordLayer<Reading>::read_each(const Match& match, const Reading& reading) const {
|
|
291
|
+
std::vector<std::u32string> words;
|
|
292
|
+
for (const auto& number : match.numbers) {
|
|
293
|
+
words.push_back(read_number(number, reading));
|
|
294
|
+
}
|
|
295
|
+
return join(words);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
template <typename Reading>
|
|
299
|
+
std::u32string Num2WordLayer<Reading>::read_international_phone(const Match& match, const Reading& reading) const {
|
|
300
|
+
// plus forty-eight, then the groups as the language reads a phone number
|
|
301
|
+
Match groups = match;
|
|
302
|
+
groups.kind = Kind::PHONE;
|
|
303
|
+
groups.numbers.erase(groups.numbers.begin());
|
|
304
|
+
groups.text = match.text.substr(groups.numbers.front().span.begin - match.span.begin);
|
|
305
|
+
return joiners_.plus + U" " + read_code_digits(match.numbers[0].integer) + U" " + read_phone(groups, reading);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
template <typename Reading>
|
|
309
|
+
std::u32string Num2WordLayer<Reading>::read_version(const Match& match, const Reading& reading) const {
|
|
310
|
+
std::vector<std::u32string> words;
|
|
311
|
+
for (const auto& part : match.numbers) {
|
|
312
|
+
words.push_back(read_number(part, reading));
|
|
313
|
+
}
|
|
314
|
+
return join(words, U" " + (match.numbers.size() == 4 ? joiners_.dot : joiners_.point) + U" ");
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
template <typename Reading>
|
|
318
|
+
std::u32string Num2WordLayer<Reading>::read_code(const Match& match, const Reading& reading) const {
|
|
319
|
+
// Letters stay, digit runs read as numbers: MP3, COVID-19, L'A380, F-16s
|
|
320
|
+
std::vector<std::u32string> out;
|
|
321
|
+
bool glue = false;
|
|
322
|
+
std::u32string_view previous;
|
|
323
|
+
|
|
324
|
+
for (auto part : code_parts(match.text)) {
|
|
325
|
+
if (part == U"'" && !out.empty()) {
|
|
326
|
+
out.back() += part; // L'A380
|
|
327
|
+
glue = true;
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
std::optional<std::u32string> word;
|
|
332
|
+
if (part == U"-" || part == U".") {
|
|
333
|
+
word = std::nullopt;
|
|
334
|
+
} else if (utils::unicode::isdigit(part.front()) && part.find(U'.') == std::u32string_view::npos) {
|
|
335
|
+
word = read_code_digits(part);
|
|
336
|
+
} else if (part.front() == U'.' && !out.empty()) {
|
|
337
|
+
word = read_code_digits(part.substr(1)); // H.264
|
|
338
|
+
} else if (utils::unicode::isdigit(part.back())) {
|
|
339
|
+
// 2.5 inside a code, or .5 starting it
|
|
340
|
+
size_t dot = part.find(U'.');
|
|
341
|
+
Number number{.integer = std::u32string(part.substr(0, dot)),
|
|
342
|
+
.fraction = std::u32string(part.substr(dot + 1)),
|
|
343
|
+
.decimal_mark = U'.'};
|
|
344
|
+
word = read_number(number, reading);
|
|
345
|
+
} else if (bool after_digits = !previous.empty() && utils::unicode::isdigit(previous.back());
|
|
346
|
+
auto words = part == U"s" && after_digits ? plural(out.back()) : std::nullopt) {
|
|
347
|
+
out.back() = *words; // F-16s
|
|
348
|
+
} else {
|
|
349
|
+
word = std::u32string(part);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
if (word) {
|
|
353
|
+
if (glue && !out.empty()) {
|
|
354
|
+
out.back() += *word;
|
|
355
|
+
} else {
|
|
356
|
+
out.push_back(*word);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
glue = false;
|
|
360
|
+
previous = part;
|
|
361
|
+
}
|
|
362
|
+
return join(out);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
template <typename Reading>
|
|
366
|
+
std::u32string Num2WordLayer<Reading>::read_amount(const Match& match, const Reading& reading,
|
|
367
|
+
AmountReading read) const {
|
|
368
|
+
Number last = match.numbers.back();
|
|
369
|
+
|
|
370
|
+
// A unit reads its number without leading zeros: 05 km
|
|
371
|
+
while (match.kind == Kind::UNIT && last.integer.size() > 1 && last.integer[0] == U'0') {
|
|
372
|
+
last.integer.erase(0, 1);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
std::u32string words = (this->*read)(last, match, reading);
|
|
376
|
+
if (match.numbers.size() == 1) {
|
|
377
|
+
return words;
|
|
378
|
+
}
|
|
379
|
+
return read_number(match.numbers[0], reading) + read_range(match.numbers[0], last, reading) + words;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
} // namespace phonemis::processor::num2word
|