react-native-executorch 0.10.5 → 0.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/android/build.gradle.kts +4 -3
- package/ios/RnExecutorch.mm +15 -17
- package/lib/module/fetcher/fetcher.js +35 -122
- package/lib/module/fetcher/fetcher.js.map +1 -1
- package/lib/module/fetcher/telemetry.js +1 -1
- package/lib/module/index.js +12 -13
- package/lib/module/index.js.map +1 -1
- package/lib/module/models.js +4 -2
- package/lib/module/models.js.map +1 -1
- package/lib/typescript/src/fetcher/fetcher.d.ts +3 -7
- package/lib/typescript/src/fetcher/fetcher.d.ts.map +1 -1
- package/lib/typescript/src/models.d.ts.map +1 -1
- package/package.json +9 -7
- package/react-native-executorch.podspec +6 -2
- package/src/fetcher/fetcher.ts +38 -131
- package/src/fetcher/telemetry.ts +1 -1
- package/src/models.ts +6 -1
- package/third-party/common/phonemis/CMakeLists.txt +70 -14
- package/third-party/common/phonemis/README.md +183 -33
- package/third-party/common/phonemis/THIRD_PARTY_NOTICES.md +57 -0
- package/third-party/common/phonemis/knowledge/code_style.md +642 -0
- package/third-party/common/phonemis/licenses/Unicode-3.0.txt +39 -0
- package/third-party/common/phonemis/licenses/misaki-Apache-2.0.txt +201 -0
- package/third-party/common/phonemis/licenses/nlohmann-json-MIT.txt +21 -0
- package/third-party/common/phonemis/licenses/xsimd-BSD-3-Clause.txt +29 -0
- package/third-party/common/phonemis/src/phonemis/base/config.h +6 -0
- package/third-party/common/phonemis/src/phonemis/base/ipipeline.cpp +75 -0
- package/third-party/common/phonemis/src/phonemis/base/ipipeline.h +21 -0
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/constants.h +2 -5
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/lexicon_phonemizer.cpp +3 -4
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/neural_phonemizer.cpp +17 -8
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/neural_phonemizer.h +4 -0
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/tokenizer.cpp +21 -6
- package/third-party/common/phonemis/src/phonemis/base/phonemizer/nn/tokenizer.h +13 -4
- package/third-party/common/phonemis/src/phonemis/base/pipeline.cpp +8 -0
- package/third-party/common/phonemis/src/phonemis/base/pipeline.h +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.cpp +37 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/conditions.h +45 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/constants.h +10 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/context.cpp +105 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/context.h +49 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/document.cpp +173 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/document.h +67 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/text.cpp +75 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/text.h +60 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/common/types.h +18 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/config.h +53 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layer.h +9 -7
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/constants.h +20 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/layer.cpp +89 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/layer.h +35 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/normalize/types.h +16 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/kind.h +40 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/layer.h +382 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/notation.h +30 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/numerals.cpp +51 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/numerals.h +31 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern.h +30 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern_set.cpp +17 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/pattern_set.h +31 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.cpp +811 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/patterns.h +285 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.cpp +64 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/primitives.h +23 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/num2word/types.h +94 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sanitizer/layer.cpp +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/{sanitizer_layer.h → layers/sanitizer/layer.h} +8 -8
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.cpp +65 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/sym2word/layer.h +29 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/layer.cpp +170 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/layer.h +40 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/term2word/types.h +25 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/trim/layer.cpp +47 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/layers/trim/layer.h +19 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/preprocessor.cpp +60 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/preprocessor.h +45 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/processor.cpp +6 -7
- package/third-party/common/phonemis/src/phonemis/base/processor/processor.h +14 -10
- package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.cpp +89 -52
- package/third-party/common/phonemis/src/phonemis/base/tagger/hmm_tagger.h +16 -9
- package/third-party/common/phonemis/src/phonemis/base/tokenizer/tokenizer.cpp +13 -12
- package/third-party/common/phonemis/src/phonemis/lang/de/constants.h +183 -19
- package/third-party/common/phonemis/src/phonemis/lang/de/num2word.cpp +355 -309
- package/third-party/common/phonemis/src/phonemis/lang/de/num2word.h +42 -17
- package/third-party/common/phonemis/src/phonemis/lang/de/patterns.cpp +56 -0
- package/third-party/common/phonemis/src/phonemis/lang/de/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/de/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/en/constants.h +181 -30
- package/third-party/common/phonemis/src/phonemis/lang/en/hmm_tagger.cpp +5 -7
- package/third-party/common/phonemis/src/phonemis/lang/en/num2word.cpp +232 -135
- package/third-party/common/phonemis/src/phonemis/lang/en/num2word.h +41 -16
- package/third-party/common/phonemis/src/phonemis/lang/en/pipeline.h +3 -10
- package/third-party/common/phonemis/src/phonemis/lang/es/constants.h +182 -60
- package/third-party/common/phonemis/src/phonemis/lang/es/num2word.cpp +247 -212
- package/third-party/common/phonemis/src/phonemis/lang/es/num2word.h +38 -18
- package/third-party/common/phonemis/src/phonemis/lang/es/patterns.cpp +50 -0
- package/third-party/common/phonemis/src/phonemis/lang/es/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/es/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/fr/constants.h +152 -26
- package/third-party/common/phonemis/src/phonemis/lang/fr/num2word.cpp +252 -179
- package/third-party/common/phonemis/src/phonemis/lang/fr/num2word.h +38 -15
- package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.cpp +83 -0
- package/third-party/common/phonemis/src/phonemis/lang/fr/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/fr/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/hi/constants.h +62 -10
- package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.cpp +74 -123
- package/third-party/common/phonemis/src/phonemis/lang/hi/num2word.h +27 -16
- package/third-party/common/phonemis/src/phonemis/lang/hi/pipeline.h +5 -3
- package/third-party/common/phonemis/src/phonemis/lang/it/constants.h +122 -42
- package/third-party/common/phonemis/src/phonemis/lang/it/num2word.cpp +200 -207
- package/third-party/common/phonemis/src/phonemis/lang/it/num2word.h +30 -18
- package/third-party/common/phonemis/src/phonemis/lang/it/patterns.cpp +25 -0
- package/third-party/common/phonemis/src/phonemis/lang/it/patterns.h +15 -0
- package/third-party/common/phonemis/src/phonemis/lang/it/pipeline.h +3 -3
- package/third-party/common/phonemis/src/phonemis/lang/pl/constants.h +346 -77
- package/third-party/common/phonemis/src/phonemis/lang/pl/num2word.cpp +631 -246
- package/third-party/common/phonemis/src/phonemis/lang/pl/num2word.h +47 -15
- package/third-party/common/phonemis/src/phonemis/lang/pl/pipeline.h +6 -6
- package/third-party/common/phonemis/src/phonemis/lang/pt/constants.h +201 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/num2word.cpp +365 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/num2word.h +53 -0
- package/third-party/common/phonemis/src/phonemis/lang/pt/pipeline.h +4 -2
- package/third-party/common/phonemis/src/phonemis/main.cpp +42 -11
- package/third-party/common/phonemis/src/phonemis/protophone/meta.h +24 -0
- package/third-party/common/phonemis/src/phonemis/protophone/numeric.h +26 -0
- package/third-party/common/phonemis/src/phonemis/protophone/protophone.cpp +525 -0
- package/third-party/common/phonemis/src/phonemis/protophone/protophone.h +80 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor.h +76 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor_utils.h +89 -0
- package/third-party/common/phonemis/src/phonemis/protophone/tensor_view.h +125 -0
- package/third-party/common/phonemis/src/phonemis/protophone/types.h +43 -0
- package/third-party/common/phonemis/src/phonemis/utils/strings.h +20 -14
- package/third-party/common/phonemis/src/phonemis/utils/unicode.h +15 -1
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_arithmetic.hpp +267 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_bit.hpp +232 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_cast.hpp +43 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_complex.hpp +112 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_details.hpp +386 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_logical.hpp +240 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_math.hpp +2601 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_memory.hpp +1006 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_rounding.hpp +72 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_swizzle.hpp +260 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/common/xsimd_common_trigo.hpp +996 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/utils/shifts.hpp +82 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx.hpp +2111 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx2.hpp +1591 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx2_128.hpp +170 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512bw.hpp +724 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512cd.hpp +28 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512dq.hpp +270 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512er.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512f.hpp +2919 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512ifma.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512pf.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vbmi.hpp +76 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vbmi2.hpp +130 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vnni_avx512bw.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx512vnni_avx512vbmi2.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avx_128.hpp +164 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_avxvnni.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_common.hpp +26 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_common_fwd.hpp +127 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_constants.hpp +393 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_emulated.hpp +787 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_avx.hpp +93 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_avx2.hpp +46 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma3_sse.hpp +97 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_fma4.hpp +92 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_i8mm_neon64.hpp +17 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_isa.hpp +148 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_neon.hpp +3580 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_neon64.hpp +1614 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_rvv.hpp +1515 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_scalar.hpp +1210 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse2.hpp +2333 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse3.hpp +66 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse4_1.hpp +510 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sse4_2.hpp +44 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_ssse3.hpp +186 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_sve.hpp +1186 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_vsx.hpp +947 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_vxe.hpp +787 -0
- package/third-party/common/phonemis/src/third-party/xsimd/arch/xsimd_wasm.hpp +1819 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_arch.hpp +242 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_config.hpp +633 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_arm.hpp +132 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_ppc.hpp +54 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_riscv.hpp +95 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_s390x.hpp +56 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpu_features_x86.hpp +1121 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_cpuid.hpp +144 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_getauxval.hpp +159 -0
- package/third-party/common/phonemis/src/third-party/xsimd/config/xsimd_macros.hpp +78 -0
- package/third-party/common/phonemis/src/third-party/xsimd/math/xsimd_rem_pio2.hpp +707 -0
- package/third-party/common/phonemis/src/third-party/xsimd/memory/xsimd_aligned_allocator.hpp +349 -0
- package/third-party/common/phonemis/src/third-party/xsimd/memory/xsimd_alignment.hpp +102 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_all_registers.hpp +57 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_api.hpp +3106 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx2_register.hpp +57 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512bw_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512cd_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512dq_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512er_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512f_register.hpp +77 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512ifma_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512pf_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vbmi2_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vbmi_register.hpp +51 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vnni_avx512bw_register.hpp +54 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx512vnni_avx512vbmi2_register.hpp +53 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avx_register.hpp +79 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_avxvnni_register.hpp +44 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch.hpp +1666 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch_constant.hpp +572 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_batch_fwd.hpp +41 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_common_arch.hpp +49 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_emulated_register.hpp +80 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_avx2_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_avx_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma3_sse_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_fma4_register.hpp +50 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_i8mm_neon64_register.hpp +55 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_neon64_register.hpp +55 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_neon_register.hpp +130 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_register.hpp +96 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_rvv_register.hpp +491 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse2_register.hpp +59 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse3_register.hpp +49 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse4_1_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sse4_2_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_ssse3_register.hpp +48 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_sve_register.hpp +216 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_traits.hpp +409 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_utils.hpp +462 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_vsx_register.hpp +84 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_vxe_register.hpp +86 -0
- package/third-party/common/phonemis/src/third-party/xsimd/types/xsimd_wasm_register.hpp +59 -0
- package/third-party/common/phonemis/src/third-party/xsimd/utils/bits.hpp +122 -0
- package/third-party/common/phonemis/src/third-party/xsimd/utils/xsimd_type_traits.hpp +145 -0
- package/third-party/common/phonemis/src/third-party/xsimd/xsimd.hpp +43 -0
- package/third-party/common/phonemis/src/phonemis/base/processor/constants.h +0 -11
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/config.h +0 -16
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/layer.cpp +0 -271
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/layer.h +0 -45
- package/third-party/common/phonemis/src/phonemis/base/processor/num2word/types.h +0 -29
- package/third-party/common/phonemis/src/phonemis/base/processor/sanitizer_layer.cpp +0 -22
- package/third-party/common/phonemis/src/phonemis/base/processor/trim_layer.cpp +0 -40
- package/third-party/common/phonemis/src/phonemis/base/processor/trim_layer.h +0 -22
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
#include "constants.h"
|
|
2
|
+
#include "num2word.h"
|
|
3
|
+
#include <phonemis/base/processor/layers/num2word/numerals.h>
|
|
4
|
+
#include <phonemis/base/processor/layers/num2word/patterns.h>
|
|
5
|
+
#include <phonemis/base/processor/layers/num2word/primitives.h>
|
|
6
|
+
#include <phonemis/utils/strings.h>
|
|
7
|
+
#include <phonemis/utils/unicode.h>
|
|
8
|
+
|
|
9
|
+
namespace phonemis::pt {
|
|
10
|
+
|
|
11
|
+
using namespace constants::num2word;
|
|
12
|
+
using namespace processor;
|
|
13
|
+
using namespace processor::num2word;
|
|
14
|
+
using namespace utils;
|
|
15
|
+
|
|
16
|
+
namespace {
|
|
17
|
+
uint64_t value(const Number& number) {
|
|
18
|
+
return value_of(number.integer).value_or(0);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// The number "1" as written, which counts a singular noun
|
|
22
|
+
bool is_one(const Number& number) {
|
|
23
|
+
return number.integer == U"1" && number.fraction.empty();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
bool is_ascii_letter(char32_t c) {
|
|
27
|
+
return unicode::is_ascii(c) && unicode::isalpha(c);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// ------------------------
|
|
31
|
+
// ----- Number words -----
|
|
32
|
+
// ------------------------
|
|
33
|
+
|
|
34
|
+
std::u32string unit(uint64_t n, Gender gender) {
|
|
35
|
+
if (gender == Gender::FEMININE && (n == 1 || n == 2)) {
|
|
36
|
+
return n == 1 ? U"uma" : U"duas";
|
|
37
|
+
}
|
|
38
|
+
return std::u32string(kOnes[n]);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
std::u32string below_hundred(uint64_t n, Gender gender) {
|
|
42
|
+
if (n < 20) {
|
|
43
|
+
return unit(n, gender);
|
|
44
|
+
}
|
|
45
|
+
return std::u32string(kTens[n / 10]) + (n % 10 ? U" e " + unit(n % 10, gender) : U"");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
std::u32string below_thousand(uint64_t n, Gender gender) {
|
|
49
|
+
uint64_t h = n / 100, r = n % 100;
|
|
50
|
+
if (h == 0) {
|
|
51
|
+
return below_hundred(r, gender);
|
|
52
|
+
}
|
|
53
|
+
if (n == 100) {
|
|
54
|
+
return U"cem";
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// duzentas pessoas
|
|
58
|
+
std::u32string head(kHundreds[h]);
|
|
59
|
+
if (gender == Gender::FEMININE && h > 1) {
|
|
60
|
+
head.replace(head.size() - 2, 2, U"as");
|
|
61
|
+
}
|
|
62
|
+
return head + (r ? U" e " + below_hundred(r, gender) : U"");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// "e" joins the last group below 100 or of round hundreds: dois mil e vinte, mil quinhentos e vinte, mil duzentos e um
|
|
66
|
+
std::u32string cardinal(uint64_t n, Gender gender = Gender::MASCULINE) {
|
|
67
|
+
std::vector<std::pair<uint64_t, std::u32string>> groups;
|
|
68
|
+
for (auto [scale, one, many] :
|
|
69
|
+
{std::tuple{1000000000000ULL, U"trilhão", U"trilhões"}, std::tuple{1000000000ULL, U"bilhão", U"bilhões"},
|
|
70
|
+
std::tuple{1000000ULL, U"milhão", U"milhões"}}) {
|
|
71
|
+
if (uint64_t q = n / scale) {
|
|
72
|
+
groups.emplace_back(q, cardinal(q) + U" " + (q == 1 ? one : many));
|
|
73
|
+
}
|
|
74
|
+
n %= scale;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Thousands agree with the noun: duas mil pessoas
|
|
78
|
+
if (uint64_t q = n / 1000) {
|
|
79
|
+
groups.emplace_back(q, q == 1 ? U"mil" : below_thousand(q, gender) + U" mil");
|
|
80
|
+
}
|
|
81
|
+
n %= 1000;
|
|
82
|
+
if (n || groups.empty()) {
|
|
83
|
+
groups.emplace_back(n, below_thousand(n, gender));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
std::u32string words = groups[0].second;
|
|
87
|
+
for (size_t i = 1; i < groups.size(); ++i) {
|
|
88
|
+
uint64_t last = groups[i].first;
|
|
89
|
+
bool joined = i + 1 == groups.size() && (last < 100 || last % 100 == 0);
|
|
90
|
+
words += (joined ? U" e " : U" ") + groups[i].second;
|
|
91
|
+
}
|
|
92
|
+
return words;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// décimo terceiro, vigésima primeira, milésimo
|
|
96
|
+
std::u32string ordinal(uint64_t n, bool feminine) {
|
|
97
|
+
char32_t ending = feminine ? U'a' : U'o';
|
|
98
|
+
std::vector<std::u32string> words;
|
|
99
|
+
if (uint64_t thousands = n / 1000) {
|
|
100
|
+
words.push_back((thousands == 1 ? U"" : cardinal(thousands) + U" ") + U"milésim" + ending);
|
|
101
|
+
}
|
|
102
|
+
for (auto [scale, stems] :
|
|
103
|
+
{std::pair{100, &kOrdinalHundreds}, std::pair{10, &kOrdinalTens}, std::pair{1, &kOrdinalOnes}}) {
|
|
104
|
+
if (uint64_t digit = n / scale % 10) {
|
|
105
|
+
words.push_back(std::u32string((*stems)[digit]) + ending);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return join(words);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
std::u32string digits(std::u32string_view text) {
|
|
112
|
+
std::vector<std::u32string> words;
|
|
113
|
+
for (char32_t c : text) {
|
|
114
|
+
words.push_back(unit(c - U'0', Gender::MASCULINE));
|
|
115
|
+
}
|
|
116
|
+
return join(words);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// um milhão de reais, but dez mil reais
|
|
120
|
+
std::u32string with_de(const std::u32string& words) {
|
|
121
|
+
bool scale = std::ranges::any_of(kScaleWords, [&](const auto& w) { return words.ends_with(w); });
|
|
122
|
+
return scale ? words + U" de" : words;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// -------------------
|
|
126
|
+
// ----- Context -----
|
|
127
|
+
// -------------------
|
|
128
|
+
|
|
129
|
+
bool feminine(std::u32string_view noun) {
|
|
130
|
+
std::u32string word = strings::to_lower(noun);
|
|
131
|
+
std::u32string singular = word;
|
|
132
|
+
if (word.ends_with(U"ões") || word.ends_with(U"ães")) {
|
|
133
|
+
singular = word.substr(0, word.size() - 3) + U"ão";
|
|
134
|
+
} else if (word.ends_with(U"res") || word.ends_with(U"zes") || word.ends_with(U"les")) {
|
|
135
|
+
singular.resize(word.size() - 2);
|
|
136
|
+
} else if (word.ends_with(U's')) {
|
|
137
|
+
singular.pop_back();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (kMasculineInA.contains(singular) || kMasculineInA.contains(word)) {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
bool feminine_ending = std::ranges::any_of(kFeminineEndings, [&](const auto& e) { return singular.ends_with(e); });
|
|
144
|
+
return kFeminineNouns.contains(singular) || kFeminineNouns.contains(word) || feminine_ending;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// The gender of the noun after a number; an hour is feminine: às duas, duas da tarde
|
|
148
|
+
Gender gender_of(const Context& context) {
|
|
149
|
+
std::u32string word = context.after();
|
|
150
|
+
if ((word == U"da" || word == U"de") && kDayParts.contains(strings::to_lower(context.after(2)))) {
|
|
151
|
+
return Gender::FEMININE;
|
|
152
|
+
}
|
|
153
|
+
if (word.empty() || !unicode::isalpha(word[0]) || kStopWords.contains(strings::to_lower(word))) {
|
|
154
|
+
return kHourWords.contains(context.before()) ? Gender::FEMININE : Gender::MASCULINE;
|
|
155
|
+
}
|
|
156
|
+
return feminine(word) ? Gender::FEMININE : Gender::MASCULINE;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// A lowercase noun follows: 500k seguidores
|
|
160
|
+
bool counted(const Context& context) {
|
|
161
|
+
std::u32string word = context.after();
|
|
162
|
+
bool lowercase_word = word.size() > 2 && std::ranges::all_of(word, [](char32_t c) { return unicode::islower(c); });
|
|
163
|
+
return lowercase_word && !kStopWords.contains(word);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// A range touching letters: A320-200, 3-4x
|
|
167
|
+
bool lettered(const Match& range, const Context& context) {
|
|
168
|
+
std::u32string_view head = context.head(), tail = context.tail();
|
|
169
|
+
size_t before = range.numbers[0].span.begin, after = range.numbers[1].span.end - range.span.end;
|
|
170
|
+
return (before > 0 && is_ascii_letter(head[before - 1])) || (after < tail.size() && is_ascii_letter(tail[after]));
|
|
171
|
+
}
|
|
172
|
+
} // namespace
|
|
173
|
+
|
|
174
|
+
Num2Word::Num2Word() : Num2WordLayer(kNotation, kJoiners) {
|
|
175
|
+
add_pattern(std::make_unique<DatePattern>(kDateFormat));
|
|
176
|
+
for (const auto& format : kPhoneFormats) {
|
|
177
|
+
add_pattern(std::make_unique<PhonePattern>(format));
|
|
178
|
+
}
|
|
179
|
+
add_pattern(std::make_unique<SuffixPattern>(Kind::ORDINAL, kOrdinalSigns, kOrdinalSignFormat));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ----------------------
|
|
183
|
+
// ----- Resolution -----
|
|
184
|
+
// ----------------------
|
|
185
|
+
|
|
186
|
+
bool Num2Word::accept(const Match& match, const Context& context) const {
|
|
187
|
+
switch (match.kind) {
|
|
188
|
+
case Kind::PHONE: {
|
|
189
|
+
// 2021-2022: a range of years; 213 456 789 is a count, unless a word says it is a phone number
|
|
190
|
+
bool years = std::ranges::all_of(match.numbers, [](const Number& group) {
|
|
191
|
+
return group.integer.size() == 4 && value(group) >= 1900 && value(group) <= 2100;
|
|
192
|
+
});
|
|
193
|
+
return !years && (match.separator != U' ' || kPhoneWords.contains(context.before()));
|
|
194
|
+
}
|
|
195
|
+
case Kind::FRACTION:
|
|
196
|
+
return kFractions.contains({value(match.numbers[0]), value(match.numbers[1])});
|
|
197
|
+
case Kind::TIME:
|
|
198
|
+
return match.numbers.size() < 2 || (value(match.numbers[0]) <= 24 && value(match.numbers[1]) <= 59);
|
|
199
|
+
case Kind::SCALE:
|
|
200
|
+
// 4K, 2K: a resolution; but 500k is quinhentos mil
|
|
201
|
+
return (match.scale != U"k" && match.scale != U"K") || value(match.numbers.back()) >= 10 || counted(context);
|
|
202
|
+
case Kind::ORDINAL:
|
|
203
|
+
return value(match.numbers[0]) > 0;
|
|
204
|
+
default:
|
|
205
|
+
return true;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
NumberReading Num2Word::resolve(const Match& match, const Context& context) const {
|
|
210
|
+
NumberReading reading{.gender = gender_of(context),
|
|
211
|
+
.day = strings::to_lower(context.after()) == U"de" &&
|
|
212
|
+
std::ranges::find(kMonths, strings::to_lower(context.after(2))) != kMonths.end(),
|
|
213
|
+
.heading = kHeadings.contains(context.before())};
|
|
214
|
+
if (match.kind == Kind::RANGE) {
|
|
215
|
+
reading.lettered = lettered(match, context);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// R$ 50 reais
|
|
219
|
+
if (match.kind == Kind::MONEY) {
|
|
220
|
+
const Forms& currency = kCurrencies.at(std::u32string(match.unit));
|
|
221
|
+
std::u32string next = strings::to_lower(context.after());
|
|
222
|
+
reading.named = next == currency.one || next == currency.many;
|
|
223
|
+
}
|
|
224
|
+
return reading;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// --------------------
|
|
228
|
+
// ----- Readings -----
|
|
229
|
+
// --------------------
|
|
230
|
+
|
|
231
|
+
std::u32string Num2Word::read_with(const Number& number, Gender gender) const {
|
|
232
|
+
if (number.fraction.empty()) {
|
|
233
|
+
auto n = value_of(number.integer);
|
|
234
|
+
return !n || spoken_digits(number, 7) ? digits(number.integer) : cardinal(*n, gender);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// The separator is read as written: dois vírgula cinco, dois ponto cinco
|
|
238
|
+
std::u32string fraction;
|
|
239
|
+
if (number.fraction.size() > 2) {
|
|
240
|
+
fraction = digits(number.fraction);
|
|
241
|
+
} else {
|
|
242
|
+
std::vector<std::u32string> words;
|
|
243
|
+
size_t zeros = number.fraction.find_first_not_of(U'0');
|
|
244
|
+
words.insert(words.end(), std::min(zeros, number.fraction.size()), U"zero");
|
|
245
|
+
if (zeros != std::u32string::npos) {
|
|
246
|
+
words.push_back(cardinal(value_of(number.fraction).value_or(0)));
|
|
247
|
+
}
|
|
248
|
+
fraction = join(words);
|
|
249
|
+
}
|
|
250
|
+
std::u32string head = number.integer.empty() ? U"" : cardinal(value(number), gender) + U" ";
|
|
251
|
+
return head + (number.decimal_mark == U',' ? U"vírgula " : U"ponto ") + fraction;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
std::u32string Num2Word::read_number(const Number& number, const NumberReading& reading) const {
|
|
255
|
+
if (number.fraction.empty() && reading.day && number.integer == U"1") {
|
|
256
|
+
return U"primeiro";
|
|
257
|
+
}
|
|
258
|
+
return read_with(number, number.fraction.empty() ? reading.gender : Gender::MASCULINE);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
std::u32string Num2Word::read_code_digits(std::u32string_view text) const {
|
|
262
|
+
auto n = value_of(text);
|
|
263
|
+
if ((text.size() > 1 && text[0] == U'0') || text.size() >= 7 || !n) {
|
|
264
|
+
return digits(text);
|
|
265
|
+
}
|
|
266
|
+
return cardinal(*n);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
std::u32string Num2Word::read_date(const Match& match, const NumberReading& reading) const {
|
|
270
|
+
uint64_t day = value(match.numbers[0]);
|
|
271
|
+
return (day == 1 ? U"primeiro" : cardinal(day)) + U" de " + kMonths[value(match.numbers[1]) - 1] + U" de " +
|
|
272
|
+
cardinal(value(match.numbers[2]));
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
std::u32string Num2Word::read_phone(const Match& match, const NumberReading& reading) const {
|
|
276
|
+
std::vector<std::u32string> groups;
|
|
277
|
+
for (const auto& group : match.numbers) {
|
|
278
|
+
groups.push_back(group.integer == U"0800" ? U"zero oitocentos" : digits(group.integer));
|
|
279
|
+
}
|
|
280
|
+
return join(groups, U", ");
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
std::u32string Num2Word::read_fraction(const Match& match, const NumberReading& reading) const {
|
|
284
|
+
return kFractions.at({value(match.numbers[0]), value(match.numbers[1])});
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
std::u32string Num2Word::read_time(const Match& match, const NumberReading& reading) const {
|
|
288
|
+
// 14h30: catorze e trinta; 21h: vinte e uma horas
|
|
289
|
+
uint64_t hours = value(match.numbers[0]);
|
|
290
|
+
uint64_t minutes = match.numbers.size() > 1 ? value(match.numbers[1]) : 0;
|
|
291
|
+
std::u32string hour = cardinal(hours, Gender::FEMININE);
|
|
292
|
+
return minutes ? hour + U" e " + cardinal(minutes) : hour + (hours == 1 ? U" hora" : U" horas");
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
std::u32string Num2Word::read_ordinal(const Match& match, const NumberReading& reading) const {
|
|
296
|
+
uint64_t n = value(match.numbers[0]);
|
|
297
|
+
return n >= 10 && reading.heading ? cardinal(n) : ordinal(n, match.suffix == U"ª");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
std::u32string Num2Word::read_compound(const Match& match, const NumberReading& reading) const {
|
|
301
|
+
return read_with(match.numbers[0], Gender::MASCULINE) + U"-" + std::u32string(match.suffix);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
std::u32string Num2Word::read_money(const Number& number, const Match& match, const NumberReading& reading) const {
|
|
305
|
+
const Forms& currency = kCurrencies.at(std::u32string(match.unit));
|
|
306
|
+
NumberReading priced = reading;
|
|
307
|
+
priced.gender = currency.one == U"libra" ? Gender::FEMININE : Gender::MASCULINE;
|
|
308
|
+
|
|
309
|
+
// R$ 37,517: prices have two decimals, so three digits are a thousands group
|
|
310
|
+
Number amount = number;
|
|
311
|
+
if (number.decimal_mark == U',' && !number.integer.empty() && number.integer.size() <= 3 &&
|
|
312
|
+
number.fraction.size() == 3) {
|
|
313
|
+
amount = {.integer = number.integer + number.fraction};
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if (reading.named) {
|
|
317
|
+
return match.scale.empty() ? read_with(amount, priced.gender) : read_scale(amount, match, priced);
|
|
318
|
+
}
|
|
319
|
+
if (!match.scale.empty()) {
|
|
320
|
+
return with_de(read_scale(amount, match, priced)) + U" " + currency.many;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// A price: vinte e nove reais e noventa centavos
|
|
324
|
+
if (amount.fraction.size() == 2 && amount.decimal_mark == U',') {
|
|
325
|
+
uint64_t cents = value_of(amount.fraction).value_or(0);
|
|
326
|
+
std::u32string centavos = cardinal(cents) + (cents == 1 ? U" centavo" : U" centavos");
|
|
327
|
+
if (value(amount) == 0) {
|
|
328
|
+
return centavos;
|
|
329
|
+
}
|
|
330
|
+
Number whole{.integer = amount.integer, .grouped = amount.grouped};
|
|
331
|
+
std::u32string words =
|
|
332
|
+
with_de(read_with(whole, priced.gender)) + U" " + (amount.integer == U"1" ? currency.one : currency.many);
|
|
333
|
+
return cents ? words + U" e " + centavos : words;
|
|
334
|
+
}
|
|
335
|
+
return with_de(read_with(amount, priced.gender)) + U" " + (is_one(amount) ? currency.one : currency.many);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
std::u32string Num2Word::read_scale(const Number& number, const Match& match, const NumberReading& reading) const {
|
|
339
|
+
const Forms& scale = kScales.at(std::u32string(match.scale));
|
|
340
|
+
if (scale.one == U"mil") {
|
|
341
|
+
return read_with(number, reading.gender) + U" mil"; // vinte e duas mil pessoas
|
|
342
|
+
}
|
|
343
|
+
return read_with(number, Gender::MASCULINE) + U" " + (value(number) < 2 ? scale.one : scale.many);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
std::u32string Num2Word::read_percent(const Number& number, const Match& match, const NumberReading& reading) const {
|
|
347
|
+
std::u32string suffix = match.suffix.empty() ? U"" : U" " + std::u32string(match.suffix);
|
|
348
|
+
return read_with(number, Gender::MASCULINE) + U" por cento" + suffix;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
std::u32string Num2Word::read_unit(const Number& number, const Match& match, const NumberReading& reading) const {
|
|
352
|
+
const Forms& unit = kUnits.at(std::u32string(match.unit));
|
|
353
|
+
std::u32string_view first_word = std::u32string_view(unit.one).substr(0, unit.one.find(U' '));
|
|
354
|
+
Gender gender = feminine(first_word) ? Gender::FEMININE : Gender::MASCULINE;
|
|
355
|
+
return read_with(number, gender) + U" " + (value(number) < 2 ? unit.one : unit.many);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
std::u32string Num2Word::read_range(const Number& from, const Number& to, const NumberReading& reading) const {
|
|
359
|
+
// 2016-2022 and 10-15 read "a"; scores and formations (4-3-3, 60-40) keep "-"
|
|
360
|
+
bool plain = from.fraction.empty() && to.fraction.empty() && !from.grouped && !to.grouped;
|
|
361
|
+
bool rising = from.integer.size() == to.integer.size() && value(to) > value(from);
|
|
362
|
+
return plain && rising && !reading.lettered ? joiners_.range : U"-";
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
} // namespace phonemis::pt
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <phonemis/base/processor/layers/num2word/layer.h>
|
|
4
|
+
|
|
5
|
+
#include <string>
|
|
6
|
+
|
|
7
|
+
namespace phonemis::pt {
|
|
8
|
+
|
|
9
|
+
using processor::num2word::Gender;
|
|
10
|
+
|
|
11
|
+
// How a Portuguese number is read, from the words around it.
|
|
12
|
+
struct NumberReading {
|
|
13
|
+
Gender gender = Gender::MASCULINE; // Of the noun after: duas pessoas; an hour is feminine (às duas)
|
|
14
|
+
bool day = false; // 1 de maio: "1" before "de" and a month is "primeiro"
|
|
15
|
+
bool heading = false; // artigo 80º: an ordinal after a legal heading reads as a cardinal
|
|
16
|
+
bool lettered = false; // A320-200, 3-4x: a range touching letters is a plain hyphen
|
|
17
|
+
bool named = false; // R$ 50 reais: the currency is already named after the amount
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
class Num2Word : public processor::num2word::Num2WordLayer<NumberReading> {
|
|
21
|
+
public:
|
|
22
|
+
Num2Word();
|
|
23
|
+
|
|
24
|
+
protected:
|
|
25
|
+
using Context = processor::Context;
|
|
26
|
+
using Match = processor::num2word::Match;
|
|
27
|
+
using Number = processor::num2word::Number;
|
|
28
|
+
|
|
29
|
+
// Resolution
|
|
30
|
+
bool accept(const Match& match, const Context& context) const override;
|
|
31
|
+
NumberReading resolve(const Match& match, const Context& context) const override;
|
|
32
|
+
|
|
33
|
+
// Readings
|
|
34
|
+
std::u32string read_number(const Number& number, const NumberReading& reading) const override;
|
|
35
|
+
std::u32string read_code_digits(std::u32string_view digits) const override;
|
|
36
|
+
std::u32string read_date(const Match& match, const NumberReading& reading) const override;
|
|
37
|
+
std::u32string read_phone(const Match& match, const NumberReading& reading) const override;
|
|
38
|
+
std::u32string read_fraction(const Match& match, const NumberReading& reading) const override;
|
|
39
|
+
std::u32string read_time(const Match& match, const NumberReading& reading) const override;
|
|
40
|
+
std::u32string read_ordinal(const Match& match, const NumberReading& reading) const override;
|
|
41
|
+
std::u32string read_compound(const Match& match, const NumberReading& reading) const override;
|
|
42
|
+
std::u32string read_money(const Number& number, const Match& match, const NumberReading& reading) const override;
|
|
43
|
+
std::u32string read_scale(const Number& number, const Match& match, const NumberReading& reading) const override;
|
|
44
|
+
std::u32string read_percent(const Number& number, const Match& match, const NumberReading& reading) const override;
|
|
45
|
+
std::u32string read_unit(const Number& number, const Match& match, const NumberReading& reading) const override;
|
|
46
|
+
std::u32string read_range(const Number& from, const Number& to, const NumberReading& reading) const override;
|
|
47
|
+
|
|
48
|
+
private:
|
|
49
|
+
// A number of a given gender, as before a currency or a unit
|
|
50
|
+
std::u32string read_with(const Number& number, Gender gender) const;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
} // namespace phonemis::pt
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
3
|
#include "constants.h"
|
|
4
|
+
#include "num2word.h"
|
|
4
5
|
|
|
5
6
|
#include <phonemis/base/config.h>
|
|
6
7
|
#include <phonemis/base/ipipeline.h>
|
|
7
8
|
#include <phonemis/base/phonemizer/hybrid_phonemizer.h>
|
|
8
9
|
#include <phonemis/base/phonemizer/lexicon_phonemizer.h>
|
|
10
|
+
#include <phonemis/base/processor/preprocessor.h>
|
|
9
11
|
#include <phonemis/base/processor/processor.h>
|
|
10
|
-
#include <phonemis/base/processor/trim_layer.h>
|
|
11
12
|
#include <phonemis/base/tokenizer/tokenizer.h>
|
|
12
13
|
|
|
13
14
|
#include <memory>
|
|
@@ -22,7 +23,8 @@ public:
|
|
|
22
23
|
phonemizer_(config.phonemizer) {
|
|
23
24
|
|
|
24
25
|
// 1. Setup Preprocessing layers
|
|
25
|
-
preprocessor_
|
|
26
|
+
preprocessor_ = processor::make_preprocessor<Num2Word>(
|
|
27
|
+
config.preprocessor, {.terms = &constants::term2word::kTerms, .symbols = &constants::sym2word::kSymbols});
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
// Performs a preprocessing stage of the pipeline.
|
|
@@ -4,18 +4,35 @@
|
|
|
4
4
|
#include <phonemis/utils/io.h>
|
|
5
5
|
#include <phonemis/utils/strings.h>
|
|
6
6
|
|
|
7
|
+
#include <chrono>
|
|
8
|
+
#include <iomanip>
|
|
7
9
|
#include <iostream>
|
|
8
10
|
#include <string>
|
|
9
11
|
#include <vector>
|
|
10
12
|
|
|
13
|
+
// Turns off a preprocessing layer by its name; false if there is no such layer.
|
|
14
|
+
bool turn_off(phonemis::processor::Config& config, const std::string& layer) {
|
|
15
|
+
for (auto [name, enabled] : {std::pair{"normalize", &config.normalize}, std::pair{"trim", &config.trim},
|
|
16
|
+
std::pair{"terms", &config.terms}, std::pair{"numbers", &config.numbers},
|
|
17
|
+
std::pair{"symbols", &config.symbols}, std::pair{"sanitize", &config.sanitize}}) {
|
|
18
|
+
if (layer == name) {
|
|
19
|
+
*enabled = false;
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
|
|
11
26
|
void print_usage(const char* prog_name) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
27
|
+
std::cout
|
|
28
|
+
<< "Usage: " << prog_name << " [options] <text>\n"
|
|
29
|
+
<< "Options:\n"
|
|
30
|
+
<< " --lexicon <path> Path to lexicon file\n"
|
|
31
|
+
<< " --model <path> Path to neural model file\n"
|
|
32
|
+
<< " --tagger <path> Path to tagger data file\n"
|
|
33
|
+
<< " --lang <lang_code> Language code (e.g., en-us, en-gb)\n"
|
|
34
|
+
<< " --no <layer> Turn off a preprocessing layer: normalize, trim, terms, numbers, symbols, sanitize\n"
|
|
35
|
+
<< std::endl;
|
|
19
36
|
}
|
|
20
37
|
|
|
21
38
|
int main(int argc, char* argv[]) {
|
|
@@ -40,12 +57,17 @@ int main(int argc, char* argv[]) {
|
|
|
40
57
|
} else if (arg == "--lang" && i + 1 < argc) {
|
|
41
58
|
config.lang = argv[++i];
|
|
42
59
|
config.phonemizer.lang = config.lang;
|
|
60
|
+
} else if (arg == "--no" && i + 1 < argc) {
|
|
61
|
+
if (!turn_off(config.preprocessor, argv[++i])) {
|
|
62
|
+
std::cerr << "Unknown preprocessing layer: " << argv[i] << std::endl;
|
|
63
|
+
return 1;
|
|
64
|
+
}
|
|
43
65
|
} else if (i == argc - 1) {
|
|
44
|
-
|
|
66
|
+
text_to_process = arg;
|
|
45
67
|
} else {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
68
|
+
std::cerr << "Unknown or incomplete option: " << arg << std::endl;
|
|
69
|
+
print_usage(argv[0]);
|
|
70
|
+
return 1;
|
|
49
71
|
}
|
|
50
72
|
}
|
|
51
73
|
|
|
@@ -57,11 +79,20 @@ int main(int argc, char* argv[]) {
|
|
|
57
79
|
|
|
58
80
|
try {
|
|
59
81
|
phonemis::Pipeline pipeline(config);
|
|
82
|
+
|
|
83
|
+
auto start = std::chrono::high_resolution_clock::now();
|
|
60
84
|
std::u32string result = pipeline(text_to_process);
|
|
85
|
+
auto end = std::chrono::high_resolution_clock::now();
|
|
86
|
+
|
|
87
|
+
auto ms = std::chrono::duration<double, std::milli>(end - start).count();
|
|
88
|
+
std::u32string preprocessed = pipeline.preprocess(phonemis::utils::conversions::utf8_to_u32(text_to_process));
|
|
61
89
|
|
|
62
90
|
std::cout << "\n\033[1;32mPhonemization Result:\033[0m" << std::endl;
|
|
63
91
|
std::cout << "\033[1;34mInput: \033[0m" << text_to_process << std::endl;
|
|
92
|
+
std::cout << "\033[1;35mPreprocessed: \033[0m" << phonemis::utils::conversions::u32_to_utf8(preprocessed)
|
|
93
|
+
<< std::endl;
|
|
64
94
|
std::cout << "\033[1;36mOutput: \033[0m" << phonemis::utils::conversions::u32_to_utf8(result) << std::endl;
|
|
95
|
+
std::cout << "\033[1;33mTime: \033[0m" << std::fixed << std::setprecision(2) << ms << " ms" << std::endl;
|
|
65
96
|
std::cout << std::endl;
|
|
66
97
|
} catch (const std::exception& e) {
|
|
67
98
|
std::cerr << "\033[1;31mError:\033[0m " << e.what() << std::endl;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include "types.h"
|
|
4
|
+
#include <cstddef>
|
|
5
|
+
#include <type_traits>
|
|
6
|
+
#include <variant>
|
|
7
|
+
|
|
8
|
+
namespace phonemis::protophone::meta {
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Converts a slicing argument to a canonical index or range variant.
|
|
12
|
+
* Maps integral types to size_t.
|
|
13
|
+
*/
|
|
14
|
+
template <typename T>
|
|
15
|
+
auto to_slice_variant(T&& arg) -> std::variant<size_t, Range<size_t>, All> {
|
|
16
|
+
using V = std::decay_t<T>;
|
|
17
|
+
if constexpr (std::is_integral_v<V>) {
|
|
18
|
+
return static_cast<size_t>(arg);
|
|
19
|
+
} else {
|
|
20
|
+
return std::forward<T>(arg);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
} // namespace phonemis::protophone::meta
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <numeric>
|
|
4
|
+
#include <vector>
|
|
5
|
+
|
|
6
|
+
namespace phonemis::protophone::numeric {
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Calculates the product of elements in a vector.
|
|
10
|
+
* Returns 0 for empty vectors.
|
|
11
|
+
* @param values The vector containing elements to multiply.
|
|
12
|
+
*/
|
|
13
|
+
template <typename T>
|
|
14
|
+
inline T product(const std::vector<T>& values) {
|
|
15
|
+
if (values.empty()) {
|
|
16
|
+
return static_cast<T>(0);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
return std::accumulate(
|
|
20
|
+
values.begin(),
|
|
21
|
+
values.end(),
|
|
22
|
+
static_cast<T>(1), std::multiplies<T>()
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
} // namespace phonemis::protophone::numeric
|