WeTextProcessing 1.0.4.1__tar.gz → 1.2.0__tar.gz

wetextprocessing-1.2.0/.clang-format

@@ -0,0 +1,93 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...

wetextprocessing-1.2.0/.flake8

@@ -0,0 +1,14 @@
+[flake8]
+select = B,C,E,F,P,T4,W,B9
+max-line-length = 80
+# C408 ignored because we like the dict keyword argument syntax
+# E501 is not flexible enough, we're using B950 instead
+ignore =
+    E203,E305,E402,E501,E721,E741,F403,F405,F821,F841,F999,W503,W504,C408,E302,W291,E303,
+    # shebang has extra meaning in fbcode lints, so I think it's not worth trying
+    # to line this up with executable bit
+    EXE001, EXE002,
+    # these ignores are from flake8-bugbear; please fix!
+    B007,B008,
+    # these ignores are from flake8-comprehensions; please fix!
+    C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415

wetextprocessing-1.2.0/.github/workflows/runtime.yml

@@ -0,0 +1,22 @@
+name: Runtime Build
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'runtime/**'
+      - '.github/workflows/runtime.yml'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build runtime
+        run: |
+          cmake -B build -S runtime -DBUILD_TESTING=OFF
+          cmake --build build -j$(nproc 2>/dev/null || sysctl -n hw.ncpu)

wetextprocessing-1.2.0/.github/workflows/unittest.yml

@@ -0,0 +1,83 @@
+name: UnitTest
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '**.py'
+      - '**.tsv'
+      - '**.toml'
+      - '**.yml'
+
+jobs:
+  unit-test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ github.event_name == 'pull_request' && fromJSON('["ubuntu-latest"]') || fromJSON('["ubuntu-latest", "macos-latest", "windows-latest"]') }}
+        python-version: ['3.9']
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }} (non-Windows)
+        if: runner.os != 'Windows'
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache Homebrew (macOS)
+        if: runner.os == 'macOS'
+        uses: actions/cache@v4
+        with:
+          path: |
+            /opt/homebrew/Cellar/openfst
+            /opt/homebrew/include/fst
+            /opt/homebrew/lib/libfst*
+          key: brew-openfst-${{ runner.os }}
+      - name: Install openfst (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install openfst || true
+          echo "CPLUS_INCLUDE_PATH=/opt/homebrew/include:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
+          echo "LIBRARY_PATH=/opt/homebrew/lib:$LIBRARY_PATH" >> $GITHUB_ENV
+      - name: Cache pip (non-Windows)
+        if: runner.os != 'Windows'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+      - name: Install dependencies (non-Windows)
+        if: runner.os != 'Windows'
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+      - name: Set up Conda (Windows)
+        if: runner.os == 'Windows'
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: test
+          use-only-tar-bz2: true
+      - name: Cache Conda (Windows)
+        if: runner.os == 'Windows'
+        uses: actions/cache@v4
+        with:
+          path: C:\Miniconda\envs\test
+          key: conda-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+      - name: Install dependencies (Windows)
+        if: runner.os == 'Windows'
+        shell: bash -el {0}
+        run: |
+          conda install -c conda-forge pynini
+          python -m pip install --upgrade pip
+          pip install .[test] --no-deps
+          pip install flake8 pytest importlib_resources
+      - name: Lint with flake8
+        shell: bash -el {0}
+        run: |
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        shell: bash -el {0}
+        run: pytest

wetextprocessing-1.2.0/.github/workflows/wheels.yml

@@ -0,0 +1,71 @@
+name: Build Wheels
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version (optional, will be determined by setuptools_scm if not provided)'
+        required: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+          pip install build twine
+
+      - name: Build Graph
+        run: |
+          python -m tn --text "2.5平方电线" --overwrite_cache --language "zh"
+          python -m tn --text "2010-03-21" --overwrite_cache --language "en"
+          python -m itn --text "二点五平方电线" --overwrite_cache
+
+      - name: Prepare Graph
+        run: |
+          mkdir graph
+          cp tn/*.fst graph
+          cp itn/*.fst graph
+
+      - name: Get version from setuptools_scm
+        id: scm_version
+        run: |
+          # Check if version is provided as input
+          if [ -n "${{ github.event.inputs.version }}" ]; then
+            echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
+          else
+            # Get version from setuptools_scm
+            pip install setuptools_scm
+            VERSION=$(python -m setuptools_scm | tail -1)
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload Graph
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-graph-v${{ steps.scm_version.outputs.VERSION }}
+          path: graph
+
+      - name: Build package
+        run: |
+          python -m build
+
+      - name: Publish on pypi.org
+        if: github.ref == 'refs/heads/master' && github.repository == 'wenet-e2e/WeTextProcessing'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          python -m twine upload dist/*

wetextprocessing-1.2.0/.gitignore

@@ -0,0 +1,47 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Visual Studio Code files
+.vscode
+.vs
+
+# PyCharm files
+.idea
+
+# Eclipse Project settings
+*.*project
+.settings
+
+# Sublime Text settings
+*.sublime-workspace
+*.sublime-project
+
+# Editor temporaries
+*.swn
+*.swo
+*.swp
+*.swm
+*~
+
+# IPython notebook checkpoints
+.ipynb_checkpoints
+
+# macOS dir files
+.DS_Store
+
+# Clangd files
+.cache
+compile_commands.json
+
+
+# Setup files
+WeTextProcessing.egg-info/
+build/
+dist/
+tn/*.far
+itn/*.far
+
+# Version file(setuptools_scm generated)
+tn/_version.py

wetextprocessing-1.2.0/.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+exclude: '.*\.(txt|tsv)$'
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    - id: trailing-whitespace
+  - repo: https://github.com/pre-commit/mirrors-yapf
+    rev: 'v0.32.0'
+    hooks:
+    - id: yapf
+  - repo: https://github.com/pycqa/flake8
+    rev: '3.8.2'
+    hooks:
+    - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: 'v17.0.6'
+    hooks:
+    - id: clang-format
+      exclude: '.*\.(json|java|js|m|mm|proto)'
+  - repo: https://github.com/cpplint/cpplint
+    rev: '1.6.1'
+    hooks:
+    - id: cpplint

wetextprocessing-1.2.0/CPPLINT.cfg

@@ -0,0 +1,2 @@
+root=runtime
+filter=-build/c++11

{WeTextProcessing-1.0.4.1 → wetextprocessing-1.2.0}/PKG-INFO

@@ -1,17 +1,25 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: WeTextProcessing
-Version: 1.0.4.1
+Version: 1.2.0
 Summary: WeTextProcessing, including TN & ITN
-Home-page: https://github.com/wenet-e2e/WeTextProcessing
-Author: Zhendong Peng, Xingchen Song
-Author-email: pzd17@tsinghua.org.cn, sxc19@tsinghua.org.cn
-License: UNKNOWN
-Platform: UNKNOWN
+Author-email: Zhendong Peng <pzd17@tsinghua.org.cn>, Xingchen Song <sxc19@tsinghua.org.cn>
+Project-URL: Homepage, https://github.com/wenet-e2e/WeTextProcessing
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pynini>=2.1.6
+Requires-Dist: importlib_resources
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: flake8; extra == "test"
+Provides-Extra: dev
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: pre-commit==3.5.0; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
 
 ## Text Normalization & Inverse Text Normalization
 
@@ -150,5 +158,3 @@ Or you can directly discuss on [Github Issues](https://github.com/wenet-e2e/WeTe
 3. Referred [Pynini](https://github.com/kylebgorman/pynini) for reading the FAR, and printing the shortest path of a lattice in the C++ runtime.
 4. Referred [TN of NeMo](https://github.com/NVIDIA/NeMo/tree/main/nemo_text_processing/text_normalization/zh) for the data to build the tagger graph.
 5. Referred [ITN of chinese_text_normalization](https://github.com/speechio/chinese_text_normalization/tree/master/thrax/src/cn) for the data to build the tagger graph.
-
-

{WeTextProcessing-1.0.4.1 → wetextprocessing-1.2.0}/WeTextProcessing.egg-info/PKG-INFO

@@ -1,17 +1,25 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: WeTextProcessing
-Version: 1.0.4.1
+Version: 1.2.0
 Summary: WeTextProcessing, including TN & ITN
-Home-page: https://github.com/wenet-e2e/WeTextProcessing
-Author: Zhendong Peng, Xingchen Song
-Author-email: pzd17@tsinghua.org.cn, sxc19@tsinghua.org.cn
-License: UNKNOWN
-Platform: UNKNOWN
+Author-email: Zhendong Peng <pzd17@tsinghua.org.cn>, Xingchen Song <sxc19@tsinghua.org.cn>
+Project-URL: Homepage, https://github.com/wenet-e2e/WeTextProcessing
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pynini>=2.1.6
+Requires-Dist: importlib_resources
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: flake8; extra == "test"
+Provides-Extra: dev
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: pre-commit==3.5.0; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
 
 ## Text Normalization & Inverse Text Normalization
 
@@ -150,5 +158,3 @@ Or you can directly discuss on [Github Issues](https://github.com/wenet-e2e/WeTe
 3. Referred [Pynini](https://github.com/kylebgorman/pynini) for reading the FAR, and printing the shortest path of a lattice in the C++ runtime.
 4. Referred [TN of NeMo](https://github.com/NVIDIA/NeMo/tree/main/nemo_text_processing/text_normalization/zh) for the data to build the tagger graph.
 5. Referred [ITN of chinese_text_normalization](https://github.com/speechio/chinese_text_normalization/tree/master/thrax/src/cn) for the data to build the tagger graph.
-
-