anyseg 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. anyseg-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +34 -0
  2. anyseg-0.1.0/.github/ISSUE_TEMPLATE/config.yml +5 -0
  3. anyseg-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml +27 -0
  4. anyseg-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +11 -0
  5. anyseg-0.1.0/.github/dependabot.yml +30 -0
  6. anyseg-0.1.0/.github/workflows/ci.yml +126 -0
  7. anyseg-0.1.0/.github/workflows/docs.yml +38 -0
  8. anyseg-0.1.0/.github/workflows/publish.yml +440 -0
  9. anyseg-0.1.0/.gitignore +32 -0
  10. anyseg-0.1.0/.pre-commit-config.yaml +93 -0
  11. anyseg-0.1.0/.python-version +1 -0
  12. anyseg-0.1.0/CITATION.cff +21 -0
  13. anyseg-0.1.0/CODE_OF_CONDUCT.md +54 -0
  14. anyseg-0.1.0/CONTRIBUTING.md +55 -0
  15. anyseg-0.1.0/Cargo.lock +155 -0
  16. anyseg-0.1.0/Cargo.toml +15 -0
  17. anyseg-0.1.0/LICENSE.txt +17 -0
  18. anyseg-0.1.0/Makefile +100 -0
  19. anyseg-0.1.0/PKG-INFO +192 -0
  20. anyseg-0.1.0/README.md +158 -0
  21. anyseg-0.1.0/crates/anyseg/Cargo.toml +24 -0
  22. anyseg-0.1.0/crates/anyseg/LICENSE.txt +17 -0
  23. anyseg-0.1.0/crates/anyseg/README.md +23 -0
  24. anyseg-0.1.0/crates/anyseg/data/ru/abbreviations.txt +126 -0
  25. anyseg-0.1.0/crates/anyseg/data/ru/dotted_names.txt +3 -0
  26. anyseg-0.1.0/crates/anyseg/data/ru/final_abbreviations.txt +10 -0
  27. anyseg-0.1.0/crates/anyseg/data/ru/homoglyphs.txt +9 -0
  28. anyseg-0.1.0/crates/anyseg/data/ru/hyphen_linkers.txt +3 -0
  29. anyseg-0.1.0/crates/anyseg/data/ru/hyphen_particles.txt +8 -0
  30. anyseg-0.1.0/crates/anyseg/data/ru/hyphen_prefixes.txt +20 -0
  31. anyseg-0.1.0/crates/anyseg/data/ru/hyphen_whole.txt +14 -0
  32. anyseg-0.1.0/crates/anyseg/data/ru/indefinite.txt +69 -0
  33. anyseg-0.1.0/crates/anyseg/data/ru/inflection_endings.txt +13 -0
  34. anyseg-0.1.0/crates/anyseg/data/ru/not_abbreviations.txt +2 -0
  35. anyseg-0.1.0/crates/anyseg/data/ru/numerals.txt +160 -0
  36. anyseg-0.1.0/crates/anyseg/src/conllu.rs +1044 -0
  37. anyseg-0.1.0/crates/anyseg/src/document.rs +746 -0
  38. anyseg-0.1.0/crates/anyseg/src/lang.rs +192 -0
  39. anyseg-0.1.0/crates/anyseg/src/layout.rs +202 -0
  40. anyseg-0.1.0/crates/anyseg/src/lib.rs +498 -0
  41. anyseg-0.1.0/crates/anyseg/src/sentences.rs +994 -0
  42. anyseg-0.1.0/crates/anyseg/src/tokens.rs +1621 -0
  43. anyseg-0.1.0/crates/anyseg-py/Cargo.toml +18 -0
  44. anyseg-0.1.0/crates/anyseg-py/src/lib.rs +1482 -0
  45. anyseg-0.1.0/docs/authors.md +15 -0
  46. anyseg-0.1.0/docs/comparison.md +112 -0
  47. anyseg-0.1.0/docs/css/extra.css +50 -0
  48. anyseg-0.1.0/docs/document.md +303 -0
  49. anyseg-0.1.0/docs/img/anyseg.png +0 -0
  50. anyseg-0.1.0/docs/img/anyseg.svg +15 -0
  51. anyseg-0.1.0/docs/index.md +100 -0
  52. anyseg-0.1.0/docs/installation.md +38 -0
  53. anyseg-0.1.0/docs/russian.md +241 -0
  54. anyseg-0.1.0/docs/segmenter.md +227 -0
  55. anyseg-0.1.0/mkdocs.yml +73 -0
  56. anyseg-0.1.0/pyproject.toml +143 -0
  57. anyseg-0.1.0/python/anyseg/__init__.py +165 -0
  58. anyseg-0.1.0/python/anyseg/_anyseg.pyi +263 -0
  59. anyseg-0.1.0/python/anyseg/py.typed +0 -0
  60. anyseg-0.1.0/scripts/smoke.py +26 -0
  61. anyseg-0.1.0/tests/test_document.py +575 -0
  62. anyseg-0.1.0/tests/test_examples.py +80 -0
  63. anyseg-0.1.0/tests/test_package.py +72 -0
  64. anyseg-0.1.0/tests/test_segmenter.py +215 -0
  65. anyseg-0.1.0/uv.lock +1264 -0
@@ -0,0 +1,34 @@
1
+ name: Bug report
2
+ description: Wrong segmentation, exception or crash
3
+ labels: ["bug"]
4
+ body:
5
+ - type: textarea
6
+ id: description
7
+ attributes:
8
+ label: What happens
9
+ description: The tokens or sentences you expected and the ones you got
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: reproduce
14
+ attributes:
15
+ label: How to reproduce
16
+ description: Minimal code with the text that triggers the bug and the full traceback, if any
17
+ render: python
18
+ validations:
19
+ required: true
20
+ - type: input
21
+ id: version
22
+ attributes:
23
+ label: anySeg version
24
+ description: "`python -c \"import anyseg; print(anyseg.__version__)\"`"
25
+ placeholder: "0.1.0"
26
+ validations:
27
+ required: true
28
+ - type: input
29
+ id: environment
30
+ attributes:
31
+ label: Python and OS
32
+ placeholder: "Python 3.12, macOS 15"
33
+ validations:
34
+ required: true
@@ -0,0 +1,5 @@
1
+ blank_issues_enabled: true
2
+ contact_links:
3
+ - name: Documentation
4
+ url: https://sergeyshk.github.io/anySeg/
5
+ about: Installation, the API and the conventions of every language
@@ -0,0 +1,27 @@
1
+ name: Feature request
2
+ description: A new language, a convention or an improvement of the API
3
+ labels: ["enhancement"]
4
+ body:
5
+ - type: textarea
6
+ id: problem
7
+ attributes:
8
+ label: What problem does it solve
9
+ description: What you want to segment or do and why the current features are not enough
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: proposal
14
+ attributes:
15
+ label: Proposed solution
16
+ description: How it could look in the API or what tokens and sentences you expect, with examples
17
+ - type: textarea
18
+ id: sources
19
+ attributes:
20
+ label: Sources
21
+ description: The Universal Dependencies guidelines or the treebank whose conventions the request follows
22
+ - type: checkboxes
23
+ id: contribution
24
+ attributes:
25
+ label: Contribution
26
+ options:
27
+ - label: I am willing to submit a pull request
@@ -0,0 +1,11 @@
1
+ ## What
2
+
3
+ <!-- What changed. If the segmentation or the API changes, say so explicitly: it goes into the release notes. If the pull request closes an issue: Closes #123 -->
4
+
5
+ ## Why
6
+
7
+ <!-- Optional: the reason, if it is not obvious from the issue -->
8
+
9
+ ## Checks
10
+
11
+ <!-- How it was checked: tests, `make lint`, `make test-cov`, `make docs-build` if the documentation changed. A change of the segmentation gives the F1 of tokens and sentences on the Universal Dependencies treebanks of the language before and after, and how they were measured -->
@@ -0,0 +1,30 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "uv"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "monthly"
7
+ open-pull-requests-limit: 5
8
+ groups:
9
+ python-dependencies:
10
+ patterns: ["*"]
11
+ update-types: ["minor", "patch"]
12
+
13
+ - package-ecosystem: "cargo"
14
+ directory: "/"
15
+ schedule:
16
+ interval: "monthly"
17
+ open-pull-requests-limit: 5
18
+ labels: ["dependencies"]
19
+ groups:
20
+ rust-dependencies:
21
+ patterns: ["*"]
22
+ update-types: ["minor", "patch"]
23
+
24
+ - package-ecosystem: "github-actions"
25
+ directory: "/"
26
+ schedule:
27
+ interval: "monthly"
28
+ groups:
29
+ github-actions:
30
+ patterns: ["*"]
@@ -0,0 +1,126 @@
1
+ name: build
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - master
7
+ pull_request:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ concurrency:
13
+ group: ${{ github.workflow }}-${{ github.ref }}
14
+ cancel-in-progress: true
15
+
16
+ jobs:
17
+ lint:
18
+ runs-on: ubuntu-latest
19
+ timeout-minutes: 30
20
+ steps:
21
+ - name: Checkout
22
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
23
+ - name: Setup uv
24
+ uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
25
+ with:
26
+ version: "0.13.5"
27
+ enable-cache: true
28
+ - name: Setup Rust
29
+ uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
30
+ with:
31
+ toolchain: stable
32
+ components: rustfmt, clippy
33
+ - name: Cache Rust
34
+ uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
35
+ - name: Install dependencies
36
+ run: uv sync --locked --no-default-groups --group lint
37
+ - name: Ruff
38
+ run: |
39
+ uv run --no-sync ruff check python scripts tests
40
+ uv run --no-sync ruff format python scripts tests --check
41
+ - name: Ruff format of the Python in Markdown
42
+ run: git ls-files -z '*.md' | xargs -0 uv run --no-sync ruff format --check
43
+ - name: Mypy
44
+ run: uv run --no-sync mypy
45
+ - name: Stubtest
46
+ run: uv run --no-sync python -m mypy.stubtest anyseg._anyseg
47
+ - name: Rustfmt
48
+ run: cargo fmt --all --check
49
+ - name: Clippy
50
+ run: |
51
+ cargo clippy --workspace --all-targets --locked -- -D warnings
52
+ cargo clippy -p anyseg --no-default-features --all-targets --locked -- -D warnings
53
+
54
+ test:
55
+ name: test (${{ matrix.python-version }}${{ matrix.os != 'ubuntu-latest' && format(', {0}', matrix.os) || '' }})
56
+ runs-on: ${{ matrix.os }}
57
+ timeout-minutes: 30
58
+ strategy:
59
+ fail-fast: false
60
+ matrix:
61
+ os: [ubuntu-latest]
62
+ python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"]
63
+ include:
64
+ - os: ubuntu-latest
65
+ python-version: "3.14t"
66
+ - os: ubuntu-latest
67
+ python-version: "3.15t"
68
+ - os: windows-latest
69
+ python-version: "3.12"
70
+ - os: macos-latest
71
+ python-version: "3.12"
72
+ steps:
73
+ - name: Checkout
74
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
75
+ - name: Setup uv
76
+ uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
77
+ with:
78
+ version: "0.13.5"
79
+ enable-cache: true
80
+ python-version: ${{ matrix.python-version }}
81
+ - name: Setup Rust
82
+ uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
83
+ with:
84
+ toolchain: stable
85
+ - name: Cache Rust
86
+ uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
87
+ - name: Install dependencies
88
+ run: uv sync --locked --no-default-groups --group test
89
+ - name: Run tests
90
+ run: uv run --no-sync pytest --cov anyseg
91
+ - name: Run Rust tests
92
+ run: |
93
+ cargo test -p anyseg --locked
94
+ cargo test -p anyseg --no-default-features --locked
95
+
96
+ msrv:
97
+ runs-on: ubuntu-latest
98
+ timeout-minutes: 30
99
+ steps:
100
+ - name: Checkout
101
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
102
+ - name: Setup Rust
103
+ uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
104
+ with:
105
+ toolchain: "1.88"
106
+ - name: Cache Rust
107
+ uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
108
+ - name: Check
109
+ run: cargo check --workspace --all-targets --locked
110
+
111
+ docs:
112
+ runs-on: ubuntu-latest
113
+ timeout-minutes: 30
114
+ steps:
115
+ - name: Checkout
116
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
117
+ - name: Setup uv
118
+ uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
119
+ with:
120
+ version: "0.13.5"
121
+ enable-cache: true
122
+ python-version: "3.13"
123
+ - name: Install dependencies
124
+ run: uv sync --locked --only-group docs
125
+ - name: Build docs
126
+ run: uv run --no-sync mkdocs build --strict
@@ -0,0 +1,38 @@
1
+ name: docs
2
+
3
+ # The site describes the released version: it is published on a release tag, not from master
4
+ on:
5
+ push:
6
+ tags:
7
+ - "[0-9]+.[0-9]+.[0-9]+"
8
+ workflow_dispatch:
9
+
10
+ concurrency:
11
+ group: docs
12
+ cancel-in-progress: false
13
+
14
+ jobs:
15
+ deploy:
16
+ runs-on: ubuntu-latest
17
+ timeout-minutes: 30
18
+ permissions:
19
+ contents: write
20
+ steps:
21
+ - name: Checkout
22
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
23
+ with:
24
+ fetch-depth: 0
25
+ - name: Setup uv
26
+ uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
27
+ with:
28
+ version: "0.13.5"
29
+ enable-cache: true
30
+ python-version: "3.13"
31
+ - name: Install dependencies
32
+ run: uv sync --locked --only-group docs
33
+ - name: Configure git
34
+ run: |
35
+ git config user.name "github-actions[bot]"
36
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
37
+ - name: Deploy
38
+ run: uv run --no-sync mkdocs gh-deploy --force --strict