anyseg 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- anyseg-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +34 -0
- anyseg-0.1.0/.github/ISSUE_TEMPLATE/config.yml +5 -0
- anyseg-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml +27 -0
- anyseg-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +11 -0
- anyseg-0.1.0/.github/dependabot.yml +30 -0
- anyseg-0.1.0/.github/workflows/ci.yml +126 -0
- anyseg-0.1.0/.github/workflows/docs.yml +38 -0
- anyseg-0.1.0/.github/workflows/publish.yml +440 -0
- anyseg-0.1.0/.gitignore +32 -0
- anyseg-0.1.0/.pre-commit-config.yaml +93 -0
- anyseg-0.1.0/.python-version +1 -0
- anyseg-0.1.0/CITATION.cff +21 -0
- anyseg-0.1.0/CODE_OF_CONDUCT.md +54 -0
- anyseg-0.1.0/CONTRIBUTING.md +55 -0
- anyseg-0.1.0/Cargo.lock +155 -0
- anyseg-0.1.0/Cargo.toml +15 -0
- anyseg-0.1.0/LICENSE.txt +17 -0
- anyseg-0.1.0/Makefile +100 -0
- anyseg-0.1.0/PKG-INFO +192 -0
- anyseg-0.1.0/README.md +158 -0
- anyseg-0.1.0/crates/anyseg/Cargo.toml +24 -0
- anyseg-0.1.0/crates/anyseg/LICENSE.txt +17 -0
- anyseg-0.1.0/crates/anyseg/README.md +23 -0
- anyseg-0.1.0/crates/anyseg/data/ru/abbreviations.txt +126 -0
- anyseg-0.1.0/crates/anyseg/data/ru/dotted_names.txt +3 -0
- anyseg-0.1.0/crates/anyseg/data/ru/final_abbreviations.txt +10 -0
- anyseg-0.1.0/crates/anyseg/data/ru/homoglyphs.txt +9 -0
- anyseg-0.1.0/crates/anyseg/data/ru/hyphen_linkers.txt +3 -0
- anyseg-0.1.0/crates/anyseg/data/ru/hyphen_particles.txt +8 -0
- anyseg-0.1.0/crates/anyseg/data/ru/hyphen_prefixes.txt +20 -0
- anyseg-0.1.0/crates/anyseg/data/ru/hyphen_whole.txt +14 -0
- anyseg-0.1.0/crates/anyseg/data/ru/indefinite.txt +69 -0
- anyseg-0.1.0/crates/anyseg/data/ru/inflection_endings.txt +13 -0
- anyseg-0.1.0/crates/anyseg/data/ru/not_abbreviations.txt +2 -0
- anyseg-0.1.0/crates/anyseg/data/ru/numerals.txt +160 -0
- anyseg-0.1.0/crates/anyseg/src/conllu.rs +1044 -0
- anyseg-0.1.0/crates/anyseg/src/document.rs +746 -0
- anyseg-0.1.0/crates/anyseg/src/lang.rs +192 -0
- anyseg-0.1.0/crates/anyseg/src/layout.rs +202 -0
- anyseg-0.1.0/crates/anyseg/src/lib.rs +498 -0
- anyseg-0.1.0/crates/anyseg/src/sentences.rs +994 -0
- anyseg-0.1.0/crates/anyseg/src/tokens.rs +1621 -0
- anyseg-0.1.0/crates/anyseg-py/Cargo.toml +18 -0
- anyseg-0.1.0/crates/anyseg-py/src/lib.rs +1482 -0
- anyseg-0.1.0/docs/authors.md +15 -0
- anyseg-0.1.0/docs/comparison.md +112 -0
- anyseg-0.1.0/docs/css/extra.css +50 -0
- anyseg-0.1.0/docs/document.md +303 -0
- anyseg-0.1.0/docs/img/anyseg.png +0 -0
- anyseg-0.1.0/docs/img/anyseg.svg +15 -0
- anyseg-0.1.0/docs/index.md +100 -0
- anyseg-0.1.0/docs/installation.md +38 -0
- anyseg-0.1.0/docs/russian.md +241 -0
- anyseg-0.1.0/docs/segmenter.md +227 -0
- anyseg-0.1.0/mkdocs.yml +73 -0
- anyseg-0.1.0/pyproject.toml +143 -0
- anyseg-0.1.0/python/anyseg/__init__.py +165 -0
- anyseg-0.1.0/python/anyseg/_anyseg.pyi +263 -0
- anyseg-0.1.0/python/anyseg/py.typed +0 -0
- anyseg-0.1.0/scripts/smoke.py +26 -0
- anyseg-0.1.0/tests/test_document.py +575 -0
- anyseg-0.1.0/tests/test_examples.py +80 -0
- anyseg-0.1.0/tests/test_package.py +72 -0
- anyseg-0.1.0/tests/test_segmenter.py +215 -0
- anyseg-0.1.0/uv.lock +1264 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Wrong segmentation, exception or crash
|
|
3
|
+
labels: ["bug"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: description
|
|
7
|
+
attributes:
|
|
8
|
+
label: What happens
|
|
9
|
+
description: The tokens or sentences you expected and the ones you got
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: reproduce
|
|
14
|
+
attributes:
|
|
15
|
+
label: How to reproduce
|
|
16
|
+
description: Minimal code with the text that triggers the bug and the full traceback, if any
|
|
17
|
+
render: python
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
- type: input
|
|
21
|
+
id: version
|
|
22
|
+
attributes:
|
|
23
|
+
label: anySeg version
|
|
24
|
+
description: "`python -c \"import anyseg; print(anyseg.__version__)\"`"
|
|
25
|
+
placeholder: "0.1.0"
|
|
26
|
+
validations:
|
|
27
|
+
required: true
|
|
28
|
+
- type: input
|
|
29
|
+
id: environment
|
|
30
|
+
attributes:
|
|
31
|
+
label: Python and OS
|
|
32
|
+
placeholder: "Python 3.12, macOS 15"
|
|
33
|
+
validations:
|
|
34
|
+
required: true
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: A new language, a convention or an improvement of the API
|
|
3
|
+
labels: ["enhancement"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: problem
|
|
7
|
+
attributes:
|
|
8
|
+
label: What problem does it solve
|
|
9
|
+
description: What you want to segment or do and why the current features are not enough
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: proposal
|
|
14
|
+
attributes:
|
|
15
|
+
label: Proposed solution
|
|
16
|
+
description: How it could look in the API or what tokens and sentences you expect, with examples
|
|
17
|
+
- type: textarea
|
|
18
|
+
id: sources
|
|
19
|
+
attributes:
|
|
20
|
+
label: Sources
|
|
21
|
+
description: The Universal Dependencies guidelines or the treebank whose conventions the request follows
|
|
22
|
+
- type: checkboxes
|
|
23
|
+
id: contribution
|
|
24
|
+
attributes:
|
|
25
|
+
label: Contribution
|
|
26
|
+
options:
|
|
27
|
+
- label: I am willing to submit a pull request
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
## What
|
|
2
|
+
|
|
3
|
+
<!-- What changed. If the segmentation or the API changes, say so explicitly: it goes into the release notes. If the pull request closes an issue: Closes #123 -->
|
|
4
|
+
|
|
5
|
+
## Why
|
|
6
|
+
|
|
7
|
+
<!-- Optional: the reason, if it is not obvious from the issue -->
|
|
8
|
+
|
|
9
|
+
## Checks
|
|
10
|
+
|
|
11
|
+
<!-- How it was checked: tests, `make lint`, `make test-cov`, `make docs-build` if the documentation changed. A change of the segmentation gives the F1 of tokens and sentences on the Universal Dependencies treebanks of the language before and after, and how they were measured -->
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
- package-ecosystem: "uv"
|
|
4
|
+
directory: "/"
|
|
5
|
+
schedule:
|
|
6
|
+
interval: "monthly"
|
|
7
|
+
open-pull-requests-limit: 5
|
|
8
|
+
groups:
|
|
9
|
+
python-dependencies:
|
|
10
|
+
patterns: ["*"]
|
|
11
|
+
update-types: ["minor", "patch"]
|
|
12
|
+
|
|
13
|
+
- package-ecosystem: "cargo"
|
|
14
|
+
directory: "/"
|
|
15
|
+
schedule:
|
|
16
|
+
interval: "monthly"
|
|
17
|
+
open-pull-requests-limit: 5
|
|
18
|
+
labels: ["dependencies"]
|
|
19
|
+
groups:
|
|
20
|
+
rust-dependencies:
|
|
21
|
+
patterns: ["*"]
|
|
22
|
+
update-types: ["minor", "patch"]
|
|
23
|
+
|
|
24
|
+
- package-ecosystem: "github-actions"
|
|
25
|
+
directory: "/"
|
|
26
|
+
schedule:
|
|
27
|
+
interval: "monthly"
|
|
28
|
+
groups:
|
|
29
|
+
github-actions:
|
|
30
|
+
patterns: ["*"]
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
name: build
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
pull_request:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
concurrency:
|
|
13
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
14
|
+
cancel-in-progress: true
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
lint:
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
timeout-minutes: 30
|
|
20
|
+
steps:
|
|
21
|
+
- name: Checkout
|
|
22
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
23
|
+
- name: Setup uv
|
|
24
|
+
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
|
|
25
|
+
with:
|
|
26
|
+
version: "0.13.5"
|
|
27
|
+
enable-cache: true
|
|
28
|
+
- name: Setup Rust
|
|
29
|
+
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
30
|
+
with:
|
|
31
|
+
toolchain: stable
|
|
32
|
+
components: rustfmt, clippy
|
|
33
|
+
- name: Cache Rust
|
|
34
|
+
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
35
|
+
- name: Install dependencies
|
|
36
|
+
run: uv sync --locked --no-default-groups --group lint
|
|
37
|
+
- name: Ruff
|
|
38
|
+
run: |
|
|
39
|
+
uv run --no-sync ruff check python scripts tests
|
|
40
|
+
uv run --no-sync ruff format python scripts tests --check
|
|
41
|
+
- name: Ruff format of the Python in Markdown
|
|
42
|
+
run: git ls-files -z '*.md' | xargs -0 uv run --no-sync ruff format --check
|
|
43
|
+
- name: Mypy
|
|
44
|
+
run: uv run --no-sync mypy
|
|
45
|
+
- name: Stubtest
|
|
46
|
+
run: uv run --no-sync python -m mypy.stubtest anyseg._anyseg
|
|
47
|
+
- name: Rustfmt
|
|
48
|
+
run: cargo fmt --all --check
|
|
49
|
+
- name: Clippy
|
|
50
|
+
run: |
|
|
51
|
+
cargo clippy --workspace --all-targets --locked -- -D warnings
|
|
52
|
+
cargo clippy -p anyseg --no-default-features --all-targets --locked -- -D warnings
|
|
53
|
+
|
|
54
|
+
test:
|
|
55
|
+
name: test (${{ matrix.python-version }}${{ matrix.os != 'ubuntu-latest' && format(', {0}', matrix.os) || '' }})
|
|
56
|
+
runs-on: ${{ matrix.os }}
|
|
57
|
+
timeout-minutes: 30
|
|
58
|
+
strategy:
|
|
59
|
+
fail-fast: false
|
|
60
|
+
matrix:
|
|
61
|
+
os: [ubuntu-latest]
|
|
62
|
+
python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"]
|
|
63
|
+
include:
|
|
64
|
+
- os: ubuntu-latest
|
|
65
|
+
python-version: "3.14t"
|
|
66
|
+
- os: ubuntu-latest
|
|
67
|
+
python-version: "3.15t"
|
|
68
|
+
- os: windows-latest
|
|
69
|
+
python-version: "3.12"
|
|
70
|
+
- os: macos-latest
|
|
71
|
+
python-version: "3.12"
|
|
72
|
+
steps:
|
|
73
|
+
- name: Checkout
|
|
74
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
75
|
+
- name: Setup uv
|
|
76
|
+
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
|
|
77
|
+
with:
|
|
78
|
+
version: "0.13.5"
|
|
79
|
+
enable-cache: true
|
|
80
|
+
python-version: ${{ matrix.python-version }}
|
|
81
|
+
- name: Setup Rust
|
|
82
|
+
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
83
|
+
with:
|
|
84
|
+
toolchain: stable
|
|
85
|
+
- name: Cache Rust
|
|
86
|
+
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
87
|
+
- name: Install dependencies
|
|
88
|
+
run: uv sync --locked --no-default-groups --group test
|
|
89
|
+
- name: Run tests
|
|
90
|
+
run: uv run --no-sync pytest --cov anyseg
|
|
91
|
+
- name: Run Rust tests
|
|
92
|
+
run: |
|
|
93
|
+
cargo test -p anyseg --locked
|
|
94
|
+
cargo test -p anyseg --no-default-features --locked
|
|
95
|
+
|
|
96
|
+
msrv:
|
|
97
|
+
runs-on: ubuntu-latest
|
|
98
|
+
timeout-minutes: 30
|
|
99
|
+
steps:
|
|
100
|
+
- name: Checkout
|
|
101
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
102
|
+
- name: Setup Rust
|
|
103
|
+
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
|
104
|
+
with:
|
|
105
|
+
toolchain: "1.88"
|
|
106
|
+
- name: Cache Rust
|
|
107
|
+
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
|
|
108
|
+
- name: Check
|
|
109
|
+
run: cargo check --workspace --all-targets --locked
|
|
110
|
+
|
|
111
|
+
docs:
|
|
112
|
+
runs-on: ubuntu-latest
|
|
113
|
+
timeout-minutes: 30
|
|
114
|
+
steps:
|
|
115
|
+
- name: Checkout
|
|
116
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
117
|
+
- name: Setup uv
|
|
118
|
+
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
|
|
119
|
+
with:
|
|
120
|
+
version: "0.13.5"
|
|
121
|
+
enable-cache: true
|
|
122
|
+
python-version: "3.13"
|
|
123
|
+
- name: Install dependencies
|
|
124
|
+
run: uv sync --locked --only-group docs
|
|
125
|
+
- name: Build docs
|
|
126
|
+
run: uv run --no-sync mkdocs build --strict
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: docs
|
|
2
|
+
|
|
3
|
+
# The site describes the released version: it is published on a release tag, not from master
|
|
4
|
+
on:
|
|
5
|
+
push:
|
|
6
|
+
tags:
|
|
7
|
+
- "[0-9]+.[0-9]+.[0-9]+"
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: docs
|
|
12
|
+
cancel-in-progress: false
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
deploy:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
timeout-minutes: 30
|
|
18
|
+
permissions:
|
|
19
|
+
contents: write
|
|
20
|
+
steps:
|
|
21
|
+
- name: Checkout
|
|
22
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
23
|
+
with:
|
|
24
|
+
fetch-depth: 0
|
|
25
|
+
- name: Setup uv
|
|
26
|
+
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
|
|
27
|
+
with:
|
|
28
|
+
version: "0.13.5"
|
|
29
|
+
enable-cache: true
|
|
30
|
+
python-version: "3.13"
|
|
31
|
+
- name: Install dependencies
|
|
32
|
+
run: uv sync --locked --only-group docs
|
|
33
|
+
- name: Configure git
|
|
34
|
+
run: |
|
|
35
|
+
git config user.name "github-actions[bot]"
|
|
36
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
37
|
+
- name: Deploy
|
|
38
|
+
run: uv run --no-sync mkdocs gh-deploy --force --strict
|