opusfilter 3.0.0rc2__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/.github/workflows/ci.yml +21 -16
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/PKG-INFO +55 -4
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/README.md +1 -1
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-autogen +9 -6
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/CHANGELOG.md +39 -5
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/CONTRIBUTING.md +3 -3
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/script_and_language_identification_filters.md +14 -5
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/downloading_and_selecting_data.md +6 -1
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/installation.md +26 -9
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/autogen.py +3 -3
- opusfilter-3.2.0/opusfilter/autogen_cluster.py +346 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/embeddings.py +9 -2
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/filters.py +61 -19
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/opusfilter.py +17 -6
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/PKG-INFO +55 -4
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/SOURCES.txt +1 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/requires.txt +20 -9
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/requirements.txt +7 -7
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/setup.py +24 -13
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_autogen.py +1 -1
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_embeddings.py +46 -3
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_filter_pipeline.py +25 -9
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_filters.py +0 -108
- opusfilter-3.2.0/tests/test_lid.py +151 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_opusfilter.py +60 -3
- opusfilter-3.0.0rc2/opusfilter/autogen_cluster.py +0 -150
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/.gitignore +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/.pylintrc +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/LICENSE +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-cmd +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-diagram +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-duplicates +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-scores +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-test +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/automatic_configuration.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/command_line_tools.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/conf.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/alignment_model_filters.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/custom_filters.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/language_model_filters.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/length_filters.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/sentence_embedding_filters.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/filtering_and_scoring.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/preprocessing_text.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/training_and_using_classifiers.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/training_language_and_alignment_models.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/using_score_files.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/index.rst +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/bpe_segmentation.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/custom_preprocessors.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/detokenizer.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/reg_exp_sub.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/tokenizer.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/references.bib +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/references.rst +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/requirements.txt +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/usage.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/README.md +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/qed_lm_langid.yaml +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/__init__.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/classifier.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/lm.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/pipeline.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/preprocessors.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/segment_hash.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/subwords.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/tokenization.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/util.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/word_alignment.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/dependency_links.txt +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/top_level.txt +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/pyproject.toml +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/setup.cfg +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_classifier.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_lm_filter.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_preprocessors.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_segment_hash.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_subwords.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_tokenization.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_util.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_wordalign_filter.py +0 -0
- {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_yaml.py +0 -0
|
@@ -11,6 +11,9 @@ on:
|
|
|
11
11
|
branches:
|
|
12
12
|
- master
|
|
13
13
|
- develop
|
|
14
|
+
schedule:
|
|
15
|
+
# Run on Tuesdays at 5:59
|
|
16
|
+
- cron: '59 5 * * 2'
|
|
14
17
|
workflow_dispatch:
|
|
15
18
|
jobs:
|
|
16
19
|
build-n-test:
|
|
@@ -18,21 +21,23 @@ jobs:
|
|
|
18
21
|
fail-fast: false
|
|
19
22
|
matrix:
|
|
20
23
|
os: [ubuntu-latest]
|
|
21
|
-
python-version: ["3.
|
|
24
|
+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
|
22
25
|
requirements-file: ["requirements.txt"]
|
|
23
26
|
runs-on: ${{ matrix.os }}
|
|
24
27
|
steps:
|
|
25
|
-
- uses: actions/checkout@
|
|
28
|
+
- uses: actions/checkout@v4
|
|
26
29
|
- name: Set up Python ${{ matrix.python-version }}
|
|
27
|
-
uses: actions/setup-python@
|
|
30
|
+
uses: actions/setup-python@v5
|
|
28
31
|
with:
|
|
29
32
|
python-version: ${{ matrix.python-version }}
|
|
30
33
|
- name: Install dependencies
|
|
31
34
|
run: |
|
|
35
|
+
python -m ensurepip --upgrade
|
|
36
|
+
python -m pip install --upgrade setuptools
|
|
32
37
|
python -m pip install --upgrade pip
|
|
33
38
|
python -m pip install flake8 pytest wheel
|
|
34
|
-
pip install -r ${{ matrix.requirements-file }}
|
|
35
|
-
python
|
|
39
|
+
python -m pip install --no-cache-dir -r ${{ matrix.requirements-file }}
|
|
40
|
+
python -m pip install .
|
|
36
41
|
- name: Lint with flake8
|
|
37
42
|
run: |
|
|
38
43
|
# stop the build if there are Python syntax errors or undefined names
|
|
@@ -46,19 +51,19 @@ jobs:
|
|
|
46
51
|
needs: build-n-test
|
|
47
52
|
runs-on: ubuntu-20.04
|
|
48
53
|
steps:
|
|
49
|
-
- uses: actions/checkout@
|
|
54
|
+
- uses: actions/checkout@v4
|
|
50
55
|
- name: Check for main release tag
|
|
51
56
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
52
57
|
id: check-tag
|
|
53
58
|
run: |
|
|
54
59
|
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]*)?$ ]]; then
|
|
55
|
-
echo
|
|
60
|
+
echo "match=true" >> $GITHUB_OUTPUT
|
|
56
61
|
fi
|
|
57
|
-
- name: Set up Python 3.
|
|
62
|
+
- name: Set up Python 3.8
|
|
58
63
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
59
|
-
uses: actions/setup-python@
|
|
64
|
+
uses: actions/setup-python@v5
|
|
60
65
|
with:
|
|
61
|
-
python-version: 3.
|
|
66
|
+
python-version: 3.8
|
|
62
67
|
- name: Install pypa/build
|
|
63
68
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
64
69
|
run: >-
|
|
@@ -91,9 +96,9 @@ jobs:
|
|
|
91
96
|
runs-on: ubuntu-latest
|
|
92
97
|
needs: [build-n-test]
|
|
93
98
|
steps:
|
|
94
|
-
- uses: actions/checkout@
|
|
99
|
+
- uses: actions/checkout@v4
|
|
95
100
|
- name: Set up Python 3.8
|
|
96
|
-
uses: actions/setup-python@
|
|
101
|
+
uses: actions/setup-python@v5
|
|
97
102
|
with:
|
|
98
103
|
python-version: 3.8
|
|
99
104
|
- name: Install dependencies to build docs
|
|
@@ -109,7 +114,7 @@ jobs:
|
|
|
109
114
|
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
|
|
110
115
|
uses: JamesIves/github-pages-deploy-action@v4
|
|
111
116
|
with:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
117
|
+
token: ${{ secrets.GITHUB_TOKEN }}
|
|
118
|
+
branch: gh-pages
|
|
119
|
+
folder: docs/build
|
|
120
|
+
clean: true
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.0
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -8,17 +8,68 @@ Author-email: mikko.aulamo@helsinki.fi
|
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: setuptools
|
|
15
|
+
Requires-Dist: opustools>=1.6.2
|
|
16
|
+
Requires-Dist: beautifulsoup4>=4.8.0
|
|
17
|
+
Requires-Dist: graphviz
|
|
18
|
+
Requires-Dist: py3langid>=0.2.2
|
|
19
|
+
Requires-Dist: matplotlib
|
|
20
|
+
Requires-Dist: morfessor
|
|
21
|
+
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
|
|
22
|
+
Requires-Dist: pandas>=1.0.0
|
|
23
|
+
Requires-Dist: xxhash>=3.2.0
|
|
24
|
+
Requires-Dist: sentence-splitter
|
|
25
|
+
Requires-Dist: rapidfuzz
|
|
26
|
+
Requires-Dist: ruamel.yaml>=0.15.0
|
|
27
|
+
Requires-Dist: regex
|
|
28
|
+
Requires-Dist: requests
|
|
29
|
+
Requires-Dist: scikit-learn
|
|
30
|
+
Requires-Dist: subword_nmt
|
|
31
|
+
Requires-Dist: tqdm
|
|
32
|
+
Requires-Dist: lingua-language-detector>=1.3.0
|
|
13
33
|
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest; extra == "test"
|
|
35
|
+
Provides-Extra: pycld2
|
|
36
|
+
Requires-Dist: pycld2; extra == "pycld2"
|
|
37
|
+
Provides-Extra: fasttext
|
|
38
|
+
Requires-Dist: py3langid<0.3.0; extra == "fasttext"
|
|
39
|
+
Requires-Dist: numpy<2.0.0; extra == "fasttext"
|
|
40
|
+
Requires-Dist: fasttext; extra == "fasttext"
|
|
14
41
|
Provides-Extra: eflomal
|
|
42
|
+
Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
|
|
15
43
|
Provides-Extra: jieba
|
|
44
|
+
Requires-Dist: jieba>=0.42; extra == "jieba"
|
|
16
45
|
Provides-Extra: mecab
|
|
46
|
+
Requires-Dist: mecab-python3>=1.0.8; extra == "mecab"
|
|
47
|
+
Requires-Dist: unidic-lite; extra == "mecab"
|
|
17
48
|
Provides-Extra: laser
|
|
49
|
+
Requires-Dist: laserembeddings; extra == "laser"
|
|
18
50
|
Provides-Extra: varikn
|
|
51
|
+
Requires-Dist: varikn; extra == "varikn"
|
|
19
52
|
Provides-Extra: docs
|
|
53
|
+
Requires-Dist: myst-parser; extra == "docs"
|
|
54
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
55
|
+
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
56
|
+
Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
|
|
20
57
|
Provides-Extra: all
|
|
21
|
-
|
|
58
|
+
Requires-Dist: pycld2; extra == "all"
|
|
59
|
+
Requires-Dist: py3langid<0.3.0; extra == "all"
|
|
60
|
+
Requires-Dist: numpy<2.0.0; extra == "all"
|
|
61
|
+
Requires-Dist: fasttext; extra == "all"
|
|
62
|
+
Requires-Dist: eflomal>=2.0.0; extra == "all"
|
|
63
|
+
Requires-Dist: jieba>=0.42; extra == "all"
|
|
64
|
+
Requires-Dist: mecab-python3>=1.0.8; extra == "all"
|
|
65
|
+
Requires-Dist: unidic-lite; extra == "all"
|
|
66
|
+
Requires-Dist: laserembeddings; extra == "all"
|
|
67
|
+
Requires-Dist: varikn; extra == "all"
|
|
68
|
+
Requires-Dist: pytest; extra == "all"
|
|
69
|
+
Requires-Dist: myst-parser; extra == "all"
|
|
70
|
+
Requires-Dist: sphinx; extra == "all"
|
|
71
|
+
Requires-Dist: sphinx-rtd-theme; extra == "all"
|
|
72
|
+
Requires-Dist: sphinxcontrib-bibtex; extra == "all"
|
|
22
73
|
|
|
23
74
|
# OpusFilter
|
|
24
75
|
|
|
@@ -47,7 +98,7 @@ Install from source:
|
|
|
47
98
|
|
|
48
99
|
### Troubleshooting
|
|
49
100
|
|
|
50
|
-
OpusFilter should generally work fine on Python 3.
|
|
101
|
+
OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
51
102
|
|
|
52
103
|
* `pip install -r requirements.txt`
|
|
53
104
|
|
|
@@ -25,7 +25,7 @@ Install from source:
|
|
|
25
25
|
|
|
26
26
|
### Troubleshooting
|
|
27
27
|
|
|
28
|
-
OpusFilter should generally work fine on Python 3.
|
|
28
|
+
OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
29
29
|
|
|
30
30
|
* `pip install -r requirements.txt`
|
|
31
31
|
|
|
@@ -43,9 +43,9 @@ parser.add_argument('--clusters', '-k', default=2, type=int, metavar='INT',
|
|
|
43
43
|
parser.add_argument('--work-dir', default='work',
|
|
44
44
|
help='Location of the source and target files for the generated configuration (default %(default)s)')
|
|
45
45
|
parser.add_argument('--inter-dir', help='Save intermediate files in this directory (use a temporary directory if not given)')
|
|
46
|
-
parser.add_argument('--plot',
|
|
47
|
-
help=('
|
|
48
|
-
'only for the clustering method'))
|
|
46
|
+
parser.add_argument('--plot', metavar='PATH', default=None, type=str,
|
|
47
|
+
help=('Create histograms of feature data distributions and a scatter plot of the clustering; '
|
|
48
|
+
'give path to plot the PDF files to, or "-" for interactive plots; only for the clustering method'))
|
|
49
49
|
parser.add_argument('--list-defaults', action='store_true', help='List default filters of the method to the output and quit')
|
|
50
50
|
parser.add_argument('--add-filter', nargs=2, action='append', default=[], metavar=('CLASS', 'JSON'),
|
|
51
51
|
help=('Instead of using default filters, add a filter of CLASS with JSON parameters object '
|
|
@@ -78,9 +78,12 @@ if args.list_defaults:
|
|
|
78
78
|
|
|
79
79
|
filters = filtergen.set_filter_thresholds()
|
|
80
80
|
|
|
81
|
-
if args.method == 'clustering' and args.plot:
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
if args.method == 'clustering' and args.plot is not None:
|
|
82
|
+
if args.plot == '-':
|
|
83
|
+
filtergen.scoredata.plot(plt)
|
|
84
|
+
plt.show()
|
|
85
|
+
else:
|
|
86
|
+
filtergen.scoredata.plot(plt, path=args.plot)
|
|
84
87
|
|
|
85
88
|
generator = ConfigurationGenerator(
|
|
86
89
|
files=[os.path.abspath(f) for f in args.files], langs=args.langs, workdir=args.work_dir)
|
|
@@ -7,7 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
-
## [3.0
|
|
10
|
+
## [3.2.0] - 2024-08-14
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- make `pycld2` and `fasttext` libraries optional
|
|
15
|
+
- replace `langid.py` library with `py3langid`
|
|
16
|
+
- update github workflows and include Python 3.12 tests
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- `OpusRead` interface using `moses` format (requires `opustools >= 1.6.2`)
|
|
21
|
+
|
|
22
|
+
## [3.1.0] - 2024-06-05
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
|
|
26
|
+
- support `lingua` based for language detection (https://github.com/Helsinki-NLP/OpusFilter/pull/65)
|
|
27
|
+
|
|
28
|
+
### Removed
|
|
29
|
+
|
|
30
|
+
- Python 3.7 support
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
|
|
34
|
+
- fix score method in `SentenceEmbeddingFilter` (https://github.com/Helsinki-NLP/OpusFilter/pull/71)
|
|
35
|
+
- fix filter and filterfalse methods in `SentenceEmbeddingFilter`
|
|
36
|
+
|
|
37
|
+
## [3.0.0] - 2023-10-11
|
|
11
38
|
|
|
12
39
|
### Added
|
|
13
40
|
|
|
@@ -16,12 +43,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
16
43
|
|
|
17
44
|
### Changed
|
|
18
45
|
|
|
46
|
+
- refactor code and move auxiliary methods to opusfilter.util
|
|
19
47
|
- update varikn installation instructions (installable from PyPI)
|
|
20
|
-
- update github workflows
|
|
48
|
+
- update github workflows and include Python 3.11 tests
|
|
49
|
+
- update library version requirements to support Python 3.11
|
|
21
50
|
- use xxhash instead of pyhash for hash functions
|
|
22
51
|
- use opus-fast-mosestokenizer instead of fast-mosestokenizer
|
|
23
52
|
- install eflomal from PyPI and use the new interface in WordAlignFilter
|
|
24
|
-
|
|
53
|
+
|
|
54
|
+
### Removed
|
|
55
|
+
|
|
56
|
+
- Python 3.6 support
|
|
25
57
|
|
|
26
58
|
### Fixed
|
|
27
59
|
|
|
@@ -184,8 +216,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
184
216
|
First tagged version.
|
|
185
217
|
|
|
186
218
|
|
|
187
|
-
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0
|
|
188
|
-
[3.0
|
|
219
|
+
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...develop
|
|
220
|
+
[3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
|
|
221
|
+
[3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
|
|
222
|
+
[3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
|
|
189
223
|
[2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
|
|
190
224
|
[2.5.1]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.0...2.5.1
|
|
191
225
|
[2.5.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.4.0...2.5.0
|
|
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
|
|
|
5
5
|
few rules for pull requests:
|
|
6
6
|
|
|
7
7
|
* Make a pull request to the `develop` branch instead of `master`.
|
|
8
|
-
* The code should support at least Python versions from 3.
|
|
8
|
+
* The code should support at least Python versions from 3.8 to 3.12.
|
|
9
9
|
* Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
|
|
10
10
|
* Especially for new features, please include test cases for unit testing.
|
|
11
11
|
|
|
@@ -19,8 +19,8 @@ work, if you have VariKN and eflomal set up as instructed - `pytest`
|
|
|
19
19
|
skips the respective tests if not.)
|
|
20
20
|
|
|
21
21
|
GitHub workflows defined in the project run automatically `flake8`
|
|
22
|
-
checks and unit testing with `pytest` using Python 3.
|
|
23
|
-
3.
|
|
22
|
+
checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
|
|
23
|
+
3.11, and 3.12.
|
|
24
24
|
|
|
25
25
|
Especially for larger contributions, consider using a code analysis
|
|
26
26
|
tool like [Pylint](https://github.com/PyCQA/pylint). Install it
|
{opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/script_and_language_identification_filters.md
RENAMED
|
@@ -35,15 +35,24 @@ Filter segments based on their language identification confidence scores.
|
|
|
35
35
|
Parameters:
|
|
36
36
|
|
|
37
37
|
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
38
|
-
* `id_method`: language indentification method (`langid
|
|
38
|
+
* `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`; default `langid`)
|
|
39
39
|
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
40
40
|
* `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
|
|
41
41
|
* `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
|
|
42
42
|
* `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
|
|
43
|
+
* `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
|
|
43
44
|
|
|
44
45
|
Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. Negative threshold can be used to skip filtering for a language.
|
|
45
46
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
Currently the following identification methods are supported:
|
|
48
|
+
|
|
49
|
+
* `langid` (default) :cite:`lui-baldwin-2012-langid`
|
|
50
|
+
* See https://github.com/adbar/py3langid
|
|
51
|
+
* `lingua`
|
|
52
|
+
* See https://github.com/pemistahl/lingua-py
|
|
53
|
+
* `cld2`
|
|
54
|
+
* See https://github.com/CLD2Owners/cld2
|
|
55
|
+
* Requires [installing optional libraries](../installation.md).
|
|
56
|
+
* `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
|
|
57
|
+
* A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
|
|
58
|
+
* Requires [installing optional libraries](../installation.md).
|
|
@@ -11,11 +11,16 @@ Parameters:
|
|
|
11
11
|
* `source_language`: language code for the source language
|
|
12
12
|
* `target_language`: language code for the target language
|
|
13
13
|
* `release`: version of the corpus in OPUS
|
|
14
|
-
* `preprocessing`: `raw` for untokenized and `xml` for tokenized segments
|
|
14
|
+
* `preprocessing`: `moses` or `raw` for untokenized and `xml` for tokenized segments
|
|
15
15
|
* `src_output`: output file for source language
|
|
16
16
|
* `tgt_output`: output file for target language
|
|
17
17
|
* `suppress_prompts`: `false` (default) prompts user to confirm before download, `true` to download without prompting
|
|
18
18
|
|
|
19
|
+
The `moses` preprocessing type (available with `OpusTools` version
|
|
20
|
+
1.6.2 and above) is recommended for those corpora for which it
|
|
21
|
+
exists. The output is equivalent to `raw`, but in some cases it can
|
|
22
|
+
significantly reduce the amount of data downloaded in the process.
|
|
23
|
+
|
|
19
24
|
## concatenate
|
|
20
25
|
|
|
21
26
|
Concatenate two or more text files.
|
|
@@ -12,20 +12,18 @@ Install from source:
|
|
|
12
12
|
|
|
13
13
|
Note that all required libraries are not available to install via PyPI
|
|
14
14
|
on Windows OS. On Linux and MacOS, it should work directly for Python
|
|
15
|
-
versions from 3.
|
|
15
|
+
versions from 3.8 to 3.12.
|
|
16
16
|
|
|
17
17
|
## Required libraries
|
|
18
18
|
|
|
19
19
|
* beautifulsoup4
|
|
20
20
|
* opus-fast-mosestokenizer
|
|
21
|
-
* fasttext
|
|
22
21
|
* graphviz
|
|
23
|
-
*
|
|
22
|
+
* py3langid
|
|
24
23
|
* matplotlib
|
|
25
24
|
* morfessor
|
|
26
25
|
* OpusTools
|
|
27
26
|
* pandas
|
|
28
|
-
* pycld2
|
|
29
27
|
* rapidfuzz
|
|
30
28
|
* ruamel.yaml
|
|
31
29
|
* regex
|
|
@@ -35,29 +33,48 @@ versions from 3.7 to 3.10.
|
|
|
35
33
|
* subword_nmt
|
|
36
34
|
* tqdm
|
|
37
35
|
* xxhash
|
|
36
|
+
* lingua-language-detector
|
|
38
37
|
|
|
39
38
|
See `setup.py` for possible version requirements.
|
|
40
39
|
|
|
41
40
|
## Optional libraries and tools
|
|
42
41
|
|
|
42
|
+
### FastText and PyCLD2 language identification
|
|
43
|
+
|
|
44
|
+
The language identification libraries currently supported out-of-the-box
|
|
45
|
+
are [py3langid](https://github.com/adbar/py3langid) and
|
|
46
|
+
[lingua](https://github.com/pemistahl/lingua-py). The support for for
|
|
47
|
+
[PyCLD2](https://github.com/aboSamoor/pycld2) and
|
|
48
|
+
[FastText models](https://fasttext.cc/docs/en/language-identification.html)
|
|
49
|
+
have been changed to optional due to the lack of support especially
|
|
50
|
+
for newer Python versions.
|
|
51
|
+
|
|
52
|
+
The PyCLD2 support can be installed automatically with pip by
|
|
53
|
+
including the extras `[pycld2]` or `[all]` (e.g.
|
|
54
|
+
`pip install opusfilter[pycld2]`).
|
|
55
|
+
|
|
56
|
+
The support for FastText models can be installed automatically with
|
|
57
|
+
pip by including the extras `[fasttext]` or `[all]` (e.g.
|
|
58
|
+
`pip install opusfilter[fasttext]`).
|
|
59
|
+
|
|
43
60
|
### Jieba and MeCab word segmentation
|
|
44
61
|
|
|
45
62
|
For Chinese tokenization (word segmentation), you can use the
|
|
46
63
|
[jieba](https://github.com/fxsjy/jieba) library. It can be installed
|
|
47
64
|
automatically with pip by including the extras `[jieba]` or `[all]`
|
|
48
|
-
(e.g. `pip install opusfilter[
|
|
65
|
+
(e.g. `pip install opusfilter[jieba]`).
|
|
49
66
|
|
|
50
67
|
For Japanese tokenization (word segmentation), you can use the
|
|
51
68
|
[MeCab](https://github.com/SamuraiT/mecab-python3) library. It can be installed
|
|
52
69
|
automatically with pip by including the extras `[mecab]` or `[all]`
|
|
53
|
-
(e.g. `pip install opusfilter[
|
|
70
|
+
(e.g. `pip install opusfilter[mecab]`).
|
|
54
71
|
|
|
55
72
|
### LASER sentence embeddings
|
|
56
73
|
|
|
57
74
|
For using sentence embeddings filters, you need to install
|
|
58
75
|
`laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
|
|
59
76
|
be installed automatically with pip by including the extras `[laser]`
|
|
60
|
-
or `[all]` (e.g. `pip install opusfilter[
|
|
77
|
+
or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
|
|
61
78
|
require a number of additional libraries, including PyTorch, jieba,
|
|
62
79
|
and MeCab. Note that you need also to download the prebuild models
|
|
63
80
|
with `python -m laserembeddings download-models`.
|
|
@@ -67,12 +84,12 @@ with `python -m laserembeddings download-models`.
|
|
|
67
84
|
For using n-gram language model filters, you need to install the
|
|
68
85
|
Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
|
|
69
86
|
be installed automatically with pip by including the extras `[varikn]`
|
|
70
|
-
or `[all]` (e.g. `pip install opusfilter[
|
|
87
|
+
or `[all]` (e.g. `pip install opusfilter[varikn]`).
|
|
71
88
|
|
|
72
89
|
### Eflomal word alignment
|
|
73
90
|
|
|
74
91
|
For using word alignment filters, you need to install elfomal
|
|
75
92
|
(https://github.com/robertostling/eflomal). It can be installed
|
|
76
93
|
automatically with pip by including the extras `[eflomal]` or `[all]`
|
|
77
|
-
(e.g. `pip install opusfilter[
|
|
94
|
+
(e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
|
|
78
95
|
for the installation.
|
|
@@ -217,7 +217,7 @@ class DefaultParameterFilters(AutoFiltersABC):
|
|
|
217
217
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
218
218
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
219
219
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
220
|
-
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': '
|
|
220
|
+
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
|
|
221
221
|
|
|
222
222
|
def set_filter_thresholds(self):
|
|
223
223
|
"""Set filter thresholds"""
|
|
@@ -272,7 +272,7 @@ class PercentileFilters(DataBasedFiltersABC):
|
|
|
272
272
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
273
273
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
274
274
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
275
|
-
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': '
|
|
275
|
+
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
|
|
276
276
|
|
|
277
277
|
def __init__(self, files, excluded_percentile=0.001, **kwargs):
|
|
278
278
|
super().__init__(files, **kwargs)
|
|
@@ -512,7 +512,7 @@ class ClusterFilters(DataBasedFiltersABC):
|
|
|
512
512
|
('LengthRatioFilter.word', {'unit': 'word'}),
|
|
513
513
|
'NonZeroNumeralsFilter',
|
|
514
514
|
'CharacterScoreFilter',
|
|
515
|
-
('LanguageIDFilter', {'id_method': '
|
|
515
|
+
('LanguageIDFilter', {'id_method': 'lingua'}),
|
|
516
516
|
'TerminalPunctuationFilter']
|
|
517
517
|
|
|
518
518
|
def __init__(self, files, k=2, max_length=150, **kwargs):
|