opusfilter 3.1.0__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opusfilter-3.1.0 → opusfilter-3.2.0}/.github/workflows/ci.yml +11 -9
- {opusfilter-3.1.0 → opusfilter-3.2.0}/PKG-INFO +16 -8
- {opusfilter-3.1.0 → opusfilter-3.2.0}/README.md +1 -1
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/CHANGELOG.md +14 -1
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/CONTRIBUTING.md +2 -2
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/script_and_language_identification_filters.md +13 -5
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/downloading_and_selecting_data.md +6 -1
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/installation.md +25 -9
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/autogen.py +3 -3
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/filters.py +13 -5
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/opusfilter.py +13 -3
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/PKG-INFO +16 -8
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/SOURCES.txt +1 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/requires.txt +16 -6
- {opusfilter-3.1.0 → opusfilter-3.2.0}/requirements.txt +4 -5
- {opusfilter-3.1.0 → opusfilter-3.2.0}/setup.py +19 -9
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_autogen.py +1 -1
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_filter_pipeline.py +25 -9
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_filters.py +0 -128
- opusfilter-3.2.0/tests/test_lid.py +151 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_opusfilter.py +21 -1
- {opusfilter-3.1.0 → opusfilter-3.2.0}/.gitignore +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/.pylintrc +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/LICENSE +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-autogen +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-cmd +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-diagram +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-duplicates +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-scores +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-test +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/automatic_configuration.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/command_line_tools.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/conf.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/alignment_model_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/custom_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/language_model_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/length_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/sentence_embedding_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/filtering_and_scoring.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/preprocessing_text.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/training_and_using_classifiers.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/training_language_and_alignment_models.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/using_score_files.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/index.rst +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/bpe_segmentation.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/custom_preprocessors.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/detokenizer.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/reg_exp_sub.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/tokenizer.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/references.bib +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/references.rst +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/requirements.txt +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/usage.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/README.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/qed_lm_langid.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/__init__.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/autogen_cluster.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/classifier.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/embeddings.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/lm.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/pipeline.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/preprocessors.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/segment_hash.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/subwords.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/tokenization.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/util.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/word_alignment.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/dependency_links.txt +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/top_level.txt +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/pyproject.toml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/setup.cfg +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_classifier.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_embeddings.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_lm_filter.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_preprocessors.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_segment_hash.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_subwords.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_tokenization.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_util.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_wordalign_filter.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_yaml.py +0 -0
|
@@ -21,21 +21,23 @@ jobs:
|
|
|
21
21
|
fail-fast: false
|
|
22
22
|
matrix:
|
|
23
23
|
os: [ubuntu-latest]
|
|
24
|
-
python-version: ["3.8", "3.9", "3.10", "3.11"]
|
|
24
|
+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
|
25
25
|
requirements-file: ["requirements.txt"]
|
|
26
26
|
runs-on: ${{ matrix.os }}
|
|
27
27
|
steps:
|
|
28
|
-
- uses: actions/checkout@
|
|
28
|
+
- uses: actions/checkout@v4
|
|
29
29
|
- name: Set up Python ${{ matrix.python-version }}
|
|
30
|
-
uses: actions/setup-python@
|
|
30
|
+
uses: actions/setup-python@v5
|
|
31
31
|
with:
|
|
32
32
|
python-version: ${{ matrix.python-version }}
|
|
33
33
|
- name: Install dependencies
|
|
34
34
|
run: |
|
|
35
|
+
python -m ensurepip --upgrade
|
|
36
|
+
python -m pip install --upgrade setuptools
|
|
35
37
|
python -m pip install --upgrade pip
|
|
36
38
|
python -m pip install flake8 pytest wheel
|
|
37
|
-
pip install -r ${{ matrix.requirements-file }}
|
|
38
|
-
python
|
|
39
|
+
python -m pip install --no-cache-dir -r ${{ matrix.requirements-file }}
|
|
40
|
+
python -m pip install .
|
|
39
41
|
- name: Lint with flake8
|
|
40
42
|
run: |
|
|
41
43
|
# stop the build if there are Python syntax errors or undefined names
|
|
@@ -49,7 +51,7 @@ jobs:
|
|
|
49
51
|
needs: build-n-test
|
|
50
52
|
runs-on: ubuntu-20.04
|
|
51
53
|
steps:
|
|
52
|
-
- uses: actions/checkout@
|
|
54
|
+
- uses: actions/checkout@v4
|
|
53
55
|
- name: Check for main release tag
|
|
54
56
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
55
57
|
id: check-tag
|
|
@@ -59,7 +61,7 @@ jobs:
|
|
|
59
61
|
fi
|
|
60
62
|
- name: Set up Python 3.8
|
|
61
63
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
62
|
-
uses: actions/setup-python@
|
|
64
|
+
uses: actions/setup-python@v5
|
|
63
65
|
with:
|
|
64
66
|
python-version: 3.8
|
|
65
67
|
- name: Install pypa/build
|
|
@@ -94,9 +96,9 @@ jobs:
|
|
|
94
96
|
runs-on: ubuntu-latest
|
|
95
97
|
needs: [build-n-test]
|
|
96
98
|
steps:
|
|
97
|
-
- uses: actions/checkout@
|
|
99
|
+
- uses: actions/checkout@v4
|
|
98
100
|
- name: Set up Python 3.8
|
|
99
|
-
uses: actions/setup-python@
|
|
101
|
+
uses: actions/setup-python@v5
|
|
100
102
|
with:
|
|
101
103
|
python-version: 3.8
|
|
102
104
|
- name: Install dependencies to build docs
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -12,16 +12,14 @@ Requires-Python: >=3.8
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: setuptools
|
|
15
|
-
Requires-Dist: opustools
|
|
15
|
+
Requires-Dist: opustools>=1.6.2
|
|
16
16
|
Requires-Dist: beautifulsoup4>=4.8.0
|
|
17
|
-
Requires-Dist: fasttext
|
|
18
17
|
Requires-Dist: graphviz
|
|
19
|
-
Requires-Dist:
|
|
18
|
+
Requires-Dist: py3langid>=0.2.2
|
|
20
19
|
Requires-Dist: matplotlib
|
|
21
20
|
Requires-Dist: morfessor
|
|
22
21
|
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
|
|
23
22
|
Requires-Dist: pandas>=1.0.0
|
|
24
|
-
Requires-Dist: pycld2
|
|
25
23
|
Requires-Dist: xxhash>=3.2.0
|
|
26
24
|
Requires-Dist: sentence-splitter
|
|
27
25
|
Requires-Dist: rapidfuzz
|
|
@@ -34,8 +32,14 @@ Requires-Dist: tqdm
|
|
|
34
32
|
Requires-Dist: lingua-language-detector>=1.3.0
|
|
35
33
|
Provides-Extra: test
|
|
36
34
|
Requires-Dist: pytest; extra == "test"
|
|
35
|
+
Provides-Extra: pycld2
|
|
36
|
+
Requires-Dist: pycld2; extra == "pycld2"
|
|
37
|
+
Provides-Extra: fasttext
|
|
38
|
+
Requires-Dist: py3langid<0.3.0; extra == "fasttext"
|
|
39
|
+
Requires-Dist: numpy<2.0.0; extra == "fasttext"
|
|
40
|
+
Requires-Dist: fasttext; extra == "fasttext"
|
|
37
41
|
Provides-Extra: eflomal
|
|
38
|
-
Requires-Dist: eflomal>=
|
|
42
|
+
Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
|
|
39
43
|
Provides-Extra: jieba
|
|
40
44
|
Requires-Dist: jieba>=0.42; extra == "jieba"
|
|
41
45
|
Provides-Extra: mecab
|
|
@@ -51,7 +55,11 @@ Requires-Dist: sphinx; extra == "docs"
|
|
|
51
55
|
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
52
56
|
Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
|
|
53
57
|
Provides-Extra: all
|
|
54
|
-
Requires-Dist:
|
|
58
|
+
Requires-Dist: pycld2; extra == "all"
|
|
59
|
+
Requires-Dist: py3langid<0.3.0; extra == "all"
|
|
60
|
+
Requires-Dist: numpy<2.0.0; extra == "all"
|
|
61
|
+
Requires-Dist: fasttext; extra == "all"
|
|
62
|
+
Requires-Dist: eflomal>=2.0.0; extra == "all"
|
|
55
63
|
Requires-Dist: jieba>=0.42; extra == "all"
|
|
56
64
|
Requires-Dist: mecab-python3>=1.0.8; extra == "all"
|
|
57
65
|
Requires-Dist: unidic-lite; extra == "all"
|
|
@@ -90,7 +98,7 @@ Install from source:
|
|
|
90
98
|
|
|
91
99
|
### Troubleshooting
|
|
92
100
|
|
|
93
|
-
OpusFilter should generally work fine on Python 3.8 to 3.
|
|
101
|
+
OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
94
102
|
|
|
95
103
|
* `pip install -r requirements.txt`
|
|
96
104
|
|
|
@@ -25,7 +25,7 @@ Install from source:
|
|
|
25
25
|
|
|
26
26
|
### Troubleshooting
|
|
27
27
|
|
|
28
|
-
OpusFilter should generally work fine on Python 3.8 to 3.
|
|
28
|
+
OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
29
29
|
|
|
30
30
|
* `pip install -r requirements.txt`
|
|
31
31
|
|
|
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [3.2.0] - 2024-08-14
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- make `pycld2` and `fasttext` libraries optional
|
|
15
|
+
- replace `langid.py` library with `py3langid`
|
|
16
|
+
- update github workflows and include Python 3.12 tests
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- `OpusRead` interface using `moses` format (requires `opustools >= 1.6.2`)
|
|
21
|
+
|
|
10
22
|
## [3.1.0] - 2024-06-05
|
|
11
23
|
|
|
12
24
|
### Added
|
|
@@ -204,7 +216,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
204
216
|
First tagged version.
|
|
205
217
|
|
|
206
218
|
|
|
207
|
-
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.
|
|
219
|
+
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...develop
|
|
220
|
+
[3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
|
|
208
221
|
[3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
|
|
209
222
|
[3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
|
|
210
223
|
[2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
|
|
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
|
|
|
5
5
|
few rules for pull requests:
|
|
6
6
|
|
|
7
7
|
* Make a pull request to the `develop` branch instead of `master`.
|
|
8
|
-
* The code should support at least Python versions from 3.8 to 3.
|
|
8
|
+
* The code should support at least Python versions from 3.8 to 3.12.
|
|
9
9
|
* Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
|
|
10
10
|
* Especially for new features, please include test cases for unit testing.
|
|
11
11
|
|
|
@@ -20,7 +20,7 @@ skips the respective tests if not.)
|
|
|
20
20
|
|
|
21
21
|
GitHub workflows defined in the project run automatically `flake8`
|
|
22
22
|
checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
|
|
23
|
-
and 3.
|
|
23
|
+
3.11, and 3.12.
|
|
24
24
|
|
|
25
25
|
Especially for larger contributions, consider using a code analysis
|
|
26
26
|
tool like [Pylint](https://github.com/PyCQA/pylint). Install it
|
{opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/script_and_language_identification_filters.md
RENAMED
|
@@ -35,7 +35,7 @@ Filter segments based on their language identification confidence scores.
|
|
|
35
35
|
Parameters:
|
|
36
36
|
|
|
37
37
|
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
38
|
-
* `id_method`: language indentification method (`langid
|
|
38
|
+
* `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`; default `langid`)
|
|
39
39
|
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
40
40
|
* `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
|
|
41
41
|
* `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
|
|
@@ -44,7 +44,15 @@ Parameters:
|
|
|
44
44
|
|
|
45
45
|
Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. Negative threshold can be used to skip filtering for a language.
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
Currently the following identification methods are supported:
|
|
48
|
+
|
|
49
|
+
* `langid` (default) :cite:`lui-baldwin-2012-langid`
|
|
50
|
+
* See https://github.com/adbar/py3langid
|
|
51
|
+
* `lingua`
|
|
52
|
+
* See https://github.com/pemistahl/lingua-py
|
|
53
|
+
* `cld2`
|
|
54
|
+
* See https://github.com/CLD2Owners/cld2
|
|
55
|
+
* Requires [installing optional libraries](../installation.md).
|
|
56
|
+
* `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
|
|
57
|
+
* A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
|
|
58
|
+
* Requires [installing optional libraries](../installation.md).
|
|
@@ -11,11 +11,16 @@ Parameters:
|
|
|
11
11
|
* `source_language`: language code for the source language
|
|
12
12
|
* `target_language`: language code for the target language
|
|
13
13
|
* `release`: version of the corpus in OPUS
|
|
14
|
-
* `preprocessing`: `raw` for untokenized and `xml` for tokenized segments
|
|
14
|
+
* `preprocessing`: `moses` or `raw` for untokenized and `xml` for tokenized segments
|
|
15
15
|
* `src_output`: output file for source language
|
|
16
16
|
* `tgt_output`: output file for target language
|
|
17
17
|
* `suppress_prompts`: `false` (default) prompts user to confirm before download, `true` to download without prompting
|
|
18
18
|
|
|
19
|
+
The `moses` preprocessing type (available with `OpusTools` version
|
|
20
|
+
1.6.2 and above) is recommended for those corpora for which it
|
|
21
|
+
exists. The output is equivalent to `raw`, but in some cases it can
|
|
22
|
+
significantly reduce the amount of data downloaded in the process.
|
|
23
|
+
|
|
19
24
|
## concatenate
|
|
20
25
|
|
|
21
26
|
Concatenate two or more text files.
|
|
@@ -12,20 +12,18 @@ Install from source:
|
|
|
12
12
|
|
|
13
13
|
Note that all required libraries are not available to install via PyPI
|
|
14
14
|
on Windows OS. On Linux and MacOS, it should work directly for Python
|
|
15
|
-
versions from 3.8 to 3.
|
|
15
|
+
versions from 3.8 to 3.12.
|
|
16
16
|
|
|
17
17
|
## Required libraries
|
|
18
18
|
|
|
19
19
|
* beautifulsoup4
|
|
20
20
|
* opus-fast-mosestokenizer
|
|
21
|
-
* fasttext
|
|
22
21
|
* graphviz
|
|
23
|
-
*
|
|
22
|
+
* py3langid
|
|
24
23
|
* matplotlib
|
|
25
24
|
* morfessor
|
|
26
25
|
* OpusTools
|
|
27
26
|
* pandas
|
|
28
|
-
* pycld2
|
|
29
27
|
* rapidfuzz
|
|
30
28
|
* ruamel.yaml
|
|
31
29
|
* regex
|
|
@@ -41,24 +39,42 @@ See `setup.py` for possible version requirements.
|
|
|
41
39
|
|
|
42
40
|
## Optional libraries and tools
|
|
43
41
|
|
|
42
|
+
### FastText and PyCLD2 language identification
|
|
43
|
+
|
|
44
|
+
The language identification libraries currently supported out-of-the-box
|
|
45
|
+
are [py3langid](https://github.com/adbar/py3langid) and
|
|
46
|
+
[lingua](https://github.com/pemistahl/lingua-py). The support for for
|
|
47
|
+
[PyCLD2](https://github.com/aboSamoor/pycld2) and
|
|
48
|
+
[FastText models](https://fasttext.cc/docs/en/language-identification.html)
|
|
49
|
+
have been changed to optional due to the lack of support especially
|
|
50
|
+
for newer Python versions.
|
|
51
|
+
|
|
52
|
+
The PyCLD2 support can be installed automatically with pip by
|
|
53
|
+
including the extras `[pycld2]` or `[all]` (e.g.
|
|
54
|
+
`pip install opusfilter[pycld2]`).
|
|
55
|
+
|
|
56
|
+
The support for FastText models can be installed automatically with
|
|
57
|
+
pip by including the extras `[fasttext]` or `[all]` (e.g.
|
|
58
|
+
`pip install opusfilter[fasttext]`).
|
|
59
|
+
|
|
44
60
|
### Jieba and MeCab word segmentation
|
|
45
61
|
|
|
46
62
|
For Chinese tokenization (word segmentation), you can use the
|
|
47
63
|
[jieba](https://github.com/fxsjy/jieba) library. It can be installed
|
|
48
64
|
automatically with pip by including the extras `[jieba]` or `[all]`
|
|
49
|
-
(e.g. `pip install opusfilter[
|
|
65
|
+
(e.g. `pip install opusfilter[jieba]`).
|
|
50
66
|
|
|
51
67
|
For Japanese tokenization (word segmentation), you can use the
|
|
52
68
|
[MeCab](https://github.com/SamuraiT/mecab-python3) library. It can be installed
|
|
53
69
|
automatically with pip by including the extras `[mecab]` or `[all]`
|
|
54
|
-
(e.g. `pip install opusfilter[
|
|
70
|
+
(e.g. `pip install opusfilter[mecab]`).
|
|
55
71
|
|
|
56
72
|
### LASER sentence embeddings
|
|
57
73
|
|
|
58
74
|
For using sentence embeddings filters, you need to install
|
|
59
75
|
`laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
|
|
60
76
|
be installed automatically with pip by including the extras `[laser]`
|
|
61
|
-
or `[all]` (e.g. `pip install opusfilter[
|
|
77
|
+
or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
|
|
62
78
|
require a number of additional libraries, including PyTorch, jieba,
|
|
63
79
|
and MeCab. Note that you need also to download the prebuild models
|
|
64
80
|
with `python -m laserembeddings download-models`.
|
|
@@ -68,12 +84,12 @@ with `python -m laserembeddings download-models`.
|
|
|
68
84
|
For using n-gram language model filters, you need to install the
|
|
69
85
|
Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
|
|
70
86
|
be installed automatically with pip by including the extras `[varikn]`
|
|
71
|
-
or `[all]` (e.g. `pip install opusfilter[
|
|
87
|
+
or `[all]` (e.g. `pip install opusfilter[varikn]`).
|
|
72
88
|
|
|
73
89
|
### Eflomal word alignment
|
|
74
90
|
|
|
75
91
|
For using word alignment filters, you need to install elfomal
|
|
76
92
|
(https://github.com/robertostling/eflomal). It can be installed
|
|
77
93
|
automatically with pip by including the extras `[eflomal]` or `[all]`
|
|
78
|
-
(e.g. `pip install opusfilter[
|
|
94
|
+
(e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
|
|
79
95
|
for the installation.
|
|
@@ -217,7 +217,7 @@ class DefaultParameterFilters(AutoFiltersABC):
|
|
|
217
217
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
218
218
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
219
219
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
220
|
-
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': '
|
|
220
|
+
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
|
|
221
221
|
|
|
222
222
|
def set_filter_thresholds(self):
|
|
223
223
|
"""Set filter thresholds"""
|
|
@@ -272,7 +272,7 @@ class PercentileFilters(DataBasedFiltersABC):
|
|
|
272
272
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
273
273
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
274
274
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
275
|
-
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': '
|
|
275
|
+
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
|
|
276
276
|
|
|
277
277
|
def __init__(self, files, excluded_percentile=0.001, **kwargs):
|
|
278
278
|
super().__init__(files, **kwargs)
|
|
@@ -512,7 +512,7 @@ class ClusterFilters(DataBasedFiltersABC):
|
|
|
512
512
|
('LengthRatioFilter.word', {'unit': 'word'}),
|
|
513
513
|
'NonZeroNumeralsFilter',
|
|
514
514
|
'CharacterScoreFilter',
|
|
515
|
-
('LanguageIDFilter', {'id_method': '
|
|
515
|
+
('LanguageIDFilter', {'id_method': 'lingua'}),
|
|
516
516
|
'TerminalPunctuationFilter']
|
|
517
517
|
|
|
518
518
|
def __init__(self, files, k=2, max_length=150, **kwargs):
|
|
@@ -334,8 +334,8 @@ class LanguageIDFilter(FilterABC):
|
|
|
334
334
|
|
|
335
335
|
def init_langid(self, langid_languages):
|
|
336
336
|
"""Initialize langid identifier"""
|
|
337
|
-
from
|
|
338
|
-
self.identifier = LanguageIdentifier.
|
|
337
|
+
from py3langid.langid import LanguageIdentifier, MODEL_FILE
|
|
338
|
+
self.identifier = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True)
|
|
339
339
|
if langid_languages:
|
|
340
340
|
self.identifier.set_languages(langid_languages)
|
|
341
341
|
|
|
@@ -344,7 +344,11 @@ class LanguageIDFilter(FilterABC):
|
|
|
344
344
|
if not fasttext_model_path:
|
|
345
345
|
raise ConfigurationError("FastText language ID method was choosen without specifying "
|
|
346
346
|
"any path to fasttext model")
|
|
347
|
-
|
|
347
|
+
try:
|
|
348
|
+
import fasttext
|
|
349
|
+
except ImportError:
|
|
350
|
+
logger.warning("Could not import fasttext. Select another id_method for LanguageIDFilter.")
|
|
351
|
+
raise
|
|
348
352
|
self.fasttext_model = fasttext.load_model(os.path.join(self.workdir, fasttext_model_path))
|
|
349
353
|
|
|
350
354
|
def init_lingua(self, lingua_mode):
|
|
@@ -366,7 +370,11 @@ class LanguageIDFilter(FilterABC):
|
|
|
366
370
|
return 1.0
|
|
367
371
|
|
|
368
372
|
if self.id_method == 'cld2':
|
|
369
|
-
|
|
373
|
+
try:
|
|
374
|
+
import pycld2
|
|
375
|
+
except ImportError:
|
|
376
|
+
logger.warning("Could not import pycld2. Select another id_method for LanguageIDFilter.")
|
|
377
|
+
raise
|
|
370
378
|
try:
|
|
371
379
|
clddetails = pycld2.detect(sentence, **self.cld2_options)
|
|
372
380
|
except pycld2.error as err:
|
|
@@ -380,7 +388,7 @@ class LanguageIDFilter(FilterABC):
|
|
|
380
388
|
|
|
381
389
|
if self.id_method == 'langid':
|
|
382
390
|
lidetails = self.identifier.classify(sentence)
|
|
383
|
-
lilan, liconf = lidetails[0], round(lidetails[1], 2)
|
|
391
|
+
lilan, liconf = lidetails[0], round(float(lidetails[1]), 2)
|
|
384
392
|
if lilan != lan:
|
|
385
393
|
liconf = 0.0
|
|
386
394
|
return liconf
|
|
@@ -334,8 +334,14 @@ class OpusFilter:
|
|
|
334
334
|
write=[src_out, tgt_out],
|
|
335
335
|
leave_non_alignments_out=True,
|
|
336
336
|
download_dir=self.output_dir)
|
|
337
|
-
|
|
338
|
-
|
|
337
|
+
try:
|
|
338
|
+
opus_reader.printPairs()
|
|
339
|
+
except Exception as err:
|
|
340
|
+
# Remove broken files
|
|
341
|
+
for outfile in [src_out, tgt_out]:
|
|
342
|
+
if os.path.isfile(outfile):
|
|
343
|
+
os.unlink(outfile)
|
|
344
|
+
raise err
|
|
339
345
|
|
|
340
346
|
@staticmethod
|
|
341
347
|
def pair_generator(*filenames, tokenizers=None):
|
|
@@ -556,7 +562,11 @@ class OpusFilter:
|
|
|
556
562
|
"""Write objects to file as JSON lines"""
|
|
557
563
|
with file_open(fname, 'w') as fobj:
|
|
558
564
|
for obj in objects:
|
|
559
|
-
|
|
565
|
+
try:
|
|
566
|
+
fobj.write(json.dumps(obj, sort_keys=True)+'\n')
|
|
567
|
+
except TypeError as err:
|
|
568
|
+
logger.error("Could not convert to JSON: %s", obj)
|
|
569
|
+
raise err
|
|
560
570
|
|
|
561
571
|
@staticmethod
|
|
562
572
|
def _read_jsonl(fname):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -12,16 +12,14 @@ Requires-Python: >=3.8
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: setuptools
|
|
15
|
-
Requires-Dist: opustools
|
|
15
|
+
Requires-Dist: opustools>=1.6.2
|
|
16
16
|
Requires-Dist: beautifulsoup4>=4.8.0
|
|
17
|
-
Requires-Dist: fasttext
|
|
18
17
|
Requires-Dist: graphviz
|
|
19
|
-
Requires-Dist:
|
|
18
|
+
Requires-Dist: py3langid>=0.2.2
|
|
20
19
|
Requires-Dist: matplotlib
|
|
21
20
|
Requires-Dist: morfessor
|
|
22
21
|
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
|
|
23
22
|
Requires-Dist: pandas>=1.0.0
|
|
24
|
-
Requires-Dist: pycld2
|
|
25
23
|
Requires-Dist: xxhash>=3.2.0
|
|
26
24
|
Requires-Dist: sentence-splitter
|
|
27
25
|
Requires-Dist: rapidfuzz
|
|
@@ -34,8 +32,14 @@ Requires-Dist: tqdm
|
|
|
34
32
|
Requires-Dist: lingua-language-detector>=1.3.0
|
|
35
33
|
Provides-Extra: test
|
|
36
34
|
Requires-Dist: pytest; extra == "test"
|
|
35
|
+
Provides-Extra: pycld2
|
|
36
|
+
Requires-Dist: pycld2; extra == "pycld2"
|
|
37
|
+
Provides-Extra: fasttext
|
|
38
|
+
Requires-Dist: py3langid<0.3.0; extra == "fasttext"
|
|
39
|
+
Requires-Dist: numpy<2.0.0; extra == "fasttext"
|
|
40
|
+
Requires-Dist: fasttext; extra == "fasttext"
|
|
37
41
|
Provides-Extra: eflomal
|
|
38
|
-
Requires-Dist: eflomal>=
|
|
42
|
+
Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
|
|
39
43
|
Provides-Extra: jieba
|
|
40
44
|
Requires-Dist: jieba>=0.42; extra == "jieba"
|
|
41
45
|
Provides-Extra: mecab
|
|
@@ -51,7 +55,11 @@ Requires-Dist: sphinx; extra == "docs"
|
|
|
51
55
|
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
52
56
|
Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
|
|
53
57
|
Provides-Extra: all
|
|
54
|
-
Requires-Dist:
|
|
58
|
+
Requires-Dist: pycld2; extra == "all"
|
|
59
|
+
Requires-Dist: py3langid<0.3.0; extra == "all"
|
|
60
|
+
Requires-Dist: numpy<2.0.0; extra == "all"
|
|
61
|
+
Requires-Dist: fasttext; extra == "all"
|
|
62
|
+
Requires-Dist: eflomal>=2.0.0; extra == "all"
|
|
55
63
|
Requires-Dist: jieba>=0.42; extra == "all"
|
|
56
64
|
Requires-Dist: mecab-python3>=1.0.8; extra == "all"
|
|
57
65
|
Requires-Dist: unidic-lite; extra == "all"
|
|
@@ -90,7 +98,7 @@ Install from source:
|
|
|
90
98
|
|
|
91
99
|
### Troubleshooting
|
|
92
100
|
|
|
93
|
-
OpusFilter should generally work fine on Python 3.8 to 3.
|
|
101
|
+
OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
94
102
|
|
|
95
103
|
* `pip install -r requirements.txt`
|
|
96
104
|
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
setuptools
|
|
2
|
-
opustools
|
|
2
|
+
opustools>=1.6.2
|
|
3
3
|
beautifulsoup4>=4.8.0
|
|
4
|
-
fasttext
|
|
5
4
|
graphviz
|
|
6
|
-
|
|
5
|
+
py3langid>=0.2.2
|
|
7
6
|
matplotlib
|
|
8
7
|
morfessor
|
|
9
8
|
opus-fast-mosestokenizer>=0.0.8.5
|
|
10
9
|
pandas>=1.0.0
|
|
11
|
-
pycld2
|
|
12
10
|
xxhash>=3.2.0
|
|
13
11
|
sentence-splitter
|
|
14
12
|
rapidfuzz
|
|
@@ -21,7 +19,11 @@ tqdm
|
|
|
21
19
|
lingua-language-detector>=1.3.0
|
|
22
20
|
|
|
23
21
|
[all]
|
|
24
|
-
|
|
22
|
+
pycld2
|
|
23
|
+
py3langid<0.3.0
|
|
24
|
+
numpy<2.0.0
|
|
25
|
+
fasttext
|
|
26
|
+
eflomal>=2.0.0
|
|
25
27
|
jieba>=0.42
|
|
26
28
|
mecab-python3>=1.0.8
|
|
27
29
|
unidic-lite
|
|
@@ -40,7 +42,12 @@ sphinx-rtd-theme
|
|
|
40
42
|
sphinxcontrib-bibtex
|
|
41
43
|
|
|
42
44
|
[eflomal]
|
|
43
|
-
eflomal>=
|
|
45
|
+
eflomal>=2.0.0
|
|
46
|
+
|
|
47
|
+
[fasttext]
|
|
48
|
+
py3langid<0.3.0
|
|
49
|
+
numpy<2.0.0
|
|
50
|
+
fasttext
|
|
44
51
|
|
|
45
52
|
[jieba]
|
|
46
53
|
jieba>=0.42
|
|
@@ -52,6 +59,9 @@ laserembeddings
|
|
|
52
59
|
mecab-python3>=1.0.8
|
|
53
60
|
unidic-lite
|
|
54
61
|
|
|
62
|
+
[pycld2]
|
|
63
|
+
pycld2
|
|
64
|
+
|
|
55
65
|
[test]
|
|
56
66
|
pytest
|
|
57
67
|
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
setuptools
|
|
2
|
-
setuptools_scm
|
|
1
|
+
setuptools>=65.5.1
|
|
2
|
+
setuptools_scm>=6.4.2
|
|
3
|
+
numpy>=1.24.4
|
|
3
4
|
opustools
|
|
4
5
|
jieba>=0.42
|
|
5
6
|
beautifulsoup4>=4.8.2
|
|
6
7
|
graphviz>=0.16
|
|
7
|
-
|
|
8
|
+
py3langid==0.3.0
|
|
8
9
|
matplotlib>=3.3.0
|
|
9
10
|
opus-fast-mosestokenizer>=0.0.8.5
|
|
10
11
|
pandas>=1.0.0
|
|
11
|
-
pycld2==0.41
|
|
12
12
|
xxhash==3.2.0
|
|
13
13
|
rapidfuzz>=2.0.5
|
|
14
14
|
regex>=2019.11.1
|
|
@@ -17,7 +17,6 @@ ruamel.yaml>=0.15.0
|
|
|
17
17
|
scikit-learn>=0.24.0
|
|
18
18
|
sentence-splitter==1.4
|
|
19
19
|
tqdm>=4.38.0
|
|
20
|
-
fasttext==0.9.2
|
|
21
20
|
mecab-python3>=1.0.8
|
|
22
21
|
unidic-lite==1.0.8
|
|
23
22
|
subword-nmt==0.3.8
|
|
@@ -5,16 +5,14 @@ with open("README.md", "r") as fh:
|
|
|
5
5
|
|
|
6
6
|
install_requires = [
|
|
7
7
|
"setuptools",
|
|
8
|
-
"opustools",
|
|
8
|
+
"opustools>=1.6.2",
|
|
9
9
|
"beautifulsoup4>=4.8.0",
|
|
10
|
-
"fasttext",
|
|
11
10
|
"graphviz",
|
|
12
|
-
"
|
|
11
|
+
"py3langid>=0.2.2",
|
|
13
12
|
"matplotlib",
|
|
14
13
|
"morfessor",
|
|
15
14
|
"opus-fast-mosestokenizer>=0.0.8.5",
|
|
16
15
|
"pandas>=1.0.0",
|
|
17
|
-
"pycld2",
|
|
18
16
|
"xxhash>=3.2.0",
|
|
19
17
|
"sentence-splitter",
|
|
20
18
|
"rapidfuzz",
|
|
@@ -27,8 +25,18 @@ install_requires = [
|
|
|
27
25
|
"lingua-language-detector>=1.3.0"
|
|
28
26
|
]
|
|
29
27
|
|
|
28
|
+
pycld2_require = [
|
|
29
|
+
"pycld2"
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
fasttext_require = [
|
|
33
|
+
"py3langid<0.3.0", # 0.3.0 requires numpy 2.0.0
|
|
34
|
+
"numpy<2.0.0",
|
|
35
|
+
"fasttext"
|
|
36
|
+
]
|
|
37
|
+
|
|
30
38
|
eflomal_require = [
|
|
31
|
-
'eflomal>=
|
|
39
|
+
'eflomal>=2.0.0'
|
|
32
40
|
]
|
|
33
41
|
|
|
34
42
|
jieba_require = [
|
|
@@ -59,7 +67,8 @@ docs_require = [
|
|
|
59
67
|
'sphinxcontrib-bibtex'
|
|
60
68
|
]
|
|
61
69
|
|
|
62
|
-
all_require =
|
|
70
|
+
all_require = pycld2_require + fasttext_require + eflomal_require + jieba_require + \
|
|
71
|
+
mecab_require + laser_require + varikn_require + tests_require + docs_require
|
|
63
72
|
|
|
64
73
|
setuptools.setup(
|
|
65
74
|
name="opusfilter",
|
|
@@ -77,9 +86,10 @@ setuptools.setup(
|
|
|
77
86
|
"bin/opusfilter-scores", "bin/opusfilter-test"],
|
|
78
87
|
install_requires=install_requires,
|
|
79
88
|
tests_require=tests_require,
|
|
80
|
-
extras_require={'test': tests_require, '
|
|
81
|
-
'
|
|
82
|
-
'
|
|
89
|
+
extras_require={'test': tests_require, 'pycld2': pycld2_require, 'fasttext': fasttext_require,
|
|
90
|
+
'eflomal': eflomal_require, 'jieba': jieba_require, 'mecab': mecab_require,
|
|
91
|
+
'laser': laser_require, 'varikn': varikn_require, 'docs': docs_require,
|
|
92
|
+
'all': all_require},
|
|
83
93
|
classifiers=(
|
|
84
94
|
"Programming Language :: Python :: 3",
|
|
85
95
|
"License :: OSI Approved :: MIT License",
|
|
@@ -103,7 +103,7 @@ class TestThresholdFinder(unittest.TestCase):
|
|
|
103
103
|
{'LengthRatioFilter': {'name': 'word', 'threshold': 1, 'unit': 'word'}},
|
|
104
104
|
{'NonZeroNumeralsFilter': {'threshold': 1}},
|
|
105
105
|
{'CharacterScoreFilter': {'scripts': ['latin', 'latin'], 'thresholds': [1, 1]}},
|
|
106
|
-
{'LanguageIDFilter': {'id_method': '
|
|
106
|
+
{'LanguageIDFilter': {'id_method': 'lingua', 'languages': ['en', 'de'], 'thresholds': [1, 1]}},
|
|
107
107
|
{'TerminalPunctuationFilter': {'threshold': 1}}
|
|
108
108
|
]
|
|
109
109
|
|