opusfilter 3.1.0__tar.gz → 3.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opusfilter-3.1.0 → opusfilter-3.3.0}/.github/workflows/ci.yml +12 -10
- {opusfilter-3.1.0 → opusfilter-3.3.0}/PKG-INFO +19 -10
- {opusfilter-3.1.0 → opusfilter-3.3.0}/README.md +1 -1
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-duplicates +13 -3
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-scores +1 -2
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/CHANGELOG.md +33 -1
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/CONTRIBUTING.md +2 -2
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/command_line_tools.md +2 -1
- opusfilter-3.3.0/docs/filters/script_and_language_identification_filters.md +204 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/downloading_and_selecting_data.md +6 -1
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/filtering_and_scoring.md +8 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/installation.md +31 -9
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/references.bib +13 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/autogen.py +3 -3
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/filters.py +43 -15
- opusfilter-3.3.0/opusfilter/lid.py +364 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/opusfilter.py +17 -4
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/segment_hash.py +29 -6
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/PKG-INFO +19 -10
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/SOURCES.txt +2 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/requires.txt +28 -8
- {opusfilter-3.1.0 → opusfilter-3.3.0}/requirements.txt +8 -7
- {opusfilter-3.1.0 → opusfilter-3.3.0}/setup.py +27 -10
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_autogen.py +4 -2
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_filter_pipeline.py +25 -9
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_filters.py +14 -130
- opusfilter-3.3.0/tests/test_lid.py +209 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_opusfilter.py +21 -1
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_segment_hash.py +18 -0
- opusfilter-3.1.0/docs/filters/script_and_language_identification_filters.md +0 -50
- {opusfilter-3.1.0 → opusfilter-3.3.0}/.gitignore +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/.pylintrc +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/LICENSE +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-autogen +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-cmd +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-diagram +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-test +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/automatic_configuration.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/conf.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/alignment_model_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/custom_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/language_model_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/length_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/sentence_embedding_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/preprocessing_text.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/training_and_using_classifiers.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/training_language_and_alignment_models.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/using_score_files.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/index.rst +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/bpe_segmentation.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/custom_preprocessors.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/detokenizer.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/reg_exp_sub.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/tokenizer.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/references.rst +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/requirements.txt +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/usage.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/README.md +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/qed_lm_langid.yaml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/__init__.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/autogen_cluster.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/classifier.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/embeddings.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/lm.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/pipeline.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/preprocessors.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/subwords.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/tokenization.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/util.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/word_alignment.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/dependency_links.txt +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/top_level.txt +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/pyproject.toml +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/setup.cfg +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_classifier.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_embeddings.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_lm_filter.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_preprocessors.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_subwords.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_tokenization.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_util.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_wordalign_filter.py +0 -0
- {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_yaml.py +0 -0
|
@@ -21,21 +21,23 @@ jobs:
|
|
|
21
21
|
fail-fast: false
|
|
22
22
|
matrix:
|
|
23
23
|
os: [ubuntu-latest]
|
|
24
|
-
python-version: ["3.8", "3.9", "3.10", "3.11"]
|
|
24
|
+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
25
25
|
requirements-file: ["requirements.txt"]
|
|
26
26
|
runs-on: ${{ matrix.os }}
|
|
27
27
|
steps:
|
|
28
|
-
- uses: actions/checkout@
|
|
28
|
+
- uses: actions/checkout@v4
|
|
29
29
|
- name: Set up Python ${{ matrix.python-version }}
|
|
30
|
-
uses: actions/setup-python@
|
|
30
|
+
uses: actions/setup-python@v5
|
|
31
31
|
with:
|
|
32
32
|
python-version: ${{ matrix.python-version }}
|
|
33
33
|
- name: Install dependencies
|
|
34
34
|
run: |
|
|
35
|
+
python -m ensurepip --upgrade
|
|
36
|
+
python -m pip install --upgrade setuptools
|
|
35
37
|
python -m pip install --upgrade pip
|
|
36
38
|
python -m pip install flake8 pytest wheel
|
|
37
|
-
pip install -r ${{ matrix.requirements-file }}
|
|
38
|
-
python
|
|
39
|
+
python -m pip install --no-cache-dir -r ${{ matrix.requirements-file }}
|
|
40
|
+
python -m pip install .
|
|
39
41
|
- name: Lint with flake8
|
|
40
42
|
run: |
|
|
41
43
|
# stop the build if there are Python syntax errors or undefined names
|
|
@@ -47,9 +49,9 @@ jobs:
|
|
|
47
49
|
build-n-publish:
|
|
48
50
|
name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
|
|
49
51
|
needs: build-n-test
|
|
50
|
-
runs-on: ubuntu-
|
|
52
|
+
runs-on: ubuntu-latest
|
|
51
53
|
steps:
|
|
52
|
-
- uses: actions/checkout@
|
|
54
|
+
- uses: actions/checkout@v4
|
|
53
55
|
- name: Check for main release tag
|
|
54
56
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
55
57
|
id: check-tag
|
|
@@ -59,7 +61,7 @@ jobs:
|
|
|
59
61
|
fi
|
|
60
62
|
- name: Set up Python 3.8
|
|
61
63
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
62
|
-
uses: actions/setup-python@
|
|
64
|
+
uses: actions/setup-python@v5
|
|
63
65
|
with:
|
|
64
66
|
python-version: 3.8
|
|
65
67
|
- name: Install pypa/build
|
|
@@ -94,9 +96,9 @@ jobs:
|
|
|
94
96
|
runs-on: ubuntu-latest
|
|
95
97
|
needs: [build-n-test]
|
|
96
98
|
steps:
|
|
97
|
-
- uses: actions/checkout@
|
|
99
|
+
- uses: actions/checkout@v4
|
|
98
100
|
- name: Set up Python 3.8
|
|
99
|
-
uses: actions/setup-python@
|
|
101
|
+
uses: actions/setup-python@v5
|
|
100
102
|
with:
|
|
101
103
|
python-version: 3.8
|
|
102
104
|
- name: Install dependencies to build docs
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.3.0
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -12,16 +12,15 @@ Requires-Python: >=3.8
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: setuptools
|
|
15
|
-
Requires-Dist: opustools
|
|
15
|
+
Requires-Dist: opustools>=1.6.2
|
|
16
16
|
Requires-Dist: beautifulsoup4>=4.8.0
|
|
17
|
-
Requires-Dist: fasttext
|
|
18
17
|
Requires-Dist: graphviz
|
|
19
|
-
Requires-Dist:
|
|
18
|
+
Requires-Dist: py3langid>=0.2.2
|
|
20
19
|
Requires-Dist: matplotlib
|
|
21
20
|
Requires-Dist: morfessor
|
|
22
|
-
Requires-Dist: opus-fast-mosestokenizer
|
|
21
|
+
Requires-Dist: opus-fast-mosestokenizer==0.0.8.7; python_version < "3.9"
|
|
22
|
+
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.7; python_version >= "3.9"
|
|
23
23
|
Requires-Dist: pandas>=1.0.0
|
|
24
|
-
Requires-Dist: pycld2
|
|
25
24
|
Requires-Dist: xxhash>=3.2.0
|
|
26
25
|
Requires-Dist: sentence-splitter
|
|
27
26
|
Requires-Dist: rapidfuzz
|
|
@@ -31,11 +30,19 @@ Requires-Dist: requests
|
|
|
31
30
|
Requires-Dist: scikit-learn
|
|
32
31
|
Requires-Dist: subword_nmt
|
|
33
32
|
Requires-Dist: tqdm
|
|
34
|
-
Requires-Dist: lingua-language-detector
|
|
33
|
+
Requires-Dist: lingua-language-detector<2.1,>=1.3.0; python_version < "3.10"
|
|
34
|
+
Requires-Dist: lingua-language-detector>=2.1.1; python_version >= "3.10"
|
|
35
|
+
Requires-Dist: iso639-lang
|
|
35
36
|
Provides-Extra: test
|
|
36
37
|
Requires-Dist: pytest; extra == "test"
|
|
38
|
+
Provides-Extra: pycld2
|
|
39
|
+
Requires-Dist: pycld2; extra == "pycld2"
|
|
40
|
+
Provides-Extra: fasttext
|
|
41
|
+
Requires-Dist: py3langid<0.3.0; extra == "fasttext"
|
|
42
|
+
Requires-Dist: numpy<2.0.0; extra == "fasttext"
|
|
43
|
+
Requires-Dist: fasttext; extra == "fasttext"
|
|
37
44
|
Provides-Extra: eflomal
|
|
38
|
-
Requires-Dist: eflomal>=
|
|
45
|
+
Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
|
|
39
46
|
Provides-Extra: jieba
|
|
40
47
|
Requires-Dist: jieba>=0.42; extra == "jieba"
|
|
41
48
|
Provides-Extra: mecab
|
|
@@ -45,13 +52,15 @@ Provides-Extra: laser
|
|
|
45
52
|
Requires-Dist: laserembeddings; extra == "laser"
|
|
46
53
|
Provides-Extra: varikn
|
|
47
54
|
Requires-Dist: varikn; extra == "varikn"
|
|
55
|
+
Provides-Extra: heliport
|
|
56
|
+
Requires-Dist: heliport>=0.10.0; extra == "heliport"
|
|
48
57
|
Provides-Extra: docs
|
|
49
58
|
Requires-Dist: myst-parser; extra == "docs"
|
|
50
59
|
Requires-Dist: sphinx; extra == "docs"
|
|
51
60
|
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
52
61
|
Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
|
|
53
62
|
Provides-Extra: all
|
|
54
|
-
Requires-Dist: eflomal>=
|
|
63
|
+
Requires-Dist: eflomal>=2.0.0; extra == "all"
|
|
55
64
|
Requires-Dist: jieba>=0.42; extra == "all"
|
|
56
65
|
Requires-Dist: mecab-python3>=1.0.8; extra == "all"
|
|
57
66
|
Requires-Dist: unidic-lite; extra == "all"
|
|
@@ -90,7 +99,7 @@ Install from source:
|
|
|
90
99
|
|
|
91
100
|
### Troubleshooting
|
|
92
101
|
|
|
93
|
-
OpusFilter should generally work fine on Python 3.8 to 3.
|
|
102
|
+
OpusFilter should generally work fine on Python 3.8 to 3.13. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
94
103
|
|
|
95
104
|
* `pip install -r requirements.txt`
|
|
96
105
|
|
|
@@ -25,7 +25,7 @@ Install from source:
|
|
|
25
25
|
|
|
26
26
|
### Troubleshooting
|
|
27
27
|
|
|
28
|
-
OpusFilter should generally work fine on Python 3.8 to 3.
|
|
28
|
+
OpusFilter should generally work fine on Python 3.8 to 3.13. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
29
29
|
|
|
30
30
|
* `pip install -r requirements.txt`
|
|
31
31
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import argparse
|
|
4
4
|
import collections
|
|
5
5
|
import logging
|
|
6
|
+
import json
|
|
6
7
|
|
|
7
8
|
from tqdm import tqdm
|
|
8
9
|
|
|
@@ -24,20 +25,29 @@ parser.add_argument('--hash', type=str, default='xxh64',
|
|
|
24
25
|
help=('hash function from xxhash library, empty string '
|
|
25
26
|
'for no hashing (default "xxh64")'))
|
|
26
27
|
parser.add_argument('--letters-only', action='store_true', default=False,
|
|
27
|
-
help='lowercase input strings before hashing')
|
|
28
|
-
parser.add_argument('--lowercase', action='store_true', default=False,
|
|
29
28
|
help='remove all non-letters from intput strings before hashing')
|
|
29
|
+
parser.add_argument('--letter-words-only', action='store_true', default=False,
|
|
30
|
+
help='remove words with non-letter characters before hashing')
|
|
31
|
+
parser.add_argument('--lowercase', action='store_true', default=False,
|
|
32
|
+
help='lowercase input strings before hashing')
|
|
33
|
+
parser.add_argument('--tokenizers', type=str, metavar='JSON', default=None,
|
|
34
|
+
help=('load tokenizer specifications from a JSON list (e.g. \'[["moses", "en"], ["jieba", "zh"]]\'); '
|
|
35
|
+
'use with --letter-words-only'))
|
|
30
36
|
|
|
31
37
|
args = parser.parse_args()
|
|
32
38
|
|
|
33
39
|
if args.overlap and len(args.overlap) != len(args.files):
|
|
34
40
|
raise ValueError("The number of the main and overlap input files should match")
|
|
35
41
|
|
|
42
|
+
tokenizers = json.loads(args.tokenizers) if args.tokenizers else None
|
|
43
|
+
|
|
36
44
|
hasher = SegmentHasher(
|
|
37
45
|
compare='all',
|
|
38
46
|
method=args.hash,
|
|
39
47
|
letters_only=args.letters_only,
|
|
40
|
-
|
|
48
|
+
letter_words_only=args.letter_words_only,
|
|
49
|
+
lowercase=args.lowercase,
|
|
50
|
+
tokenizers=tokenizers
|
|
41
51
|
)
|
|
42
52
|
|
|
43
53
|
total = 0
|
|
@@ -14,8 +14,7 @@ from tqdm import tqdm
|
|
|
14
14
|
|
|
15
15
|
from pandas.plotting import scatter_matrix
|
|
16
16
|
|
|
17
|
-
from opusfilter.util import file_open
|
|
18
|
-
from opusfilter.classifier import lists_to_dicts
|
|
17
|
+
from opusfilter.util import file_open, lists_to_dicts
|
|
19
18
|
|
|
20
19
|
|
|
21
20
|
logger = logging.getLogger(__name__)
|
|
@@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [3.3.0] - 2025-07-02
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `letter_words_only` option for duplicate detection
|
|
15
|
+
- support for selection of possible languages for `lingua` language detection
|
|
16
|
+
- support for language detection based on `heliport` (a port of HeLI-OTS)
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- update library requirements and include Python 3.13 tests
|
|
21
|
+
- removed pycld2 and fasttext from the `[all]` extras
|
|
22
|
+
- refactored all language identification filters to have their own classes and marked `LanguageIDFilter` as deprecated
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- fix broken import in `opusfilter-scores`
|
|
27
|
+
|
|
28
|
+
## [3.2.0] - 2024-08-14
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- make `pycld2` and `fasttext` libraries optional
|
|
33
|
+
- replace `langid.py` library with `py3langid`
|
|
34
|
+
- update github workflows and include Python 3.12 tests
|
|
35
|
+
|
|
36
|
+
### Fixed
|
|
37
|
+
|
|
38
|
+
- `OpusRead` interface using `moses` format (requires `opustools >= 1.6.2`)
|
|
39
|
+
|
|
10
40
|
## [3.1.0] - 2024-06-05
|
|
11
41
|
|
|
12
42
|
### Added
|
|
@@ -204,7 +234,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
204
234
|
First tagged version.
|
|
205
235
|
|
|
206
236
|
|
|
207
|
-
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.
|
|
237
|
+
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.0...develop
|
|
238
|
+
[3.3.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...3.3.0
|
|
239
|
+
[3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
|
|
208
240
|
[3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
|
|
209
241
|
[3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
|
|
210
242
|
[2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
|
|
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
|
|
|
5
5
|
few rules for pull requests:
|
|
6
6
|
|
|
7
7
|
* Make a pull request to the `develop` branch instead of `master`.
|
|
8
|
-
* The code should support at least Python versions from 3.8 to 3.
|
|
8
|
+
* The code should support at least Python versions from 3.8 to 3.13.
|
|
9
9
|
* Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
|
|
10
10
|
* Especially for new features, please include test cases for unit testing.
|
|
11
11
|
|
|
@@ -20,7 +20,7 @@ skips the respective tests if not.)
|
|
|
20
20
|
|
|
21
21
|
GitHub workflows defined in the project run automatically `flake8`
|
|
22
22
|
checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
|
|
23
|
-
and 3.
|
|
23
|
+
3.11, 3.12, and 3.13.
|
|
24
24
|
|
|
25
25
|
Especially for larger contributions, consider using a code analysis
|
|
26
26
|
tool like [Pylint](https://github.com/PyCQA/pylint). Install it
|
|
@@ -30,7 +30,8 @@ number of duplicates in it, or two corpora for calculating the overlap
|
|
|
30
30
|
between them. The syntax for the `opusfilter-duplicates` is:
|
|
31
31
|
|
|
32
32
|
```
|
|
33
|
-
opusfilter-duplicates [--overlap FILE [FILE ...]] [--hash HASH] [--letters-only]
|
|
33
|
+
opusfilter-duplicates [--overlap FILE [FILE ...]] [--hash HASH] [--letters-only]
|
|
34
|
+
[--letter-words-only] [--lowercase] [--tokenizers JSON] FILE [FILE ...]
|
|
34
35
|
```
|
|
35
36
|
|
|
36
37
|
The options are essentially the same as for [`remove_duplicates`](remove_duplicates).
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Script and language identification filters
|
|
2
|
+
|
|
3
|
+
## AlphabetRatioFilter
|
|
4
|
+
|
|
5
|
+
Filter segments based on what proportion of the characters are alphabetic characters.
|
|
6
|
+
|
|
7
|
+
Parameters:
|
|
8
|
+
|
|
9
|
+
* `threshold`: minimum proportion of alphabets in a segment (default 0.75)
|
|
10
|
+
* `exclude_whitespace`: whether to exclude whitespace characters from the ratio (default `false`)
|
|
11
|
+
|
|
12
|
+
Returned scores are proportions of alphabetic characters in the
|
|
13
|
+
segments (after removing whitespace if `exclude_whitespace` is
|
|
14
|
+
true). In filtering, all values have to be equal to or greater than
|
|
15
|
+
the minimum threshold.
|
|
16
|
+
|
|
17
|
+
In order to allow have different thresholds per language, the
|
|
18
|
+
threshold parameter can also be given as a list.
|
|
19
|
+
|
|
20
|
+
## CharacterScoreFilter
|
|
21
|
+
|
|
22
|
+
Filter segments based on what proportion of their alphabetic characters are in a given script. For a list of valid scripts, see e.g. [www.regular-expressions.info/unicode.html](https://www.regular-expressions.info/unicode.html).
|
|
23
|
+
|
|
24
|
+
Parameters:
|
|
25
|
+
|
|
26
|
+
* `scripts`: scripts for input segments
|
|
27
|
+
* `thresholds`: minimum proportion of characters in a script (default 1)
|
|
28
|
+
|
|
29
|
+
Returned scores are proportions of valid characters in the segments. In filtering, all values have to be equal to or greater than the minimum thresholds.
|
|
30
|
+
|
|
31
|
+
## LanguageIDFilter (deprecated)
|
|
32
|
+
|
|
33
|
+
*Note: The common LanguageIDFilter is now deprecated. Please see the method-specific language identification filters below.*
|
|
34
|
+
|
|
35
|
+
Filter segments based on their language identification confidence scores.
|
|
36
|
+
|
|
37
|
+
Parameters:
|
|
38
|
+
|
|
39
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
40
|
+
* `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`, `heliport`; default `langid`)
|
|
41
|
+
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
42
|
+
* `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
|
|
43
|
+
* `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages (valid only for the `langid` and `lingua` methods; default `null`)
|
|
44
|
+
* `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
|
|
45
|
+
* `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
|
|
46
|
+
|
|
47
|
+
Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. A negative threshold can be used to skip filtering for a language.
|
|
48
|
+
|
|
49
|
+
Currently the following identification methods are supported:
|
|
50
|
+
|
|
51
|
+
* `langid` (default) :cite:`lui-baldwin-2012-langid`
|
|
52
|
+
* See https://github.com/adbar/py3langid
|
|
53
|
+
* `lingua`
|
|
54
|
+
* See https://github.com/pemistahl/lingua-py
|
|
55
|
+
* `cld2`
|
|
56
|
+
* See https://github.com/CLD2Owners/cld2
|
|
57
|
+
* Requires [installing optional libraries](../installation.md).
|
|
58
|
+
* `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
|
|
59
|
+
* A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
|
|
60
|
+
* Requires [installing optional libraries](../installation.md).
|
|
61
|
+
|
|
62
|
+
## LangidFilter
|
|
63
|
+
|
|
64
|
+
Filter segments based on the langid method :cite:`lui-baldwin-2012-langid`; see
|
|
65
|
+
https://github.com/adbar/py3langid
|
|
66
|
+
|
|
67
|
+
Parameters:
|
|
68
|
+
|
|
69
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
70
|
+
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
71
|
+
* `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages
|
|
72
|
+
|
|
73
|
+
Returned scores are the language identification confidence scores for
|
|
74
|
+
the segments. The scores range from 0 to 1. In filtering, all values
|
|
75
|
+
have to be greater than the minimum thresholds. A negative threshold
|
|
76
|
+
can be used to skip filtering for a language.
|
|
77
|
+
|
|
78
|
+
## Cld2Filter
|
|
79
|
+
|
|
80
|
+
Filter segments based on the CLD2 method; see https://github.com/CLD2Owners/cld2
|
|
81
|
+
|
|
82
|
+
Requires [installing optional libraries](../installation.md).
|
|
83
|
+
|
|
84
|
+
Parameters:
|
|
85
|
+
|
|
86
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
87
|
+
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
88
|
+
* `options`: a dictionary of options for the method (see https://github.com/CLD2Owners/cld2; default `null`)
|
|
89
|
+
|
|
90
|
+
Returned scores are the language identification confidence scores from
|
|
91
|
+
a given identification method for the segments. The scores range from
|
|
92
|
+
0 to 1. In filtering, all values have to be greater than the minimum
|
|
93
|
+
thresholds. A negative threshold can be used to skip filtering for a
|
|
94
|
+
language.
|
|
95
|
+
|
|
96
|
+
## FastTextFilter
|
|
97
|
+
|
|
98
|
+
Filter segments based on the Fasttext language identification models
|
|
99
|
+
(:cite:`joulin-etal-2016-fasttext`, :cite:`joulin-etal-2017-bag`), see
|
|
100
|
+
https://github.com/CLD2Owners/cld2.
|
|
101
|
+
|
|
102
|
+
Requires [installing optional libraries](../installation.md).
|
|
103
|
+
|
|
104
|
+
Parameters:
|
|
105
|
+
|
|
106
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
107
|
+
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
108
|
+
* `model_path`: path for a `fasttext` model
|
|
109
|
+
|
|
110
|
+
Returned scores are the language identification confidence scores from
|
|
111
|
+
a given identification method for the segments. The scores range from
|
|
112
|
+
0 to 1. In filtering, all values have to be greater than the minimum
|
|
113
|
+
thresholds. A negative threshold can be used to skip filtering for a
|
|
114
|
+
language.
|
|
115
|
+
|
|
116
|
+
## LinguaFilter
|
|
117
|
+
|
|
118
|
+
Filter segments based on the Lingua method; see https://github.com/pemistahl/lingua-py
|
|
119
|
+
|
|
120
|
+
Parameters:
|
|
121
|
+
|
|
122
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
123
|
+
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
124
|
+
* `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages (default `null`)
|
|
125
|
+
* `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode (default `low`)
|
|
126
|
+
|
|
127
|
+
Returned scores are the language identification confidence scores from
|
|
128
|
+
a given identification method for the segments. The scores range from
|
|
129
|
+
0 to 1. In filtering, all values have to be greater than the minimum
|
|
130
|
+
thresholds. A negative threshold can be used to skip filtering for a
|
|
131
|
+
language.
|
|
132
|
+
|
|
133
|
+
## HeliportSimpleFilter
|
|
134
|
+
|
|
135
|
+
Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
|
|
136
|
+
https://github.com/ZJaume/heliport
|
|
137
|
+
|
|
138
|
+
Requires [installing optional libraries](../installation.md).
|
|
139
|
+
|
|
140
|
+
Parameters:
|
|
141
|
+
|
|
142
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
143
|
+
* `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
|
|
144
|
+
|
|
145
|
+
This version uses three distinct score values. Score 1.0 is used if
|
|
146
|
+
the method predicts the correct language. If the low-confidence label
|
|
147
|
+
`und` is returned, the language is considered correct with score
|
|
148
|
+
0.5. Finally, if a wrong language is predicted, the score will
|
|
149
|
+
0.0. A negative threshold can be used to skip filtering for a language.
|
|
150
|
+
|
|
151
|
+
## HeliportConfidenceFilter
|
|
152
|
+
|
|
153
|
+
Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
|
|
154
|
+
https://github.com/ZJaume/heliport
|
|
155
|
+
|
|
156
|
+
Requires [installing optional libraries](../installation.md).
|
|
157
|
+
|
|
158
|
+
Parameters:
|
|
159
|
+
|
|
160
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
161
|
+
* `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
|
|
162
|
+
|
|
163
|
+
This version uses the confidence value returned by the method. It is
|
|
164
|
+
the difference between raw scores of the the best (lowest) and second
|
|
165
|
+
best language. The maximum value is 7.0, which is a penalty value for
|
|
166
|
+
the languages that do not match. The minimum value 0 is returned if
|
|
167
|
+
the best language does not match. The low-confidence label `und` is
|
|
168
|
+
considered as a match. A negative threshold can be used to skip
|
|
169
|
+
filtering for a language.
|
|
170
|
+
|
|
171
|
+
## HeliportRawScoreFilter
|
|
172
|
+
|
|
173
|
+
Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
|
|
174
|
+
https://github.com/ZJaume/heliport
|
|
175
|
+
|
|
176
|
+
Requires [installing optional libraries](../installation.md).
|
|
177
|
+
|
|
178
|
+
Parameters:
|
|
179
|
+
|
|
180
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
181
|
+
* `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
|
|
182
|
+
* `topk`: the number of top languages to consider (default 50)
|
|
183
|
+
|
|
184
|
+
This version uses the raw scores returned by the method. A lower score
|
|
185
|
+
is better. If the correct language is among the top-k languages, the
|
|
186
|
+
score if that language is returned. Otherwise, the penalty value 7 is
|
|
187
|
+
returned. A threshold over the penalty value can be used to skip
|
|
188
|
+
filtering for a language.
|
|
189
|
+
|
|
190
|
+
## HeliportProbabilityFilter
|
|
191
|
+
|
|
192
|
+
Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
|
|
193
|
+
https://github.com/ZJaume/heliport
|
|
194
|
+
|
|
195
|
+
Requires [installing optional libraries](../installation.md).
|
|
196
|
+
|
|
197
|
+
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
198
|
+
* `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
|
|
199
|
+
* `topk`: the number of top languages to consider (default 50)
|
|
200
|
+
|
|
201
|
+
This version transforms the raw scores of of the top-k languages into
|
|
202
|
+
probability-like values. If the correct language is not in the top-k
|
|
203
|
+
list, zero is returned. A negative threshold can be used to skip
|
|
204
|
+
filtering for a language.
|
|
@@ -11,11 +11,16 @@ Parameters:
|
|
|
11
11
|
* `source_language`: language code for the source language
|
|
12
12
|
* `target_language`: language code for the target language
|
|
13
13
|
* `release`: version of the corpus in OPUS
|
|
14
|
-
* `preprocessing`: `raw` for untokenized and `xml` for tokenized segments
|
|
14
|
+
* `preprocessing`: `moses` or `raw` for untokenized and `xml` for tokenized segments
|
|
15
15
|
* `src_output`: output file for source language
|
|
16
16
|
* `tgt_output`: output file for target language
|
|
17
17
|
* `suppress_prompts`: `false` (default) prompts user to confirm before download, `true` to download without prompting
|
|
18
18
|
|
|
19
|
+
The `moses` preprocessing type (available with `OpusTools` version
|
|
20
|
+
1.6.2 and above) is recommended for those corpora for which it
|
|
21
|
+
exists. The output is equivalent to `raw`, but in some cases it can
|
|
22
|
+
significantly reduce the amount of data downloaded in the process.
|
|
23
|
+
|
|
19
24
|
## concatenate
|
|
20
25
|
|
|
21
26
|
Concatenate two or more text files.
|
|
@@ -11,6 +11,10 @@ Parameters:
|
|
|
11
11
|
* `compare`: select files for duplicate comparison (optional; default `all` or a list of indices)
|
|
12
12
|
* `hash`: select hash algorithm from xxhash (optional; default `xxh64`)
|
|
13
13
|
* `overlap`: remove overlap with a second set of files (optional; default `null`)
|
|
14
|
+
* `letters_only`: remove all non-letters from intput strings before hashing (optional; default `false`)
|
|
15
|
+
* `letter_words_only`: remove words with non-letter characters before hashing (optional; default `false`)
|
|
16
|
+
* `lowercase`: lowercase input strings before hashing (optional; default `false`)
|
|
17
|
+
* `tokenizers`: load tokenizer specifications from a list (optional; use with `letter_words_only`)
|
|
14
18
|
|
|
15
19
|
Duplicate filtering is recommended as a first step especially if you
|
|
16
20
|
combine different corpus collections (e.g. data crawled from web) and
|
|
@@ -38,6 +42,10 @@ sizes, but if do not care about memory use and want to be extra sure
|
|
|
38
42
|
there are no collisions, you can disable hashing by setting the `hash`
|
|
39
43
|
parameter as empty string or null.
|
|
40
44
|
|
|
45
|
+
For the `letter_words_only`, unless the input is pre-tokenized, you
|
|
46
|
+
should select tokenizers for each of the input files. See
|
|
47
|
+
[Tokenizer](tokenizer.md) for description of the tokenizer parameters.
|
|
48
|
+
|
|
41
49
|
## filter
|
|
42
50
|
|
|
43
51
|
Filter parallel data with a combination of filters.
|
|
@@ -12,20 +12,18 @@ Install from source:
|
|
|
12
12
|
|
|
13
13
|
Note that all required libraries are not available to install via PyPI
|
|
14
14
|
on Windows OS. On Linux and MacOS, it should work directly for Python
|
|
15
|
-
versions from 3.8 to 3.
|
|
15
|
+
versions from 3.8 to 3.13.
|
|
16
16
|
|
|
17
17
|
## Required libraries
|
|
18
18
|
|
|
19
19
|
* beautifulsoup4
|
|
20
20
|
* opus-fast-mosestokenizer
|
|
21
|
-
* fasttext
|
|
22
21
|
* graphviz
|
|
23
|
-
*
|
|
22
|
+
* py3langid
|
|
24
23
|
* matplotlib
|
|
25
24
|
* morfessor
|
|
26
25
|
* OpusTools
|
|
27
26
|
* pandas
|
|
28
|
-
* pycld2
|
|
29
27
|
* rapidfuzz
|
|
30
28
|
* ruamel.yaml
|
|
31
29
|
* regex
|
|
@@ -41,24 +39,48 @@ See `setup.py` for possible version requirements.
|
|
|
41
39
|
|
|
42
40
|
## Optional libraries and tools
|
|
43
41
|
|
|
42
|
+
### FastText, PyCLD2, and heliport language identification
|
|
43
|
+
|
|
44
|
+
The language identification libraries currently supported out-of-the-box
|
|
45
|
+
are [py3langid](https://github.com/adbar/py3langid) and
|
|
46
|
+
[lingua](https://github.com/pemistahl/lingua-py). The support for
|
|
47
|
+
[PyCLD2](https://github.com/aboSamoor/pycld2) and
|
|
48
|
+
[FastText models](https://fasttext.cc/docs/en/language-identification.html)
|
|
49
|
+
have been changed to optional due to the lack of support especially
|
|
50
|
+
for newer Python versions. Also the support for
|
|
51
|
+
[heliport](https://github.com/ZJaume/heliport) is optional. Please
|
|
52
|
+
note that `heliport` has GPL-3.0 license.
|
|
53
|
+
|
|
54
|
+
The PyCLD2 support can be installed automatically with pip by
|
|
55
|
+
including the extras `[pycld2]` (e.g. `pip install
|
|
56
|
+
opusfilter[pycld2]`).
|
|
57
|
+
|
|
58
|
+
The support for FastText models can be installed automatically with
|
|
59
|
+
pip by including the extras `[fasttext]` (e.g. `pip install
|
|
60
|
+
opusfilter[fasttext]`).
|
|
61
|
+
|
|
62
|
+
The support for `heliport` can be installed automatically with pip by
|
|
63
|
+
including the extras `[heliport]` (e.g. `pip install
|
|
64
|
+
opusfilter[heliport]`).
|
|
65
|
+
|
|
44
66
|
### Jieba and MeCab word segmentation
|
|
45
67
|
|
|
46
68
|
For Chinese tokenization (word segmentation), you can use the
|
|
47
69
|
[jieba](https://github.com/fxsjy/jieba) library. It can be installed
|
|
48
70
|
automatically with pip by including the extras `[jieba]` or `[all]`
|
|
49
|
-
(e.g. `pip install opusfilter[
|
|
71
|
+
(e.g. `pip install opusfilter[jieba]`).
|
|
50
72
|
|
|
51
73
|
For Japanese tokenization (word segmentation), you can use the
|
|
52
74
|
[MeCab](https://github.com/SamuraiT/mecab-python3) library. It can be installed
|
|
53
75
|
automatically with pip by including the extras `[mecab]` or `[all]`
|
|
54
|
-
(e.g. `pip install opusfilter[
|
|
76
|
+
(e.g. `pip install opusfilter[mecab]`).
|
|
55
77
|
|
|
56
78
|
### LASER sentence embeddings
|
|
57
79
|
|
|
58
80
|
For using sentence embeddings filters, you need to install
|
|
59
81
|
`laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
|
|
60
82
|
be installed automatically with pip by including the extras `[laser]`
|
|
61
|
-
or `[all]` (e.g. `pip install opusfilter[
|
|
83
|
+
or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
|
|
62
84
|
require a number of additional libraries, including PyTorch, jieba,
|
|
63
85
|
and MeCab. Note that you need also to download the prebuild models
|
|
64
86
|
with `python -m laserembeddings download-models`.
|
|
@@ -68,12 +90,12 @@ with `python -m laserembeddings download-models`.
|
|
|
68
90
|
For using n-gram language model filters, you need to install the
|
|
69
91
|
Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
|
|
70
92
|
be installed automatically with pip by including the extras `[varikn]`
|
|
71
|
-
or `[all]` (e.g. `pip install opusfilter[
|
|
93
|
+
or `[all]` (e.g. `pip install opusfilter[varikn]`).
|
|
72
94
|
|
|
73
95
|
### Eflomal word alignment
|
|
74
96
|
|
|
75
97
|
For using word alignment filters, you need to install elfomal
|
|
76
98
|
(https://github.com/robertostling/eflomal). It can be installed
|
|
77
99
|
automatically with pip by including the extras `[eflomal]` or `[all]`
|
|
78
|
-
(e.g. `pip install opusfilter[
|
|
100
|
+
(e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
|
|
79
101
|
for the installation.
|
|
@@ -68,6 +68,19 @@
|
|
|
68
68
|
pages = "261--266"
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
% Jauhiainen et al. (2022)
|
|
72
|
+
@inproceedings{jauhiainen-etal-2022-heli,
|
|
73
|
+
title = "{H}e{LI}-{OTS}, Off-the-shelf Language Identifier for Text",
|
|
74
|
+
author = "Jauhiainen, Tommi and Jauhiainen, Heidi and Lind{\'e}n, Krister",
|
|
75
|
+
booktitle = "Proceedings of the Thirteenth Language Resources and Evaluation Conference",
|
|
76
|
+
month = jun,
|
|
77
|
+
year = "2022",
|
|
78
|
+
address = "Marseille, France",
|
|
79
|
+
publisher = "European Language Resources Association",
|
|
80
|
+
url = "https://aclanthology.org/2022.lrec-1.416/",
|
|
81
|
+
pages = "3912--3922"
|
|
82
|
+
}
|
|
83
|
+
|
|
71
84
|
% Joulin et al. (2016)
|
|
72
85
|
@article{joulin-etal-2016-fasttext,
|
|
73
86
|
author = {Armand Joulin and Edouard Grave and Piotr Bojanowski and Matthijs Douze and Herv{\'{e}} J{\'{e}}gou and Tom{\'{a}}s Mikolov},
|
|
@@ -217,7 +217,7 @@ class DefaultParameterFilters(AutoFiltersABC):
|
|
|
217
217
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
218
218
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
219
219
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
220
|
-
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': '
|
|
220
|
+
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
|
|
221
221
|
|
|
222
222
|
def set_filter_thresholds(self):
|
|
223
223
|
"""Set filter thresholds"""
|
|
@@ -272,7 +272,7 @@ class PercentileFilters(DataBasedFiltersABC):
|
|
|
272
272
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
273
273
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
274
274
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
275
|
-
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': '
|
|
275
|
+
'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
|
|
276
276
|
|
|
277
277
|
def __init__(self, files, excluded_percentile=0.001, **kwargs):
|
|
278
278
|
super().__init__(files, **kwargs)
|
|
@@ -512,7 +512,7 @@ class ClusterFilters(DataBasedFiltersABC):
|
|
|
512
512
|
('LengthRatioFilter.word', {'unit': 'word'}),
|
|
513
513
|
'NonZeroNumeralsFilter',
|
|
514
514
|
'CharacterScoreFilter',
|
|
515
|
-
('LanguageIDFilter', {'id_method': '
|
|
515
|
+
('LanguageIDFilter', {'id_method': 'lingua'}),
|
|
516
516
|
'TerminalPunctuationFilter']
|
|
517
517
|
|
|
518
518
|
def __init__(self, files, k=2, max_length=150, **kwargs):
|