opusfilter 3.3.0__tar.gz → 3.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opusfilter-3.3.0/opusfilter.egg-info → opusfilter-3.3.1}/PKG-INFO +2 -3
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/CHANGELOG.md +13 -1
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/script_and_language_identification_filters.md +20 -20
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/special_character_and_similarity_filters.md +6 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/filtering_and_scoring.md +2 -1
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/installation.md +9 -3
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/autogen.py +11 -2
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/filters.py +1 -1
- {opusfilter-3.3.0 → opusfilter-3.3.1/opusfilter.egg-info}/PKG-INFO +2 -3
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/requires.txt +1 -6
- {opusfilter-3.3.0 → opusfilter-3.3.1}/setup.py +1 -2
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_filters.py +2 -12
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_lid.py +4 -1
- {opusfilter-3.3.0 → opusfilter-3.3.1}/.github/workflows/ci.yml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/.gitignore +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/.pylintrc +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/LICENSE +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/README.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-autogen +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-cmd +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-diagram +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-duplicates +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-scores +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-test +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/CONTRIBUTING.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/automatic_configuration.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/command_line_tools.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/conf.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/alignment_model_filters.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/custom_filters.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/language_model_filters.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/length_filters.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/sentence_embedding_filters.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/downloading_and_selecting_data.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/preprocessing_text.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/training_and_using_classifiers.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/training_language_and_alignment_models.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/using_score_files.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/index.rst +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/bpe_segmentation.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/custom_preprocessors.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/detokenizer.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/morfessor_segmentation.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/reg_exp_sub.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/tokenizer.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/whitespaceNormalizer.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/references.bib +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/references.rst +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/requirements.txt +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/usage.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/README.md +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/qed_lm_langid.yaml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/__init__.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/autogen_cluster.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/classifier.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/embeddings.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/lid.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/lm.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/opusfilter.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/pipeline.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/preprocessors.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/segment_hash.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/subwords.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/tokenization.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/util.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/word_alignment.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/SOURCES.txt +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/dependency_links.txt +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/top_level.txt +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/pyproject.toml +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/requirements.txt +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/setup.cfg +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_autogen.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_classifier.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_embeddings.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_filter_pipeline.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_lm_filter.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_opusfilter.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_preprocessors.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_segment_hash.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_subwords.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_tokenization.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_util.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_wordalign_filter.py +0 -0
- {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_yaml.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.3.
|
|
3
|
+
Version: 3.3.1
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -18,8 +18,7 @@ Requires-Dist: graphviz
|
|
|
18
18
|
Requires-Dist: py3langid>=0.2.2
|
|
19
19
|
Requires-Dist: matplotlib
|
|
20
20
|
Requires-Dist: morfessor
|
|
21
|
-
Requires-Dist: opus-fast-mosestokenizer
|
|
22
|
-
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.7; python_version >= "3.9"
|
|
21
|
+
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.11
|
|
23
22
|
Requires-Dist: pandas>=1.0.0
|
|
24
23
|
Requires-Dist: xxhash>=3.2.0
|
|
25
24
|
Requires-Dist: sentence-splitter
|
|
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [3.3.1] - 2025-12-10
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- changed `HtmlTagFilter` to accept broken tags in order to avoid inconsistent results
|
|
15
|
+
- update required `opus-fast-mosestokenizer` version for better platform support
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- broken citations and links in the documentation
|
|
20
|
+
|
|
10
21
|
## [3.3.0] - 2025-07-02
|
|
11
22
|
|
|
12
23
|
### Added
|
|
@@ -234,7 +245,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
234
245
|
First tagged version.
|
|
235
246
|
|
|
236
247
|
|
|
237
|
-
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.
|
|
248
|
+
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.1...develop
|
|
249
|
+
[3.3.1]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.0...3.3.1
|
|
238
250
|
[3.3.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...3.3.0
|
|
239
251
|
[3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
|
|
240
252
|
[3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/script_and_language_identification_filters.md
RENAMED
|
@@ -48,21 +48,21 @@ Returned scores are the language identification confidence scores from a given i
|
|
|
48
48
|
|
|
49
49
|
Currently the following identification methods are supported:
|
|
50
50
|
|
|
51
|
-
* `langid` (default) :
|
|
52
|
-
* See https://github.com/adbar/py3langid
|
|
51
|
+
* `langid` (default) {cite:p}`lui-baldwin-2012-langid`
|
|
52
|
+
* See <https://github.com/adbar/py3langid>
|
|
53
53
|
* `lingua`
|
|
54
|
-
* See https://github.com/pemistahl/lingua-py
|
|
54
|
+
* See <https://github.com/pemistahl/lingua-py>
|
|
55
55
|
* `cld2`
|
|
56
|
-
* See https://github.com/CLD2Owners/cld2
|
|
56
|
+
* See <https://github.com/CLD2Owners/cld2>
|
|
57
57
|
* Requires [installing optional libraries](../installation.md).
|
|
58
|
-
* `fasttext` :
|
|
58
|
+
* `fasttext` {cite:p}`joulin-etal-2016-fasttext` and {cite:p}`joulin-etal-2017-bag`
|
|
59
59
|
* A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
|
|
60
60
|
* Requires [installing optional libraries](../installation.md).
|
|
61
61
|
|
|
62
62
|
## LangidFilter
|
|
63
63
|
|
|
64
|
-
Filter segments based on the langid method :
|
|
65
|
-
https://github.com/adbar/py3langid
|
|
64
|
+
Filter segments based on the langid method {cite:p}`lui-baldwin-2012-langid`; see
|
|
65
|
+
<https://github.com/adbar/py3langid>.
|
|
66
66
|
|
|
67
67
|
Parameters:
|
|
68
68
|
|
|
@@ -77,7 +77,7 @@ can be used to skip filtering for a language.
|
|
|
77
77
|
|
|
78
78
|
## Cld2Filter
|
|
79
79
|
|
|
80
|
-
Filter segments based on the CLD2 method; see https://github.com/CLD2Owners/cld2
|
|
80
|
+
Filter segments based on the CLD2 method; see <https://github.com/CLD2Owners/cld2>.
|
|
81
81
|
|
|
82
82
|
Requires [installing optional libraries](../installation.md).
|
|
83
83
|
|
|
@@ -85,7 +85,7 @@ Parameters:
|
|
|
85
85
|
|
|
86
86
|
* `languages`: expected languages (ISO639 language codes) for the segments
|
|
87
87
|
* `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
|
|
88
|
-
* `options`: a dictionary of options for the method (see https://github.com/CLD2Owners/cld2
|
|
88
|
+
* `options`: a dictionary of options for the method (see <https://github.com/CLD2Owners/cld2>; default `null`)
|
|
89
89
|
|
|
90
90
|
Returned scores are the language identification confidence scores from
|
|
91
91
|
a given identification method for the segments. The scores range from
|
|
@@ -96,8 +96,8 @@ language.
|
|
|
96
96
|
## FastTextFilter
|
|
97
97
|
|
|
98
98
|
Filter segments based on the Fasttext language identification models
|
|
99
|
-
(:
|
|
100
|
-
https://github.com/CLD2Owners/cld2
|
|
99
|
+
({cite:t}`joulin-etal-2016-fasttext`, {cite:t}`joulin-etal-2017-bag`), see
|
|
100
|
+
<https://github.com/CLD2Owners/cld2>.
|
|
101
101
|
|
|
102
102
|
Requires [installing optional libraries](../installation.md).
|
|
103
103
|
|
|
@@ -115,7 +115,7 @@ language.
|
|
|
115
115
|
|
|
116
116
|
## LinguaFilter
|
|
117
117
|
|
|
118
|
-
Filter segments based on the Lingua method; see https://github.com/pemistahl/lingua-py
|
|
118
|
+
Filter segments based on the Lingua method; see <https://github.com/pemistahl/lingua-py>.
|
|
119
119
|
|
|
120
120
|
Parameters:
|
|
121
121
|
|
|
@@ -132,8 +132,8 @@ language.
|
|
|
132
132
|
|
|
133
133
|
## HeliportSimpleFilter
|
|
134
134
|
|
|
135
|
-
Filter segments based on the HeLI-OTS method :
|
|
136
|
-
https://github.com/ZJaume/heliport
|
|
135
|
+
Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
|
|
136
|
+
<https://github.com/ZJaume/heliport>.
|
|
137
137
|
|
|
138
138
|
Requires [installing optional libraries](../installation.md).
|
|
139
139
|
|
|
@@ -150,8 +150,8 @@ the method predicts the correct language. If the low-confidence label
|
|
|
150
150
|
|
|
151
151
|
## HeliportConfidenceFilter
|
|
152
152
|
|
|
153
|
-
Filter segments based on the HeLI-OTS method :
|
|
154
|
-
https://github.com/ZJaume/heliport
|
|
153
|
+
Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
|
|
154
|
+
<https://github.com/ZJaume/heliport>.
|
|
155
155
|
|
|
156
156
|
Requires [installing optional libraries](../installation.md).
|
|
157
157
|
|
|
@@ -170,8 +170,8 @@ filtering for a language.
|
|
|
170
170
|
|
|
171
171
|
## HeliportRawScoreFilter
|
|
172
172
|
|
|
173
|
-
Filter segments based on the HeLI-OTS method :
|
|
174
|
-
https://github.com/ZJaume/heliport
|
|
173
|
+
Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
|
|
174
|
+
<https://github.com/ZJaume/heliport>.
|
|
175
175
|
|
|
176
176
|
Requires [installing optional libraries](../installation.md).
|
|
177
177
|
|
|
@@ -189,8 +189,8 @@ filtering for a language.
|
|
|
189
189
|
|
|
190
190
|
## HeliportProbabilityFilter
|
|
191
191
|
|
|
192
|
-
Filter segments based on the HeLI-OTS method :
|
|
193
|
-
https://github.com/ZJaume/heliport
|
|
192
|
+
Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
|
|
193
|
+
<https://github.com/ZJaume/heliport>.
|
|
194
194
|
|
|
195
195
|
Requires [installing optional libraries](../installation.md).
|
|
196
196
|
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/special_character_and_similarity_filters.md
RENAMED
|
@@ -8,6 +8,12 @@ The returned scores are two boolean values indicating whether the
|
|
|
8
8
|
segments contain HTML tags. In filtering, a segment pair is accepted
|
|
9
9
|
if none of the segments contains HTML tags.
|
|
10
10
|
|
|
11
|
+
The beautifulsoup4 library is used for detecting HTML. Up to
|
|
12
|
+
OpusFilter version 3.3.0, the behavior on broken HTML tags varies
|
|
13
|
+
depending on the input and installed library versions. For newer
|
|
14
|
+
versions, broken HTML is not considered as HTML, i.e., it is accepted
|
|
15
|
+
by the filter.
|
|
16
|
+
|
|
11
17
|
## TerminalPunctuationFilter
|
|
12
18
|
|
|
13
19
|
Filter segments based on a penalty score with respect to the
|
|
@@ -44,7 +44,8 @@ parameter as empty string or null.
|
|
|
44
44
|
|
|
45
45
|
For the `letter_words_only`, unless the input is pre-tokenized, you
|
|
46
46
|
should select tokenizers for each of the input files. See
|
|
47
|
-
[Tokenizer](tokenizer.md) for description of the
|
|
47
|
+
[Tokenizer](../preprocessors/tokenizer.md) for description of the
|
|
48
|
+
tokenizer parameters.
|
|
48
49
|
|
|
49
50
|
## filter
|
|
50
51
|
|
|
@@ -78,24 +78,30 @@ automatically with pip by including the extras `[mecab]` or `[all]`
|
|
|
78
78
|
### LASER sentence embeddings
|
|
79
79
|
|
|
80
80
|
For using sentence embeddings filters, you need to install
|
|
81
|
-
`laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
|
|
81
|
+
`laserembeddings` (<https://github.com/yannvgn/laserembeddings>). It can
|
|
82
82
|
be installed automatically with pip by including the extras `[laser]`
|
|
83
83
|
or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
|
|
84
84
|
require a number of additional libraries, including PyTorch, jieba,
|
|
85
85
|
and MeCab. Note that you need also to download the prebuild models
|
|
86
86
|
with `python -m laserembeddings download-models`.
|
|
87
87
|
|
|
88
|
+
**Note:** The current latest release `laserembeddings==1.1.2` does not
|
|
89
|
+
support `torch>=2.0.0` and thus fails on Python 3.12 and above. The
|
|
90
|
+
code itself works on Python 3.12 and 3.13, but you need to fix
|
|
91
|
+
`pyproject.toml` to allow later `torch` versions, and install it
|
|
92
|
+
manually.
|
|
93
|
+
|
|
88
94
|
### VariKN n-gram models
|
|
89
95
|
|
|
90
96
|
For using n-gram language model filters, you need to install the
|
|
91
|
-
Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
|
|
97
|
+
Python wrapper for VariKN (<https://github.com/vsiivola/variKN>). It can
|
|
92
98
|
be installed automatically with pip by including the extras `[varikn]`
|
|
93
99
|
or `[all]` (e.g. `pip install opusfilter[varikn]`).
|
|
94
100
|
|
|
95
101
|
### Eflomal word alignment
|
|
96
102
|
|
|
97
103
|
For using word alignment filters, you need to install elfomal
|
|
98
|
-
(https://github.com/robertostling/eflomal). It can be installed
|
|
104
|
+
(<https://github.com/robertostling/eflomal>). It can be installed
|
|
99
105
|
automatically with pip by including the extras `[eflomal]` or `[all]`
|
|
100
106
|
(e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
|
|
101
107
|
for the installation.
|
|
@@ -14,6 +14,7 @@ import ruamel.yaml
|
|
|
14
14
|
from . import CLEAN_LOW, CLEAN_HIGH, CLEAN_BETWEEN, CLEAN_TRUE, CLEAN_FALSE
|
|
15
15
|
from . import OpusFilterError, ConfigurationError
|
|
16
16
|
from . import filters as filtermodule
|
|
17
|
+
from . import lid as lidmodule
|
|
17
18
|
from .autogen_cluster import ScoreClusters
|
|
18
19
|
from .classifier import load_dataframe
|
|
19
20
|
from .opusfilter import OpusFilter
|
|
@@ -171,6 +172,13 @@ def parse_filter_specs(specs):
|
|
|
171
172
|
return name, params
|
|
172
173
|
|
|
173
174
|
|
|
175
|
+
def get_lang_id_filters():
|
|
176
|
+
"""Return all language identification filters"""
|
|
177
|
+
names = ['LanguageIDFilter'] # old deprecated filter
|
|
178
|
+
names += [var for var in dir(lidmodule) if var.endswith('Filter')]
|
|
179
|
+
return names
|
|
180
|
+
|
|
181
|
+
|
|
174
182
|
class AutoFiltersABC(metaclass=abc.ABCMeta):
|
|
175
183
|
"""Abstract base class for automatic filter configuration"""
|
|
176
184
|
|
|
@@ -181,13 +189,14 @@ class AutoFiltersABC(metaclass=abc.ABCMeta):
|
|
|
181
189
|
filters = self.DEFAULT_FILTERS
|
|
182
190
|
filters = [parse_filter_specs(spec) for spec in filters]
|
|
183
191
|
self.filters_to_add = []
|
|
192
|
+
lang_id_filters = get_lang_id_filters()
|
|
184
193
|
for filter_name, filter_params in filters:
|
|
185
194
|
if filter_name == 'CharacterScoreFilter' and 'scripts' not in filter_params:
|
|
186
195
|
if not scripts:
|
|
187
196
|
logger.warning('Cannot add CharacterScoreFilter (no scripts provided)')
|
|
188
197
|
continue
|
|
189
198
|
filter_params['scripts'] = scripts
|
|
190
|
-
if filter_name
|
|
199
|
+
if filter_name in lang_id_filters and 'languages' not in filter_params:
|
|
191
200
|
if not langs:
|
|
192
201
|
logger.warning('Cannot add LanguageIDFilter (no languages provided)')
|
|
193
202
|
continue
|
|
@@ -217,7 +226,7 @@ class DefaultParameterFilters(AutoFiltersABC):
|
|
|
217
226
|
'AverageWordLengthFilter', 'AlphabetRatioFilter',
|
|
218
227
|
'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
|
|
219
228
|
'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
|
|
220
|
-
'CharacterScoreFilter',
|
|
229
|
+
'CharacterScoreFilter', 'LinguaFilter']
|
|
221
230
|
|
|
222
231
|
def set_filter_thresholds(self):
|
|
223
232
|
"""Set filter thresholds"""
|
|
@@ -164,7 +164,7 @@ class HtmlTagFilter(FilterABC):
|
|
|
164
164
|
except (TypeError, UnboundLocalError, NotImplementedError,
|
|
165
165
|
AssertionError, bs4.builder.ParserRejectedMarkup) as err:
|
|
166
166
|
logger.warning("BeautifulSoup parsing failed for %s: %s", repr(segment), err)
|
|
167
|
-
found =
|
|
167
|
+
found = False
|
|
168
168
|
return found
|
|
169
169
|
|
|
170
170
|
def score(self, pairs):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.3.
|
|
3
|
+
Version: 3.3.1
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -18,8 +18,7 @@ Requires-Dist: graphviz
|
|
|
18
18
|
Requires-Dist: py3langid>=0.2.2
|
|
19
19
|
Requires-Dist: matplotlib
|
|
20
20
|
Requires-Dist: morfessor
|
|
21
|
-
Requires-Dist: opus-fast-mosestokenizer
|
|
22
|
-
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.7; python_version >= "3.9"
|
|
21
|
+
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.11
|
|
23
22
|
Requires-Dist: pandas>=1.0.0
|
|
24
23
|
Requires-Dist: xxhash>=3.2.0
|
|
25
24
|
Requires-Dist: sentence-splitter
|
|
@@ -5,6 +5,7 @@ graphviz
|
|
|
5
5
|
py3langid>=0.2.2
|
|
6
6
|
matplotlib
|
|
7
7
|
morfessor
|
|
8
|
+
opus-fast-mosestokenizer>=0.0.8.11
|
|
8
9
|
pandas>=1.0.0
|
|
9
10
|
xxhash>=3.2.0
|
|
10
11
|
sentence-splitter
|
|
@@ -20,15 +21,9 @@ iso639-lang
|
|
|
20
21
|
[:python_version < "3.10"]
|
|
21
22
|
lingua-language-detector<2.1,>=1.3.0
|
|
22
23
|
|
|
23
|
-
[:python_version < "3.9"]
|
|
24
|
-
opus-fast-mosestokenizer==0.0.8.7
|
|
25
|
-
|
|
26
24
|
[:python_version >= "3.10"]
|
|
27
25
|
lingua-language-detector>=2.1.1
|
|
28
26
|
|
|
29
|
-
[:python_version >= "3.9"]
|
|
30
|
-
opus-fast-mosestokenizer>=0.0.8.7
|
|
31
|
-
|
|
32
27
|
[all]
|
|
33
28
|
eflomal>=2.0.0
|
|
34
29
|
jieba>=0.42
|
|
@@ -11,8 +11,7 @@ install_requires = [
|
|
|
11
11
|
"py3langid>=0.2.2",
|
|
12
12
|
"matplotlib",
|
|
13
13
|
"morfessor",
|
|
14
|
-
'opus-fast-mosestokenizer
|
|
15
|
-
'opus-fast-mosestokenizer>=0.0.8.7 ; python_version >= "3.9"',
|
|
14
|
+
'opus-fast-mosestokenizer>=0.0.8.11',
|
|
16
15
|
"pandas>=1.0.0",
|
|
17
16
|
"xxhash>=3.2.0",
|
|
18
17
|
"sentence-splitter",
|
|
@@ -104,20 +104,10 @@ class TestHtmlTagFilter(unittest.TestCase):
|
|
|
104
104
|
def test_broken(self):
|
|
105
105
|
testfilter = HtmlTagFilter()
|
|
106
106
|
cases = [['<aaa bee', 'cee dee'], ['aa br> bee', 'cee deee'],
|
|
107
|
-
['<p aaa bee</p', 'p>cee dee /p>'], ['', '']]
|
|
107
|
+
['<p aaa bee</p', 'p>cee dee /p>'], ['<![ foo', 'foo'], ['', '']]
|
|
108
108
|
expected = [([False, False], True), ([False, False], True),
|
|
109
109
|
([False, False], True), ([False, False], True),
|
|
110
|
-
([True, False], False)]
|
|
111
|
-
results = [(x, testfilter.accept(x)) for x in testfilter.score(cases)]
|
|
112
|
-
logging.warning(results)
|
|
113
|
-
for result, correct in zip(results, expected):
|
|
114
|
-
self.assertSequenceEqual(result, correct)
|
|
115
|
-
|
|
116
|
-
@pytest.mark.skipif(sys.version_info >= (3, 13), reason="different output for py3.13")
|
|
117
|
-
def test_broken_exception(self):
|
|
118
|
-
testfilter = HtmlTagFilter()
|
|
119
|
-
cases = [['<![ foo', 'foo']]
|
|
120
|
-
expected = [([True, False], False)]
|
|
110
|
+
([False, False], True), ([True, False], False)]
|
|
121
111
|
results = [(x, testfilter.accept(x)) for x in testfilter.score(cases)]
|
|
122
112
|
logging.warning(results)
|
|
123
113
|
for result, correct in zip(results, expected):
|
|
@@ -93,7 +93,7 @@ class TestFasttext(TestLangIDMethod):
|
|
|
93
93
|
def setUpClass(self):
|
|
94
94
|
self.tempdir = tempfile.mkdtemp()
|
|
95
95
|
if 'fasttext' not in globals():
|
|
96
|
-
|
|
96
|
+
return
|
|
97
97
|
self.testmodel = os.path.join(self.tempdir, 'model.ftz')
|
|
98
98
|
try:
|
|
99
99
|
file_download(self.model_url, self.testmodel)
|
|
@@ -104,11 +104,13 @@ class TestFasttext(TestLangIDMethod):
|
|
|
104
104
|
def tearDownClass(self):
|
|
105
105
|
shutil.rmtree(self.tempdir)
|
|
106
106
|
|
|
107
|
+
@unittest.skipIf('fasttext' not in globals(), 'fasttext not installed')
|
|
107
108
|
def test_missing_model(self):
|
|
108
109
|
with self.assertRaises(ConfigurationError):
|
|
109
110
|
model = FastTextFilter(
|
|
110
111
|
languages=['en', 'fr'], thresholds=[0.8, 0.99])
|
|
111
112
|
|
|
113
|
+
@unittest.skipIf('fasttext' not in globals(), 'fasttext not installed')
|
|
112
114
|
def test_fasttext_predict_lang(self):
|
|
113
115
|
if self.testmodel is None:
|
|
114
116
|
self.skipTest("Failed to download test resources")
|
|
@@ -120,6 +122,7 @@ class TestFasttext(TestLangIDMethod):
|
|
|
120
122
|
for fasttext_input in self.fasttext_inputs]
|
|
121
123
|
self.assertSequenceEqual(expected, results)
|
|
122
124
|
|
|
125
|
+
@unittest.skipIf('fasttext' not in globals(), 'fasttext not installed')
|
|
123
126
|
def test_accept(self):
|
|
124
127
|
if self.testmodel is None:
|
|
125
128
|
self.skipTest("Failed to download test resources")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/training_language_and_alignment_models.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_domain_sets.yaml
RENAMED
|
File without changes
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml
RENAMED
|
File without changes
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl
RENAMED
|
File without changes
|
{opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|