opusfilter 3.3.0__tar.gz → 3.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {opusfilter-3.3.0/opusfilter.egg-info → opusfilter-3.3.1}/PKG-INFO +2 -3
  2. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/CHANGELOG.md +13 -1
  3. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/script_and_language_identification_filters.md +20 -20
  4. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/special_character_and_similarity_filters.md +6 -0
  5. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/filtering_and_scoring.md +2 -1
  6. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/installation.md +9 -3
  7. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/autogen.py +11 -2
  8. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/filters.py +1 -1
  9. {opusfilter-3.3.0 → opusfilter-3.3.1/opusfilter.egg-info}/PKG-INFO +2 -3
  10. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/requires.txt +1 -6
  11. {opusfilter-3.3.0 → opusfilter-3.3.1}/setup.py +1 -2
  12. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_filters.py +2 -12
  13. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_lid.py +4 -1
  14. {opusfilter-3.3.0 → opusfilter-3.3.1}/.github/workflows/ci.yml +0 -0
  15. {opusfilter-3.3.0 → opusfilter-3.3.1}/.gitignore +0 -0
  16. {opusfilter-3.3.0 → opusfilter-3.3.1}/.pylintrc +0 -0
  17. {opusfilter-3.3.0 → opusfilter-3.3.1}/LICENSE +0 -0
  18. {opusfilter-3.3.0 → opusfilter-3.3.1}/README.md +0 -0
  19. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter +0 -0
  20. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-autogen +0 -0
  21. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-cmd +0 -0
  22. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-diagram +0 -0
  23. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-duplicates +0 -0
  24. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-scores +0 -0
  25. {opusfilter-3.3.0 → opusfilter-3.3.1}/bin/opusfilter-test +0 -0
  26. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/CONTRIBUTING.md +0 -0
  27. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/automatic_configuration.md +0 -0
  28. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/command_line_tools.md +0 -0
  29. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/conf.py +0 -0
  30. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/alignment_model_filters.md +0 -0
  31. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/custom_filters.md +0 -0
  32. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/language_model_filters.md +0 -0
  33. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/length_filters.md +0 -0
  34. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/filters/sentence_embedding_filters.md +0 -0
  35. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/downloading_and_selecting_data.md +0 -0
  36. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/preprocessing_text.md +0 -0
  37. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/training_and_using_classifiers.md +0 -0
  38. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/training_language_and_alignment_models.md +0 -0
  39. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/functions/using_score_files.md +0 -0
  40. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/index.rst +0 -0
  41. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/bpe_segmentation.md +0 -0
  42. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/custom_preprocessors.md +0 -0
  43. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/detokenizer.md +0 -0
  44. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
  45. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/morfessor_segmentation.md +0 -0
  46. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/reg_exp_sub.md +0 -0
  47. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/tokenizer.md +0 -0
  48. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/preprocessors/whitespaceNormalizer.md +0 -0
  49. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/references.bib +0 -0
  50. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/references.rst +0 -0
  51. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/requirements.txt +0 -0
  52. {opusfilter-3.3.0 → opusfilter-3.3.1}/docs/usage.md +0 -0
  53. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/README.md +0 -0
  54. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
  55. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
  56. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
  57. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
  58. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
  59. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
  60. {opusfilter-3.3.0 → opusfilter-3.3.1}/example_configs/qed_lm_langid.yaml +0 -0
  61. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/__init__.py +0 -0
  62. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/autogen_cluster.py +0 -0
  63. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/classifier.py +0 -0
  64. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/embeddings.py +0 -0
  65. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/lid.py +0 -0
  66. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/lm.py +0 -0
  67. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/opusfilter.py +0 -0
  68. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/pipeline.py +0 -0
  69. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/preprocessors.py +0 -0
  70. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/segment_hash.py +0 -0
  71. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/subwords.py +0 -0
  72. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/tokenization.py +0 -0
  73. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/util.py +0 -0
  74. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter/word_alignment.py +0 -0
  75. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/SOURCES.txt +0 -0
  76. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/dependency_links.txt +0 -0
  77. {opusfilter-3.3.0 → opusfilter-3.3.1}/opusfilter.egg-info/top_level.txt +0 -0
  78. {opusfilter-3.3.0 → opusfilter-3.3.1}/pyproject.toml +0 -0
  79. {opusfilter-3.3.0 → opusfilter-3.3.1}/requirements.txt +0 -0
  80. {opusfilter-3.3.0 → opusfilter-3.3.1}/setup.cfg +0 -0
  81. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_autogen.py +0 -0
  82. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_classifier.py +0 -0
  83. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_embeddings.py +0 -0
  84. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_filter_pipeline.py +0 -0
  85. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_lm_filter.py +0 -0
  86. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_opusfilter.py +0 -0
  87. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_preprocessors.py +0 -0
  88. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_segment_hash.py +0 -0
  89. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_subwords.py +0 -0
  90. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_tokenization.py +0 -0
  91. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_util.py +0 -0
  92. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_wordalign_filter.py +0 -0
  93. {opusfilter-3.3.0 → opusfilter-3.3.1}/tests/test_yaml.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.3.0
3
+ Version: 3.3.1
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -18,8 +18,7 @@ Requires-Dist: graphviz
18
18
  Requires-Dist: py3langid>=0.2.2
19
19
  Requires-Dist: matplotlib
20
20
  Requires-Dist: morfessor
21
- Requires-Dist: opus-fast-mosestokenizer==0.0.8.7; python_version < "3.9"
22
- Requires-Dist: opus-fast-mosestokenizer>=0.0.8.7; python_version >= "3.9"
21
+ Requires-Dist: opus-fast-mosestokenizer>=0.0.8.11
23
22
  Requires-Dist: pandas>=1.0.0
24
23
  Requires-Dist: xxhash>=3.2.0
25
24
  Requires-Dist: sentence-splitter
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [3.3.1] - 2025-12-10
11
+
12
+ ### Changed
13
+
14
+ - changed `HtmlTagFilter` to accept broken tags in order to avoid inconsistent results
15
+ - update required `opus-fast-mosestokenizer` version for better platform support
16
+
17
+ ### Fixed
18
+
19
+ - broken citations and links in the documentation
20
+
10
21
  ## [3.3.0] - 2025-07-02
11
22
 
12
23
  ### Added
@@ -234,7 +245,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
234
245
  First tagged version.
235
246
 
236
247
 
237
- [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.0...develop
248
+ [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.1...develop
249
+ [3.3.1]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.0...3.3.1
238
250
  [3.3.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...3.3.0
239
251
  [3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
240
252
  [3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
@@ -48,21 +48,21 @@ Returned scores are the language identification confidence scores from a given i
48
48
 
49
49
  Currently the following identification methods are supported:
50
50
 
51
- * `langid` (default) :cite:`lui-baldwin-2012-langid`
52
- * See https://github.com/adbar/py3langid
51
+ * `langid` (default) {cite:p}`lui-baldwin-2012-langid`
52
+ * See <https://github.com/adbar/py3langid>
53
53
  * `lingua`
54
- * See https://github.com/pemistahl/lingua-py
54
+ * See <https://github.com/pemistahl/lingua-py>
55
55
  * `cld2`
56
- * See https://github.com/CLD2Owners/cld2
56
+ * See <https://github.com/CLD2Owners/cld2>
57
57
  * Requires [installing optional libraries](../installation.md).
58
- * `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
58
+ * `fasttext` {cite:p}`joulin-etal-2016-fasttext` and {cite:p}`joulin-etal-2017-bag`
59
59
  * A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
60
60
  * Requires [installing optional libraries](../installation.md).
61
61
 
62
62
  ## LangidFilter
63
63
 
64
- Filter segments based on the langid method :cite:`lui-baldwin-2012-langid`; see
65
- https://github.com/adbar/py3langid
64
+ Filter segments based on the langid method {cite:p}`lui-baldwin-2012-langid`; see
65
+ <https://github.com/adbar/py3langid>.
66
66
 
67
67
  Parameters:
68
68
 
@@ -77,7 +77,7 @@ can be used to skip filtering for a language.
77
77
 
78
78
  ## Cld2Filter
79
79
 
80
- Filter segments based on the CLD2 method; see https://github.com/CLD2Owners/cld2
80
+ Filter segments based on the CLD2 method; see <https://github.com/CLD2Owners/cld2>.
81
81
 
82
82
  Requires [installing optional libraries](../installation.md).
83
83
 
@@ -85,7 +85,7 @@ Parameters:
85
85
 
86
86
  * `languages`: expected languages (ISO639 language codes) for the segments
87
87
  * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
88
- * `options`: a dictionary of options for the method (see https://github.com/CLD2Owners/cld2; default `null`)
88
+ * `options`: a dictionary of options for the method (see <https://github.com/CLD2Owners/cld2>; default `null`)
89
89
 
90
90
  Returned scores are the language identification confidence scores from
91
91
  a given identification method for the segments. The scores range from
@@ -96,8 +96,8 @@ language.
96
96
  ## FastTextFilter
97
97
 
98
98
  Filter segments based on the Fasttext language identification models
99
- (:cite:`joulin-etal-2016-fasttext`, :cite:`joulin-etal-2017-bag`), see
100
- https://github.com/CLD2Owners/cld2.
99
+ ({cite:t}`joulin-etal-2016-fasttext`, {cite:t}`joulin-etal-2017-bag`), see
100
+ <https://github.com/CLD2Owners/cld2>.
101
101
 
102
102
  Requires [installing optional libraries](../installation.md).
103
103
 
@@ -115,7 +115,7 @@ language.
115
115
 
116
116
  ## LinguaFilter
117
117
 
118
- Filter segments based on the Lingua method; see https://github.com/pemistahl/lingua-py
118
+ Filter segments based on the Lingua method; see <https://github.com/pemistahl/lingua-py>.
119
119
 
120
120
  Parameters:
121
121
 
@@ -132,8 +132,8 @@ language.
132
132
 
133
133
  ## HeliportSimpleFilter
134
134
 
135
- Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
136
- https://github.com/ZJaume/heliport
135
+ Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
136
+ <https://github.com/ZJaume/heliport>.
137
137
 
138
138
  Requires [installing optional libraries](../installation.md).
139
139
 
@@ -150,8 +150,8 @@ the method predicts the correct language. If the low-confidence label
150
150
 
151
151
  ## HeliportConfidenceFilter
152
152
 
153
- Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
154
- https://github.com/ZJaume/heliport
153
+ Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
154
+ <https://github.com/ZJaume/heliport>.
155
155
 
156
156
  Requires [installing optional libraries](../installation.md).
157
157
 
@@ -170,8 +170,8 @@ filtering for a language.
170
170
 
171
171
  ## HeliportRawScoreFilter
172
172
 
173
- Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
174
- https://github.com/ZJaume/heliport
173
+ Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
174
+ <https://github.com/ZJaume/heliport>.
175
175
 
176
176
  Requires [installing optional libraries](../installation.md).
177
177
 
@@ -189,8 +189,8 @@ filtering for a language.
189
189
 
190
190
  ## HeliportProbabilityFilter
191
191
 
192
- Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
193
- https://github.com/ZJaume/heliport
192
+ Filter segments based on the HeLI-OTS method {cite:p}`jauhiainen-etal-2022-heli`; see
193
+ <https://github.com/ZJaume/heliport>.
194
194
 
195
195
  Requires [installing optional libraries](../installation.md).
196
196
 
@@ -8,6 +8,12 @@ The returned scores are two boolean values indicating whether the
8
8
  segments contain HTML tags. In filtering, a segment pair is accepted
9
9
  if none of the segments contains HTML tags.
10
10
 
11
+ The beautifulsoup4 library is used for detecting HTML. Up to
12
+ OpusFilter version 3.3.0, the behavior on broken HTML tags varies
13
+ depending on the input and installed library versions. For newer
14
+ versions, broken HTML is not considered as HTML, i.e., it is accepted
15
+ by the filter.
16
+
11
17
  ## TerminalPunctuationFilter
12
18
 
13
19
  Filter segments based on a penalty score with respect to the
@@ -44,7 +44,8 @@ parameter as empty string or null.
44
44
 
45
45
  For the `letter_words_only`, unless the input is pre-tokenized, you
46
46
  should select tokenizers for each of the input files. See
47
- [Tokenizer](tokenizer.md) for description of the tokenizer parameters.
47
+ [Tokenizer](../preprocessors/tokenizer.md) for description of the
48
+ tokenizer parameters.
48
49
 
49
50
  ## filter
50
51
 
@@ -78,24 +78,30 @@ automatically with pip by including the extras `[mecab]` or `[all]`
78
78
  ### LASER sentence embeddings
79
79
 
80
80
  For using sentence embeddings filters, you need to install
81
- `laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
81
+ `laserembeddings` (<https://github.com/yannvgn/laserembeddings>). It can
82
82
  be installed automatically with pip by including the extras `[laser]`
83
83
  or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
84
84
  require a number of additional libraries, including PyTorch, jieba,
85
85
  and MeCab. Note that you need also to download the prebuild models
86
86
  with `python -m laserembeddings download-models`.
87
87
 
88
+ **Note:** The current latest release `laserembeddings==1.1.2` does not
89
+ support `torch>=2.0.0` and thus fails on Python 3.12 and above. The
90
+ code itself works on Python 3.12 and 3.13, but you need to fix
91
+ `pyproject.toml` to allow later `torch` versions, and install it
92
+ manually.
93
+
88
94
  ### VariKN n-gram models
89
95
 
90
96
  For using n-gram language model filters, you need to install the
91
- Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
97
+ Python wrapper for VariKN (<https://github.com/vsiivola/variKN>). It can
92
98
  be installed automatically with pip by including the extras `[varikn]`
93
99
  or `[all]` (e.g. `pip install opusfilter[varikn]`).
94
100
 
95
101
  ### Eflomal word alignment
96
102
 
97
103
  For using word alignment filters, you need to install elfomal
98
- (https://github.com/robertostling/eflomal). It can be installed
104
+ (<https://github.com/robertostling/eflomal>). It can be installed
99
105
  automatically with pip by including the extras `[eflomal]` or `[all]`
100
106
  (e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
101
107
  for the installation.
@@ -14,6 +14,7 @@ import ruamel.yaml
14
14
  from . import CLEAN_LOW, CLEAN_HIGH, CLEAN_BETWEEN, CLEAN_TRUE, CLEAN_FALSE
15
15
  from . import OpusFilterError, ConfigurationError
16
16
  from . import filters as filtermodule
17
+ from . import lid as lidmodule
17
18
  from .autogen_cluster import ScoreClusters
18
19
  from .classifier import load_dataframe
19
20
  from .opusfilter import OpusFilter
@@ -171,6 +172,13 @@ def parse_filter_specs(specs):
171
172
  return name, params
172
173
 
173
174
 
175
+ def get_lang_id_filters():
176
+ """Return all language identification filters"""
177
+ names = ['LanguageIDFilter'] # old deprecated filter
178
+ names += [var for var in dir(lidmodule) if var.endswith('Filter')]
179
+ return names
180
+
181
+
174
182
  class AutoFiltersABC(metaclass=abc.ABCMeta):
175
183
  """Abstract base class for automatic filter configuration"""
176
184
 
@@ -181,13 +189,14 @@ class AutoFiltersABC(metaclass=abc.ABCMeta):
181
189
  filters = self.DEFAULT_FILTERS
182
190
  filters = [parse_filter_specs(spec) for spec in filters]
183
191
  self.filters_to_add = []
192
+ lang_id_filters = get_lang_id_filters()
184
193
  for filter_name, filter_params in filters:
185
194
  if filter_name == 'CharacterScoreFilter' and 'scripts' not in filter_params:
186
195
  if not scripts:
187
196
  logger.warning('Cannot add CharacterScoreFilter (no scripts provided)')
188
197
  continue
189
198
  filter_params['scripts'] = scripts
190
- if filter_name == 'LanguageIDFilter' and 'languages' not in filter_params:
199
+ if filter_name in lang_id_filters and 'languages' not in filter_params:
191
200
  if not langs:
192
201
  logger.warning('Cannot add LanguageIDFilter (no languages provided)')
193
202
  continue
@@ -217,7 +226,7 @@ class DefaultParameterFilters(AutoFiltersABC):
217
226
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
218
227
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
219
228
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
220
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
229
+ 'CharacterScoreFilter', 'LinguaFilter']
221
230
 
222
231
  def set_filter_thresholds(self):
223
232
  """Set filter thresholds"""
@@ -164,7 +164,7 @@ class HtmlTagFilter(FilterABC):
164
164
  except (TypeError, UnboundLocalError, NotImplementedError,
165
165
  AssertionError, bs4.builder.ParserRejectedMarkup) as err:
166
166
  logger.warning("BeautifulSoup parsing failed for %s: %s", repr(segment), err)
167
- found = True
167
+ found = False
168
168
  return found
169
169
 
170
170
  def score(self, pairs):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.3.0
3
+ Version: 3.3.1
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -18,8 +18,7 @@ Requires-Dist: graphviz
18
18
  Requires-Dist: py3langid>=0.2.2
19
19
  Requires-Dist: matplotlib
20
20
  Requires-Dist: morfessor
21
- Requires-Dist: opus-fast-mosestokenizer==0.0.8.7; python_version < "3.9"
22
- Requires-Dist: opus-fast-mosestokenizer>=0.0.8.7; python_version >= "3.9"
21
+ Requires-Dist: opus-fast-mosestokenizer>=0.0.8.11
23
22
  Requires-Dist: pandas>=1.0.0
24
23
  Requires-Dist: xxhash>=3.2.0
25
24
  Requires-Dist: sentence-splitter
@@ -5,6 +5,7 @@ graphviz
5
5
  py3langid>=0.2.2
6
6
  matplotlib
7
7
  morfessor
8
+ opus-fast-mosestokenizer>=0.0.8.11
8
9
  pandas>=1.0.0
9
10
  xxhash>=3.2.0
10
11
  sentence-splitter
@@ -20,15 +21,9 @@ iso639-lang
20
21
  [:python_version < "3.10"]
21
22
  lingua-language-detector<2.1,>=1.3.0
22
23
 
23
- [:python_version < "3.9"]
24
- opus-fast-mosestokenizer==0.0.8.7
25
-
26
24
  [:python_version >= "3.10"]
27
25
  lingua-language-detector>=2.1.1
28
26
 
29
- [:python_version >= "3.9"]
30
- opus-fast-mosestokenizer>=0.0.8.7
31
-
32
27
  [all]
33
28
  eflomal>=2.0.0
34
29
  jieba>=0.42
@@ -11,8 +11,7 @@ install_requires = [
11
11
  "py3langid>=0.2.2",
12
12
  "matplotlib",
13
13
  "morfessor",
14
- 'opus-fast-mosestokenizer==0.0.8.7 ; python_version < "3.9"',
15
- 'opus-fast-mosestokenizer>=0.0.8.7 ; python_version >= "3.9"',
14
+ 'opus-fast-mosestokenizer>=0.0.8.11',
16
15
  "pandas>=1.0.0",
17
16
  "xxhash>=3.2.0",
18
17
  "sentence-splitter",
@@ -104,20 +104,10 @@ class TestHtmlTagFilter(unittest.TestCase):
104
104
  def test_broken(self):
105
105
  testfilter = HtmlTagFilter()
106
106
  cases = [['<aaa bee', 'cee dee'], ['aa br> bee', 'cee deee'],
107
- ['<p aaa bee</p', 'p>cee dee /p>'], ['', '']]
107
+ ['<p aaa bee</p', 'p>cee dee /p>'], ['<![ foo', 'foo'], ['', '']]
108
108
  expected = [([False, False], True), ([False, False], True),
109
109
  ([False, False], True), ([False, False], True),
110
- ([True, False], False)]
111
- results = [(x, testfilter.accept(x)) for x in testfilter.score(cases)]
112
- logging.warning(results)
113
- for result, correct in zip(results, expected):
114
- self.assertSequenceEqual(result, correct)
115
-
116
- @pytest.mark.skipif(sys.version_info >= (3, 13), reason="different output for py3.13")
117
- def test_broken_exception(self):
118
- testfilter = HtmlTagFilter()
119
- cases = [['<![ foo', 'foo']]
120
- expected = [([True, False], False)]
110
+ ([False, False], True), ([True, False], False)]
121
111
  results = [(x, testfilter.accept(x)) for x in testfilter.score(cases)]
122
112
  logging.warning(results)
123
113
  for result, correct in zip(results, expected):
@@ -93,7 +93,7 @@ class TestFasttext(TestLangIDMethod):
93
93
  def setUpClass(self):
94
94
  self.tempdir = tempfile.mkdtemp()
95
95
  if 'fasttext' not in globals():
96
- raise unittest.SkipTest('fasttext not installed')
96
+ return
97
97
  self.testmodel = os.path.join(self.tempdir, 'model.ftz')
98
98
  try:
99
99
  file_download(self.model_url, self.testmodel)
@@ -104,11 +104,13 @@ class TestFasttext(TestLangIDMethod):
104
104
  def tearDownClass(self):
105
105
  shutil.rmtree(self.tempdir)
106
106
 
107
+ @unittest.skipIf('fasttext' not in globals(), 'fasttext not installed')
107
108
  def test_missing_model(self):
108
109
  with self.assertRaises(ConfigurationError):
109
110
  model = FastTextFilter(
110
111
  languages=['en', 'fr'], thresholds=[0.8, 0.99])
111
112
 
113
+ @unittest.skipIf('fasttext' not in globals(), 'fasttext not installed')
112
114
  def test_fasttext_predict_lang(self):
113
115
  if self.testmodel is None:
114
116
  self.skipTest("Failed to download test resources")
@@ -120,6 +122,7 @@ class TestFasttext(TestLangIDMethod):
120
122
  for fasttext_input in self.fasttext_inputs]
121
123
  self.assertSequenceEqual(expected, results)
122
124
 
125
+ @unittest.skipIf('fasttext' not in globals(), 'fasttext not installed')
123
126
  def test_accept(self):
124
127
  if self.testmodel is None:
125
128
  self.skipTest("Failed to download test resources")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes