opusfilter 3.1.0__tar.gz → 3.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {opusfilter-3.1.0 → opusfilter-3.2.0}/.github/workflows/ci.yml +11 -9
  2. {opusfilter-3.1.0 → opusfilter-3.2.0}/PKG-INFO +16 -8
  3. {opusfilter-3.1.0 → opusfilter-3.2.0}/README.md +1 -1
  4. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/CHANGELOG.md +14 -1
  5. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/CONTRIBUTING.md +2 -2
  6. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/script_and_language_identification_filters.md +13 -5
  7. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/downloading_and_selecting_data.md +6 -1
  8. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/installation.md +25 -9
  9. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/autogen.py +3 -3
  10. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/filters.py +13 -5
  11. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/opusfilter.py +13 -3
  12. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/PKG-INFO +16 -8
  13. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/SOURCES.txt +1 -0
  14. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/requires.txt +16 -6
  15. {opusfilter-3.1.0 → opusfilter-3.2.0}/requirements.txt +4 -5
  16. {opusfilter-3.1.0 → opusfilter-3.2.0}/setup.py +19 -9
  17. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_autogen.py +1 -1
  18. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_filter_pipeline.py +25 -9
  19. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_filters.py +0 -128
  20. opusfilter-3.2.0/tests/test_lid.py +151 -0
  21. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_opusfilter.py +21 -1
  22. {opusfilter-3.1.0 → opusfilter-3.2.0}/.gitignore +0 -0
  23. {opusfilter-3.1.0 → opusfilter-3.2.0}/.pylintrc +0 -0
  24. {opusfilter-3.1.0 → opusfilter-3.2.0}/LICENSE +0 -0
  25. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter +0 -0
  26. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-autogen +0 -0
  27. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-cmd +0 -0
  28. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-diagram +0 -0
  29. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-duplicates +0 -0
  30. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-scores +0 -0
  31. {opusfilter-3.1.0 → opusfilter-3.2.0}/bin/opusfilter-test +0 -0
  32. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/automatic_configuration.md +0 -0
  33. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/command_line_tools.md +0 -0
  34. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/conf.py +0 -0
  35. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/alignment_model_filters.md +0 -0
  36. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/custom_filters.md +0 -0
  37. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/language_model_filters.md +0 -0
  38. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/length_filters.md +0 -0
  39. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/sentence_embedding_filters.md +0 -0
  40. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
  41. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/filtering_and_scoring.md +0 -0
  42. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/preprocessing_text.md +0 -0
  43. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/training_and_using_classifiers.md +0 -0
  44. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/training_language_and_alignment_models.md +0 -0
  45. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/functions/using_score_files.md +0 -0
  46. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/index.rst +0 -0
  47. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/bpe_segmentation.md +0 -0
  48. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/custom_preprocessors.md +0 -0
  49. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/detokenizer.md +0 -0
  50. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
  51. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
  52. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/reg_exp_sub.md +0 -0
  53. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/tokenizer.md +0 -0
  54. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
  55. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/references.bib +0 -0
  56. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/references.rst +0 -0
  57. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/requirements.txt +0 -0
  58. {opusfilter-3.1.0 → opusfilter-3.2.0}/docs/usage.md +0 -0
  59. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/README.md +0 -0
  60. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
  61. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
  62. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
  63. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
  64. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
  65. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
  66. {opusfilter-3.1.0 → opusfilter-3.2.0}/example_configs/qed_lm_langid.yaml +0 -0
  67. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/__init__.py +0 -0
  68. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/autogen_cluster.py +0 -0
  69. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/classifier.py +0 -0
  70. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/embeddings.py +0 -0
  71. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/lm.py +0 -0
  72. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/pipeline.py +0 -0
  73. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/preprocessors.py +0 -0
  74. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/segment_hash.py +0 -0
  75. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/subwords.py +0 -0
  76. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/tokenization.py +0 -0
  77. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/util.py +0 -0
  78. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter/word_alignment.py +0 -0
  79. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/dependency_links.txt +0 -0
  80. {opusfilter-3.1.0 → opusfilter-3.2.0}/opusfilter.egg-info/top_level.txt +0 -0
  81. {opusfilter-3.1.0 → opusfilter-3.2.0}/pyproject.toml +0 -0
  82. {opusfilter-3.1.0 → opusfilter-3.2.0}/setup.cfg +0 -0
  83. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_classifier.py +0 -0
  84. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_embeddings.py +0 -0
  85. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_lm_filter.py +0 -0
  86. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_preprocessors.py +0 -0
  87. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_segment_hash.py +0 -0
  88. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_subwords.py +0 -0
  89. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_tokenization.py +0 -0
  90. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_util.py +0 -0
  91. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_wordalign_filter.py +0 -0
  92. {opusfilter-3.1.0 → opusfilter-3.2.0}/tests/test_yaml.py +0 -0
@@ -21,21 +21,23 @@ jobs:
21
21
  fail-fast: false
22
22
  matrix:
23
23
  os: [ubuntu-latest]
24
- python-version: ["3.8", "3.9", "3.10", "3.11"]
24
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
25
25
  requirements-file: ["requirements.txt"]
26
26
  runs-on: ${{ matrix.os }}
27
27
  steps:
28
- - uses: actions/checkout@v3
28
+ - uses: actions/checkout@v4
29
29
  - name: Set up Python ${{ matrix.python-version }}
30
- uses: actions/setup-python@v4
30
+ uses: actions/setup-python@v5
31
31
  with:
32
32
  python-version: ${{ matrix.python-version }}
33
33
  - name: Install dependencies
34
34
  run: |
35
+ python -m ensurepip --upgrade
36
+ python -m pip install --upgrade setuptools
35
37
  python -m pip install --upgrade pip
36
38
  python -m pip install flake8 pytest wheel
37
- pip install -r ${{ matrix.requirements-file }}
38
- python setup.py install
39
+ python -m pip install --no-cache-dir -r ${{ matrix.requirements-file }}
40
+ python -m pip install .
39
41
  - name: Lint with flake8
40
42
  run: |
41
43
  # stop the build if there are Python syntax errors or undefined names
@@ -49,7 +51,7 @@ jobs:
49
51
  needs: build-n-test
50
52
  runs-on: ubuntu-20.04
51
53
  steps:
52
- - uses: actions/checkout@v3
54
+ - uses: actions/checkout@v4
53
55
  - name: Check for main release tag
54
56
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
55
57
  id: check-tag
@@ -59,7 +61,7 @@ jobs:
59
61
  fi
60
62
  - name: Set up Python 3.8
61
63
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
62
- uses: actions/setup-python@v4
64
+ uses: actions/setup-python@v5
63
65
  with:
64
66
  python-version: 3.8
65
67
  - name: Install pypa/build
@@ -94,9 +96,9 @@ jobs:
94
96
  runs-on: ubuntu-latest
95
97
  needs: [build-n-test]
96
98
  steps:
97
- - uses: actions/checkout@v3
99
+ - uses: actions/checkout@v4
98
100
  - name: Set up Python 3.8
99
- uses: actions/setup-python@v4
101
+ uses: actions/setup-python@v5
100
102
  with:
101
103
  python-version: 3.8
102
104
  - name: Install dependencies to build docs
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.1.0
3
+ Version: 3.2.0
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -12,16 +12,14 @@ Requires-Python: >=3.8
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
14
  Requires-Dist: setuptools
15
- Requires-Dist: opustools
15
+ Requires-Dist: opustools>=1.6.2
16
16
  Requires-Dist: beautifulsoup4>=4.8.0
17
- Requires-Dist: fasttext
18
17
  Requires-Dist: graphviz
19
- Requires-Dist: langid
18
+ Requires-Dist: py3langid>=0.2.2
20
19
  Requires-Dist: matplotlib
21
20
  Requires-Dist: morfessor
22
21
  Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
23
22
  Requires-Dist: pandas>=1.0.0
24
- Requires-Dist: pycld2
25
23
  Requires-Dist: xxhash>=3.2.0
26
24
  Requires-Dist: sentence-splitter
27
25
  Requires-Dist: rapidfuzz
@@ -34,8 +32,14 @@ Requires-Dist: tqdm
34
32
  Requires-Dist: lingua-language-detector>=1.3.0
35
33
  Provides-Extra: test
36
34
  Requires-Dist: pytest; extra == "test"
35
+ Provides-Extra: pycld2
36
+ Requires-Dist: pycld2; extra == "pycld2"
37
+ Provides-Extra: fasttext
38
+ Requires-Dist: py3langid<0.3.0; extra == "fasttext"
39
+ Requires-Dist: numpy<2.0.0; extra == "fasttext"
40
+ Requires-Dist: fasttext; extra == "fasttext"
37
41
  Provides-Extra: eflomal
38
- Requires-Dist: eflomal>=1.0.0b1; extra == "eflomal"
42
+ Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
39
43
  Provides-Extra: jieba
40
44
  Requires-Dist: jieba>=0.42; extra == "jieba"
41
45
  Provides-Extra: mecab
@@ -51,7 +55,11 @@ Requires-Dist: sphinx; extra == "docs"
51
55
  Requires-Dist: sphinx-rtd-theme; extra == "docs"
52
56
  Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
53
57
  Provides-Extra: all
54
- Requires-Dist: eflomal>=1.0.0b1; extra == "all"
58
+ Requires-Dist: pycld2; extra == "all"
59
+ Requires-Dist: py3langid<0.3.0; extra == "all"
60
+ Requires-Dist: numpy<2.0.0; extra == "all"
61
+ Requires-Dist: fasttext; extra == "all"
62
+ Requires-Dist: eflomal>=2.0.0; extra == "all"
55
63
  Requires-Dist: jieba>=0.42; extra == "all"
56
64
  Requires-Dist: mecab-python3>=1.0.8; extra == "all"
57
65
  Requires-Dist: unidic-lite; extra == "all"
@@ -90,7 +98,7 @@ Install from source:
90
98
 
91
99
  ### Troubleshooting
92
100
 
93
- OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
101
+ OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
94
102
 
95
103
  * `pip install -r requirements.txt`
96
104
 
@@ -25,7 +25,7 @@ Install from source:
25
25
 
26
26
  ### Troubleshooting
27
27
 
28
- OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
28
+ OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
29
29
 
30
30
  * `pip install -r requirements.txt`
31
31
 
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [3.2.0] - 2024-08-14
11
+
12
+ ### Changed
13
+
14
+ - make `pycld2` and `fasttext` libraries optional
15
+ - replace `langid.py` library with `py3langid`
16
+ - update github workflows and include Python 3.12 tests
17
+
18
+ ### Fixed
19
+
20
+ - `OpusRead` interface using `moses` format (requires `opustools >= 1.6.2`)
21
+
10
22
  ## [3.1.0] - 2024-06-05
11
23
 
12
24
  ### Added
@@ -204,7 +216,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
204
216
  First tagged version.
205
217
 
206
218
 
207
- [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...develop
219
+ [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...develop
220
+ [3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
208
221
  [3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
209
222
  [3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
210
223
  [2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
5
5
  few rules for pull requests:
6
6
 
7
7
  * Make a pull request to the `develop` branch instead of `master`.
8
- * The code should support at least Python versions from 3.8 to 3.11.
8
+ * The code should support at least Python versions from 3.8 to 3.12.
9
9
  * Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
10
10
  * Especially for new features, please include test cases for unit testing.
11
11
 
@@ -20,7 +20,7 @@ skips the respective tests if not.)
20
20
 
21
21
  GitHub workflows defined in the project run automatically `flake8`
22
22
  checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
23
- and 3.11.
23
+ 3.11, and 3.12.
24
24
 
25
25
  Especially for larger contributions, consider using a code analysis
26
26
  tool like [Pylint](https://github.com/PyCQA/pylint). Install it
@@ -35,7 +35,7 @@ Filter segments based on their language identification confidence scores.
35
35
  Parameters:
36
36
 
37
37
  * `languages`: expected languages (ISO639 language codes) for the segments
38
- * `id_method`: language indentification method (`langid` for using the `langid` library, `cld2` for using the `cld2` library, or `fasttext` for using a `fasttext` model; the default is `langid`)
38
+ * `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`; default `langid`)
39
39
  * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
40
40
  * `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
41
41
  * `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
@@ -44,7 +44,15 @@ Parameters:
44
44
 
45
45
  Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. Negative threshold can be used to skip filtering for a language.
46
46
 
47
- See [langid.py](https://github.com/saffsd/langid.py) and
48
- [pycld2](https://github.com/aboSamoor/pycld2) for the method-specific
49
- options. A pretrained `fasttext` model can be downloaded from
50
- [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
47
+ Currently the following identification methods are supported:
48
+
49
+ * `langid` (default) :cite:`lui-baldwin-2012-langid`
50
+ * See https://github.com/adbar/py3langid
51
+ * `lingua`
52
+ * See https://github.com/pemistahl/lingua-py
53
+ * `cld2`
54
+ * See https://github.com/CLD2Owners/cld2
55
+ * Requires [installing optional libraries](../installation.md).
56
+ * `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
57
+ * A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
58
+ * Requires [installing optional libraries](../installation.md).
@@ -11,11 +11,16 @@ Parameters:
11
11
  * `source_language`: language code for the source language
12
12
  * `target_language`: language code for the target language
13
13
  * `release`: version of the corpus in OPUS
14
- * `preprocessing`: `raw` for untokenized and `xml` for tokenized segments
14
+ * `preprocessing`: `moses` or `raw` for untokenized and `xml` for tokenized segments
15
15
  * `src_output`: output file for source language
16
16
  * `tgt_output`: output file for target language
17
17
  * `suppress_prompts`: `false` (default) prompts user to confirm before download, `true` to download without prompting
18
18
 
19
+ The `moses` preprocessing type (available with `OpusTools` version
20
+ 1.6.2 and above) is recommended for those corpora for which it
21
+ exists. The output is equivalent to `raw`, but in some cases it can
22
+ significantly reduce the amount of data downloaded in the process.
23
+
19
24
  ## concatenate
20
25
 
21
26
  Concatenate two or more text files.
@@ -12,20 +12,18 @@ Install from source:
12
12
 
13
13
  Note that all required libraries are not available to install via PyPI
14
14
  on Windows OS. On Linux and MacOS, it should work directly for Python
15
- versions from 3.8 to 3.11.
15
+ versions from 3.8 to 3.12.
16
16
 
17
17
  ## Required libraries
18
18
 
19
19
  * beautifulsoup4
20
20
  * opus-fast-mosestokenizer
21
- * fasttext
22
21
  * graphviz
23
- * langid
22
+ * py3langid
24
23
  * matplotlib
25
24
  * morfessor
26
25
  * OpusTools
27
26
  * pandas
28
- * pycld2
29
27
  * rapidfuzz
30
28
  * ruamel.yaml
31
29
  * regex
@@ -41,24 +39,42 @@ See `setup.py` for possible version requirements.
41
39
 
42
40
  ## Optional libraries and tools
43
41
 
42
+ ### FastText and PyCLD2 language identification
43
+
44
+ The language identification libraries currently supported out-of-the-box
45
+ are [py3langid](https://github.com/adbar/py3langid) and
46
+ [lingua](https://github.com/pemistahl/lingua-py). The support for for
47
+ [PyCLD2](https://github.com/aboSamoor/pycld2) and
48
+ [FastText models](https://fasttext.cc/docs/en/language-identification.html)
49
+ have been changed to optional due to the lack of support especially
50
+ for newer Python versions.
51
+
52
+ The PyCLD2 support can be installed automatically with pip by
53
+ including the extras `[pycld2]` or `[all]` (e.g.
54
+ `pip install opusfilter[pycld2]`).
55
+
56
+ The support for FastText models can be installed automatically with
57
+ pip by including the extras `[fasttext]` or `[all]` (e.g.
58
+ `pip install opusfilter[fasttext]`).
59
+
44
60
  ### Jieba and MeCab word segmentation
45
61
 
46
62
  For Chinese tokenization (word segmentation), you can use the
47
63
  [jieba](https://github.com/fxsjy/jieba) library. It can be installed
48
64
  automatically with pip by including the extras `[jieba]` or `[all]`
49
- (e.g. `pip install opusfilter[all]`).
65
+ (e.g. `pip install opusfilter[jieba]`).
50
66
 
51
67
  For Japanese tokenization (word segmentation), you can use the
52
68
  [MeCab](https://github.com/SamuraiT/mecab-python3) library. It can be installed
53
69
  automatically with pip by including the extras `[mecab]` or `[all]`
54
- (e.g. `pip install opusfilter[all]`).
70
+ (e.g. `pip install opusfilter[mecab]`).
55
71
 
56
72
  ### LASER sentence embeddings
57
73
 
58
74
  For using sentence embeddings filters, you need to install
59
75
  `laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
60
76
  be installed automatically with pip by including the extras `[laser]`
61
- or `[all]` (e.g. `pip install opusfilter[all]`). The package will also
77
+ or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
62
78
  require a number of additional libraries, including PyTorch, jieba,
63
79
  and MeCab. Note that you need also to download the prebuild models
64
80
  with `python -m laserembeddings download-models`.
@@ -68,12 +84,12 @@ with `python -m laserembeddings download-models`.
68
84
  For using n-gram language model filters, you need to install the
69
85
  Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
70
86
  be installed automatically with pip by including the extras `[varikn]`
71
- or `[all]` (e.g. `pip install opusfilter[all]`).
87
+ or `[all]` (e.g. `pip install opusfilter[varikn]`).
72
88
 
73
89
  ### Eflomal word alignment
74
90
 
75
91
  For using word alignment filters, you need to install elfomal
76
92
  (https://github.com/robertostling/eflomal). It can be installed
77
93
  automatically with pip by including the extras `[eflomal]` or `[all]`
78
- (e.g. `pip install opusfilter[all]`). Note that you will need `Cython`
94
+ (e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
79
95
  for the installation.
@@ -217,7 +217,7 @@ class DefaultParameterFilters(AutoFiltersABC):
217
217
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
218
218
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
219
219
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
220
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'cld2'})]
220
+ 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
221
221
 
222
222
  def set_filter_thresholds(self):
223
223
  """Set filter thresholds"""
@@ -272,7 +272,7 @@ class PercentileFilters(DataBasedFiltersABC):
272
272
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
273
273
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
274
274
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
275
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'cld2'})]
275
+ 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
276
276
 
277
277
  def __init__(self, files, excluded_percentile=0.001, **kwargs):
278
278
  super().__init__(files, **kwargs)
@@ -512,7 +512,7 @@ class ClusterFilters(DataBasedFiltersABC):
512
512
  ('LengthRatioFilter.word', {'unit': 'word'}),
513
513
  'NonZeroNumeralsFilter',
514
514
  'CharacterScoreFilter',
515
- ('LanguageIDFilter', {'id_method': 'cld2'}),
515
+ ('LanguageIDFilter', {'id_method': 'lingua'}),
516
516
  'TerminalPunctuationFilter']
517
517
 
518
518
  def __init__(self, files, k=2, max_length=150, **kwargs):
@@ -334,8 +334,8 @@ class LanguageIDFilter(FilterABC):
334
334
 
335
335
  def init_langid(self, langid_languages):
336
336
  """Initialize langid identifier"""
337
- from langid.langid import LanguageIdentifier, model
338
- self.identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
337
+ from py3langid.langid import LanguageIdentifier, MODEL_FILE
338
+ self.identifier = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True)
339
339
  if langid_languages:
340
340
  self.identifier.set_languages(langid_languages)
341
341
 
@@ -344,7 +344,11 @@ class LanguageIDFilter(FilterABC):
344
344
  if not fasttext_model_path:
345
345
  raise ConfigurationError("FastText language ID method was choosen without specifying "
346
346
  "any path to fasttext model")
347
- import fasttext
347
+ try:
348
+ import fasttext
349
+ except ImportError:
350
+ logger.warning("Could not import fasttext. Select another id_method for LanguageIDFilter.")
351
+ raise
348
352
  self.fasttext_model = fasttext.load_model(os.path.join(self.workdir, fasttext_model_path))
349
353
 
350
354
  def init_lingua(self, lingua_mode):
@@ -366,7 +370,11 @@ class LanguageIDFilter(FilterABC):
366
370
  return 1.0
367
371
 
368
372
  if self.id_method == 'cld2':
369
- import pycld2
373
+ try:
374
+ import pycld2
375
+ except ImportError:
376
+ logger.warning("Could not import pycld2. Select another id_method for LanguageIDFilter.")
377
+ raise
370
378
  try:
371
379
  clddetails = pycld2.detect(sentence, **self.cld2_options)
372
380
  except pycld2.error as err:
@@ -380,7 +388,7 @@ class LanguageIDFilter(FilterABC):
380
388
 
381
389
  if self.id_method == 'langid':
382
390
  lidetails = self.identifier.classify(sentence)
383
- lilan, liconf = lidetails[0], round(lidetails[1], 2)
391
+ lilan, liconf = lidetails[0], round(float(lidetails[1]), 2)
384
392
  if lilan != lan:
385
393
  liconf = 0.0
386
394
  return liconf
@@ -334,8 +334,14 @@ class OpusFilter:
334
334
  write=[src_out, tgt_out],
335
335
  leave_non_alignments_out=True,
336
336
  download_dir=self.output_dir)
337
-
338
- opus_reader.printPairs()
337
+ try:
338
+ opus_reader.printPairs()
339
+ except Exception as err:
340
+ # Remove broken files
341
+ for outfile in [src_out, tgt_out]:
342
+ if os.path.isfile(outfile):
343
+ os.unlink(outfile)
344
+ raise err
339
345
 
340
346
  @staticmethod
341
347
  def pair_generator(*filenames, tokenizers=None):
@@ -556,7 +562,11 @@ class OpusFilter:
556
562
  """Write objects to file as JSON lines"""
557
563
  with file_open(fname, 'w') as fobj:
558
564
  for obj in objects:
559
- fobj.write(json.dumps(obj, sort_keys=True)+'\n')
565
+ try:
566
+ fobj.write(json.dumps(obj, sort_keys=True)+'\n')
567
+ except TypeError as err:
568
+ logger.error("Could not convert to JSON: %s", obj)
569
+ raise err
560
570
 
561
571
  @staticmethod
562
572
  def _read_jsonl(fname):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.1.0
3
+ Version: 3.2.0
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -12,16 +12,14 @@ Requires-Python: >=3.8
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
14
  Requires-Dist: setuptools
15
- Requires-Dist: opustools
15
+ Requires-Dist: opustools>=1.6.2
16
16
  Requires-Dist: beautifulsoup4>=4.8.0
17
- Requires-Dist: fasttext
18
17
  Requires-Dist: graphviz
19
- Requires-Dist: langid
18
+ Requires-Dist: py3langid>=0.2.2
20
19
  Requires-Dist: matplotlib
21
20
  Requires-Dist: morfessor
22
21
  Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
23
22
  Requires-Dist: pandas>=1.0.0
24
- Requires-Dist: pycld2
25
23
  Requires-Dist: xxhash>=3.2.0
26
24
  Requires-Dist: sentence-splitter
27
25
  Requires-Dist: rapidfuzz
@@ -34,8 +32,14 @@ Requires-Dist: tqdm
34
32
  Requires-Dist: lingua-language-detector>=1.3.0
35
33
  Provides-Extra: test
36
34
  Requires-Dist: pytest; extra == "test"
35
+ Provides-Extra: pycld2
36
+ Requires-Dist: pycld2; extra == "pycld2"
37
+ Provides-Extra: fasttext
38
+ Requires-Dist: py3langid<0.3.0; extra == "fasttext"
39
+ Requires-Dist: numpy<2.0.0; extra == "fasttext"
40
+ Requires-Dist: fasttext; extra == "fasttext"
37
41
  Provides-Extra: eflomal
38
- Requires-Dist: eflomal>=1.0.0b1; extra == "eflomal"
42
+ Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
39
43
  Provides-Extra: jieba
40
44
  Requires-Dist: jieba>=0.42; extra == "jieba"
41
45
  Provides-Extra: mecab
@@ -51,7 +55,11 @@ Requires-Dist: sphinx; extra == "docs"
51
55
  Requires-Dist: sphinx-rtd-theme; extra == "docs"
52
56
  Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
53
57
  Provides-Extra: all
54
- Requires-Dist: eflomal>=1.0.0b1; extra == "all"
58
+ Requires-Dist: pycld2; extra == "all"
59
+ Requires-Dist: py3langid<0.3.0; extra == "all"
60
+ Requires-Dist: numpy<2.0.0; extra == "all"
61
+ Requires-Dist: fasttext; extra == "all"
62
+ Requires-Dist: eflomal>=2.0.0; extra == "all"
55
63
  Requires-Dist: jieba>=0.42; extra == "all"
56
64
  Requires-Dist: mecab-python3>=1.0.8; extra == "all"
57
65
  Requires-Dist: unidic-lite; extra == "all"
@@ -90,7 +98,7 @@ Install from source:
90
98
 
91
99
  ### Troubleshooting
92
100
 
93
- OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
101
+ OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
94
102
 
95
103
  * `pip install -r requirements.txt`
96
104
 
@@ -79,6 +79,7 @@ tests/test_classifier.py
79
79
  tests/test_embeddings.py
80
80
  tests/test_filter_pipeline.py
81
81
  tests/test_filters.py
82
+ tests/test_lid.py
82
83
  tests/test_lm_filter.py
83
84
  tests/test_opusfilter.py
84
85
  tests/test_preprocessors.py
@@ -1,14 +1,12 @@
1
1
  setuptools
2
- opustools
2
+ opustools>=1.6.2
3
3
  beautifulsoup4>=4.8.0
4
- fasttext
5
4
  graphviz
6
- langid
5
+ py3langid>=0.2.2
7
6
  matplotlib
8
7
  morfessor
9
8
  opus-fast-mosestokenizer>=0.0.8.5
10
9
  pandas>=1.0.0
11
- pycld2
12
10
  xxhash>=3.2.0
13
11
  sentence-splitter
14
12
  rapidfuzz
@@ -21,7 +19,11 @@ tqdm
21
19
  lingua-language-detector>=1.3.0
22
20
 
23
21
  [all]
24
- eflomal>=1.0.0b1
22
+ pycld2
23
+ py3langid<0.3.0
24
+ numpy<2.0.0
25
+ fasttext
26
+ eflomal>=2.0.0
25
27
  jieba>=0.42
26
28
  mecab-python3>=1.0.8
27
29
  unidic-lite
@@ -40,7 +42,12 @@ sphinx-rtd-theme
40
42
  sphinxcontrib-bibtex
41
43
 
42
44
  [eflomal]
43
- eflomal>=1.0.0b1
45
+ eflomal>=2.0.0
46
+
47
+ [fasttext]
48
+ py3langid<0.3.0
49
+ numpy<2.0.0
50
+ fasttext
44
51
 
45
52
  [jieba]
46
53
  jieba>=0.42
@@ -52,6 +59,9 @@ laserembeddings
52
59
  mecab-python3>=1.0.8
53
60
  unidic-lite
54
61
 
62
+ [pycld2]
63
+ pycld2
64
+
55
65
  [test]
56
66
  pytest
57
67
 
@@ -1,14 +1,14 @@
1
- setuptools==65.5.1
2
- setuptools_scm==6.4.2
1
+ setuptools>=65.5.1
2
+ setuptools_scm>=6.4.2
3
+ numpy>=1.24.4
3
4
  opustools
4
5
  jieba>=0.42
5
6
  beautifulsoup4>=4.8.2
6
7
  graphviz>=0.16
7
- langid==1.1.6
8
+ py3langid==0.3.0
8
9
  matplotlib>=3.3.0
9
10
  opus-fast-mosestokenizer>=0.0.8.5
10
11
  pandas>=1.0.0
11
- pycld2==0.41
12
12
  xxhash==3.2.0
13
13
  rapidfuzz>=2.0.5
14
14
  regex>=2019.11.1
@@ -17,7 +17,6 @@ ruamel.yaml>=0.15.0
17
17
  scikit-learn>=0.24.0
18
18
  sentence-splitter==1.4
19
19
  tqdm>=4.38.0
20
- fasttext==0.9.2
21
20
  mecab-python3>=1.0.8
22
21
  unidic-lite==1.0.8
23
22
  subword-nmt==0.3.8
@@ -5,16 +5,14 @@ with open("README.md", "r") as fh:
5
5
 
6
6
  install_requires = [
7
7
  "setuptools",
8
- "opustools",
8
+ "opustools>=1.6.2",
9
9
  "beautifulsoup4>=4.8.0",
10
- "fasttext",
11
10
  "graphviz",
12
- "langid",
11
+ "py3langid>=0.2.2",
13
12
  "matplotlib",
14
13
  "morfessor",
15
14
  "opus-fast-mosestokenizer>=0.0.8.5",
16
15
  "pandas>=1.0.0",
17
- "pycld2",
18
16
  "xxhash>=3.2.0",
19
17
  "sentence-splitter",
20
18
  "rapidfuzz",
@@ -27,8 +25,18 @@ install_requires = [
27
25
  "lingua-language-detector>=1.3.0"
28
26
  ]
29
27
 
28
+ pycld2_require = [
29
+ "pycld2"
30
+ ]
31
+
32
+ fasttext_require = [
33
+ "py3langid<0.3.0", # 0.3.0 requires numpy 2.0.0
34
+ "numpy<2.0.0",
35
+ "fasttext"
36
+ ]
37
+
30
38
  eflomal_require = [
31
- 'eflomal>=1.0.0b1'
39
+ 'eflomal>=2.0.0'
32
40
  ]
33
41
 
34
42
  jieba_require = [
@@ -59,7 +67,8 @@ docs_require = [
59
67
  'sphinxcontrib-bibtex'
60
68
  ]
61
69
 
62
- all_require = eflomal_require + jieba_require + mecab_require + laser_require + varikn_require + tests_require + docs_require
70
+ all_require = pycld2_require + fasttext_require + eflomal_require + jieba_require + \
71
+ mecab_require + laser_require + varikn_require + tests_require + docs_require
63
72
 
64
73
  setuptools.setup(
65
74
  name="opusfilter",
@@ -77,9 +86,10 @@ setuptools.setup(
77
86
  "bin/opusfilter-scores", "bin/opusfilter-test"],
78
87
  install_requires=install_requires,
79
88
  tests_require=tests_require,
80
- extras_require={'test': tests_require, 'eflomal': eflomal_require, 'jieba': jieba_require,
81
- 'mecab': mecab_require, 'laser': laser_require, 'varikn': varikn_require,
82
- 'docs': docs_require, 'all': all_require},
89
+ extras_require={'test': tests_require, 'pycld2': pycld2_require, 'fasttext': fasttext_require,
90
+ 'eflomal': eflomal_require, 'jieba': jieba_require, 'mecab': mecab_require,
91
+ 'laser': laser_require, 'varikn': varikn_require, 'docs': docs_require,
92
+ 'all': all_require},
83
93
  classifiers=(
84
94
  "Programming Language :: Python :: 3",
85
95
  "License :: OSI Approved :: MIT License",
@@ -103,7 +103,7 @@ class TestThresholdFinder(unittest.TestCase):
103
103
  {'LengthRatioFilter': {'name': 'word', 'threshold': 1, 'unit': 'word'}},
104
104
  {'NonZeroNumeralsFilter': {'threshold': 1}},
105
105
  {'CharacterScoreFilter': {'scripts': ['latin', 'latin'], 'thresholds': [1, 1]}},
106
- {'LanguageIDFilter': {'id_method': 'cld2', 'languages': ['en', 'de'], 'thresholds': [1, 1]}},
106
+ {'LanguageIDFilter': {'id_method': 'lingua', 'languages': ['en', 'de'], 'thresholds': [1, 1]}},
107
107
  {'TerminalPunctuationFilter': {'threshold': 1}}
108
108
  ]
109
109