opusfilter 3.1.0__tar.gz → 3.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {opusfilter-3.1.0 → opusfilter-3.3.0}/.github/workflows/ci.yml +12 -10
  2. {opusfilter-3.1.0 → opusfilter-3.3.0}/PKG-INFO +19 -10
  3. {opusfilter-3.1.0 → opusfilter-3.3.0}/README.md +1 -1
  4. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-duplicates +13 -3
  5. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-scores +1 -2
  6. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/CHANGELOG.md +33 -1
  7. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/CONTRIBUTING.md +2 -2
  8. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/command_line_tools.md +2 -1
  9. opusfilter-3.3.0/docs/filters/script_and_language_identification_filters.md +204 -0
  10. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/downloading_and_selecting_data.md +6 -1
  11. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/filtering_and_scoring.md +8 -0
  12. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/installation.md +31 -9
  13. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/references.bib +13 -0
  14. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/autogen.py +3 -3
  15. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/filters.py +43 -15
  16. opusfilter-3.3.0/opusfilter/lid.py +364 -0
  17. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/opusfilter.py +17 -4
  18. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/segment_hash.py +29 -6
  19. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/PKG-INFO +19 -10
  20. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/SOURCES.txt +2 -0
  21. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/requires.txt +28 -8
  22. {opusfilter-3.1.0 → opusfilter-3.3.0}/requirements.txt +8 -7
  23. {opusfilter-3.1.0 → opusfilter-3.3.0}/setup.py +27 -10
  24. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_autogen.py +4 -2
  25. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_filter_pipeline.py +25 -9
  26. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_filters.py +14 -130
  27. opusfilter-3.3.0/tests/test_lid.py +209 -0
  28. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_opusfilter.py +21 -1
  29. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_segment_hash.py +18 -0
  30. opusfilter-3.1.0/docs/filters/script_and_language_identification_filters.md +0 -50
  31. {opusfilter-3.1.0 → opusfilter-3.3.0}/.gitignore +0 -0
  32. {opusfilter-3.1.0 → opusfilter-3.3.0}/.pylintrc +0 -0
  33. {opusfilter-3.1.0 → opusfilter-3.3.0}/LICENSE +0 -0
  34. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter +0 -0
  35. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-autogen +0 -0
  36. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-cmd +0 -0
  37. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-diagram +0 -0
  38. {opusfilter-3.1.0 → opusfilter-3.3.0}/bin/opusfilter-test +0 -0
  39. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/automatic_configuration.md +0 -0
  40. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/conf.py +0 -0
  41. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/alignment_model_filters.md +0 -0
  42. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/custom_filters.md +0 -0
  43. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/language_model_filters.md +0 -0
  44. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/length_filters.md +0 -0
  45. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/sentence_embedding_filters.md +0 -0
  46. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
  47. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/preprocessing_text.md +0 -0
  48. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/training_and_using_classifiers.md +0 -0
  49. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/training_language_and_alignment_models.md +0 -0
  50. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/functions/using_score_files.md +0 -0
  51. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/index.rst +0 -0
  52. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/bpe_segmentation.md +0 -0
  53. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/custom_preprocessors.md +0 -0
  54. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/detokenizer.md +0 -0
  55. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
  56. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
  57. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/reg_exp_sub.md +0 -0
  58. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/tokenizer.md +0 -0
  59. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
  60. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/references.rst +0 -0
  61. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/requirements.txt +0 -0
  62. {opusfilter-3.1.0 → opusfilter-3.3.0}/docs/usage.md +0 -0
  63. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/README.md +0 -0
  64. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
  65. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
  66. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
  67. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
  68. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
  69. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
  70. {opusfilter-3.1.0 → opusfilter-3.3.0}/example_configs/qed_lm_langid.yaml +0 -0
  71. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/__init__.py +0 -0
  72. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/autogen_cluster.py +0 -0
  73. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/classifier.py +0 -0
  74. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/embeddings.py +0 -0
  75. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/lm.py +0 -0
  76. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/pipeline.py +0 -0
  77. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/preprocessors.py +0 -0
  78. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/subwords.py +0 -0
  79. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/tokenization.py +0 -0
  80. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/util.py +0 -0
  81. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter/word_alignment.py +0 -0
  82. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/dependency_links.txt +0 -0
  83. {opusfilter-3.1.0 → opusfilter-3.3.0}/opusfilter.egg-info/top_level.txt +0 -0
  84. {opusfilter-3.1.0 → opusfilter-3.3.0}/pyproject.toml +0 -0
  85. {opusfilter-3.1.0 → opusfilter-3.3.0}/setup.cfg +0 -0
  86. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_classifier.py +0 -0
  87. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_embeddings.py +0 -0
  88. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_lm_filter.py +0 -0
  89. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_preprocessors.py +0 -0
  90. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_subwords.py +0 -0
  91. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_tokenization.py +0 -0
  92. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_util.py +0 -0
  93. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_wordalign_filter.py +0 -0
  94. {opusfilter-3.1.0 → opusfilter-3.3.0}/tests/test_yaml.py +0 -0
@@ -21,21 +21,23 @@ jobs:
21
21
  fail-fast: false
22
22
  matrix:
23
23
  os: [ubuntu-latest]
24
- python-version: ["3.8", "3.9", "3.10", "3.11"]
24
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
25
25
  requirements-file: ["requirements.txt"]
26
26
  runs-on: ${{ matrix.os }}
27
27
  steps:
28
- - uses: actions/checkout@v3
28
+ - uses: actions/checkout@v4
29
29
  - name: Set up Python ${{ matrix.python-version }}
30
- uses: actions/setup-python@v4
30
+ uses: actions/setup-python@v5
31
31
  with:
32
32
  python-version: ${{ matrix.python-version }}
33
33
  - name: Install dependencies
34
34
  run: |
35
+ python -m ensurepip --upgrade
36
+ python -m pip install --upgrade setuptools
35
37
  python -m pip install --upgrade pip
36
38
  python -m pip install flake8 pytest wheel
37
- pip install -r ${{ matrix.requirements-file }}
38
- python setup.py install
39
+ python -m pip install --no-cache-dir -r ${{ matrix.requirements-file }}
40
+ python -m pip install .
39
41
  - name: Lint with flake8
40
42
  run: |
41
43
  # stop the build if there are Python syntax errors or undefined names
@@ -47,9 +49,9 @@ jobs:
47
49
  build-n-publish:
48
50
  name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
49
51
  needs: build-n-test
50
- runs-on: ubuntu-20.04
52
+ runs-on: ubuntu-latest
51
53
  steps:
52
- - uses: actions/checkout@v3
54
+ - uses: actions/checkout@v4
53
55
  - name: Check for main release tag
54
56
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
55
57
  id: check-tag
@@ -59,7 +61,7 @@ jobs:
59
61
  fi
60
62
  - name: Set up Python 3.8
61
63
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
62
- uses: actions/setup-python@v4
64
+ uses: actions/setup-python@v5
63
65
  with:
64
66
  python-version: 3.8
65
67
  - name: Install pypa/build
@@ -94,9 +96,9 @@ jobs:
94
96
  runs-on: ubuntu-latest
95
97
  needs: [build-n-test]
96
98
  steps:
97
- - uses: actions/checkout@v3
99
+ - uses: actions/checkout@v4
98
100
  - name: Set up Python 3.8
99
- uses: actions/setup-python@v4
101
+ uses: actions/setup-python@v5
100
102
  with:
101
103
  python-version: 3.8
102
104
  - name: Install dependencies to build docs
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.1.0
3
+ Version: 3.3.0
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -12,16 +12,15 @@ Requires-Python: >=3.8
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
14
  Requires-Dist: setuptools
15
- Requires-Dist: opustools
15
+ Requires-Dist: opustools>=1.6.2
16
16
  Requires-Dist: beautifulsoup4>=4.8.0
17
- Requires-Dist: fasttext
18
17
  Requires-Dist: graphviz
19
- Requires-Dist: langid
18
+ Requires-Dist: py3langid>=0.2.2
20
19
  Requires-Dist: matplotlib
21
20
  Requires-Dist: morfessor
22
- Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
21
+ Requires-Dist: opus-fast-mosestokenizer==0.0.8.7; python_version < "3.9"
22
+ Requires-Dist: opus-fast-mosestokenizer>=0.0.8.7; python_version >= "3.9"
23
23
  Requires-Dist: pandas>=1.0.0
24
- Requires-Dist: pycld2
25
24
  Requires-Dist: xxhash>=3.2.0
26
25
  Requires-Dist: sentence-splitter
27
26
  Requires-Dist: rapidfuzz
@@ -31,11 +30,19 @@ Requires-Dist: requests
31
30
  Requires-Dist: scikit-learn
32
31
  Requires-Dist: subword_nmt
33
32
  Requires-Dist: tqdm
34
- Requires-Dist: lingua-language-detector>=1.3.0
33
+ Requires-Dist: lingua-language-detector<2.1,>=1.3.0; python_version < "3.10"
34
+ Requires-Dist: lingua-language-detector>=2.1.1; python_version >= "3.10"
35
+ Requires-Dist: iso639-lang
35
36
  Provides-Extra: test
36
37
  Requires-Dist: pytest; extra == "test"
38
+ Provides-Extra: pycld2
39
+ Requires-Dist: pycld2; extra == "pycld2"
40
+ Provides-Extra: fasttext
41
+ Requires-Dist: py3langid<0.3.0; extra == "fasttext"
42
+ Requires-Dist: numpy<2.0.0; extra == "fasttext"
43
+ Requires-Dist: fasttext; extra == "fasttext"
37
44
  Provides-Extra: eflomal
38
- Requires-Dist: eflomal>=1.0.0b1; extra == "eflomal"
45
+ Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
39
46
  Provides-Extra: jieba
40
47
  Requires-Dist: jieba>=0.42; extra == "jieba"
41
48
  Provides-Extra: mecab
@@ -45,13 +52,15 @@ Provides-Extra: laser
45
52
  Requires-Dist: laserembeddings; extra == "laser"
46
53
  Provides-Extra: varikn
47
54
  Requires-Dist: varikn; extra == "varikn"
55
+ Provides-Extra: heliport
56
+ Requires-Dist: heliport>=0.10.0; extra == "heliport"
48
57
  Provides-Extra: docs
49
58
  Requires-Dist: myst-parser; extra == "docs"
50
59
  Requires-Dist: sphinx; extra == "docs"
51
60
  Requires-Dist: sphinx-rtd-theme; extra == "docs"
52
61
  Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
53
62
  Provides-Extra: all
54
- Requires-Dist: eflomal>=1.0.0b1; extra == "all"
63
+ Requires-Dist: eflomal>=2.0.0; extra == "all"
55
64
  Requires-Dist: jieba>=0.42; extra == "all"
56
65
  Requires-Dist: mecab-python3>=1.0.8; extra == "all"
57
66
  Requires-Dist: unidic-lite; extra == "all"
@@ -90,7 +99,7 @@ Install from source:
90
99
 
91
100
  ### Troubleshooting
92
101
 
93
- OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
102
+ OpusFilter should generally work fine on Python 3.8 to 3.13. In the case of troubles, try installing the exact versions in `requirements.txt`:
94
103
 
95
104
  * `pip install -r requirements.txt`
96
105
 
@@ -25,7 +25,7 @@ Install from source:
25
25
 
26
26
  ### Troubleshooting
27
27
 
28
- OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
28
+ OpusFilter should generally work fine on Python 3.8 to 3.13. In the case of troubles, try installing the exact versions in `requirements.txt`:
29
29
 
30
30
  * `pip install -r requirements.txt`
31
31
 
@@ -3,6 +3,7 @@
3
3
  import argparse
4
4
  import collections
5
5
  import logging
6
+ import json
6
7
 
7
8
  from tqdm import tqdm
8
9
 
@@ -24,20 +25,29 @@ parser.add_argument('--hash', type=str, default='xxh64',
24
25
  help=('hash function from xxhash library, empty string '
25
26
  'for no hashing (default "xxh64")'))
26
27
  parser.add_argument('--letters-only', action='store_true', default=False,
27
- help='lowercase input strings before hashing')
28
- parser.add_argument('--lowercase', action='store_true', default=False,
29
28
  help='remove all non-letters from intput strings before hashing')
29
+ parser.add_argument('--letter-words-only', action='store_true', default=False,
30
+ help='remove words with non-letter characters before hashing')
31
+ parser.add_argument('--lowercase', action='store_true', default=False,
32
+ help='lowercase input strings before hashing')
33
+ parser.add_argument('--tokenizers', type=str, metavar='JSON', default=None,
34
+ help=('load tokenizer specifications from a JSON list (e.g. \'[["moses", "en"], ["jieba", "zh"]]\'); '
35
+ 'use with --letter-words-only'))
30
36
 
31
37
  args = parser.parse_args()
32
38
 
33
39
  if args.overlap and len(args.overlap) != len(args.files):
34
40
  raise ValueError("The number of the main and overlap input files should match")
35
41
 
42
+ tokenizers = json.loads(args.tokenizers) if args.tokenizers else None
43
+
36
44
  hasher = SegmentHasher(
37
45
  compare='all',
38
46
  method=args.hash,
39
47
  letters_only=args.letters_only,
40
- lowercase=args.lowercase
48
+ letter_words_only=args.letter_words_only,
49
+ lowercase=args.lowercase,
50
+ tokenizers=tokenizers
41
51
  )
42
52
 
43
53
  total = 0
@@ -14,8 +14,7 @@ from tqdm import tqdm
14
14
 
15
15
  from pandas.plotting import scatter_matrix
16
16
 
17
- from opusfilter.util import file_open
18
- from opusfilter.classifier import lists_to_dicts
17
+ from opusfilter.util import file_open, lists_to_dicts
19
18
 
20
19
 
21
20
  logger = logging.getLogger(__name__)
@@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [3.3.0] - 2025-07-02
11
+
12
+ ### Added
13
+
14
+ - `letter_words_only` option for duplicate detection
15
+ - support for selection of possible languages for `lingua` language detection
16
+ - support for language detection based on `heliport` (a port of HeLI-OTS)
17
+
18
+ ### Changed
19
+
20
+ - update library requirements and include Python 3.13 tests
21
+ - removed pycld2 and fasttext from the `[all]` extras
22
+ - refactored all language identification filters to have their own classes and marked `LanguageIDFilter` as deprecated
23
+
24
+ ### Fixed
25
+
26
+ - fix broken import in `opusfilter-scores`
27
+
28
+ ## [3.2.0] - 2024-08-14
29
+
30
+ ### Changed
31
+
32
+ - make `pycld2` and `fasttext` libraries optional
33
+ - replace `langid.py` library with `py3langid`
34
+ - update github workflows and include Python 3.12 tests
35
+
36
+ ### Fixed
37
+
38
+ - `OpusRead` interface using `moses` format (requires `opustools >= 1.6.2`)
39
+
10
40
  ## [3.1.0] - 2024-06-05
11
41
 
12
42
  ### Added
@@ -204,7 +234,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
204
234
  First tagged version.
205
235
 
206
236
 
207
- [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...develop
237
+ [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.3.0...develop
238
+ [3.3.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...3.3.0
239
+ [3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
208
240
  [3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
209
241
  [3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
210
242
  [2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
5
5
  few rules for pull requests:
6
6
 
7
7
  * Make a pull request to the `develop` branch instead of `master`.
8
- * The code should support at least Python versions from 3.8 to 3.11.
8
+ * The code should support at least Python versions from 3.8 to 3.13.
9
9
  * Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
10
10
  * Especially for new features, please include test cases for unit testing.
11
11
 
@@ -20,7 +20,7 @@ skips the respective tests if not.)
20
20
 
21
21
  GitHub workflows defined in the project run automatically `flake8`
22
22
  checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
23
- and 3.11.
23
+ 3.11, 3.12, and 3.13.
24
24
 
25
25
  Especially for larger contributions, consider using a code analysis
26
26
  tool like [Pylint](https://github.com/PyCQA/pylint). Install it
@@ -30,7 +30,8 @@ number of duplicates in it, or two corpora for calculating the overlap
30
30
  between them. The syntax for the `opusfilter-duplicates` is:
31
31
 
32
32
  ```
33
- opusfilter-duplicates [--overlap FILE [FILE ...]] [--hash HASH] [--letters-only] [--lowercase] FILE [FILE ...]
33
+ opusfilter-duplicates [--overlap FILE [FILE ...]] [--hash HASH] [--letters-only]
34
+ [--letter-words-only] [--lowercase] [--tokenizers JSON] FILE [FILE ...]
34
35
  ```
35
36
 
36
37
  The options are essentially the same as for [`remove_duplicates`](remove_duplicates).
@@ -0,0 +1,204 @@
1
+ # Script and language identification filters
2
+
3
+ ## AlphabetRatioFilter
4
+
5
+ Filter segments based on what proportion of the characters are alphabetic characters.
6
+
7
+ Parameters:
8
+
9
+ * `threshold`: minimum proportion of alphabets in a segment (default 0.75)
10
+ * `exclude_whitespace`: whether to exclude whitespace characters from the ratio (default `false`)
11
+
12
+ Returned scores are proportions of alphabetic characters in the
13
+ segments (after removing whitespace if `exclude_whitespace` is
14
+ true). In filtering, all values have to be equal to or greater than
15
+ the minimum threshold.
16
+
17
+ In order to allow have different thresholds per language, the
18
+ threshold parameter can also be given as a list.
19
+
20
+ ## CharacterScoreFilter
21
+
22
+ Filter segments based on what proportion of their alphabetic characters are in a given script. For a list of valid scripts, see e.g. [www.regular-expressions.info/unicode.html](https://www.regular-expressions.info/unicode.html).
23
+
24
+ Parameters:
25
+
26
+ * `scripts`: scripts for input segments
27
+ * `thresholds`: minimum proportion of characters in a script (default 1)
28
+
29
+ Returned scores are proportions of valid characters in the segments. In filtering, all values have to be equal to or greater than the minimum thresholds.
30
+
31
+ ## LanguageIDFilter (deprecated)
32
+
33
+ *Note: The common LanguageIDFilter is now deprecated. Please see the method-specific language identification filters below.*
34
+
35
+ Filter segments based on their language identification confidence scores.
36
+
37
+ Parameters:
38
+
39
+ * `languages`: expected languages (ISO639 language codes) for the segments
40
+ * `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`, `heliport`; default `langid`)
41
+ * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
42
+ * `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
43
+ * `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages (valid only for the `langid` and `lingua` methods; default `null`)
44
+ * `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
45
+ * `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
46
+
47
+ Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. A negative threshold can be used to skip filtering for a language.
48
+
49
+ Currently the following identification methods are supported:
50
+
51
+ * `langid` (default) :cite:`lui-baldwin-2012-langid`
52
+ * See https://github.com/adbar/py3langid
53
+ * `lingua`
54
+ * See https://github.com/pemistahl/lingua-py
55
+ * `cld2`
56
+ * See https://github.com/CLD2Owners/cld2
57
+ * Requires [installing optional libraries](../installation.md).
58
+ * `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
59
+ * A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
60
+ * Requires [installing optional libraries](../installation.md).
61
+
62
+ ## LangidFilter
63
+
64
+ Filter segments based on the langid method :cite:`lui-baldwin-2012-langid`; see
65
+ https://github.com/adbar/py3langid
66
+
67
+ Parameters:
68
+
69
+ * `languages`: expected languages (ISO639 language codes) for the segments
70
+ * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
71
+ * `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages
72
+
73
+ Returned scores are the language identification confidence scores for
74
+ the segments. The scores range from 0 to 1. In filtering, all values
75
+ have to be greater than the minimum thresholds. A negative threshold
76
+ can be used to skip filtering for a language.
77
+
78
+ ## Cld2Filter
79
+
80
+ Filter segments based on the CLD2 method; see https://github.com/CLD2Owners/cld2
81
+
82
+ Requires [installing optional libraries](../installation.md).
83
+
84
+ Parameters:
85
+
86
+ * `languages`: expected languages (ISO639 language codes) for the segments
87
+ * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
88
+ * `options`: a dictionary of options for the method (see https://github.com/CLD2Owners/cld2; default `null`)
89
+
90
+ Returned scores are the language identification confidence scores from
91
+ a given identification method for the segments. The scores range from
92
+ 0 to 1. In filtering, all values have to be greater than the minimum
93
+ thresholds. A negative threshold can be used to skip filtering for a
94
+ language.
95
+
96
+ ## FastTextFilter
97
+
98
+ Filter segments based on the Fasttext language identification models
99
+ (:cite:`joulin-etal-2016-fasttext`, :cite:`joulin-etal-2017-bag`), see
100
+ https://github.com/CLD2Owners/cld2.
101
+
102
+ Requires [installing optional libraries](../installation.md).
103
+
104
+ Parameters:
105
+
106
+ * `languages`: expected languages (ISO639 language codes) for the segments
107
+ * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
108
+ * `model_path`: path for a `fasttext` model
109
+
110
+ Returned scores are the language identification confidence scores from
111
+ a given identification method for the segments. The scores range from
112
+ 0 to 1. In filtering, all values have to be greater than the minimum
113
+ thresholds. A negative threshold can be used to skip filtering for a
114
+ language.
115
+
116
+ ## LinguaFilter
117
+
118
+ Filter segments based on the Lingua method; see https://github.com/pemistahl/lingua-py
119
+
120
+ Parameters:
121
+
122
+ * `languages`: expected languages (ISO639 language codes) for the segments
123
+ * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
124
+ * `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages (default `null`)
125
+ * `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode (default `low`)
126
+
127
+ Returned scores are the language identification confidence scores from
128
+ a given identification method for the segments. The scores range from
129
+ 0 to 1. In filtering, all values have to be greater than the minimum
130
+ thresholds. A negative threshold can be used to skip filtering for a
131
+ language.
132
+
133
+ ## HeliportSimpleFilter
134
+
135
+ Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
136
+ https://github.com/ZJaume/heliport
137
+
138
+ Requires [installing optional libraries](../installation.md).
139
+
140
+ Parameters:
141
+
142
+ * `languages`: expected languages (ISO639 language codes) for the segments
143
+ * `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
144
+
145
+ This version uses three distinct score values. Score 1.0 is used if
146
+ the method predicts the correct language. If the low-confidence label
147
+ `und` is returned, the language is considered correct with score
148
+ 0.5. Finally, if a wrong language is predicted, the score will
149
+ 0.0. A negative threshold can be used to skip filtering for a language.
150
+
151
+ ## HeliportConfidenceFilter
152
+
153
+ Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
154
+ https://github.com/ZJaume/heliport
155
+
156
+ Requires [installing optional libraries](../installation.md).
157
+
158
+ Parameters:
159
+
160
+ * `languages`: expected languages (ISO639 language codes) for the segments
161
+ * `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
162
+
163
+ This version uses the confidence value returned by the method. It is
164
+ the difference between raw scores of the the best (lowest) and second
165
+ best language. The maximum value is 7.0, which is a penalty value for
166
+ the languages that do not match. The minimum value 0 is returned if
167
+ the best language does not match. The low-confidence label `und` is
168
+ considered as a match. A negative threshold can be used to skip
169
+ filtering for a language.
170
+
171
+ ## HeliportRawScoreFilter
172
+
173
+ Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
174
+ https://github.com/ZJaume/heliport
175
+
176
+ Requires [installing optional libraries](../installation.md).
177
+
178
+ Parameters:
179
+
180
+ * `languages`: expected languages (ISO639 language codes) for the segments
181
+ * `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
182
+ * `topk`: the number of top languages to consider (default 50)
183
+
184
+ This version uses the raw scores returned by the method. A lower score
185
+ is better. If the correct language is among the top-k languages, the
186
+ score if that language is returned. Otherwise, the penalty value 7 is
187
+ returned. A threshold over the penalty value can be used to skip
188
+ filtering for a language.
189
+
190
+ ## HeliportProbabilityFilter
191
+
192
+ Filter segments based on the HeLI-OTS method :cite:`jauhiainen-etal-2022-heli`; see
193
+ https://github.com/ZJaume/heliport
194
+
195
+ Requires [installing optional libraries](../installation.md).
196
+
197
+ * `languages`: expected languages (ISO639 language codes) for the segments
198
+ * `thresholds`: minimum identification score for the segments (a single float or a list of floats per language)
199
+ * `topk`: the number of top languages to consider (default 50)
200
+
201
+ This version transforms the raw scores of of the top-k languages into
202
+ probability-like values. If the correct language is not in the top-k
203
+ list, zero is returned. A negative threshold can be used to skip
204
+ filtering for a language.
@@ -11,11 +11,16 @@ Parameters:
11
11
  * `source_language`: language code for the source language
12
12
  * `target_language`: language code for the target language
13
13
  * `release`: version of the corpus in OPUS
14
- * `preprocessing`: `raw` for untokenized and `xml` for tokenized segments
14
+ * `preprocessing`: `moses` or `raw` for untokenized and `xml` for tokenized segments
15
15
  * `src_output`: output file for source language
16
16
  * `tgt_output`: output file for target language
17
17
  * `suppress_prompts`: `false` (default) prompts user to confirm before download, `true` to download without prompting
18
18
 
19
+ The `moses` preprocessing type (available with `OpusTools` version
20
+ 1.6.2 and above) is recommended for those corpora for which it
21
+ exists. The output is equivalent to `raw`, but in some cases it can
22
+ significantly reduce the amount of data downloaded in the process.
23
+
19
24
  ## concatenate
20
25
 
21
26
  Concatenate two or more text files.
@@ -11,6 +11,10 @@ Parameters:
11
11
  * `compare`: select files for duplicate comparison (optional; default `all` or a list of indices)
12
12
  * `hash`: select hash algorithm from xxhash (optional; default `xxh64`)
13
13
  * `overlap`: remove overlap with a second set of files (optional; default `null`)
14
+ * `letters_only`: remove all non-letters from intput strings before hashing (optional; default `false`)
15
+ * `letter_words_only`: remove words with non-letter characters before hashing (optional; default `false`)
16
+ * `lowercase`: lowercase input strings before hashing (optional; default `false`)
17
+ * `tokenizers`: load tokenizer specifications from a list (optional; use with `letter_words_only`)
14
18
 
15
19
  Duplicate filtering is recommended as a first step especially if you
16
20
  combine different corpus collections (e.g. data crawled from web) and
@@ -38,6 +42,10 @@ sizes, but if do not care about memory use and want to be extra sure
38
42
  there are no collisions, you can disable hashing by setting the `hash`
39
43
  parameter as empty string or null.
40
44
 
45
+ For the `letter_words_only`, unless the input is pre-tokenized, you
46
+ should select tokenizers for each of the input files. See
47
+ [Tokenizer](tokenizer.md) for description of the tokenizer parameters.
48
+
41
49
  ## filter
42
50
 
43
51
  Filter parallel data with a combination of filters.
@@ -12,20 +12,18 @@ Install from source:
12
12
 
13
13
  Note that all required libraries are not available to install via PyPI
14
14
  on Windows OS. On Linux and MacOS, it should work directly for Python
15
- versions from 3.8 to 3.11.
15
+ versions from 3.8 to 3.13.
16
16
 
17
17
  ## Required libraries
18
18
 
19
19
  * beautifulsoup4
20
20
  * opus-fast-mosestokenizer
21
- * fasttext
22
21
  * graphviz
23
- * langid
22
+ * py3langid
24
23
  * matplotlib
25
24
  * morfessor
26
25
  * OpusTools
27
26
  * pandas
28
- * pycld2
29
27
  * rapidfuzz
30
28
  * ruamel.yaml
31
29
  * regex
@@ -41,24 +39,48 @@ See `setup.py` for possible version requirements.
41
39
 
42
40
  ## Optional libraries and tools
43
41
 
42
+ ### FastText, PyCLD2, and heliport language identification
43
+
44
+ The language identification libraries currently supported out-of-the-box
45
+ are [py3langid](https://github.com/adbar/py3langid) and
46
+ [lingua](https://github.com/pemistahl/lingua-py). The support for
47
+ [PyCLD2](https://github.com/aboSamoor/pycld2) and
48
+ [FastText models](https://fasttext.cc/docs/en/language-identification.html)
49
+ have been changed to optional due to the lack of support especially
50
+ for newer Python versions. Also the support for
51
+ [heliport](https://github.com/ZJaume/heliport) is optional. Please
52
+ note that `heliport` has GPL-3.0 license.
53
+
54
+ The PyCLD2 support can be installed automatically with pip by
55
+ including the extras `[pycld2]` (e.g. `pip install
56
+ opusfilter[pycld2]`).
57
+
58
+ The support for FastText models can be installed automatically with
59
+ pip by including the extras `[fasttext]` (e.g. `pip install
60
+ opusfilter[fasttext]`).
61
+
62
+ The support for `heliport` can be installed automatically with pip by
63
+ including the extras `[heliport]` (e.g. `pip install
64
+ opusfilter[heliport]`).
65
+
44
66
  ### Jieba and MeCab word segmentation
45
67
 
46
68
  For Chinese tokenization (word segmentation), you can use the
47
69
  [jieba](https://github.com/fxsjy/jieba) library. It can be installed
48
70
  automatically with pip by including the extras `[jieba]` or `[all]`
49
- (e.g. `pip install opusfilter[all]`).
71
+ (e.g. `pip install opusfilter[jieba]`).
50
72
 
51
73
  For Japanese tokenization (word segmentation), you can use the
52
74
  [MeCab](https://github.com/SamuraiT/mecab-python3) library. It can be installed
53
75
  automatically with pip by including the extras `[mecab]` or `[all]`
54
- (e.g. `pip install opusfilter[all]`).
76
+ (e.g. `pip install opusfilter[mecab]`).
55
77
 
56
78
  ### LASER sentence embeddings
57
79
 
58
80
  For using sentence embeddings filters, you need to install
59
81
  `laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
60
82
  be installed automatically with pip by including the extras `[laser]`
61
- or `[all]` (e.g. `pip install opusfilter[all]`). The package will also
83
+ or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
62
84
  require a number of additional libraries, including PyTorch, jieba,
63
85
  and MeCab. Note that you need also to download the prebuild models
64
86
  with `python -m laserembeddings download-models`.
@@ -68,12 +90,12 @@ with `python -m laserembeddings download-models`.
68
90
  For using n-gram language model filters, you need to install the
69
91
  Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
70
92
  be installed automatically with pip by including the extras `[varikn]`
71
- or `[all]` (e.g. `pip install opusfilter[all]`).
93
+ or `[all]` (e.g. `pip install opusfilter[varikn]`).
72
94
 
73
95
  ### Eflomal word alignment
74
96
 
75
97
  For using word alignment filters, you need to install elfomal
76
98
  (https://github.com/robertostling/eflomal). It can be installed
77
99
  automatically with pip by including the extras `[eflomal]` or `[all]`
78
- (e.g. `pip install opusfilter[all]`). Note that you will need `Cython`
100
+ (e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
79
101
  for the installation.
@@ -68,6 +68,19 @@
68
68
  pages = "261--266"
69
69
  }
70
70
 
71
+ % Jauhiainen et al. (2022)
72
+ @inproceedings{jauhiainen-etal-2022-heli,
73
+ title = "{H}e{LI}-{OTS}, Off-the-shelf Language Identifier for Text",
74
+ author = "Jauhiainen, Tommi and Jauhiainen, Heidi and Lind{\'e}n, Krister",
75
+ booktitle = "Proceedings of the Thirteenth Language Resources and Evaluation Conference",
76
+ month = jun,
77
+ year = "2022",
78
+ address = "Marseille, France",
79
+ publisher = "European Language Resources Association",
80
+ url = "https://aclanthology.org/2022.lrec-1.416/",
81
+ pages = "3912--3922"
82
+ }
83
+
71
84
  % Joulin et al. (2016)
72
85
  @article{joulin-etal-2016-fasttext,
73
86
  author = {Armand Joulin and Edouard Grave and Piotr Bojanowski and Matthijs Douze and Herv{\'{e}} J{\'{e}}gou and Tom{\'{a}}s Mikolov},
@@ -217,7 +217,7 @@ class DefaultParameterFilters(AutoFiltersABC):
217
217
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
218
218
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
219
219
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
220
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'cld2'})]
220
+ 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
221
221
 
222
222
  def set_filter_thresholds(self):
223
223
  """Set filter thresholds"""
@@ -272,7 +272,7 @@ class PercentileFilters(DataBasedFiltersABC):
272
272
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
273
273
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
274
274
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
275
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'cld2'})]
275
+ 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
276
276
 
277
277
  def __init__(self, files, excluded_percentile=0.001, **kwargs):
278
278
  super().__init__(files, **kwargs)
@@ -512,7 +512,7 @@ class ClusterFilters(DataBasedFiltersABC):
512
512
  ('LengthRatioFilter.word', {'unit': 'word'}),
513
513
  'NonZeroNumeralsFilter',
514
514
  'CharacterScoreFilter',
515
- ('LanguageIDFilter', {'id_method': 'cld2'}),
515
+ ('LanguageIDFilter', {'id_method': 'lingua'}),
516
516
  'TerminalPunctuationFilter']
517
517
 
518
518
  def __init__(self, files, k=2, max_length=150, **kwargs):