opusfilter 3.0.0rc2__tar.gz → 3.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/.github/workflows/ci.yml +21 -16
  2. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/PKG-INFO +55 -4
  3. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/README.md +1 -1
  4. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-autogen +9 -6
  5. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/CHANGELOG.md +39 -5
  6. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/CONTRIBUTING.md +3 -3
  7. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/script_and_language_identification_filters.md +14 -5
  8. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/downloading_and_selecting_data.md +6 -1
  9. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/installation.md +26 -9
  10. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/autogen.py +3 -3
  11. opusfilter-3.2.0/opusfilter/autogen_cluster.py +346 -0
  12. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/embeddings.py +9 -2
  13. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/filters.py +61 -19
  14. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/opusfilter.py +17 -6
  15. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/PKG-INFO +55 -4
  16. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/SOURCES.txt +1 -0
  17. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/requires.txt +20 -9
  18. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/requirements.txt +7 -7
  19. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/setup.py +24 -13
  20. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_autogen.py +1 -1
  21. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_embeddings.py +46 -3
  22. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_filter_pipeline.py +25 -9
  23. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_filters.py +0 -108
  24. opusfilter-3.2.0/tests/test_lid.py +151 -0
  25. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_opusfilter.py +60 -3
  26. opusfilter-3.0.0rc2/opusfilter/autogen_cluster.py +0 -150
  27. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/.gitignore +0 -0
  28. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/.pylintrc +0 -0
  29. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/LICENSE +0 -0
  30. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter +0 -0
  31. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-cmd +0 -0
  32. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-diagram +0 -0
  33. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-duplicates +0 -0
  34. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-scores +0 -0
  35. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/bin/opusfilter-test +0 -0
  36. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/automatic_configuration.md +0 -0
  37. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/command_line_tools.md +0 -0
  38. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/conf.py +0 -0
  39. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/alignment_model_filters.md +0 -0
  40. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/custom_filters.md +0 -0
  41. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/language_model_filters.md +0 -0
  42. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/length_filters.md +0 -0
  43. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/sentence_embedding_filters.md +0 -0
  44. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
  45. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/filtering_and_scoring.md +0 -0
  46. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/preprocessing_text.md +0 -0
  47. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/training_and_using_classifiers.md +0 -0
  48. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/training_language_and_alignment_models.md +0 -0
  49. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/functions/using_score_files.md +0 -0
  50. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/index.rst +0 -0
  51. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/bpe_segmentation.md +0 -0
  52. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/custom_preprocessors.md +0 -0
  53. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/detokenizer.md +0 -0
  54. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
  55. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
  56. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/reg_exp_sub.md +0 -0
  57. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/tokenizer.md +0 -0
  58. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
  59. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/references.bib +0 -0
  60. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/references.rst +0 -0
  61. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/requirements.txt +0 -0
  62. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/docs/usage.md +0 -0
  63. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/README.md +0 -0
  64. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
  65. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
  66. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
  67. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
  68. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
  69. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
  70. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/example_configs/qed_lm_langid.yaml +0 -0
  71. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/__init__.py +0 -0
  72. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/classifier.py +0 -0
  73. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/lm.py +0 -0
  74. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/pipeline.py +0 -0
  75. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/preprocessors.py +0 -0
  76. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/segment_hash.py +0 -0
  77. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/subwords.py +0 -0
  78. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/tokenization.py +0 -0
  79. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/util.py +0 -0
  80. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter/word_alignment.py +0 -0
  81. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/dependency_links.txt +0 -0
  82. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/opusfilter.egg-info/top_level.txt +0 -0
  83. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/pyproject.toml +0 -0
  84. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/setup.cfg +0 -0
  85. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_classifier.py +0 -0
  86. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_lm_filter.py +0 -0
  87. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_preprocessors.py +0 -0
  88. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_segment_hash.py +0 -0
  89. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_subwords.py +0 -0
  90. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_tokenization.py +0 -0
  91. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_util.py +0 -0
  92. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_wordalign_filter.py +0 -0
  93. {opusfilter-3.0.0rc2 → opusfilter-3.2.0}/tests/test_yaml.py +0 -0
@@ -11,6 +11,9 @@ on:
11
11
  branches:
12
12
  - master
13
13
  - develop
14
+ schedule:
15
+ # Run on Tuesdays at 5:59
16
+ - cron: '59 5 * * 2'
14
17
  workflow_dispatch:
15
18
  jobs:
16
19
  build-n-test:
@@ -18,21 +21,23 @@ jobs:
18
21
  fail-fast: false
19
22
  matrix:
20
23
  os: [ubuntu-latest]
21
- python-version: ["3.7", "3.8", "3.9", "3.10"]
24
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
22
25
  requirements-file: ["requirements.txt"]
23
26
  runs-on: ${{ matrix.os }}
24
27
  steps:
25
- - uses: actions/checkout@v3
28
+ - uses: actions/checkout@v4
26
29
  - name: Set up Python ${{ matrix.python-version }}
27
- uses: actions/setup-python@v4
30
+ uses: actions/setup-python@v5
28
31
  with:
29
32
  python-version: ${{ matrix.python-version }}
30
33
  - name: Install dependencies
31
34
  run: |
35
+ python -m ensurepip --upgrade
36
+ python -m pip install --upgrade setuptools
32
37
  python -m pip install --upgrade pip
33
38
  python -m pip install flake8 pytest wheel
34
- pip install -r ${{ matrix.requirements-file }}
35
- python setup.py install
39
+ python -m pip install --no-cache-dir -r ${{ matrix.requirements-file }}
40
+ python -m pip install .
36
41
  - name: Lint with flake8
37
42
  run: |
38
43
  # stop the build if there are Python syntax errors or undefined names
@@ -46,19 +51,19 @@ jobs:
46
51
  needs: build-n-test
47
52
  runs-on: ubuntu-20.04
48
53
  steps:
49
- - uses: actions/checkout@v3
54
+ - uses: actions/checkout@v4
50
55
  - name: Check for main release tag
51
56
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
52
57
  id: check-tag
53
58
  run: |
54
59
  if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]*)?$ ]]; then
55
- echo ::set-output name=match::true
60
+ echo "match=true" >> $GITHUB_OUTPUT
56
61
  fi
57
- - name: Set up Python 3.7
62
+ - name: Set up Python 3.8
58
63
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
59
- uses: actions/setup-python@v4
64
+ uses: actions/setup-python@v5
60
65
  with:
61
- python-version: 3.7
66
+ python-version: 3.8
62
67
  - name: Install pypa/build
63
68
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
64
69
  run: >-
@@ -91,9 +96,9 @@ jobs:
91
96
  runs-on: ubuntu-latest
92
97
  needs: [build-n-test]
93
98
  steps:
94
- - uses: actions/checkout@v3
99
+ - uses: actions/checkout@v4
95
100
  - name: Set up Python 3.8
96
- uses: actions/setup-python@v4
101
+ uses: actions/setup-python@v5
97
102
  with:
98
103
  python-version: 3.8
99
104
  - name: Install dependencies to build docs
@@ -109,7 +114,7 @@ jobs:
109
114
  if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
110
115
  uses: JamesIves/github-pages-deploy-action@v4
111
116
  with:
112
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
113
- BRANCH: gh-pages
114
- FOLDER: docs/build
115
- CLEAN: true
117
+ token: ${{ secrets.GITHUB_TOKEN }}
118
+ branch: gh-pages
119
+ folder: docs/build
120
+ clean: true
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.0.0rc2
3
+ Version: 3.2.0
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -8,17 +8,68 @@ Author-email: mikko.aulamo@helsinki.fi
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.7
11
+ Requires-Python: >=3.8
12
12
  Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: setuptools
15
+ Requires-Dist: opustools>=1.6.2
16
+ Requires-Dist: beautifulsoup4>=4.8.0
17
+ Requires-Dist: graphviz
18
+ Requires-Dist: py3langid>=0.2.2
19
+ Requires-Dist: matplotlib
20
+ Requires-Dist: morfessor
21
+ Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
22
+ Requires-Dist: pandas>=1.0.0
23
+ Requires-Dist: xxhash>=3.2.0
24
+ Requires-Dist: sentence-splitter
25
+ Requires-Dist: rapidfuzz
26
+ Requires-Dist: ruamel.yaml>=0.15.0
27
+ Requires-Dist: regex
28
+ Requires-Dist: requests
29
+ Requires-Dist: scikit-learn
30
+ Requires-Dist: subword_nmt
31
+ Requires-Dist: tqdm
32
+ Requires-Dist: lingua-language-detector>=1.3.0
13
33
  Provides-Extra: test
34
+ Requires-Dist: pytest; extra == "test"
35
+ Provides-Extra: pycld2
36
+ Requires-Dist: pycld2; extra == "pycld2"
37
+ Provides-Extra: fasttext
38
+ Requires-Dist: py3langid<0.3.0; extra == "fasttext"
39
+ Requires-Dist: numpy<2.0.0; extra == "fasttext"
40
+ Requires-Dist: fasttext; extra == "fasttext"
14
41
  Provides-Extra: eflomal
42
+ Requires-Dist: eflomal>=2.0.0; extra == "eflomal"
15
43
  Provides-Extra: jieba
44
+ Requires-Dist: jieba>=0.42; extra == "jieba"
16
45
  Provides-Extra: mecab
46
+ Requires-Dist: mecab-python3>=1.0.8; extra == "mecab"
47
+ Requires-Dist: unidic-lite; extra == "mecab"
17
48
  Provides-Extra: laser
49
+ Requires-Dist: laserembeddings; extra == "laser"
18
50
  Provides-Extra: varikn
51
+ Requires-Dist: varikn; extra == "varikn"
19
52
  Provides-Extra: docs
53
+ Requires-Dist: myst-parser; extra == "docs"
54
+ Requires-Dist: sphinx; extra == "docs"
55
+ Requires-Dist: sphinx-rtd-theme; extra == "docs"
56
+ Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
20
57
  Provides-Extra: all
21
- License-File: LICENSE
58
+ Requires-Dist: pycld2; extra == "all"
59
+ Requires-Dist: py3langid<0.3.0; extra == "all"
60
+ Requires-Dist: numpy<2.0.0; extra == "all"
61
+ Requires-Dist: fasttext; extra == "all"
62
+ Requires-Dist: eflomal>=2.0.0; extra == "all"
63
+ Requires-Dist: jieba>=0.42; extra == "all"
64
+ Requires-Dist: mecab-python3>=1.0.8; extra == "all"
65
+ Requires-Dist: unidic-lite; extra == "all"
66
+ Requires-Dist: laserembeddings; extra == "all"
67
+ Requires-Dist: varikn; extra == "all"
68
+ Requires-Dist: pytest; extra == "all"
69
+ Requires-Dist: myst-parser; extra == "all"
70
+ Requires-Dist: sphinx; extra == "all"
71
+ Requires-Dist: sphinx-rtd-theme; extra == "all"
72
+ Requires-Dist: sphinxcontrib-bibtex; extra == "all"
22
73
 
23
74
  # OpusFilter
24
75
 
@@ -47,7 +98,7 @@ Install from source:
47
98
 
48
99
  ### Troubleshooting
49
100
 
50
- OpusFilter should generally work fine on Python 3.7 to 3.10. In the case of troubles, try installing the exact versions in `requirements.txt`:
101
+ OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
51
102
 
52
103
  * `pip install -r requirements.txt`
53
104
 
@@ -25,7 +25,7 @@ Install from source:
25
25
 
26
26
  ### Troubleshooting
27
27
 
28
- OpusFilter should generally work fine on Python 3.7 to 3.10. In the case of troubles, try installing the exact versions in `requirements.txt`:
28
+ OpusFilter should generally work fine on Python 3.8 to 3.12. In the case of troubles, try installing the exact versions in `requirements.txt`:
29
29
 
30
30
  * `pip install -r requirements.txt`
31
31
 
@@ -43,9 +43,9 @@ parser.add_argument('--clusters', '-k', default=2, type=int, metavar='INT',
43
43
  parser.add_argument('--work-dir', default='work',
44
44
  help='Location of the source and target files for the generated configuration (default %(default)s)')
45
45
  parser.add_argument('--inter-dir', help='Save intermediate files in this directory (use a temporary directory if not given)')
46
- parser.add_argument('--plot', action='store_true',
47
- help=('Show a scatter plot of the clustering and histograms of feature data distributions; '
48
- 'only for the clustering method'))
46
+ parser.add_argument('--plot', metavar='PATH', default=None, type=str,
47
+ help=('Create histograms of feature data distributions and a scatter plot of the clustering; '
48
+ 'give path to plot the PDF files to, or "-" for interactive plots; only for the clustering method'))
49
49
  parser.add_argument('--list-defaults', action='store_true', help='List default filters of the method to the output and quit')
50
50
  parser.add_argument('--add-filter', nargs=2, action='append', default=[], metavar=('CLASS', 'JSON'),
51
51
  help=('Instead of using default filters, add a filter of CLASS with JSON parameters object '
@@ -78,9 +78,12 @@ if args.list_defaults:
78
78
 
79
79
  filters = filtergen.set_filter_thresholds()
80
80
 
81
- if args.method == 'clustering' and args.plot:
82
- filtergen.scoredata.plot(plt)
83
- plt.show()
81
+ if args.method == 'clustering' and args.plot is not None:
82
+ if args.plot == '-':
83
+ filtergen.scoredata.plot(plt)
84
+ plt.show()
85
+ else:
86
+ filtergen.scoredata.plot(plt, path=args.plot)
84
87
 
85
88
  generator = ConfigurationGenerator(
86
89
  files=[os.path.abspath(f) for f in args.files], langs=args.langs, workdir=args.work_dir)
@@ -7,7 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
- ## [3.0.0rc1] - 2023-09-20
10
+ ## [3.2.0] - 2024-08-14
11
+
12
+ ### Changed
13
+
14
+ - make `pycld2` and `fasttext` libraries optional
15
+ - replace `langid.py` library with `py3langid`
16
+ - update github workflows and include Python 3.12 tests
17
+
18
+ ### Fixed
19
+
20
+ - `OpusRead` interface using `moses` format (requires `opustools >= 1.6.2`)
21
+
22
+ ## [3.1.0] - 2024-06-05
23
+
24
+ ### Added
25
+
26
+ - support `lingua` based for language detection (https://github.com/Helsinki-NLP/OpusFilter/pull/65)
27
+
28
+ ### Removed
29
+
30
+ - Python 3.7 support
31
+
32
+ ### Fixed
33
+
34
+ - fix score method in `SentenceEmbeddingFilter` (https://github.com/Helsinki-NLP/OpusFilter/pull/71)
35
+ - fix filter and filterfalse methods in `SentenceEmbeddingFilter`
36
+
37
+ ## [3.0.0] - 2023-10-11
11
38
 
12
39
  ### Added
13
40
 
@@ -16,12 +43,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
16
43
 
17
44
  ### Changed
18
45
 
46
+ - refactor code and move auxiliary methods to opusfilter.util
19
47
  - update varikn installation instructions (installable from PyPI)
20
- - update github workflows
48
+ - update github workflows and include Python 3.11 tests
49
+ - update library version requirements to support Python 3.11
21
50
  - use xxhash instead of pyhash for hash functions
22
51
  - use opus-fast-mosestokenizer instead of fast-mosestokenizer
23
52
  - install eflomal from PyPI and use the new interface in WordAlignFilter
24
- - remove Python 3.6 support and tests
53
+
54
+ ### Removed
55
+
56
+ - Python 3.6 support
25
57
 
26
58
  ### Fixed
27
59
 
@@ -184,8 +216,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
184
216
  First tagged version.
185
217
 
186
218
 
187
- [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0rc1...develop
188
- [3.0.0rc1]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0rc1
219
+ [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.2.0...develop
220
+ [3.2.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...3.2.0
221
+ [3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
222
+ [3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
189
223
  [2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
190
224
  [2.5.1]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.0...2.5.1
191
225
  [2.5.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.4.0...2.5.0
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
5
5
  few rules for pull requests:
6
6
 
7
7
  * Make a pull request to the `develop` branch instead of `master`.
8
- * The code should support at least Python versions from 3.7 to 3.10.
8
+ * The code should support at least Python versions from 3.8 to 3.12.
9
9
  * Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
10
10
  * Especially for new features, please include test cases for unit testing.
11
11
 
@@ -19,8 +19,8 @@ work, if you have VariKN and eflomal set up as instructed - `pytest`
19
19
  skips the respective tests if not.)
20
20
 
21
21
  GitHub workflows defined in the project run automatically `flake8`
22
- checks and unit testing with `pytest` using Python 3.7, 3.8, 3.9, and
23
- 3.10.
22
+ checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
23
+ 3.11, and 3.12.
24
24
 
25
25
  Especially for larger contributions, consider using a code analysis
26
26
  tool like [Pylint](https://github.com/PyCQA/pylint). Install it
@@ -35,15 +35,24 @@ Filter segments based on their language identification confidence scores.
35
35
  Parameters:
36
36
 
37
37
  * `languages`: expected languages (ISO639 language codes) for the segments
38
- * `id_method`: language indentification method (`langid` for using the `langid` library, `cld2` for using the `cld2` library, or `fasttext` for using a `fasttext` model; the default is `langid`)
38
+ * `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`; default `langid`)
39
39
  * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
40
40
  * `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
41
41
  * `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
42
42
  * `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
43
+ * `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
43
44
 
44
45
  Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. Negative threshold can be used to skip filtering for a language.
45
46
 
46
- See [langid.py](https://github.com/saffsd/langid.py) and
47
- [pycld2](https://github.com/aboSamoor/pycld2) for the method-specific
48
- options. A pretrained `fasttext` model can be downloaded from
49
- [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
47
+ Currently the following identification methods are supported:
48
+
49
+ * `langid` (default) :cite:`lui-baldwin-2012-langid`
50
+ * See https://github.com/adbar/py3langid
51
+ * `lingua`
52
+ * See https://github.com/pemistahl/lingua-py
53
+ * `cld2`
54
+ * See https://github.com/CLD2Owners/cld2
55
+ * Requires [installing optional libraries](../installation.md).
56
+ * `fasttext` :cite:`joulin-etal-2016-fasttext` and :cite:`joulin-etal-2017-bag`
57
+ * A pretrained model can be downloaded from [fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html).
58
+ * Requires [installing optional libraries](../installation.md).
@@ -11,11 +11,16 @@ Parameters:
11
11
  * `source_language`: language code for the source language
12
12
  * `target_language`: language code for the target language
13
13
  * `release`: version of the corpus in OPUS
14
- * `preprocessing`: `raw` for untokenized and `xml` for tokenized segments
14
+ * `preprocessing`: `moses` or `raw` for untokenized and `xml` for tokenized segments
15
15
  * `src_output`: output file for source language
16
16
  * `tgt_output`: output file for target language
17
17
  * `suppress_prompts`: `false` (default) prompts user to confirm before download, `true` to download without prompting
18
18
 
19
+ The `moses` preprocessing type (available with `OpusTools` version
20
+ 1.6.2 and above) is recommended for those corpora for which it
21
+ exists. The output is equivalent to `raw`, but in some cases it can
22
+ significantly reduce the amount of data downloaded in the process.
23
+
19
24
  ## concatenate
20
25
 
21
26
  Concatenate two or more text files.
@@ -12,20 +12,18 @@ Install from source:
12
12
 
13
13
  Note that all required libraries are not available to install via PyPI
14
14
  on Windows OS. On Linux and MacOS, it should work directly for Python
15
- versions from 3.7 to 3.10.
15
+ versions from 3.8 to 3.12.
16
16
 
17
17
  ## Required libraries
18
18
 
19
19
  * beautifulsoup4
20
20
  * opus-fast-mosestokenizer
21
- * fasttext
22
21
  * graphviz
23
- * langid
22
+ * py3langid
24
23
  * matplotlib
25
24
  * morfessor
26
25
  * OpusTools
27
26
  * pandas
28
- * pycld2
29
27
  * rapidfuzz
30
28
  * ruamel.yaml
31
29
  * regex
@@ -35,29 +33,48 @@ versions from 3.7 to 3.10.
35
33
  * subword_nmt
36
34
  * tqdm
37
35
  * xxhash
36
+ * lingua-language-detector
38
37
 
39
38
  See `setup.py` for possible version requirements.
40
39
 
41
40
  ## Optional libraries and tools
42
41
 
42
+ ### FastText and PyCLD2 language identification
43
+
44
+ The language identification libraries currently supported out-of-the-box
45
+ are [py3langid](https://github.com/adbar/py3langid) and
46
+ [lingua](https://github.com/pemistahl/lingua-py). The support for for
47
+ [PyCLD2](https://github.com/aboSamoor/pycld2) and
48
+ [FastText models](https://fasttext.cc/docs/en/language-identification.html)
49
+ have been changed to optional due to the lack of support especially
50
+ for newer Python versions.
51
+
52
+ The PyCLD2 support can be installed automatically with pip by
53
+ including the extras `[pycld2]` or `[all]` (e.g.
54
+ `pip install opusfilter[pycld2]`).
55
+
56
+ The support for FastText models can be installed automatically with
57
+ pip by including the extras `[fasttext]` or `[all]` (e.g.
58
+ `pip install opusfilter[fasttext]`).
59
+
43
60
  ### Jieba and MeCab word segmentation
44
61
 
45
62
  For Chinese tokenization (word segmentation), you can use the
46
63
  [jieba](https://github.com/fxsjy/jieba) library. It can be installed
47
64
  automatically with pip by including the extras `[jieba]` or `[all]`
48
- (e.g. `pip install opusfilter[all]`).
65
+ (e.g. `pip install opusfilter[jieba]`).
49
66
 
50
67
  For Japanese tokenization (word segmentation), you can use the
51
68
  [MeCab](https://github.com/SamuraiT/mecab-python3) library. It can be installed
52
69
  automatically with pip by including the extras `[mecab]` or `[all]`
53
- (e.g. `pip install opusfilter[all]`).
70
+ (e.g. `pip install opusfilter[mecab]`).
54
71
 
55
72
  ### LASER sentence embeddings
56
73
 
57
74
  For using sentence embeddings filters, you need to install
58
75
  `laserembeddings` (https://github.com/yannvgn/laserembeddings). It can
59
76
  be installed automatically with pip by including the extras `[laser]`
60
- or `[all]` (e.g. `pip install opusfilter[all]`). The package will also
77
+ or `[all]` (e.g. `pip install opusfilter[laser]`). The package will also
61
78
  require a number of additional libraries, including PyTorch, jieba,
62
79
  and MeCab. Note that you need also to download the prebuild models
63
80
  with `python -m laserembeddings download-models`.
@@ -67,12 +84,12 @@ with `python -m laserembeddings download-models`.
67
84
  For using n-gram language model filters, you need to install the
68
85
  Python wrapper for VariKN (https://github.com/vsiivola/variKN). It can
69
86
  be installed automatically with pip by including the extras `[varikn]`
70
- or `[all]` (e.g. `pip install opusfilter[all]`).
87
+ or `[all]` (e.g. `pip install opusfilter[varikn]`).
71
88
 
72
89
  ### Eflomal word alignment
73
90
 
74
91
  For using word alignment filters, you need to install elfomal
75
92
  (https://github.com/robertostling/eflomal). It can be installed
76
93
  automatically with pip by including the extras `[eflomal]` or `[all]`
77
- (e.g. `pip install opusfilter[all]`). Note that you will need `Cython`
94
+ (e.g. `pip install opusfilter[eflomal]`). Note that you will need `Cython`
78
95
  for the installation.
@@ -217,7 +217,7 @@ class DefaultParameterFilters(AutoFiltersABC):
217
217
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
218
218
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
219
219
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
220
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'cld2'})]
220
+ 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
221
221
 
222
222
  def set_filter_thresholds(self):
223
223
  """Set filter thresholds"""
@@ -272,7 +272,7 @@ class PercentileFilters(DataBasedFiltersABC):
272
272
  'AverageWordLengthFilter', 'AlphabetRatioFilter',
273
273
  'TerminalPunctuationFilter', 'NonZeroNumeralsFilter',
274
274
  'LongestCommonSubstringFilter', 'SimilarityFilter', 'RepetitionFilter',
275
- 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'cld2'})]
275
+ 'CharacterScoreFilter', ('LanguageIDFilter', {'id_method': 'lingua'})]
276
276
 
277
277
  def __init__(self, files, excluded_percentile=0.001, **kwargs):
278
278
  super().__init__(files, **kwargs)
@@ -512,7 +512,7 @@ class ClusterFilters(DataBasedFiltersABC):
512
512
  ('LengthRatioFilter.word', {'unit': 'word'}),
513
513
  'NonZeroNumeralsFilter',
514
514
  'CharacterScoreFilter',
515
- ('LanguageIDFilter', {'id_method': 'cld2'}),
515
+ ('LanguageIDFilter', {'id_method': 'lingua'}),
516
516
  'TerminalPunctuationFilter']
517
517
 
518
518
  def __init__(self, files, k=2, max_length=150, **kwargs):