opusfilter 3.0.0__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opusfilter-3.0.0 → opusfilter-3.1.0}/.github/workflows/ci.yml +12 -9
- {opusfilter-3.0.0/opusfilter.egg-info → opusfilter-3.1.0}/PKG-INFO +47 -4
- {opusfilter-3.0.0 → opusfilter-3.1.0}/README.md +1 -1
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-autogen +9 -6
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/CHANGELOG.md +17 -1
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/CONTRIBUTING.md +3 -3
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/script_and_language_identification_filters.md +1 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/installation.md +2 -1
- opusfilter-3.1.0/opusfilter/autogen_cluster.py +346 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/embeddings.py +9 -2
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/filters.py +51 -17
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/opusfilter.py +4 -3
- {opusfilter-3.0.0 → opusfilter-3.1.0/opusfilter.egg-info}/PKG-INFO +47 -4
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/requires.txt +1 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/requirements.txt +1 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/setup.py +3 -2
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_embeddings.py +46 -3
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_filters.py +20 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_opusfilter.py +39 -2
- opusfilter-3.0.0/opusfilter/autogen_cluster.py +0 -150
- {opusfilter-3.0.0 → opusfilter-3.1.0}/.gitignore +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/.pylintrc +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/LICENSE +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-cmd +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-diagram +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-duplicates +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-scores +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-test +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/automatic_configuration.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/command_line_tools.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/conf.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/alignment_model_filters.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/custom_filters.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/language_model_filters.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/length_filters.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/sentence_embedding_filters.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/downloading_and_selecting_data.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/filtering_and_scoring.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/preprocessing_text.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/training_and_using_classifiers.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/training_language_and_alignment_models.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/using_score_files.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/index.rst +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/bpe_segmentation.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/custom_preprocessors.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/detokenizer.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/reg_exp_sub.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/tokenizer.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/references.bib +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/references.rst +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/requirements.txt +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/usage.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/README.md +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/qed_lm_langid.yaml +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/__init__.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/autogen.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/classifier.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/lm.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/pipeline.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/preprocessors.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/segment_hash.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/subwords.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/tokenization.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/util.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/word_alignment.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/SOURCES.txt +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/dependency_links.txt +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/top_level.txt +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/pyproject.toml +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/setup.cfg +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_autogen.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_classifier.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_filter_pipeline.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_lm_filter.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_preprocessors.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_segment_hash.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_subwords.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_tokenization.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_util.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_wordalign_filter.py +0 -0
- {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_yaml.py +0 -0
|
@@ -11,6 +11,9 @@ on:
|
|
|
11
11
|
branches:
|
|
12
12
|
- master
|
|
13
13
|
- develop
|
|
14
|
+
schedule:
|
|
15
|
+
# Run on Tuesdays at 5:59
|
|
16
|
+
- cron: '59 5 * * 2'
|
|
14
17
|
workflow_dispatch:
|
|
15
18
|
jobs:
|
|
16
19
|
build-n-test:
|
|
@@ -18,7 +21,7 @@ jobs:
|
|
|
18
21
|
fail-fast: false
|
|
19
22
|
matrix:
|
|
20
23
|
os: [ubuntu-latest]
|
|
21
|
-
python-version: ["3.
|
|
24
|
+
python-version: ["3.8", "3.9", "3.10", "3.11"]
|
|
22
25
|
requirements-file: ["requirements.txt"]
|
|
23
26
|
runs-on: ${{ matrix.os }}
|
|
24
27
|
steps:
|
|
@@ -54,11 +57,11 @@ jobs:
|
|
|
54
57
|
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]*)?$ ]]; then
|
|
55
58
|
echo "match=true" >> $GITHUB_OUTPUT
|
|
56
59
|
fi
|
|
57
|
-
- name: Set up Python 3.
|
|
60
|
+
- name: Set up Python 3.8
|
|
58
61
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
59
62
|
uses: actions/setup-python@v4
|
|
60
63
|
with:
|
|
61
|
-
python-version: 3.
|
|
64
|
+
python-version: 3.8
|
|
62
65
|
- name: Install pypa/build
|
|
63
66
|
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
64
67
|
run: >-
|
|
@@ -75,12 +78,12 @@ jobs:
|
|
|
75
78
|
--wheel
|
|
76
79
|
--outdir dist/
|
|
77
80
|
.
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
- name: Publish distribution 📦 to Test PyPI
|
|
82
|
+
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
|
|
83
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
84
|
+
with:
|
|
85
|
+
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
|
86
|
+
repository-url: https://test.pypi.org/legacy/
|
|
84
87
|
- name: Publish distribution 📦 to PyPI
|
|
85
88
|
if: github.event_name == 'push' && steps.check-tag.outputs.match == 'true'
|
|
86
89
|
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: opusfilter
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.1.0
|
|
4
4
|
Summary: Toolbox for filtering parallel corpora
|
|
5
5
|
Home-page: https://github.com/Helsinki-NLP/OpusFilter
|
|
6
6
|
Author: Mikko Aulamo, Sami Virpioja
|
|
@@ -8,17 +8,60 @@ Author-email: mikko.aulamo@helsinki.fi
|
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: setuptools
|
|
15
|
+
Requires-Dist: opustools
|
|
16
|
+
Requires-Dist: beautifulsoup4>=4.8.0
|
|
17
|
+
Requires-Dist: fasttext
|
|
18
|
+
Requires-Dist: graphviz
|
|
19
|
+
Requires-Dist: langid
|
|
20
|
+
Requires-Dist: matplotlib
|
|
21
|
+
Requires-Dist: morfessor
|
|
22
|
+
Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
|
|
23
|
+
Requires-Dist: pandas>=1.0.0
|
|
24
|
+
Requires-Dist: pycld2
|
|
25
|
+
Requires-Dist: xxhash>=3.2.0
|
|
26
|
+
Requires-Dist: sentence-splitter
|
|
27
|
+
Requires-Dist: rapidfuzz
|
|
28
|
+
Requires-Dist: ruamel.yaml>=0.15.0
|
|
29
|
+
Requires-Dist: regex
|
|
30
|
+
Requires-Dist: requests
|
|
31
|
+
Requires-Dist: scikit-learn
|
|
32
|
+
Requires-Dist: subword_nmt
|
|
33
|
+
Requires-Dist: tqdm
|
|
34
|
+
Requires-Dist: lingua-language-detector>=1.3.0
|
|
13
35
|
Provides-Extra: test
|
|
36
|
+
Requires-Dist: pytest; extra == "test"
|
|
14
37
|
Provides-Extra: eflomal
|
|
38
|
+
Requires-Dist: eflomal>=1.0.0b1; extra == "eflomal"
|
|
15
39
|
Provides-Extra: jieba
|
|
40
|
+
Requires-Dist: jieba>=0.42; extra == "jieba"
|
|
16
41
|
Provides-Extra: mecab
|
|
42
|
+
Requires-Dist: mecab-python3>=1.0.8; extra == "mecab"
|
|
43
|
+
Requires-Dist: unidic-lite; extra == "mecab"
|
|
17
44
|
Provides-Extra: laser
|
|
45
|
+
Requires-Dist: laserembeddings; extra == "laser"
|
|
18
46
|
Provides-Extra: varikn
|
|
47
|
+
Requires-Dist: varikn; extra == "varikn"
|
|
19
48
|
Provides-Extra: docs
|
|
49
|
+
Requires-Dist: myst-parser; extra == "docs"
|
|
50
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
51
|
+
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
52
|
+
Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
|
|
20
53
|
Provides-Extra: all
|
|
21
|
-
|
|
54
|
+
Requires-Dist: eflomal>=1.0.0b1; extra == "all"
|
|
55
|
+
Requires-Dist: jieba>=0.42; extra == "all"
|
|
56
|
+
Requires-Dist: mecab-python3>=1.0.8; extra == "all"
|
|
57
|
+
Requires-Dist: unidic-lite; extra == "all"
|
|
58
|
+
Requires-Dist: laserembeddings; extra == "all"
|
|
59
|
+
Requires-Dist: varikn; extra == "all"
|
|
60
|
+
Requires-Dist: pytest; extra == "all"
|
|
61
|
+
Requires-Dist: myst-parser; extra == "all"
|
|
62
|
+
Requires-Dist: sphinx; extra == "all"
|
|
63
|
+
Requires-Dist: sphinx-rtd-theme; extra == "all"
|
|
64
|
+
Requires-Dist: sphinxcontrib-bibtex; extra == "all"
|
|
22
65
|
|
|
23
66
|
# OpusFilter
|
|
24
67
|
|
|
@@ -47,7 +90,7 @@ Install from source:
|
|
|
47
90
|
|
|
48
91
|
### Troubleshooting
|
|
49
92
|
|
|
50
|
-
OpusFilter should generally work fine on Python 3.
|
|
93
|
+
OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
51
94
|
|
|
52
95
|
* `pip install -r requirements.txt`
|
|
53
96
|
|
|
@@ -25,7 +25,7 @@ Install from source:
|
|
|
25
25
|
|
|
26
26
|
### Troubleshooting
|
|
27
27
|
|
|
28
|
-
OpusFilter should generally work fine on Python 3.
|
|
28
|
+
OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
|
|
29
29
|
|
|
30
30
|
* `pip install -r requirements.txt`
|
|
31
31
|
|
|
@@ -43,9 +43,9 @@ parser.add_argument('--clusters', '-k', default=2, type=int, metavar='INT',
|
|
|
43
43
|
parser.add_argument('--work-dir', default='work',
|
|
44
44
|
help='Location of the source and target files for the generated configuration (default %(default)s)')
|
|
45
45
|
parser.add_argument('--inter-dir', help='Save intermediate files in this directory (use a temporary directory if not given)')
|
|
46
|
-
parser.add_argument('--plot',
|
|
47
|
-
help=('
|
|
48
|
-
'only for the clustering method'))
|
|
46
|
+
parser.add_argument('--plot', metavar='PATH', default=None, type=str,
|
|
47
|
+
help=('Create histograms of feature data distributions and a scatter plot of the clustering; '
|
|
48
|
+
'give path to plot the PDF files to, or "-" for interactive plots; only for the clustering method'))
|
|
49
49
|
parser.add_argument('--list-defaults', action='store_true', help='List default filters of the method to the output and quit')
|
|
50
50
|
parser.add_argument('--add-filter', nargs=2, action='append', default=[], metavar=('CLASS', 'JSON'),
|
|
51
51
|
help=('Instead of using default filters, add a filter of CLASS with JSON parameters object '
|
|
@@ -78,9 +78,12 @@ if args.list_defaults:
|
|
|
78
78
|
|
|
79
79
|
filters = filtergen.set_filter_thresholds()
|
|
80
80
|
|
|
81
|
-
if args.method == 'clustering' and args.plot:
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
if args.method == 'clustering' and args.plot is not None:
|
|
82
|
+
if args.plot == '-':
|
|
83
|
+
filtergen.scoredata.plot(plt)
|
|
84
|
+
plt.show()
|
|
85
|
+
else:
|
|
86
|
+
filtergen.scoredata.plot(plt, path=args.plot)
|
|
84
87
|
|
|
85
88
|
generator = ConfigurationGenerator(
|
|
86
89
|
files=[os.path.abspath(f) for f in args.files], langs=args.langs, workdir=args.work_dir)
|
|
@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [3.1.0] - 2024-06-05
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- support `lingua` based for language detection (https://github.com/Helsinki-NLP/OpusFilter/pull/65)
|
|
15
|
+
|
|
16
|
+
### Removed
|
|
17
|
+
|
|
18
|
+
- Python 3.7 support
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- fix score method in `SentenceEmbeddingFilter` (https://github.com/Helsinki-NLP/OpusFilter/pull/71)
|
|
23
|
+
- fix filter and filterfalse methods in `SentenceEmbeddingFilter`
|
|
24
|
+
|
|
10
25
|
## [3.0.0] - 2023-10-11
|
|
11
26
|
|
|
12
27
|
### Added
|
|
@@ -189,7 +204,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
189
204
|
First tagged version.
|
|
190
205
|
|
|
191
206
|
|
|
192
|
-
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.
|
|
207
|
+
[Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...develop
|
|
208
|
+
[3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
|
|
193
209
|
[3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
|
|
194
210
|
[2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
|
|
195
211
|
[2.5.1]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.0...2.5.1
|
|
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
|
|
|
5
5
|
few rules for pull requests:
|
|
6
6
|
|
|
7
7
|
* Make a pull request to the `develop` branch instead of `master`.
|
|
8
|
-
* The code should support at least Python versions from 3.
|
|
8
|
+
* The code should support at least Python versions from 3.8 to 3.11.
|
|
9
9
|
* Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
|
|
10
10
|
* Especially for new features, please include test cases for unit testing.
|
|
11
11
|
|
|
@@ -19,8 +19,8 @@ work, if you have VariKN and eflomal set up as instructed - `pytest`
|
|
|
19
19
|
skips the respective tests if not.)
|
|
20
20
|
|
|
21
21
|
GitHub workflows defined in the project run automatically `flake8`
|
|
22
|
-
checks and unit testing with `pytest` using Python 3.
|
|
23
|
-
|
|
22
|
+
checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
|
|
23
|
+
and 3.11.
|
|
24
24
|
|
|
25
25
|
Especially for larger contributions, consider using a code analysis
|
|
26
26
|
tool like [Pylint](https://github.com/PyCQA/pylint). Install it
|
{opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/script_and_language_identification_filters.md
RENAMED
|
@@ -40,6 +40,7 @@ Parameters:
|
|
|
40
40
|
* `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
|
|
41
41
|
* `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
|
|
42
42
|
* `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
|
|
43
|
+
* `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
|
|
43
44
|
|
|
44
45
|
Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. Negative threshold can be used to skip filtering for a language.
|
|
45
46
|
|
|
@@ -12,7 +12,7 @@ Install from source:
|
|
|
12
12
|
|
|
13
13
|
Note that all required libraries are not available to install via PyPI
|
|
14
14
|
on Windows OS. On Linux and MacOS, it should work directly for Python
|
|
15
|
-
versions from 3.
|
|
15
|
+
versions from 3.8 to 3.11.
|
|
16
16
|
|
|
17
17
|
## Required libraries
|
|
18
18
|
|
|
@@ -35,6 +35,7 @@ versions from 3.7 to 3.11.
|
|
|
35
35
|
* subword_nmt
|
|
36
36
|
* tqdm
|
|
37
37
|
* xxhash
|
|
38
|
+
* lingua-language-detector
|
|
38
39
|
|
|
39
40
|
See `setup.py` for possible version requirements.
|
|
40
41
|
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""Unsupervised threshold selection for filters"""
|
|
2
|
+
|
|
3
|
+
from collections import Counter
|
|
4
|
+
import itertools
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from sklearn.base import BaseEstimator, TransformerMixin
|
|
10
|
+
from sklearn.cluster import KMeans
|
|
11
|
+
from sklearn import decomposition, preprocessing, random_projection
|
|
12
|
+
from sklearn.ensemble import RandomForestClassifier
|
|
13
|
+
from sklearn.inspection import permutation_importance
|
|
14
|
+
from sklearn.utils.validation import check_is_fitted
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from . import CLEAN_LOW
|
|
18
|
+
from . import filters as filtermodule
|
|
19
|
+
from .classifier import load_dataframe
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# allow scikit-learn style code here
|
|
26
|
+
# pylint: disable=C0103,W0613
|
|
27
|
+
class ArcProjection(BaseEstimator, TransformerMixin):
|
|
28
|
+
"""Project data to two dimensions using evenly distributed unit vectors
|
|
29
|
+
|
|
30
|
+
The assigment of unit vectors to the original vectors is optimized
|
|
31
|
+
so that L2 norm between the correlation coefficients of the
|
|
32
|
+
original features and dot product of the vectors is minimized.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
|
|
37
|
+
arc : float, 'auto' or 'full', default='auto'
|
|
38
|
+
Length of the arc used for projection vectors. For 'auto', the
|
|
39
|
+
length is set as the arccosine of the minimum correlation of
|
|
40
|
+
the input features. For 'full', full circle (2 * pi) is used.
|
|
41
|
+
|
|
42
|
+
Attributes
|
|
43
|
+
----------
|
|
44
|
+
n_components_ : int
|
|
45
|
+
Number of components, always 2.
|
|
46
|
+
|
|
47
|
+
components_ : ndarray of shape (n_components, n_features)
|
|
48
|
+
Matrix used for the projection.
|
|
49
|
+
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(self, arc='auto'):
|
|
53
|
+
self.components_ = None
|
|
54
|
+
self.n_components_ = 2
|
|
55
|
+
self.arc = arc
|
|
56
|
+
|
|
57
|
+
def _best_permutation(self, current, n_features, eval_func):
|
|
58
|
+
"""Greedy search for best permutation of projection matrix"""
|
|
59
|
+
permutation = np.arange(n_features)
|
|
60
|
+
while True:
|
|
61
|
+
logger.debug("current permutation: %s", permutation)
|
|
62
|
+
minpair, mincost = self._best_single_swap(current, eval_func)
|
|
63
|
+
logger.debug("best swap: %s (%s)", minpair, mincost)
|
|
64
|
+
if minpair is None:
|
|
65
|
+
return permutation, current, mincost
|
|
66
|
+
fwd, rev = list(minpair), list(reversed(minpair))
|
|
67
|
+
current[fwd] = current[rev]
|
|
68
|
+
permutation[fwd] = permutation[rev]
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def _best_single_swap(matrix, eval_func):
|
|
72
|
+
"""Find best swap of matrix rows based in eval_func"""
|
|
73
|
+
mincost = eval_func(matrix)
|
|
74
|
+
minpair = None
|
|
75
|
+
for pair in itertools.combinations(range(matrix.shape[0]), 2):
|
|
76
|
+
fwd, rev = list(pair), list(reversed(pair))
|
|
77
|
+
matrix[fwd] = matrix[rev]
|
|
78
|
+
cost = eval_func(matrix)
|
|
79
|
+
if cost < mincost:
|
|
80
|
+
mincost = cost
|
|
81
|
+
minpair = pair
|
|
82
|
+
matrix[fwd] = matrix[rev]
|
|
83
|
+
return minpair, mincost
|
|
84
|
+
|
|
85
|
+
def _make_matrix(self, corrmat, n_features):
|
|
86
|
+
"""Generate the random projection matrix.
|
|
87
|
+
|
|
88
|
+
Parameters
|
|
89
|
+
----------
|
|
90
|
+
corrmat : ndarray of shape (n_features, n_features),
|
|
91
|
+
Correlation matrix for the original features
|
|
92
|
+
|
|
93
|
+
n_features : int,
|
|
94
|
+
Dimensionality of the original source space.
|
|
95
|
+
|
|
96
|
+
Returns
|
|
97
|
+
-------
|
|
98
|
+
components : ndarray of shape (n_components, n_features)
|
|
99
|
+
The generated random matrix.
|
|
100
|
+
"""
|
|
101
|
+
def costf(matrix):
|
|
102
|
+
return ((corrmat - matrix @ matrix.T)**2).sum()
|
|
103
|
+
|
|
104
|
+
if self.arc == 'full':
|
|
105
|
+
dist = 2 * np.pi / n_features
|
|
106
|
+
elif self.arc == 'auto':
|
|
107
|
+
dist = np.arccos(corrmat.min()) / (n_features - 1)
|
|
108
|
+
else:
|
|
109
|
+
dist = self.arc / (n_features - 1)
|
|
110
|
+
matrix = np.array([
|
|
111
|
+
[np.cos(idx * dist), np.sin(idx * dist)] for idx in range(n_features)
|
|
112
|
+
])
|
|
113
|
+
_, best_m, best_cost = self._best_permutation(matrix.copy(), n_features, costf)
|
|
114
|
+
logger.debug("fcorr:\n%s", corrmat.round(3))
|
|
115
|
+
logger.debug("vdist:\n%s", (best_m @ best_m.T).round(3))
|
|
116
|
+
logger.debug("diff:\n%s", (corrmat - best_m @ best_m.T).round(3))
|
|
117
|
+
logger.debug("cost: %s", best_cost)
|
|
118
|
+
return best_m.T
|
|
119
|
+
|
|
120
|
+
def fit(self, X: np.array, y=None):
|
|
121
|
+
"""Generate a projection matrix.
|
|
122
|
+
|
|
123
|
+
Parameters
|
|
124
|
+
----------
|
|
125
|
+
X : {ndarray, sparse matrix} of shape (n_samples, n_features)
|
|
126
|
+
Training set: only the shape is used to find matrix dimensions.
|
|
127
|
+
|
|
128
|
+
y : Ignored
|
|
129
|
+
Not used, present here for API consistency by convention.
|
|
130
|
+
|
|
131
|
+
Returns
|
|
132
|
+
-------
|
|
133
|
+
self : object
|
|
134
|
+
BaseRandomProjection class instance.
|
|
135
|
+
"""
|
|
136
|
+
X = self._validate_data(
|
|
137
|
+
X, accept_sparse=["csr", "csc"], dtype=[np.float64, np.float32]
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
n_features = X.shape[1]
|
|
141
|
+
corrmat = np.corrcoef(X.T)
|
|
142
|
+
self.components_ = self._make_matrix(
|
|
143
|
+
corrmat, n_features
|
|
144
|
+
).astype(X.dtype, copy=False)
|
|
145
|
+
|
|
146
|
+
return self
|
|
147
|
+
|
|
148
|
+
def transform(self, X):
|
|
149
|
+
"""Project the data by using matrix product with the random matrix.
|
|
150
|
+
|
|
151
|
+
Parameters
|
|
152
|
+
----------
|
|
153
|
+
X : {ndarray, sparse matrix} of shape (n_samples, n_features)
|
|
154
|
+
The input data to project into a smaller dimensional space.
|
|
155
|
+
|
|
156
|
+
Returns
|
|
157
|
+
-------
|
|
158
|
+
X_new : ndarray of shape (n_samples, n_components)
|
|
159
|
+
Projected array.
|
|
160
|
+
"""
|
|
161
|
+
check_is_fitted(self)
|
|
162
|
+
X = self._validate_data(
|
|
163
|
+
X, accept_sparse=["csr", "csc"], reset=False, dtype=[np.float64, np.float32]
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
return X @ self.components_.T
|
|
167
|
+
# pylint: enable=C0103,W0613
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# Would need some refactoring, but disable pylint's "too many" warnings for now.
|
|
171
|
+
# pylint: disable=R0902,R0912,R0914,R0915
|
|
172
|
+
class ScoreClusters:
|
|
173
|
+
"""Cluster segments by filter scores
|
|
174
|
+
|
|
175
|
+
Train k-means clustering and take thresholds based on the noisy cluster center.
|
|
176
|
+
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
def __init__(self, score_file, k=2):
|
|
180
|
+
self.k = k
|
|
181
|
+
self.df = load_dataframe(score_file)
|
|
182
|
+
self.filters = {}
|
|
183
|
+
for name in self.df.columns:
|
|
184
|
+
first_part = name.split('.')[0]
|
|
185
|
+
filter_cls = getattr(filtermodule, first_part)
|
|
186
|
+
self.filters[name] = filter_cls
|
|
187
|
+
self.scaler = preprocessing.StandardScaler()
|
|
188
|
+
self.standard_data = self.scaler.fit_transform(self.df.mul(self.direction_vector))
|
|
189
|
+
|
|
190
|
+
logger.info('Training KMeans with %s clusters', self.k)
|
|
191
|
+
self.kmeans = KMeans(n_clusters=self.k, random_state=0, init='k-means++', n_init=1)
|
|
192
|
+
self.kmeans.fit(self.standard_data)
|
|
193
|
+
self.labels = self.kmeans.labels_
|
|
194
|
+
self.cluster_centers = self.scaler.inverse_transform(self.kmeans.cluster_centers_) * self.direction_vector
|
|
195
|
+
self._noisy_label = self._get_noisy_label()
|
|
196
|
+
self.rejects = None
|
|
197
|
+
self.thresholds = None
|
|
198
|
+
|
|
199
|
+
@property
|
|
200
|
+
def noisy_label(self):
|
|
201
|
+
"""Cluster label for noisy data"""
|
|
202
|
+
return self._noisy_label
|
|
203
|
+
|
|
204
|
+
@property
|
|
205
|
+
def clean_labels(self):
|
|
206
|
+
"""Cluster labels for clean data"""
|
|
207
|
+
return [idx for idx in range(self.k) if idx != self._noisy_label]
|
|
208
|
+
|
|
209
|
+
@property
|
|
210
|
+
def direction_vector(self):
|
|
211
|
+
"""Direction vector for the features (1 for CLEAN_LOW, -1 for CLEAN_HIGH)"""
|
|
212
|
+
return np.array([1 if self.filters[name].score_direction == CLEAN_LOW else -1
|
|
213
|
+
for name in self.df.columns])
|
|
214
|
+
|
|
215
|
+
def _get_noisy_label(self):
|
|
216
|
+
"""Find label for the noisy cluster"""
|
|
217
|
+
means = np.mean(self.kmeans.cluster_centers_, axis=1)
|
|
218
|
+
|
|
219
|
+
# Output some cluster information
|
|
220
|
+
nlabels = Counter(self.labels)
|
|
221
|
+
for i, (center, inv_center, mean) in enumerate(zip(self.kmeans.cluster_centers_, self.cluster_centers, means)):
|
|
222
|
+
logger.info('Cluster #%s', i)
|
|
223
|
+
logger.info('* number of samples: %s', nlabels[i])
|
|
224
|
+
logger.info('* centroid (score, scaled value, original value):')
|
|
225
|
+
for j, val in enumerate(center):
|
|
226
|
+
logger.info(' %s\t%s\t%s', self.df.columns[j].ljust(25), round(val, 2), round(inv_center[j], 2))
|
|
227
|
+
logger.info('Average center\t%s', np.round(mean, 2))
|
|
228
|
+
|
|
229
|
+
# Cluster center of the noisiest cluster based on average features
|
|
230
|
+
noisy_mean = np.max(means)
|
|
231
|
+
noisy_label = np.argmax(means)
|
|
232
|
+
logger.info('Cluster center of the noisiest cluster (%s)', np.round(noisy_mean, 2))
|
|
233
|
+
logger.info('Noisy label: %s', noisy_label)
|
|
234
|
+
noisy_labels = np.where(self.labels == noisy_label)[0]
|
|
235
|
+
logger.info('Number of noisy labels: %s',
|
|
236
|
+
f'{len(noisy_labels)}/{len(self.labels)} ({round(100*len(noisy_labels)/len(self.labels), 2)}%)')
|
|
237
|
+
return noisy_label
|
|
238
|
+
|
|
239
|
+
def get_columns(self):
|
|
240
|
+
"""Return data column names"""
|
|
241
|
+
return self.df.columns
|
|
242
|
+
|
|
243
|
+
def get_thresholds(self, method='noisy_center', precision=6):
|
|
244
|
+
"""Return a list of thresholds for noisy samples"""
|
|
245
|
+
if method != 'noisy_center':
|
|
246
|
+
raise ValueError(f'Method {method} for thresholds not implemented')
|
|
247
|
+
return self.cluster_centers[self.noisy_label].round(precision).tolist()
|
|
248
|
+
|
|
249
|
+
def get_rejects(self):
|
|
250
|
+
"""Train random forest classifier to find important features
|
|
251
|
+
|
|
252
|
+
Returns a list of booleans (True = reject).
|
|
253
|
+
|
|
254
|
+
"""
|
|
255
|
+
logger.info('Training random forest')
|
|
256
|
+
clf = RandomForestClassifier(random_state=1)
|
|
257
|
+
clf.fit(self.standard_data, self.labels)
|
|
258
|
+
logger.info('Finding important features')
|
|
259
|
+
feature_importances = permutation_importance(clf, self.standard_data, self.labels)
|
|
260
|
+
importance_mean_mean = np.mean(feature_importances.importances_mean)
|
|
261
|
+
rej_coef = 0.1
|
|
262
|
+
logger.info('* mean importance: %s', round(importance_mean_mean, 3))
|
|
263
|
+
logger.info('* rejection coefficient: %s', rej_coef)
|
|
264
|
+
logger.info('* decisions:')
|
|
265
|
+
rejects = []
|
|
266
|
+
for i, col in enumerate(self.df.columns):
|
|
267
|
+
importance = feature_importances['importances_mean'][i]
|
|
268
|
+
reject = importance < importance_mean_mean * rej_coef
|
|
269
|
+
logger.info(' %s\t%s\t%s', col.ljust(25), round(importance, 3), 'reject' if reject else 'keep')
|
|
270
|
+
rejects.append(reject)
|
|
271
|
+
return rejects
|
|
272
|
+
|
|
273
|
+
def get_result_df(self):
|
|
274
|
+
"""Return dataframe containing the thresholds and reject booleans"""
|
|
275
|
+
self.rejects = self.get_rejects()
|
|
276
|
+
self.thresholds = self.get_thresholds()
|
|
277
|
+
return pd.DataFrame.from_dict(
|
|
278
|
+
{'name': self.get_columns(), 'threshold': self.thresholds, 'reject': self.rejects})
|
|
279
|
+
|
|
280
|
+
def plot(self, plt, path=None, apply_rejects=True, projection='arc'):
|
|
281
|
+
"""Plot clustering and histograms"""
|
|
282
|
+
if projection == 'pca':
|
|
283
|
+
proj = decomposition.PCA(n_components=2)
|
|
284
|
+
elif projection == 'random':
|
|
285
|
+
proj = random_projection.GaussianRandomProjection(n_components=2)
|
|
286
|
+
elif projection == 'arc':
|
|
287
|
+
proj = ArcProjection()
|
|
288
|
+
else:
|
|
289
|
+
raise ValueError(f"Unknown projection: {projection}")
|
|
290
|
+
col_index = {col: idx for idx, col in enumerate(self.get_columns())}
|
|
291
|
+
index_col = dict(enumerate(self.get_columns()))
|
|
292
|
+
if apply_rejects and self.rejects:
|
|
293
|
+
indices = [idx for idx, reject in enumerate(self.rejects) if not reject]
|
|
294
|
+
cols = [index_col[idx] for idx in indices]
|
|
295
|
+
index_col = dict(enumerate(cols))
|
|
296
|
+
data_t = proj.fit_transform(self.standard_data[:, indices])
|
|
297
|
+
centroids = proj.transform(self.kmeans.cluster_centers_[:, indices])
|
|
298
|
+
else:
|
|
299
|
+
data_t = proj.fit_transform(self.standard_data)
|
|
300
|
+
centroids = proj.transform(self.kmeans.cluster_centers_)
|
|
301
|
+
plt.figure()
|
|
302
|
+
for idx in range(proj.components_.shape[1]):
|
|
303
|
+
plt.arrow(0, 0, proj.components_[0, idx], proj.components_[1, idx], head_width=0.01, fc='k')
|
|
304
|
+
plt.text(proj.components_[0, idx], proj.components_[1, idx], index_col[idx])
|
|
305
|
+
plt.gca().set_aspect('equal', adjustable='box')
|
|
306
|
+
plt.title('Projection vectors for clustering')
|
|
307
|
+
if path is not None:
|
|
308
|
+
plt.savefig(os.path.join(path, 'projection.pdf'))
|
|
309
|
+
plt.figure(figsize=(10, 10))
|
|
310
|
+
for label_id in range(self.k):
|
|
311
|
+
points = np.where(self.labels == label_id)
|
|
312
|
+
plt.scatter(data_t[points, 0], data_t[points, 1],
|
|
313
|
+
c='orange' if label_id == self.noisy_label else 'blue',
|
|
314
|
+
label='noisy' if label_id == self.noisy_label else 'clean',
|
|
315
|
+
marker=',', s=1, alpha=0.1)
|
|
316
|
+
for label_id in range(self.k):
|
|
317
|
+
plt.scatter(centroids[label_id, 0], centroids[label_id, 1], s=100, alpha=1,
|
|
318
|
+
marker='+', c='brown' if label_id == self.noisy_label else 'darkblue',
|
|
319
|
+
label='noisy centroid' if label_id == self.noisy_label else 'clean centroid')
|
|
320
|
+
plt.legend()
|
|
321
|
+
plt.title('Clustering')
|
|
322
|
+
if path is not None:
|
|
323
|
+
plt.savefig(os.path.join(path, 'clustering.pdf'))
|
|
324
|
+
noisy_samples = self.df.iloc[np.where(self.labels == self.noisy_label)]
|
|
325
|
+
clean_samples = self.df.iloc[np.where(self.labels != self.noisy_label)]
|
|
326
|
+
subplots_n = noisy_samples.hist(bins=100, figsize=(10, 10))
|
|
327
|
+
fig_noisy = plt.gcf()
|
|
328
|
+
plt.suptitle('Histograms for noisy samples')
|
|
329
|
+
subplots_c = clean_samples.hist(bins=100, figsize=(10, 10))
|
|
330
|
+
fig_clean = plt.gcf()
|
|
331
|
+
plt.suptitle('Histograms for clean samples')
|
|
332
|
+
if apply_rejects and self.rejects:
|
|
333
|
+
for axes in ([axes for sublist in subplots_c for axes in sublist] +
|
|
334
|
+
[axes for sublist in subplots_n for axes in sublist]):
|
|
335
|
+
title = axes.get_title()
|
|
336
|
+
if not title:
|
|
337
|
+
continue
|
|
338
|
+
idx = col_index[title]
|
|
339
|
+
if self.rejects[idx]:
|
|
340
|
+
axes.text(0.5, 1, 'REJECTED', horizontalalignment='center',
|
|
341
|
+
verticalalignment='top', transform=axes.transAxes, color='r')
|
|
342
|
+
axes.axvline(self.thresholds[idx], color='k', linestyle='--', alpha=0.5)
|
|
343
|
+
if path is not None:
|
|
344
|
+
fig_clean.savefig(os.path.join(path, 'histogram_clean.pdf'))
|
|
345
|
+
fig_noisy.savefig(os.path.join(path, 'histogram_noisy.pdf'))
|
|
346
|
+
# pylint: enable=R0902,R0912,R0914,R0915
|
|
@@ -133,13 +133,20 @@ class SentenceEmbeddingFilter(FilterABC):
|
|
|
133
133
|
|
|
134
134
|
def score(self, pairs):
|
|
135
135
|
for chunk in grouper(pairs, self.chunksize):
|
|
136
|
-
|
|
136
|
+
for score in self._score_chunk(chunk):
|
|
137
|
+
yield score
|
|
137
138
|
|
|
138
139
|
def accept(self, score):
|
|
139
140
|
return all(similarity >= self.threshold for similarity in score)
|
|
140
141
|
|
|
141
142
|
def filter(self, pairs):
|
|
142
143
|
for chunk in grouper(pairs, self.chunksize):
|
|
143
|
-
for pair, score in zip(
|
|
144
|
+
for pair, score in zip(chunk, self._score_chunk(chunk)):
|
|
144
145
|
if self.accept(score):
|
|
145
146
|
yield pair
|
|
147
|
+
|
|
148
|
+
def filterfalse(self, pairs):
|
|
149
|
+
for chunk in grouper(pairs, self.chunksize):
|
|
150
|
+
for pair, score in zip(chunk, self._score_chunk(chunk)):
|
|
151
|
+
if not self.accept(score):
|
|
152
|
+
yield pair
|