opusfilter 3.0.0__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {opusfilter-3.0.0 → opusfilter-3.1.0}/.github/workflows/ci.yml +12 -9
  2. {opusfilter-3.0.0/opusfilter.egg-info → opusfilter-3.1.0}/PKG-INFO +47 -4
  3. {opusfilter-3.0.0 → opusfilter-3.1.0}/README.md +1 -1
  4. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-autogen +9 -6
  5. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/CHANGELOG.md +17 -1
  6. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/CONTRIBUTING.md +3 -3
  7. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/script_and_language_identification_filters.md +1 -0
  8. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/installation.md +2 -1
  9. opusfilter-3.1.0/opusfilter/autogen_cluster.py +346 -0
  10. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/embeddings.py +9 -2
  11. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/filters.py +51 -17
  12. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/opusfilter.py +4 -3
  13. {opusfilter-3.0.0 → opusfilter-3.1.0/opusfilter.egg-info}/PKG-INFO +47 -4
  14. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/requires.txt +1 -0
  15. {opusfilter-3.0.0 → opusfilter-3.1.0}/requirements.txt +1 -0
  16. {opusfilter-3.0.0 → opusfilter-3.1.0}/setup.py +3 -2
  17. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_embeddings.py +46 -3
  18. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_filters.py +20 -0
  19. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_opusfilter.py +39 -2
  20. opusfilter-3.0.0/opusfilter/autogen_cluster.py +0 -150
  21. {opusfilter-3.0.0 → opusfilter-3.1.0}/.gitignore +0 -0
  22. {opusfilter-3.0.0 → opusfilter-3.1.0}/.pylintrc +0 -0
  23. {opusfilter-3.0.0 → opusfilter-3.1.0}/LICENSE +0 -0
  24. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter +0 -0
  25. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-cmd +0 -0
  26. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-diagram +0 -0
  27. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-duplicates +0 -0
  28. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-scores +0 -0
  29. {opusfilter-3.0.0 → opusfilter-3.1.0}/bin/opusfilter-test +0 -0
  30. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/automatic_configuration.md +0 -0
  31. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/command_line_tools.md +0 -0
  32. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/conf.py +0 -0
  33. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/alignment_model_filters.md +0 -0
  34. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/custom_filters.md +0 -0
  35. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/language_model_filters.md +0 -0
  36. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/length_filters.md +0 -0
  37. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/sentence_embedding_filters.md +0 -0
  38. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/filters/special_character_and_similarity_filters.md +0 -0
  39. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/downloading_and_selecting_data.md +0 -0
  40. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/filtering_and_scoring.md +0 -0
  41. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/preprocessing_text.md +0 -0
  42. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/training_and_using_classifiers.md +0 -0
  43. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/training_language_and_alignment_models.md +0 -0
  44. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/functions/using_score_files.md +0 -0
  45. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/index.rst +0 -0
  46. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/bpe_segmentation.md +0 -0
  47. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/custom_preprocessors.md +0 -0
  48. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/detokenizer.md +0 -0
  49. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/monolingual_sentence_splitter.md +0 -0
  50. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/morfessor_segmentation.md +0 -0
  51. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/reg_exp_sub.md +0 -0
  52. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/tokenizer.md +0 -0
  53. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/preprocessors/whitespaceNormalizer.md +0 -0
  54. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/references.bib +0 -0
  55. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/references.rst +0 -0
  56. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/requirements.txt +0 -0
  57. {opusfilter-3.0.0 → opusfilter-3.1.0}/docs/usage.md +0 -0
  58. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/README.md +0 -0
  59. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/create_ce_sets.yaml +0 -0
  60. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/create_domain_sets.yaml +0 -0
  61. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/create_roc_auc_sets.yaml +0 -0
  62. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/devset_100_1_labels.jsonl +0 -0
  63. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/devset_100_2_labels.jsonl +0 -0
  64. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/paracrawl_fi-en/prepare_data.yaml +0 -0
  65. {opusfilter-3.0.0 → opusfilter-3.1.0}/example_configs/qed_lm_langid.yaml +0 -0
  66. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/__init__.py +0 -0
  67. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/autogen.py +0 -0
  68. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/classifier.py +0 -0
  69. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/lm.py +0 -0
  70. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/pipeline.py +0 -0
  71. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/preprocessors.py +0 -0
  72. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/segment_hash.py +0 -0
  73. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/subwords.py +0 -0
  74. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/tokenization.py +0 -0
  75. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/util.py +0 -0
  76. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter/word_alignment.py +0 -0
  77. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/SOURCES.txt +0 -0
  78. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/dependency_links.txt +0 -0
  79. {opusfilter-3.0.0 → opusfilter-3.1.0}/opusfilter.egg-info/top_level.txt +0 -0
  80. {opusfilter-3.0.0 → opusfilter-3.1.0}/pyproject.toml +0 -0
  81. {opusfilter-3.0.0 → opusfilter-3.1.0}/setup.cfg +0 -0
  82. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_autogen.py +0 -0
  83. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_classifier.py +0 -0
  84. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_filter_pipeline.py +0 -0
  85. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_lm_filter.py +0 -0
  86. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_preprocessors.py +0 -0
  87. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_segment_hash.py +0 -0
  88. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_subwords.py +0 -0
  89. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_tokenization.py +0 -0
  90. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_util.py +0 -0
  91. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_wordalign_filter.py +0 -0
  92. {opusfilter-3.0.0 → opusfilter-3.1.0}/tests/test_yaml.py +0 -0
@@ -11,6 +11,9 @@ on:
11
11
  branches:
12
12
  - master
13
13
  - develop
14
+ schedule:
15
+ # Run on Tuesdays at 5:59
16
+ - cron: '59 5 * * 2'
14
17
  workflow_dispatch:
15
18
  jobs:
16
19
  build-n-test:
@@ -18,7 +21,7 @@ jobs:
18
21
  fail-fast: false
19
22
  matrix:
20
23
  os: [ubuntu-latest]
21
- python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
24
+ python-version: ["3.8", "3.9", "3.10", "3.11"]
22
25
  requirements-file: ["requirements.txt"]
23
26
  runs-on: ${{ matrix.os }}
24
27
  steps:
@@ -54,11 +57,11 @@ jobs:
54
57
  if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]*)?$ ]]; then
55
58
  echo "match=true" >> $GITHUB_OUTPUT
56
59
  fi
57
- - name: Set up Python 3.7
60
+ - name: Set up Python 3.8
58
61
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
59
62
  uses: actions/setup-python@v4
60
63
  with:
61
- python-version: 3.7
64
+ python-version: 3.8
62
65
  - name: Install pypa/build
63
66
  if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
64
67
  run: >-
@@ -75,12 +78,12 @@ jobs:
75
78
  --wheel
76
79
  --outdir dist/
77
80
  .
78
- # - name: Publish distribution 📦 to Test PyPI
79
- # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
80
- # uses: pypa/gh-action-pypi-publish@release/v1
81
- # with:
82
- # password: ${{ secrets.TEST_PYPI_API_TOKEN }}
83
- # repository-url: https://test.pypi.org/legacy/
81
+ - name: Publish distribution 📦 to Test PyPI
82
+ if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
83
+ uses: pypa/gh-action-pypi-publish@release/v1
84
+ with:
85
+ password: ${{ secrets.TEST_PYPI_API_TOKEN }}
86
+ repository-url: https://test.pypi.org/legacy/
84
87
  - name: Publish distribution 📦 to PyPI
85
88
  if: github.event_name == 'push' && steps.check-tag.outputs.match == 'true'
86
89
  uses: pypa/gh-action-pypi-publish@release/v1
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: opusfilter
3
- Version: 3.0.0
3
+ Version: 3.1.0
4
4
  Summary: Toolbox for filtering parallel corpora
5
5
  Home-page: https://github.com/Helsinki-NLP/OpusFilter
6
6
  Author: Mikko Aulamo, Sami Virpioja
@@ -8,17 +8,60 @@ Author-email: mikko.aulamo@helsinki.fi
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.7
11
+ Requires-Python: >=3.8
12
12
  Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: setuptools
15
+ Requires-Dist: opustools
16
+ Requires-Dist: beautifulsoup4>=4.8.0
17
+ Requires-Dist: fasttext
18
+ Requires-Dist: graphviz
19
+ Requires-Dist: langid
20
+ Requires-Dist: matplotlib
21
+ Requires-Dist: morfessor
22
+ Requires-Dist: opus-fast-mosestokenizer>=0.0.8.5
23
+ Requires-Dist: pandas>=1.0.0
24
+ Requires-Dist: pycld2
25
+ Requires-Dist: xxhash>=3.2.0
26
+ Requires-Dist: sentence-splitter
27
+ Requires-Dist: rapidfuzz
28
+ Requires-Dist: ruamel.yaml>=0.15.0
29
+ Requires-Dist: regex
30
+ Requires-Dist: requests
31
+ Requires-Dist: scikit-learn
32
+ Requires-Dist: subword_nmt
33
+ Requires-Dist: tqdm
34
+ Requires-Dist: lingua-language-detector>=1.3.0
13
35
  Provides-Extra: test
36
+ Requires-Dist: pytest; extra == "test"
14
37
  Provides-Extra: eflomal
38
+ Requires-Dist: eflomal>=1.0.0b1; extra == "eflomal"
15
39
  Provides-Extra: jieba
40
+ Requires-Dist: jieba>=0.42; extra == "jieba"
16
41
  Provides-Extra: mecab
42
+ Requires-Dist: mecab-python3>=1.0.8; extra == "mecab"
43
+ Requires-Dist: unidic-lite; extra == "mecab"
17
44
  Provides-Extra: laser
45
+ Requires-Dist: laserembeddings; extra == "laser"
18
46
  Provides-Extra: varikn
47
+ Requires-Dist: varikn; extra == "varikn"
19
48
  Provides-Extra: docs
49
+ Requires-Dist: myst-parser; extra == "docs"
50
+ Requires-Dist: sphinx; extra == "docs"
51
+ Requires-Dist: sphinx-rtd-theme; extra == "docs"
52
+ Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
20
53
  Provides-Extra: all
21
- License-File: LICENSE
54
+ Requires-Dist: eflomal>=1.0.0b1; extra == "all"
55
+ Requires-Dist: jieba>=0.42; extra == "all"
56
+ Requires-Dist: mecab-python3>=1.0.8; extra == "all"
57
+ Requires-Dist: unidic-lite; extra == "all"
58
+ Requires-Dist: laserembeddings; extra == "all"
59
+ Requires-Dist: varikn; extra == "all"
60
+ Requires-Dist: pytest; extra == "all"
61
+ Requires-Dist: myst-parser; extra == "all"
62
+ Requires-Dist: sphinx; extra == "all"
63
+ Requires-Dist: sphinx-rtd-theme; extra == "all"
64
+ Requires-Dist: sphinxcontrib-bibtex; extra == "all"
22
65
 
23
66
  # OpusFilter
24
67
 
@@ -47,7 +90,7 @@ Install from source:
47
90
 
48
91
  ### Troubleshooting
49
92
 
50
- OpusFilter should generally work fine on Python 3.7 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
93
+ OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
51
94
 
52
95
  * `pip install -r requirements.txt`
53
96
 
@@ -25,7 +25,7 @@ Install from source:
25
25
 
26
26
  ### Troubleshooting
27
27
 
28
- OpusFilter should generally work fine on Python 3.7 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
28
+ OpusFilter should generally work fine on Python 3.8 to 3.11. In the case of troubles, try installing the exact versions in `requirements.txt`:
29
29
 
30
30
  * `pip install -r requirements.txt`
31
31
 
@@ -43,9 +43,9 @@ parser.add_argument('--clusters', '-k', default=2, type=int, metavar='INT',
43
43
  parser.add_argument('--work-dir', default='work',
44
44
  help='Location of the source and target files for the generated configuration (default %(default)s)')
45
45
  parser.add_argument('--inter-dir', help='Save intermediate files in this directory (use a temporary directory if not given)')
46
- parser.add_argument('--plot', action='store_true',
47
- help=('Show a scatter plot of the clustering and histograms of feature data distributions; '
48
- 'only for the clustering method'))
46
+ parser.add_argument('--plot', metavar='PATH', default=None, type=str,
47
+ help=('Create histograms of feature data distributions and a scatter plot of the clustering; '
48
+ 'give path to plot the PDF files to, or "-" for interactive plots; only for the clustering method'))
49
49
  parser.add_argument('--list-defaults', action='store_true', help='List default filters of the method to the output and quit')
50
50
  parser.add_argument('--add-filter', nargs=2, action='append', default=[], metavar=('CLASS', 'JSON'),
51
51
  help=('Instead of using default filters, add a filter of CLASS with JSON parameters object '
@@ -78,9 +78,12 @@ if args.list_defaults:
78
78
 
79
79
  filters = filtergen.set_filter_thresholds()
80
80
 
81
- if args.method == 'clustering' and args.plot:
82
- filtergen.scoredata.plot(plt)
83
- plt.show()
81
+ if args.method == 'clustering' and args.plot is not None:
82
+ if args.plot == '-':
83
+ filtergen.scoredata.plot(plt)
84
+ plt.show()
85
+ else:
86
+ filtergen.scoredata.plot(plt, path=args.plot)
84
87
 
85
88
  generator = ConfigurationGenerator(
86
89
  files=[os.path.abspath(f) for f in args.files], langs=args.langs, workdir=args.work_dir)
@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [3.1.0] - 2024-06-05
11
+
12
+ ### Added
13
+
14
+ - support `lingua` based for language detection (https://github.com/Helsinki-NLP/OpusFilter/pull/65)
15
+
16
+ ### Removed
17
+
18
+ - Python 3.7 support
19
+
20
+ ### Fixed
21
+
22
+ - fix score method in `SentenceEmbeddingFilter` (https://github.com/Helsinki-NLP/OpusFilter/pull/71)
23
+ - fix filter and filterfalse methods in `SentenceEmbeddingFilter`
24
+
10
25
  ## [3.0.0] - 2023-10-11
11
26
 
12
27
  ### Added
@@ -189,7 +204,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
189
204
  First tagged version.
190
205
 
191
206
 
192
- [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...develop
207
+ [Unreleased]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.1.0...develop
208
+ [3.1.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/3.0.0...3.1.0
193
209
  [3.0.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.6.0...3.0.0
194
210
  [2.6.0]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.1...2.6.0
195
211
  [2.5.1]: https://github.com/Helsinki-NLP/OpusFilter/compare/2.5.0...2.5.1
@@ -5,7 +5,7 @@ issues page. We are also happy to consider pull requests. There are a
5
5
  few rules for pull requests:
6
6
 
7
7
  * Make a pull request to the `develop` branch instead of `master`.
8
- * The code should support at least Python versions from 3.7 to 3.11.
8
+ * The code should support at least Python versions from 3.8 to 3.11.
9
9
  * Please follow [PEP 8](https://www.python.org/dev/peps/pep-0008/). Exception: The maximum line length is 127 characters instead of 79.
10
10
  * Especially for new features, please include test cases for unit testing.
11
11
 
@@ -19,8 +19,8 @@ work, if you have VariKN and eflomal set up as instructed - `pytest`
19
19
  skips the respective tests if not.)
20
20
 
21
21
  GitHub workflows defined in the project run automatically `flake8`
22
- checks and unit testing with `pytest` using Python 3.7, 3.8, 3.9,
23
- 3.10, and 3.11.
22
+ checks and unit testing with `pytest` using Python 3.8, 3.9, 3.10,
23
+ and 3.11.
24
24
 
25
25
  Especially for larger contributions, consider using a code analysis
26
26
  tool like [Pylint](https://github.com/PyCQA/pylint). Install it
@@ -40,6 +40,7 @@ Parameters:
40
40
  * `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
41
41
  * `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
42
42
  * `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
43
+ * `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
43
44
 
44
45
  Returned scores are the language identification confidence scores from a given identification method for the segments. The scores range from 0 to 1. In filtering, all values have to be greater than the minimum thresholds. Negative threshold can be used to skip filtering for a language.
45
46
 
@@ -12,7 +12,7 @@ Install from source:
12
12
 
13
13
  Note that all required libraries are not available to install via PyPI
14
14
  on Windows OS. On Linux and MacOS, it should work directly for Python
15
- versions from 3.7 to 3.11.
15
+ versions from 3.8 to 3.11.
16
16
 
17
17
  ## Required libraries
18
18
 
@@ -35,6 +35,7 @@ versions from 3.7 to 3.11.
35
35
  * subword_nmt
36
36
  * tqdm
37
37
  * xxhash
38
+ * lingua-language-detector
38
39
 
39
40
  See `setup.py` for possible version requirements.
40
41
 
@@ -0,0 +1,346 @@
1
+ """Unsupervised threshold selection for filters"""
2
+
3
+ from collections import Counter
4
+ import itertools
5
+ import logging
6
+ import os
7
+
8
+ import pandas as pd
9
+ from sklearn.base import BaseEstimator, TransformerMixin
10
+ from sklearn.cluster import KMeans
11
+ from sklearn import decomposition, preprocessing, random_projection
12
+ from sklearn.ensemble import RandomForestClassifier
13
+ from sklearn.inspection import permutation_importance
14
+ from sklearn.utils.validation import check_is_fitted
15
+ import numpy as np
16
+
17
+ from . import CLEAN_LOW
18
+ from . import filters as filtermodule
19
+ from .classifier import load_dataframe
20
+
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ # allow scikit-learn style code here
26
+ # pylint: disable=C0103,W0613
27
+ class ArcProjection(BaseEstimator, TransformerMixin):
28
+ """Project data to two dimensions using evenly distributed unit vectors
29
+
30
+ The assigment of unit vectors to the original vectors is optimized
31
+ so that L2 norm between the correlation coefficients of the
32
+ original features and dot product of the vectors is minimized.
33
+
34
+ Parameters
35
+ ----------
36
+
37
+ arc : float, 'auto' or 'full', default='auto'
38
+ Length of the arc used for projection vectors. For 'auto', the
39
+ length is set as the arccosine of the minimum correlation of
40
+ the input features. For 'full', full circle (2 * pi) is used.
41
+
42
+ Attributes
43
+ ----------
44
+ n_components_ : int
45
+ Number of components, always 2.
46
+
47
+ components_ : ndarray of shape (n_components, n_features)
48
+ Matrix used for the projection.
49
+
50
+ """
51
+
52
+ def __init__(self, arc='auto'):
53
+ self.components_ = None
54
+ self.n_components_ = 2
55
+ self.arc = arc
56
+
57
+ def _best_permutation(self, current, n_features, eval_func):
58
+ """Greedy search for best permutation of projection matrix"""
59
+ permutation = np.arange(n_features)
60
+ while True:
61
+ logger.debug("current permutation: %s", permutation)
62
+ minpair, mincost = self._best_single_swap(current, eval_func)
63
+ logger.debug("best swap: %s (%s)", minpair, mincost)
64
+ if minpair is None:
65
+ return permutation, current, mincost
66
+ fwd, rev = list(minpair), list(reversed(minpair))
67
+ current[fwd] = current[rev]
68
+ permutation[fwd] = permutation[rev]
69
+
70
+ @staticmethod
71
+ def _best_single_swap(matrix, eval_func):
72
+ """Find best swap of matrix rows based in eval_func"""
73
+ mincost = eval_func(matrix)
74
+ minpair = None
75
+ for pair in itertools.combinations(range(matrix.shape[0]), 2):
76
+ fwd, rev = list(pair), list(reversed(pair))
77
+ matrix[fwd] = matrix[rev]
78
+ cost = eval_func(matrix)
79
+ if cost < mincost:
80
+ mincost = cost
81
+ minpair = pair
82
+ matrix[fwd] = matrix[rev]
83
+ return minpair, mincost
84
+
85
+ def _make_matrix(self, corrmat, n_features):
86
+ """Generate the random projection matrix.
87
+
88
+ Parameters
89
+ ----------
90
+ corrmat : ndarray of shape (n_features, n_features),
91
+ Correlation matrix for the original features
92
+
93
+ n_features : int,
94
+ Dimensionality of the original source space.
95
+
96
+ Returns
97
+ -------
98
+ components : ndarray of shape (n_components, n_features)
99
+ The generated random matrix.
100
+ """
101
+ def costf(matrix):
102
+ return ((corrmat - matrix @ matrix.T)**2).sum()
103
+
104
+ if self.arc == 'full':
105
+ dist = 2 * np.pi / n_features
106
+ elif self.arc == 'auto':
107
+ dist = np.arccos(corrmat.min()) / (n_features - 1)
108
+ else:
109
+ dist = self.arc / (n_features - 1)
110
+ matrix = np.array([
111
+ [np.cos(idx * dist), np.sin(idx * dist)] for idx in range(n_features)
112
+ ])
113
+ _, best_m, best_cost = self._best_permutation(matrix.copy(), n_features, costf)
114
+ logger.debug("fcorr:\n%s", corrmat.round(3))
115
+ logger.debug("vdist:\n%s", (best_m @ best_m.T).round(3))
116
+ logger.debug("diff:\n%s", (corrmat - best_m @ best_m.T).round(3))
117
+ logger.debug("cost: %s", best_cost)
118
+ return best_m.T
119
+
120
+ def fit(self, X: np.array, y=None):
121
+ """Generate a projection matrix.
122
+
123
+ Parameters
124
+ ----------
125
+ X : {ndarray, sparse matrix} of shape (n_samples, n_features)
126
+ Training set: only the shape is used to find matrix dimensions.
127
+
128
+ y : Ignored
129
+ Not used, present here for API consistency by convention.
130
+
131
+ Returns
132
+ -------
133
+ self : object
134
+ BaseRandomProjection class instance.
135
+ """
136
+ X = self._validate_data(
137
+ X, accept_sparse=["csr", "csc"], dtype=[np.float64, np.float32]
138
+ )
139
+
140
+ n_features = X.shape[1]
141
+ corrmat = np.corrcoef(X.T)
142
+ self.components_ = self._make_matrix(
143
+ corrmat, n_features
144
+ ).astype(X.dtype, copy=False)
145
+
146
+ return self
147
+
148
+ def transform(self, X):
149
+ """Project the data by using matrix product with the random matrix.
150
+
151
+ Parameters
152
+ ----------
153
+ X : {ndarray, sparse matrix} of shape (n_samples, n_features)
154
+ The input data to project into a smaller dimensional space.
155
+
156
+ Returns
157
+ -------
158
+ X_new : ndarray of shape (n_samples, n_components)
159
+ Projected array.
160
+ """
161
+ check_is_fitted(self)
162
+ X = self._validate_data(
163
+ X, accept_sparse=["csr", "csc"], reset=False, dtype=[np.float64, np.float32]
164
+ )
165
+
166
+ return X @ self.components_.T
167
+ # pylint: enable=C0103,W0613
168
+
169
+
170
+ # Would need some refactoring, but disable pylint's "too many" warnings for now.
171
+ # pylint: disable=R0902,R0912,R0914,R0915
172
+ class ScoreClusters:
173
+ """Cluster segments by filter scores
174
+
175
+ Train k-means clustering and take thresholds based on the noisy cluster center.
176
+
177
+ """
178
+
179
+ def __init__(self, score_file, k=2):
180
+ self.k = k
181
+ self.df = load_dataframe(score_file)
182
+ self.filters = {}
183
+ for name in self.df.columns:
184
+ first_part = name.split('.')[0]
185
+ filter_cls = getattr(filtermodule, first_part)
186
+ self.filters[name] = filter_cls
187
+ self.scaler = preprocessing.StandardScaler()
188
+ self.standard_data = self.scaler.fit_transform(self.df.mul(self.direction_vector))
189
+
190
+ logger.info('Training KMeans with %s clusters', self.k)
191
+ self.kmeans = KMeans(n_clusters=self.k, random_state=0, init='k-means++', n_init=1)
192
+ self.kmeans.fit(self.standard_data)
193
+ self.labels = self.kmeans.labels_
194
+ self.cluster_centers = self.scaler.inverse_transform(self.kmeans.cluster_centers_) * self.direction_vector
195
+ self._noisy_label = self._get_noisy_label()
196
+ self.rejects = None
197
+ self.thresholds = None
198
+
199
+ @property
200
+ def noisy_label(self):
201
+ """Cluster label for noisy data"""
202
+ return self._noisy_label
203
+
204
+ @property
205
+ def clean_labels(self):
206
+ """Cluster labels for clean data"""
207
+ return [idx for idx in range(self.k) if idx != self._noisy_label]
208
+
209
+ @property
210
+ def direction_vector(self):
211
+ """Direction vector for the features (1 for CLEAN_LOW, -1 for CLEAN_HIGH)"""
212
+ return np.array([1 if self.filters[name].score_direction == CLEAN_LOW else -1
213
+ for name in self.df.columns])
214
+
215
+ def _get_noisy_label(self):
216
+ """Find label for the noisy cluster"""
217
+ means = np.mean(self.kmeans.cluster_centers_, axis=1)
218
+
219
+ # Output some cluster information
220
+ nlabels = Counter(self.labels)
221
+ for i, (center, inv_center, mean) in enumerate(zip(self.kmeans.cluster_centers_, self.cluster_centers, means)):
222
+ logger.info('Cluster #%s', i)
223
+ logger.info('* number of samples: %s', nlabels[i])
224
+ logger.info('* centroid (score, scaled value, original value):')
225
+ for j, val in enumerate(center):
226
+ logger.info(' %s\t%s\t%s', self.df.columns[j].ljust(25), round(val, 2), round(inv_center[j], 2))
227
+ logger.info('Average center\t%s', np.round(mean, 2))
228
+
229
+ # Cluster center of the noisiest cluster based on average features
230
+ noisy_mean = np.max(means)
231
+ noisy_label = np.argmax(means)
232
+ logger.info('Cluster center of the noisiest cluster (%s)', np.round(noisy_mean, 2))
233
+ logger.info('Noisy label: %s', noisy_label)
234
+ noisy_labels = np.where(self.labels == noisy_label)[0]
235
+ logger.info('Number of noisy labels: %s',
236
+ f'{len(noisy_labels)}/{len(self.labels)} ({round(100*len(noisy_labels)/len(self.labels), 2)}%)')
237
+ return noisy_label
238
+
239
+ def get_columns(self):
240
+ """Return data column names"""
241
+ return self.df.columns
242
+
243
+ def get_thresholds(self, method='noisy_center', precision=6):
244
+ """Return a list of thresholds for noisy samples"""
245
+ if method != 'noisy_center':
246
+ raise ValueError(f'Method {method} for thresholds not implemented')
247
+ return self.cluster_centers[self.noisy_label].round(precision).tolist()
248
+
249
+ def get_rejects(self):
250
+ """Train random forest classifier to find important features
251
+
252
+ Returns a list of booleans (True = reject).
253
+
254
+ """
255
+ logger.info('Training random forest')
256
+ clf = RandomForestClassifier(random_state=1)
257
+ clf.fit(self.standard_data, self.labels)
258
+ logger.info('Finding important features')
259
+ feature_importances = permutation_importance(clf, self.standard_data, self.labels)
260
+ importance_mean_mean = np.mean(feature_importances.importances_mean)
261
+ rej_coef = 0.1
262
+ logger.info('* mean importance: %s', round(importance_mean_mean, 3))
263
+ logger.info('* rejection coefficient: %s', rej_coef)
264
+ logger.info('* decisions:')
265
+ rejects = []
266
+ for i, col in enumerate(self.df.columns):
267
+ importance = feature_importances['importances_mean'][i]
268
+ reject = importance < importance_mean_mean * rej_coef
269
+ logger.info(' %s\t%s\t%s', col.ljust(25), round(importance, 3), 'reject' if reject else 'keep')
270
+ rejects.append(reject)
271
+ return rejects
272
+
273
+ def get_result_df(self):
274
+ """Return dataframe containing the thresholds and reject booleans"""
275
+ self.rejects = self.get_rejects()
276
+ self.thresholds = self.get_thresholds()
277
+ return pd.DataFrame.from_dict(
278
+ {'name': self.get_columns(), 'threshold': self.thresholds, 'reject': self.rejects})
279
+
280
+ def plot(self, plt, path=None, apply_rejects=True, projection='arc'):
281
+ """Plot clustering and histograms"""
282
+ if projection == 'pca':
283
+ proj = decomposition.PCA(n_components=2)
284
+ elif projection == 'random':
285
+ proj = random_projection.GaussianRandomProjection(n_components=2)
286
+ elif projection == 'arc':
287
+ proj = ArcProjection()
288
+ else:
289
+ raise ValueError(f"Unknown projection: {projection}")
290
+ col_index = {col: idx for idx, col in enumerate(self.get_columns())}
291
+ index_col = dict(enumerate(self.get_columns()))
292
+ if apply_rejects and self.rejects:
293
+ indices = [idx for idx, reject in enumerate(self.rejects) if not reject]
294
+ cols = [index_col[idx] for idx in indices]
295
+ index_col = dict(enumerate(cols))
296
+ data_t = proj.fit_transform(self.standard_data[:, indices])
297
+ centroids = proj.transform(self.kmeans.cluster_centers_[:, indices])
298
+ else:
299
+ data_t = proj.fit_transform(self.standard_data)
300
+ centroids = proj.transform(self.kmeans.cluster_centers_)
301
+ plt.figure()
302
+ for idx in range(proj.components_.shape[1]):
303
+ plt.arrow(0, 0, proj.components_[0, idx], proj.components_[1, idx], head_width=0.01, fc='k')
304
+ plt.text(proj.components_[0, idx], proj.components_[1, idx], index_col[idx])
305
+ plt.gca().set_aspect('equal', adjustable='box')
306
+ plt.title('Projection vectors for clustering')
307
+ if path is not None:
308
+ plt.savefig(os.path.join(path, 'projection.pdf'))
309
+ plt.figure(figsize=(10, 10))
310
+ for label_id in range(self.k):
311
+ points = np.where(self.labels == label_id)
312
+ plt.scatter(data_t[points, 0], data_t[points, 1],
313
+ c='orange' if label_id == self.noisy_label else 'blue',
314
+ label='noisy' if label_id == self.noisy_label else 'clean',
315
+ marker=',', s=1, alpha=0.1)
316
+ for label_id in range(self.k):
317
+ plt.scatter(centroids[label_id, 0], centroids[label_id, 1], s=100, alpha=1,
318
+ marker='+', c='brown' if label_id == self.noisy_label else 'darkblue',
319
+ label='noisy centroid' if label_id == self.noisy_label else 'clean centroid')
320
+ plt.legend()
321
+ plt.title('Clustering')
322
+ if path is not None:
323
+ plt.savefig(os.path.join(path, 'clustering.pdf'))
324
+ noisy_samples = self.df.iloc[np.where(self.labels == self.noisy_label)]
325
+ clean_samples = self.df.iloc[np.where(self.labels != self.noisy_label)]
326
+ subplots_n = noisy_samples.hist(bins=100, figsize=(10, 10))
327
+ fig_noisy = plt.gcf()
328
+ plt.suptitle('Histograms for noisy samples')
329
+ subplots_c = clean_samples.hist(bins=100, figsize=(10, 10))
330
+ fig_clean = plt.gcf()
331
+ plt.suptitle('Histograms for clean samples')
332
+ if apply_rejects and self.rejects:
333
+ for axes in ([axes for sublist in subplots_c for axes in sublist] +
334
+ [axes for sublist in subplots_n for axes in sublist]):
335
+ title = axes.get_title()
336
+ if not title:
337
+ continue
338
+ idx = col_index[title]
339
+ if self.rejects[idx]:
340
+ axes.text(0.5, 1, 'REJECTED', horizontalalignment='center',
341
+ verticalalignment='top', transform=axes.transAxes, color='r')
342
+ axes.axvline(self.thresholds[idx], color='k', linestyle='--', alpha=0.5)
343
+ if path is not None:
344
+ fig_clean.savefig(os.path.join(path, 'histogram_clean.pdf'))
345
+ fig_noisy.savefig(os.path.join(path, 'histogram_noisy.pdf'))
346
+ # pylint: enable=R0902,R0912,R0914,R0915
@@ -133,13 +133,20 @@ class SentenceEmbeddingFilter(FilterABC):
133
133
 
134
134
  def score(self, pairs):
135
135
  for chunk in grouper(pairs, self.chunksize):
136
- return self._score_chunk(chunk)
136
+ for score in self._score_chunk(chunk):
137
+ yield score
137
138
 
138
139
  def accept(self, score):
139
140
  return all(similarity >= self.threshold for similarity in score)
140
141
 
141
142
  def filter(self, pairs):
142
143
  for chunk in grouper(pairs, self.chunksize):
143
- for pair, score in zip(pairs, self._score_chunk(chunk)):
144
+ for pair, score in zip(chunk, self._score_chunk(chunk)):
144
145
  if self.accept(score):
145
146
  yield pair
147
+
148
+ def filterfalse(self, pairs):
149
+ for chunk in grouper(pairs, self.chunksize):
150
+ for pair, score in zip(chunk, self._score_chunk(chunk)):
151
+ if not self.accept(score):
152
+ yield pair