shorttext 4.0.2__tar.gz → 4.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {shorttext-4.0.2/src/shorttext.egg-info → shorttext-4.0.4}/PKG-INFO +4 -2
- {shorttext-4.0.2 → shorttext-4.0.4}/README.md +2 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/pyproject.toml +2 -2
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/dtm.py +8 -6
- {shorttext-4.0.2 → shorttext-4.0.4/src/shorttext.egg-info}/PKG-INFO +4 -2
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/requires.txt +1 -1
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_dtm.py +1 -1
- {shorttext-4.0.2 → shorttext-4.0.4}/LICENSE +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/MANIFEST.in +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/setup.cfg +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/base.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/maxent/MaxEntClassification.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/maxent/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/SkLearnClassification.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/TopicVectorDistanceClassification.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/nnlib/VarNNEmbedVecClassification.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/nnlib/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/nnlib/frameworks.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/SumEmbedVecClassification.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/VarNNSumEmbedVecClassification.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/frameworks.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/cli/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/cli/categorization.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/cli/wordembedsim.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/data/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/data/data_retrieval.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/data/shorttext_exampledata.csv +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/AutoEncodingTopicModeling.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/GensimTopicModeling.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/LatentTopicModeling.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/charbase/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/charbase/char2vec.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/seq2seq/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/seq2seq/charbaseS2S.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/seq2seq/s2skeras.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/dldist.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/jaccard.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/lcp.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/embedfuzzy/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/embedfuzzy/jaccard.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/wasserstein/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/wasserstein/wordmoverdist.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/schemas/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/schemas/models.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/smartload.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/basespellcorrector.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/editor.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/norvig.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/stack/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/stack/stacking.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/__init__.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/classification_exceptions.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/compactmodel_io.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/compute.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/gensim_corpora.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/kerasmodel_io.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/misc.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/nonneg_stopwords.txt +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/stopwords.txt +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/textpreprocessing.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/wordembed.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/SOURCES.txt +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/dependency_links.txt +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/entry_points.txt +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/not-zip-safe +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/top_level.txt +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_charonehot.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_compute.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_fuzzylogic.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_norvigspell.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_stacking.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_textpreprocessing.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_topicmodeling.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_var_nn_embedded_vec_classifier.py +0 -0
- {shorttext-4.0.2 → shorttext-4.0.4}/test/test_wmd.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shorttext
|
|
3
|
-
Version: 4.0.
|
|
3
|
+
Version: 4.0.4
|
|
4
4
|
Summary: Short Text Mining
|
|
5
5
|
Author-email: Kwan Yuet Stephen Ho <stephenhky@yahoo.com.hk>
|
|
6
6
|
License: MIT
|
|
@@ -35,7 +35,7 @@ Requires-Dist: pandas>=1.2.0
|
|
|
35
35
|
Requires-Dist: snowballstemmer>=3.0.0
|
|
36
36
|
Requires-Dist: numba>=0.57.0
|
|
37
37
|
Requires-Dist: deprecation>=2.0.0
|
|
38
|
-
Requires-Dist: npdict>=0.0.
|
|
38
|
+
Requires-Dist: npdict>=0.0.11
|
|
39
39
|
Requires-Dist: sparse>=0.10.0
|
|
40
40
|
Requires-Dist: orjson>=3.0.0
|
|
41
41
|
Requires-Dist: loguru>=0.6.0
|
|
@@ -125,6 +125,8 @@ You can talk to me in advance through e-mails or the [Issues](https://github.com
|
|
|
125
125
|
|
|
126
126
|
## News
|
|
127
127
|
|
|
128
|
+
* 07/02/2026: `shorttext` 4.0.4 released.
|
|
129
|
+
* 05/26/2026: `shorttext` 4.0.3 released.
|
|
128
130
|
* 05/18/2026: `shorttext` 4.0.2 released.
|
|
129
131
|
* 05/14/2026: `shorttext` 4.0.1 released.
|
|
130
132
|
* 04/19/2026: `shorttext` 4.0.0 released.
|
|
@@ -80,6 +80,8 @@ You can talk to me in advance through e-mails or the [Issues](https://github.com
|
|
|
80
80
|
|
|
81
81
|
## News
|
|
82
82
|
|
|
83
|
+
* 07/02/2026: `shorttext` 4.0.4 released.
|
|
84
|
+
* 05/26/2026: `shorttext` 4.0.3 released.
|
|
83
85
|
* 05/18/2026: `shorttext` 4.0.2 released.
|
|
84
86
|
* 05/14/2026: `shorttext` 4.0.1 released.
|
|
85
87
|
* 04/19/2026: `shorttext` 4.0.0 released.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "shorttext"
|
|
7
|
-
version = "4.0.
|
|
7
|
+
version = "4.0.4"
|
|
8
8
|
authors = [
|
|
9
9
|
{name = "Kwan Yuet Stephen Ho", email = "stephenhky@yahoo.com.hk"}
|
|
10
10
|
]
|
|
@@ -40,7 +40,7 @@ dependencies = [
|
|
|
40
40
|
"snowballstemmer>=3.0.0",
|
|
41
41
|
"numba>=0.57.0",
|
|
42
42
|
"deprecation>=2.0.0",
|
|
43
|
-
"npdict>=0.0.
|
|
43
|
+
"npdict>=0.0.11",
|
|
44
44
|
"sparse>=0.10.0",
|
|
45
45
|
"orjson>=3.0.0",
|
|
46
46
|
"loguru>=0.6.0"
|
|
@@ -173,7 +173,7 @@ def compute_document_frequency(
|
|
|
173
173
|
Returns:
|
|
174
174
|
Array of document frequencies for each token.
|
|
175
175
|
"""
|
|
176
|
-
if
|
|
176
|
+
if npdtm.is_sparse():
|
|
177
177
|
return np.sum(npdtm.to_coo() > 0, axis=0).todense()
|
|
178
178
|
else:
|
|
179
179
|
return np.sum(npdtm.to_numpy() > 0, axis=0)
|
|
@@ -194,7 +194,7 @@ def compute_tfidf_document_term_matrix(
|
|
|
194
194
|
"""
|
|
195
195
|
doc_frequencies = compute_document_frequency(npdtm)
|
|
196
196
|
nbdocs = npdtm.dimension_sizes[0]
|
|
197
|
-
if
|
|
197
|
+
if npdtm.is_sparse():
|
|
198
198
|
new_dtm_sparray = npdtm.to_coo() * np.log(nbdocs / doc_frequencies)
|
|
199
199
|
return npdtm.generate_dict(new_dtm_sparray, dense=not sparse)
|
|
200
200
|
|
|
@@ -288,11 +288,13 @@ class NumpyDocumentTermMatrix(CompactIOMachine):
|
|
|
288
288
|
Total term frequency.
|
|
289
289
|
"""
|
|
290
290
|
token_index = self.npdtm._keystrings_to_indices[1][token]
|
|
291
|
-
if
|
|
291
|
+
if self.npdtm.is_sparse():
|
|
292
292
|
matrix = self.npdtm.to_coo()
|
|
293
|
+
array = matrix[:, token_index].todense()
|
|
293
294
|
else:
|
|
294
295
|
matrix = self.npdtm.to_numpy()
|
|
295
|
-
|
|
296
|
+
array = matrix[:, token_index]
|
|
297
|
+
return np.sum(array)
|
|
296
298
|
|
|
297
299
|
def get_doc_frequency(self, token) -> int:
|
|
298
300
|
"""Get document frequency of a token.
|
|
@@ -304,8 +306,8 @@ class NumpyDocumentTermMatrix(CompactIOMachine):
|
|
|
304
306
|
Number of documents containing the token.
|
|
305
307
|
"""
|
|
306
308
|
token_index = self.npdtm._keystrings_to_indices[1][token]
|
|
307
|
-
if
|
|
308
|
-
freq_array = self.npdtm.to_coo()[:, token_index]
|
|
309
|
+
if self.npdtm.is_sparse():
|
|
310
|
+
freq_array = self.npdtm.to_coo()[:, token_index].todense()
|
|
309
311
|
else:
|
|
310
312
|
freq_array = self.npdtm.to_numpy()[:, token_index]
|
|
311
313
|
return np.sum(freq_array > 0, axis=0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shorttext
|
|
3
|
-
Version: 4.0.
|
|
3
|
+
Version: 4.0.4
|
|
4
4
|
Summary: Short Text Mining
|
|
5
5
|
Author-email: Kwan Yuet Stephen Ho <stephenhky@yahoo.com.hk>
|
|
6
6
|
License: MIT
|
|
@@ -35,7 +35,7 @@ Requires-Dist: pandas>=1.2.0
|
|
|
35
35
|
Requires-Dist: snowballstemmer>=3.0.0
|
|
36
36
|
Requires-Dist: numba>=0.57.0
|
|
37
37
|
Requires-Dist: deprecation>=2.0.0
|
|
38
|
-
Requires-Dist: npdict>=0.0.
|
|
38
|
+
Requires-Dist: npdict>=0.0.11
|
|
39
39
|
Requires-Dist: sparse>=0.10.0
|
|
40
40
|
Requires-Dist: orjson>=3.0.0
|
|
41
41
|
Requires-Dist: loguru>=0.6.0
|
|
@@ -125,6 +125,8 @@ You can talk to me in advance through e-mails or the [Issues](https://github.com
|
|
|
125
125
|
|
|
126
126
|
## News
|
|
127
127
|
|
|
128
|
+
* 07/02/2026: `shorttext` 4.0.4 released.
|
|
129
|
+
* 05/26/2026: `shorttext` 4.0.3 released.
|
|
128
130
|
* 05/18/2026: `shorttext` 4.0.2 released.
|
|
129
131
|
* 05/14/2026: `shorttext` 4.0.1 released.
|
|
130
132
|
* 04/19/2026: `shorttext` 4.0.0 released.
|
|
@@ -24,5 +24,5 @@ def test_inaugural():
|
|
|
24
24
|
# check results
|
|
25
25
|
assert dtm.get_token_occurences(stemword('change'))['2009-Obama'] == pytest.approx(0.9400072584914713)
|
|
26
26
|
assert dtm.nbdocs == 56
|
|
27
|
-
assert dtm.nbtokens ==
|
|
27
|
+
assert dtm.nbtokens == 5077
|
|
28
28
|
assert dtm.get_total_termfreq(stemword('government')) == pytest.approx(37.82606692473982)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/maxent/MaxEntClassification.py
RENAMED
|
File without changes
|
|
File without changes
|
{shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/SkLearnClassification.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/AutoEncodingTopicModeling.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|