shorttext 4.0.2__tar.gz → 4.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {shorttext-4.0.2/src/shorttext.egg-info → shorttext-4.0.4}/PKG-INFO +4 -2
  2. {shorttext-4.0.2 → shorttext-4.0.4}/README.md +2 -0
  3. {shorttext-4.0.2 → shorttext-4.0.4}/pyproject.toml +2 -2
  4. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/dtm.py +8 -6
  5. {shorttext-4.0.2 → shorttext-4.0.4/src/shorttext.egg-info}/PKG-INFO +4 -2
  6. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/requires.txt +1 -1
  7. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_dtm.py +1 -1
  8. {shorttext-4.0.2 → shorttext-4.0.4}/LICENSE +0 -0
  9. {shorttext-4.0.2 → shorttext-4.0.4}/MANIFEST.in +0 -0
  10. {shorttext-4.0.2 → shorttext-4.0.4}/setup.cfg +0 -0
  11. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/__init__.py +0 -0
  12. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/__init__.py +0 -0
  13. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/base.py +0 -0
  14. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/__init__.py +0 -0
  15. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/maxent/MaxEntClassification.py +0 -0
  16. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/maxent/__init__.py +0 -0
  17. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/SkLearnClassification.py +0 -0
  18. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/TopicVectorDistanceClassification.py +0 -0
  19. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/bow/topic/__init__.py +0 -0
  20. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/__init__.py +0 -0
  21. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/nnlib/VarNNEmbedVecClassification.py +0 -0
  22. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/nnlib/__init__.py +0 -0
  23. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/nnlib/frameworks.py +0 -0
  24. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/SumEmbedVecClassification.py +0 -0
  25. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/VarNNSumEmbedVecClassification.py +0 -0
  26. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/__init__.py +0 -0
  27. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/classifiers/embed/sumvec/frameworks.py +0 -0
  28. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/cli/__init__.py +0 -0
  29. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/cli/categorization.py +0 -0
  30. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/cli/wordembedsim.py +0 -0
  31. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/data/__init__.py +0 -0
  32. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/data/data_retrieval.py +0 -0
  33. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/data/shorttext_exampledata.csv +0 -0
  34. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/__init__.py +0 -0
  35. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/AutoEncodingTopicModeling.py +0 -0
  36. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/GensimTopicModeling.py +0 -0
  37. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/LatentTopicModeling.py +0 -0
  38. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/bow/__init__.py +0 -0
  39. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/charbase/__init__.py +0 -0
  40. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/charbase/char2vec.py +0 -0
  41. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/seq2seq/__init__.py +0 -0
  42. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/seq2seq/charbaseS2S.py +0 -0
  43. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/generators/seq2seq/s2skeras.py +0 -0
  44. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/__init__.py +0 -0
  45. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/__init__.py +0 -0
  46. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/dldist.py +0 -0
  47. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/jaccard.py +0 -0
  48. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/dynprog/lcp.py +0 -0
  49. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/embedfuzzy/__init__.py +0 -0
  50. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/embedfuzzy/jaccard.py +0 -0
  51. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/wasserstein/__init__.py +0 -0
  52. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/metrics/wasserstein/wordmoverdist.py +0 -0
  53. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/schemas/__init__.py +0 -0
  54. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/schemas/models.py +0 -0
  55. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/smartload.py +0 -0
  56. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/__init__.py +0 -0
  57. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/basespellcorrector.py +0 -0
  58. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/editor.py +0 -0
  59. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/spell/norvig.py +0 -0
  60. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/stack/__init__.py +0 -0
  61. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/stack/stacking.py +0 -0
  62. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/__init__.py +0 -0
  63. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/classification_exceptions.py +0 -0
  64. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/compactmodel_io.py +0 -0
  65. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/compute.py +0 -0
  66. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/gensim_corpora.py +0 -0
  67. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/kerasmodel_io.py +0 -0
  68. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/misc.py +0 -0
  69. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/nonneg_stopwords.txt +0 -0
  70. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/stopwords.txt +0 -0
  71. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/textpreprocessing.py +0 -0
  72. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext/utils/wordembed.py +0 -0
  73. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/SOURCES.txt +0 -0
  74. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/dependency_links.txt +0 -0
  75. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/entry_points.txt +0 -0
  76. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/not-zip-safe +0 -0
  77. {shorttext-4.0.2 → shorttext-4.0.4}/src/shorttext.egg-info/top_level.txt +0 -0
  78. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_charonehot.py +0 -0
  79. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_compute.py +0 -0
  80. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_fuzzylogic.py +0 -0
  81. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_norvigspell.py +0 -0
  82. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_stacking.py +0 -0
  83. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_textpreprocessing.py +0 -0
  84. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_topicmodeling.py +0 -0
  85. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_var_nn_embedded_vec_classifier.py +0 -0
  86. {shorttext-4.0.2 → shorttext-4.0.4}/test/test_wmd.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: shorttext
3
- Version: 4.0.2
3
+ Version: 4.0.4
4
4
  Summary: Short Text Mining
5
5
  Author-email: Kwan Yuet Stephen Ho <stephenhky@yahoo.com.hk>
6
6
  License: MIT
@@ -35,7 +35,7 @@ Requires-Dist: pandas>=1.2.0
35
35
  Requires-Dist: snowballstemmer>=3.0.0
36
36
  Requires-Dist: numba>=0.57.0
37
37
  Requires-Dist: deprecation>=2.0.0
38
- Requires-Dist: npdict>=0.0.10
38
+ Requires-Dist: npdict>=0.0.11
39
39
  Requires-Dist: sparse>=0.10.0
40
40
  Requires-Dist: orjson>=3.0.0
41
41
  Requires-Dist: loguru>=0.6.0
@@ -125,6 +125,8 @@ You can talk to me in advance through e-mails or the [Issues](https://github.com
125
125
 
126
126
  ## News
127
127
 
128
+ * 07/02/2026: `shorttext` 4.0.4 released.
129
+ * 05/26/2026: `shorttext` 4.0.3 released.
128
130
  * 05/18/2026: `shorttext` 4.0.2 released.
129
131
  * 05/14/2026: `shorttext` 4.0.1 released.
130
132
  * 04/19/2026: `shorttext` 4.0.0 released.
@@ -80,6 +80,8 @@ You can talk to me in advance through e-mails or the [Issues](https://github.com
80
80
 
81
81
  ## News
82
82
 
83
+ * 07/02/2026: `shorttext` 4.0.4 released.
84
+ * 05/26/2026: `shorttext` 4.0.3 released.
83
85
  * 05/18/2026: `shorttext` 4.0.2 released.
84
86
  * 05/14/2026: `shorttext` 4.0.1 released.
85
87
  * 04/19/2026: `shorttext` 4.0.0 released.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "shorttext"
7
- version = "4.0.2"
7
+ version = "4.0.4"
8
8
  authors = [
9
9
  {name = "Kwan Yuet Stephen Ho", email = "stephenhky@yahoo.com.hk"}
10
10
  ]
@@ -40,7 +40,7 @@ dependencies = [
40
40
  "snowballstemmer>=3.0.0",
41
41
  "numba>=0.57.0",
42
42
  "deprecation>=2.0.0",
43
- "npdict>=0.0.10",
43
+ "npdict>=0.0.11",
44
44
  "sparse>=0.10.0",
45
45
  "orjson>=3.0.0",
46
46
  "loguru>=0.6.0"
@@ -173,7 +173,7 @@ def compute_document_frequency(
173
173
  Returns:
174
174
  Array of document frequencies for each token.
175
175
  """
176
- if isinstance(npdtm, npdict.SparseArrayWrappedDict):
176
+ if npdtm.is_sparse():
177
177
  return np.sum(npdtm.to_coo() > 0, axis=0).todense()
178
178
  else:
179
179
  return np.sum(npdtm.to_numpy() > 0, axis=0)
@@ -194,7 +194,7 @@ def compute_tfidf_document_term_matrix(
194
194
  """
195
195
  doc_frequencies = compute_document_frequency(npdtm)
196
196
  nbdocs = npdtm.dimension_sizes[0]
197
- if isinstance(npdtm, npdict.SparseArrayWrappedDict):
197
+ if npdtm.is_sparse():
198
198
  new_dtm_sparray = npdtm.to_coo() * np.log(nbdocs / doc_frequencies)
199
199
  return npdtm.generate_dict(new_dtm_sparray, dense=not sparse)
200
200
 
@@ -288,11 +288,13 @@ class NumpyDocumentTermMatrix(CompactIOMachine):
288
288
  Total term frequency.
289
289
  """
290
290
  token_index = self.npdtm._keystrings_to_indices[1][token]
291
- if isinstance(self.npdtm, npdict.SparseArrayWrappedDict):
291
+ if self.npdtm.is_sparse():
292
292
  matrix = self.npdtm.to_coo()
293
+ array = matrix[:, token_index].todense()
293
294
  else:
294
295
  matrix = self.npdtm.to_numpy()
295
- return np.sum(matrix[:, token_index])
296
+ array = matrix[:, token_index]
297
+ return np.sum(array)
296
298
 
297
299
  def get_doc_frequency(self, token) -> int:
298
300
  """Get document frequency of a token.
@@ -304,8 +306,8 @@ class NumpyDocumentTermMatrix(CompactIOMachine):
304
306
  Number of documents containing the token.
305
307
  """
306
308
  token_index = self.npdtm._keystrings_to_indices[1][token]
307
- if isinstance(self.npdtm, npdict.SparseArrayWrappedDict):
308
- freq_array = self.npdtm.to_coo()[:, token_index]
309
+ if self.npdtm.is_sparse():
310
+ freq_array = self.npdtm.to_coo()[:, token_index].todense()
309
311
  else:
310
312
  freq_array = self.npdtm.to_numpy()[:, token_index]
311
313
  return np.sum(freq_array > 0, axis=0)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: shorttext
3
- Version: 4.0.2
3
+ Version: 4.0.4
4
4
  Summary: Short Text Mining
5
5
  Author-email: Kwan Yuet Stephen Ho <stephenhky@yahoo.com.hk>
6
6
  License: MIT
@@ -35,7 +35,7 @@ Requires-Dist: pandas>=1.2.0
35
35
  Requires-Dist: snowballstemmer>=3.0.0
36
36
  Requires-Dist: numba>=0.57.0
37
37
  Requires-Dist: deprecation>=2.0.0
38
- Requires-Dist: npdict>=0.0.10
38
+ Requires-Dist: npdict>=0.0.11
39
39
  Requires-Dist: sparse>=0.10.0
40
40
  Requires-Dist: orjson>=3.0.0
41
41
  Requires-Dist: loguru>=0.6.0
@@ -125,6 +125,8 @@ You can talk to me in advance through e-mails or the [Issues](https://github.com
125
125
 
126
126
  ## News
127
127
 
128
+ * 07/02/2026: `shorttext` 4.0.4 released.
129
+ * 05/26/2026: `shorttext` 4.0.3 released.
128
130
  * 05/18/2026: `shorttext` 4.0.2 released.
129
131
  * 05/14/2026: `shorttext` 4.0.1 released.
130
132
  * 04/19/2026: `shorttext` 4.0.0 released.
@@ -9,7 +9,7 @@ pandas>=1.2.0
9
9
  snowballstemmer>=3.0.0
10
10
  numba>=0.57.0
11
11
  deprecation>=2.0.0
12
- npdict>=0.0.10
12
+ npdict>=0.0.11
13
13
  sparse>=0.10.0
14
14
  orjson>=3.0.0
15
15
  loguru>=0.6.0
@@ -24,5 +24,5 @@ def test_inaugural():
24
24
  # check results
25
25
  assert dtm.get_token_occurences(stemword('change'))['2009-Obama'] == pytest.approx(0.9400072584914713)
26
26
  assert dtm.nbdocs == 56
27
- assert dtm.nbtokens == 5075
27
+ assert dtm.nbtokens == 5077
28
28
  assert dtm.get_total_termfreq(stemword('government')) == pytest.approx(37.82606692473982)
File without changes
File without changes
File without changes
File without changes