x-transformers 2.2.2__py3-none-any.whl → 2.2.3__py3-none-any.whl

x_transformers/entropy_based_tokenizer.py

@@ -89,15 +89,15 @@ class EntropyBasedTokenizer(Module):
 
         num_tokens = boundaries.sum(dim = -1) # number of tokens
 
-        boundaries = arange_plus_one[boundaries].split(num_tokens.tolist())
+        indices = arange_plus_one[boundaries].split(num_tokens.tolist())
 
         # get the token lengths
 
         token_lengths = []
 
-        for one_boundary in boundaries:
-            padded_boundary = F.pad(one_boundary, (1, 0), value = 0.)
-            one_token_lengths = padded_boundary[1:] - padded_boundary[:-1]
+        for one_indices in indices:
+            padded_indices = F.pad(one_indices, (1, 0), value = 0.)
+            one_token_lengths = padded_indices[1:] - padded_indices[:-1]
 
             token_lengths.append(one_token_lengths)
 
@@ -120,7 +120,6 @@ class EntropyBasedTokenizer(Module):
 
             one_token_length = one_token_length[one_token_length > 0]
 
-            print(one_seq.shape, one_token_length)
             splitted_seq = one_seq.split(one_token_length.tolist())
             segmented_seq.append(splitted_seq)
 

{x_transformers-2.2.2.dist-info → x_transformers-2.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.2.2
+Version: 2.2.3
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.2.2.dist-info → x_transformers-2.2.3.dist-info}/RECORD

@@ -4,14 +4,14 @@ x_transformers/autoregressive_wrapper.py,sha256=reLCno9Z9pchVU79tBF8OMo21LwSZ67K
 x_transformers/belief_state_wrapper.py,sha256=YLUMk6t2MhFBEw5lHDDHJHcoCxTIkHvxTNY__GGZEKU,13374
 x_transformers/continuous.py,sha256=p0sCAiH1na236ygwgL1Yyhu36eZBf9cZvoW1JyP_fFE,7073
 x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
-x_transformers/entropy_based_tokenizer.py,sha256=hdYfw8GqMj0YVWY_gpaCCzhkMALnQB9yAUaCg8RWMss,3624
+x_transformers/entropy_based_tokenizer.py,sha256=sEOf_J_9PGNFKPZ9Gks3MwNjiTa_JljjO1OU3ubIziQ,3562
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
 x_transformers/x_transformers.py,sha256=Fl2CuAKTxJDOQvqwQo2FK8eO2s1iNLO-P1PP2Yw64rQ,111826
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.2.2.dist-info/METADATA,sha256=V0g-qeMS5RoayZAttx8bGdkIh0ZAsVmDDfTiPNI5qHM,88686
-x_transformers-2.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.2.2.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.2.2.dist-info/RECORD,,
+x_transformers-2.2.3.dist-info/METADATA,sha256=aHA3vvgyUsxooOXzLLPtw0FmRGYrqdzMXe2QVoRJk04,88686
+x_transformers-2.2.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.2.3.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.2.3.dist-info/RECORD,,