hjxdl 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.7'
-__version_tuple__ = version_tuple = (0, 1, 7)
+__version__ = version = '0.1.8'
+__version_tuple__ = version_tuple = (0, 1, 8)

hdl/utils/llm/embs.py

@@ -1,4 +1,4 @@
-from sentence_transformers import SentenceTransformer
+import re
 
 
 class BEEmbedder():
@@ -118,6 +118,8 @@ class HFEmbedder():
         Returns:
             None
         """
+
+        from sentence_transformers import SentenceTransformer
         
         self.device = device
         self.emb_dir = emb_dir
@@ -190,7 +192,7 @@ class HFEmbedder():
 
 def get_n_tokens(
     paragraph,
-    model: str = None
+    model: str = ""
 ):
     """Get the number of tokens in a paragraph using a specified model.
     
@@ -201,7 +203,7 @@ def get_n_tokens(
     Returns:
         int: The number of tokens in the paragraph based on the specified model or default CJK tokenization.
     """
-    if model is None:
+    if model == "":
         cjk_regex = re.compile(u'[\u1100-\uFFFDh]+?')
         trimed_cjk = cjk_regex.sub( ' a ', paragraph, 0)
         return len(trimed_cjk.split())

{hjxdl-0.1.7.dist-info → hjxdl-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.1.7
+Version: 0.1.8
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.1.7.dist-info → hjxdl-0.1.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=5sZZNySv08wwfzJcSDssGTqUn9wlmDsR6R4XB8J8mFM,70
-hdl/_version.py,sha256=0A08Kvw-SYs_CkLPEV1KmD8lb9IPH1psSxb5iQEGtI8,411
+hdl/_version.py,sha256=PdJ7dZoz_SyEgX0MdrMfQYBFlGcwpemv6ibF8NKALBY,411
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -85,11 +85,11 @@ hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 hdl/utils/general/glob.py,sha256=8-RCnt6L297wMIfn34ZAMCsGCZUjHG3MGglGZI1cX0g,491
 hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/llm/chat.py,sha256=gsbqWh8fTcJUENU6ZuMClZAuSOLFnD5VP8kXOxGh3Zw,13776
-hdl/utils/llm/embs.py,sha256=ntT9Noax4ao7Rm1kjVMtXa3j0ykbRq0cABAci147Bu0,7159
+hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
-hjxdl-0.1.7.dist-info/METADATA,sha256=MMDp3uVtFocfPk5vDF056atHLM0dBDzPu6GrLZXbGDg,542
-hjxdl-0.1.7.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-hjxdl-0.1.7.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.1.7.dist-info/RECORD,,
+hjxdl-0.1.8.dist-info/METADATA,sha256=a9BaE0EGy5G9EM3Tbsi4LMmIrCMFJUuDjFnmmu_nBW4,542
+hjxdl-0.1.8.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+hjxdl-0.1.8.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.1.8.dist-info/RECORD,,