bella-openapi 1.0.2.4__tar.gz → 1.0.2.5__tar.gz

{bella_openapi-1.0.2.4/src/bella_openapi.egg-info → bella_openapi-1.0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bella-openapi
-Version: 1.0.2.4
+Version: 1.0.2.5
 Summary: client for openapi service.
 Home-page: 
 Author: ['tangxiaolong', 'fanqiangwei', 'zhangxiaojia', 'liumin', 'wangyukun']

{bella_openapi-1.0.2.4 → bella_openapi-1.0.2.5}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 SHORT = "client for openapi service."
 
-__version__ = "1.0.2.4"
+__version__ = "1.0.2.5"
 __author__ = ["tangxiaolong", "fanqiangwei", "zhangxiaojia", 'liumin', 'wangyukun']
 __email__ = ''
 readme_path = 'README.md'

{bella_openapi-1.0.2.4 → bella_openapi-1.0.2.5}/src/bella_openapi/entity/standard_domtree.py

@@ -31,16 +31,7 @@ layout_type_mapping = {
     "TableNote": "Text",  # 目前实际解析出来没有
 }
 
-# 自研模型均用gpt-4计算（可能有误差，可忽略）
-def count_tokens(text: str, model: str = "gpt-4") -> int:
-    if not text:
-        return 0
-    encoding = tiktoken.encoding_for_model(model)
-    tokens = encoding.encode(text)
-    # 计算标记列表的长度，即标记的数量
-    token_count = len(tokens)
-    # 返回标记的数量
-    return token_count
+
 
 class SourceFile(BaseModel):
     id: str  # 文件ID，唯一标识符，类型为string
@@ -440,7 +431,7 @@ class StandardDomTree(BaseModel):
                                       cell_data['end_col']],
                                 text=cell_text,
                                 # 目前只会有一个元素,且是Text类型，Path重新从头编号，相对cell是root
-                                nodes=[StandardNode(summary="", tokens=count_tokens(cell_text), path=[1], children=[],
+                                nodes=[StandardNode(summary="", tokens=cls.count_tokens(cell_text), path=[1], children=[],
                                                     element=StandardElement(
                                                         type='Text',
                                                         positions=[],
@@ -491,7 +482,7 @@ class StandardDomTree(BaseModel):
                     standard_node.children.append(standard_child)
 
         # 计算 token 数量：自身 text 的 token 数量 + 子节点 token 数量
-        tokens = count_tokens(text)
+        tokens = cls.count_tokens(text)
         for child in standard_node.children:
             tokens += child.tokens
 
@@ -500,3 +491,23 @@ class StandardDomTree(BaseModel):
 
         return standard_node
 
+    @classmethod
+    def count_tokens(cls, text: str, model: str = "gpt-4") -> int:
+        """
+        计算文本的token数量
+
+        Args:
+            text: 要计算的文本
+            model: 使用的模型名称，默认为gpt-4
+
+        Returns:
+            int: token数量
+        """
+        if not text:
+            return 0
+        encoding = tiktoken.encoding_for_model(model)
+        tokens = encoding.encode(text)
+        # 计算标记列表的长度，即标记的数量
+        token_count = len(tokens)
+        # 返回标记的数量
+        return token_count

{bella_openapi-1.0.2.4 → bella_openapi-1.0.2.5/src/bella_openapi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bella-openapi
-Version: 1.0.2.4
+Version: 1.0.2.5
 Summary: client for openapi service.
 Home-page: 
 Author: ['tangxiaolong', 'fanqiangwei', 'zhangxiaojia', 'liumin', 'wangyukun']