dashscope 1.13.6__py3-none-any.whl → 1.14.1__py3-none-any.whl

dashscope/tokenizers/__init__.py

@@ -1,5 +1,5 @@
 from .tokenization import Tokenization
+from .tokenizer import get_tokenizer, list_tokenizers
+from .tokenizer_base import Tokenizer
 
-__all__ = [
-    Tokenization,
-]
+__all__ = [Tokenization, Tokenizer, get_tokenizer, list_tokenizers]

dashscope/tokenizers/qwen_tokenizer.py

@@ -0,0 +1,109 @@
+import base64
+import unicodedata
+from typing import Collection, Dict, List, Set, Union
+
+from .tokenizer_base import Tokenizer
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""  # noqa E501
+ENDOFTEXT = '<|endoftext|>'
+IMSTART = '<|im_start|>'
+IMEND = '<|im_end|>'
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f'<|extra_{i}|>' for i in range(205)))
+# changed to use actual index to avoid misconfiguration with vocabulary expansion
+SPECIAL_START_ID = 151643
+SPECIAL_TOKENS = tuple(
+    enumerate(
+        ((
+            ENDOFTEXT,
+            IMSTART,
+            IMEND,
+        ) + EXTRAS),
+        start=SPECIAL_START_ID,
+    ))
+SPECIAL_TOKENS_SET = set(t for i, t in SPECIAL_TOKENS)
+
+
+class QwenTokenizer(Tokenizer):
+    @staticmethod
+    def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+        with open(tiktoken_bpe_file, 'rb') as f:
+            contents = f.read()
+        return {
+            base64.b64decode(token): int(rank)
+            for token, rank in (line.split() for line in contents.splitlines()
+                                if line)
+        }
+
+    def __init__(self, vocab_file, errors='replace', extra_vocab_file=None):
+        self._errors = errors
+        self._vocab_file = vocab_file
+        self._extra_vocab_file = extra_vocab_file
+
+        self._mergeable_ranks = QwenTokenizer._load_tiktoken_bpe(
+            vocab_file)  # type: Dict[bytes, int]
+        self._special_tokens = {
+            token: index
+            for index, token in SPECIAL_TOKENS
+        }
+
+        # try load extra vocab from file
+        if extra_vocab_file is not None:
+            used_ids = set(self._mergeable_ranks.values()) | set(
+                self._special_tokens.values())
+            extra_mergeable_ranks = self._load_tiktoken_bpe(extra_vocab_file)
+            for token, index in extra_mergeable_ranks.items():
+                if token in self._mergeable_ranks:
+                    continue
+                if index in used_ids:
+                    continue
+                self._mergeable_ranks[token] = index
+            # the index may be sparse after this, but don't worry tiktoken.Encoding will handle this
+        import tiktoken
+        enc = tiktoken.Encoding(
+            'Qwen',
+            pat_str=PAT_STR,
+            mergeable_ranks=self._mergeable_ranks,
+            special_tokens=self._special_tokens,
+        )
+        assert (
+            len(self._mergeable_ranks) +
+            len(self._special_tokens) == enc.n_vocab
+        ), f'{len(self._mergeable_ranks) + len(self._special_tokens)} != {enc.n_vocab} in encoding'
+
+        self.decoder = {v: k
+                        for k, v in self._mergeable_ranks.items()
+                        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self._special_tokens.items()})
+
+        self._tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self._tokenizer.eot_token
+        self.im_start_id = self._special_tokens[IMSTART]
+        self.im_end_id = self._special_tokens[IMEND]
+
+    def encode(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = 'all',
+        disallowed_special: Union[Collection, str] = (),
+    ) -> Union[List[List], List]:
+        text = unicodedata.normalize('NFC', text)
+        return self._tokenizer.encode(text,
+                                      allowed_special=allowed_special,
+                                      disallowed_special=disallowed_special)
+
+    def decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if skip_special_tokens:
+            token_ids = [i for i in token_ids if i < self.eod_id]
+        return self._tokenizer.decode(token_ids, errors=errors or self._errors)

dashscope/tokenizers/tokenization.py

@@ -25,6 +25,8 @@ class Tokenization(BaseApi):
         qwen_14b_chat = 'qwen-14b-chat'
         llama2_7b_chat_v2 = 'llama2-7b-chat-v2'
         llama2_13b_chat_v2 = 'llama2-13b-chat-v2'
+        text_embedding_v2 = 'text-embedding-v2'
+        qwen_72b_chat = 'qwen-72b-chat'
 
     @classmethod
     def call(cls,

dashscope/tokenizers/tokenizer.py

@@ -0,0 +1,43 @@
+import os
+from typing import List
+
+from dashscope.common.error import UnsupportedModel
+from dashscope.tokenizers.qwen_tokenizer import QwenTokenizer
+
+from .tokenizer_base import Tokenizer
+
+QWEN_SERIALS = ['qwen-7b-chat', 'qwen-turbo', 'qwen-plus', 'qwen-max']
+current_path = os.path.dirname(os.path.abspath(__file__))
+root_path = os.path.dirname(current_path)
+
+
+def get_tokenizer(model: str) -> Tokenizer:
+    """Get a tokenizer based on model name.
+
+    Args:
+        model (str): The model name.
+
+    Raises:
+        UnsupportedModel: Not support model
+
+    Returns:
+        Tokenizer: The  `Tokenizer` of the model.
+    """
+    if model in QWEN_SERIALS:
+        return QwenTokenizer(
+            os.path.join(root_path, 'resources', 'qwen.tiktoken'))
+    elif model.startswith('qwen'):
+        return QwenTokenizer(
+            os.path.join(root_path, 'resources', 'qwen.tiktoken'))
+    else:
+        raise UnsupportedModel(
+            f'Not support model: {model}, currently only support qwen models.')
+
+
+def list_tokenizers() -> List[str]:
+    """List support models
+
+    Returns:
+        List[str]: The model list.
+    """
+    return QWEN_SERIALS

dashscope/tokenizers/tokenizer_base.py

@@ -0,0 +1,30 @@
+from typing import List
+
+
+class Tokenizer:
+    """Base tokenizer interface for local tokenizers.
+    """
+    def __init__(self):
+        pass
+
+    def encode(self, text: str, **kwargs) -> List[int]:
+        """Encode input text string to token ids.
+
+        Args:
+            text (str): The string to be encoded.
+
+        Returns:
+            List[int]: The token ids.
+        """
+        pass
+
+    def decode(self, token_ids: List[int], **kwargs) -> str:
+        """Decode token ids to string.
+
+        Args:
+            token_ids (List[int]): The input token ids.
+
+        Returns:
+            str: The string of the token ids.
+        """
+        pass

dashscope/version.py

@@ -1 +1 @@
-__version__ = '1.13.6'
+__version__ = '1.14.1'

{dashscope-1.13.6.dist-info → dashscope-1.14.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dashscope
-Version: 1.13.6
+Version: 1.14.1
 Summary: dashscope client sdk library
 Home-page: https://dashscope.aliyun.com/
 Author: Alibaba Cloud
@@ -21,6 +21,8 @@ Requires-Python: >=3.7.0
 Description-Content-Type: text/markdown
 Requires-Dist: aiohttp
 Requires-Dist: requests
+Provides-Extra: tokenizer
+Requires-Dist: tiktoken ; extra == 'tokenizer'
 
 <h4 align="center">
     <p>
@@ -44,6 +46,10 @@ If you clone the code from github, you can install from  source by running:
 pip install -e .
 ```
 
+To use tokenizer in local mode without downloading any files, run:
+```shell
+pip install dashscope[tokenizer]
+```
 
 
 ## QuickStart
@@ -157,50 +163,33 @@ if __name__ == '__main__':
     sample_sync_call_stream()
 
 ```
-#### Streaming with History
+#### Stream with Messages
 ```python
 from http import HTTPStatus
+from dashscope import Generation
+from dashscope.api_entities.dashscope_response import Role
 
-from dashscope import Conversation, History, HistoryItem
-def conversation_stream_example():
-    history = History()
-    item = HistoryItem('user', text='Is the weather good today?')
-    history.append(item)
-    item = HistoryItem('bot', text='The weather is nice today, do you want to go out and play?')
-    history.append(item)
-
-    item = HistoryItem('user', text='Do you have any places to recommend?')
-    history.append(item)
-    item = HistoryItem('bot', text='I suggest you go to the park. Spring is here, and the flowers are blooming. It is beautiful.')
-    history.append(item)
-    chat = Conversation(history)
-    response = chat.call(Conversation.Models.qwen_turbo,
-                         prompt='Recommend a nearby park',
-                         stream=True)
-    for part in response:
-        if part.status_code == HTTPStatus.OK:
-            print(part.output)
-        else:
-            print('Failed request_id: %s, status_code: %s code: %s, message:%s' %
-                  (part.id, part.status_code, part.code, part.message))
-    response = chat.call(
-        Conversation.Models.qwen_turbo,
-        prompt='I have been to that park many times, how about a more distant one?',
-        auto_history=True,
+
+def stream_with_messages():
+    messages = [{'role': Role.SYSTEM, 'content': 'You are a helpful assistant.'},
+                {'role': Role.USER, 'content': '如何做西红柿炖牛腩？'}]
+    responses = Generation.call(
+        Generation.Models.qwen_turbo,
+        messages=messages,
+        result_format='message',  # set the result to be "message" format.
         stream=True,
     )
-    for part in response:
-        if part.status_code == HTTPStatus.OK:
-            print(part.output.text)
-            print(part.usage)
-        else:
-            print('Failed request_id: %s, status_code: %s, code: %s, message:%s' %
-                  (part.id, part.status_code, part.code, part.message))
-
+    for response in responses:
+       if response.status_code == HTTPStatus.OK:
+           print(response)
+       else:
+           print('Request id: %s, Status code: %s, error code: %s, error message: %s' % (
+               response.request_id, response.status_code,
+               response.code, response.message
+           ))
 
 if __name__ == '__main__':
-    conversation_stream_example()
-
+    stream_with_messages()
 
 ```
 ## Logging

{dashscope-1.13.6.dist-info → dashscope-1.14.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
-dashscope/__init__.py,sha256=ZxCknpY8JWS4tnlSiH-2rIRnjSJwuCgeY7P5wb_nJgs,1897
+dashscope/__init__.py,sha256=UExSJnogmqVAzRL6R4HhTp-yTz9GwoDIh5xAf7E_BEI,2018
 dashscope/cli.py,sha256=7CnUZqaaUc0FxbmXHMiqOALl9YzZmSrFS23A7hbR0w0,23917
 dashscope/deployment.py,sha256=uhlQYOiu1baMQxQnFmRwI6ces0I-_DKp8uej_L0v1eY,4400
 dashscope/file.py,sha256=Dv2Fz3DLbcye2uuQxyQwRM7ky27OthouLXIpSQagQy4,3324
 dashscope/finetune.py,sha256=5QlIcnFCfSjN3gn9A2FO0kLUDa9Rx_jSc44ezT3aBEo,5118
 dashscope/model.py,sha256=iuIfal-vOo0yav0HXPdA7f93vd5JNTGIAdCG_WIYkW8,1158
-dashscope/version.py,sha256=ZvZac_TGhBsmAT7mxeokg8Dq4epRltEzBfVWYw6TAIM,23
+dashscope/version.py,sha256=eyARjsqZfiB7blnKzG-sgKU2HEsI143aQSUAfhkeA8U,23
 dashscope/aigc/__init__.py,sha256=s-MCA87KYiVumYtKtJi5IMN7xelSF6TqEU3s3_7RF-Y,327
 dashscope/aigc/code_generation.py,sha256=bizJb3zGZx3pB74FKMIcnOi_6jkxpKgx__6urzqhQ_E,10627
 dashscope/aigc/conversation.py,sha256=_sAWhQjLgJENqRAXh-i2dw8bt_i69fJv9KZYVIi4CEg,14090
@@ -12,13 +12,13 @@ dashscope/aigc/generation.py,sha256=SgOtVsd0DaMOuHmJ2EcA3E_hcU1Bqah1Q-Ztnb3TcC8,
 dashscope/aigc/image_synthesis.py,sha256=Y20B_hpCe-uLKR661hvFpjQVk9jFfNtiPpiBkYiHAA0,8996
 dashscope/aigc/multimodal_conversation.py,sha256=D5JYLLxwdGsgCrYeF0gNvE1kR5Vbmq1g0MiRU-4hlZQ,5228
 dashscope/api_entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dashscope/api_entities/aiohttp_request.py,sha256=1WuvODWcm-merZnAFpxV2Y9QohcB3hSGS_YkwK18O7g,10359
+dashscope/api_entities/aiohttp_request.py,sha256=ZFbdpJh7SwHnBzbYLhqr_FdcDVRgLVMLhLUS_vXbUGs,10228
 dashscope/api_entities/api_request_data.py,sha256=JUMcfpJjKXEZLCBSFIDpgoaeQYk5uK9-CwhM4OHIHFQ,5463
-dashscope/api_entities/api_request_factory.py,sha256=yRCY9OVGagsEvid6RXwvwiQv_8ZTEqYaKXnVLd9F9pA,4281
+dashscope/api_entities/api_request_factory.py,sha256=MCJOGgUzaWTaLZ5d8m8dck7QdSUNX7XLbKYyB3pT23E,4067
 dashscope/api_entities/base_request.py,sha256=cXUL7xqSV8wBr5d-1kx65AO3IsRR9A_ps6Lok-v-MKM,926
 dashscope/api_entities/dashscope_response.py,sha256=1KdBib5xOftcKXLzTETe5LElGy2mB8IG8E3JZrGOaAY,15814
-dashscope/api_entities/http_request.py,sha256=8bNtylkE98h1SeCrxTPow4Dqy3wPXWp9wHPdeWAIDe8,9637
-dashscope/api_entities/websocket_request.py,sha256=VK4488_3Qi2Rwx7i1FGb-lPu_BSNZAoedOqnHzOxTH8,15844
+dashscope/api_entities/http_request.py,sha256=MzaTznVJUbyA8u6cLegxVSEM3JWlAhPHbSh4uDCX0-A,9506
+dashscope/api_entities/websocket_request.py,sha256=zU7OskAHaNlKcR9s_6Hlnhr5N3oam4w0sUD5E9N-4BQ,14586
 dashscope/audio/__init__.py,sha256=vlw0TFVRdeRWfzmJxhzarVUqkMs-DZNf4GiMtm3C8XE,45
 dashscope/audio/asr/__init__.py,sha256=-s180qWn_JPSpCo1q0aDJJ5HQ3zTzD4z5yUwsRqH4aU,275
 dashscope/audio/asr/asr_phrase_manager.py,sha256=BNrRBGGoAW_rcxoi_euh6bRktJO6asTiGroE8PMh1Xg,6641
@@ -30,16 +30,16 @@ dashscope/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 dashscope/client/base_api.py,sha256=1YTlwY2_tlvMLaz6oGh1BNQt5GT_Xx3obS6nKOvnpkY,31749
 dashscope/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dashscope/common/api_key.py,sha256=5Stp0odL5JSuIO3qJBp23QNppuGbqhhvKPS66qbMs0I,1986
-dashscope/common/constants.py,sha256=2zF1l2BDnDDa7oFSfg3gLV34A-6Lr-3_o8QfX60rkCk,2640
+dashscope/common/constants.py,sha256=OCe3UNGH05sTEsffipWDYpUL_bz__AaYm9lSDB-ZkRI,2297
 dashscope/common/env.py,sha256=oQOZW5JyEeTSde394un2lpDJ5RBh4fMU9hBfbtrKKkc,869
-dashscope/common/error.py,sha256=ihgDx08P2fw0cB-0HsA6MMgO0vY-6yIhxXHlIIF8c0Y,1641
+dashscope/common/error.py,sha256=5Ndi6w38fSHX6qcPgr9udKISf1RbRJVL8hjDPtU4lJ4,1896
 dashscope/common/logging.py,sha256=ecGxylG3bWES_Xv5-BD6ep4_0Ciu7F6ZPBjiZtu9Jx4,984
 dashscope/common/message_manager.py,sha256=i5149WzDk6nWmdFaHzYx4USXMBeX18GKSI-F4fLwbN0,1097
-dashscope/common/utils.py,sha256=UZSK6AStBVriVTR5wdxfcKTBDzCEN6PLyEv3avZ8YwQ,6679
+dashscope/common/utils.py,sha256=xsY36zXGnProdkgarbFu30rX9rmdQNZHcHbvEuVXoss,6697
 dashscope/embeddings/__init__.py,sha256=-dxHaoxZZVuP-wAGUIa3sNNh8CQwaeWj2UlqsDy1sV4,240
 dashscope/embeddings/batch_text_embedding.py,sha256=M8jvdz304Oh5ZwegHWGwTda9AezYsgVC59HnweuCagY,7977
 dashscope/embeddings/batch_text_embedding_response.py,sha256=WziXlQsFIkL1kPc_7lRG1HtqgkO5vVThtnNqExJggNU,2000
-dashscope/embeddings/multimodal_embedding.py,sha256=a6aQYR1Ikmh4ZOGldfVTP0KDAlCVzM6pxDj2pNd5w3M,3779
+dashscope/embeddings/multimodal_embedding.py,sha256=Wgh80k-mI0DJiapWxM27H1rwkCPoFqunca2JHvwTE68,3824
 dashscope/embeddings/text_embedding.py,sha256=X7_FBM57gQQCu8xb-W-jTlSsathaCiAtW0mdb19N-PM,1714
 dashscope/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dashscope/io/input_output.py,sha256=iZ1X1x1btdoZK2VeC9JsKkag2eaXwqfNT3Q6SrmRi2w,3941
@@ -47,13 +47,17 @@ dashscope/nlp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dashscope/nlp/understanding.py,sha256=uVkf_4iULnlnuGmEX3taKiHgzxu173vvgVwY5EO2C2Y,2803
 dashscope/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dashscope/protocol/websocket.py,sha256=z-v6PGx3L4zYBANuC48s7SWSQSwRCDoh0zcfhv9Bf8U,561
-dashscope/tokenizers/__init__.py,sha256=zRNBe3ONNOA3T2gnxhlG3PTjkVOQBUYotMMJAruyk70,71
-dashscope/tokenizers/tokenization.py,sha256=WhSdiH0Bu-pGQCsJfC91LL_y5tVDzCtQ7tTs2Tox-HY,4543
+dashscope/resources/qwen.tiktoken,sha256=srG437XMXwJLr8NzEhxquj9m-aWgJp4kNHCh3hajMYY,2561218
+dashscope/tokenizers/__init__.py,sha256=Oy5FMT37Non6e1YxdHQ89U93Dy3CG1Ez0gBa771KZo0,200
+dashscope/tokenizers/qwen_tokenizer.py,sha256=dCnT9-9NrqPS85bEhjlPULUfDADVRhlleYwM_ILgCeI,4111
+dashscope/tokenizers/tokenization.py,sha256=q-mx2ufZAXyelLyPZdjsKieYHC6tJU19t7lC5bEz6kU,4631
+dashscope/tokenizers/tokenizer.py,sha256=y6P91qTCYo__pEx_0VHAcj9YECfbUdRqZU1fdGTjF4o,1154
+dashscope/tokenizers/tokenizer_base.py,sha256=REDhzRyDT13iequ61-a6_KcTy0GFKlihQve5HkyoyRs,656
 dashscope/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dashscope/utils/oss_utils.py,sha256=iCF5fLqIXDCS2zkE96t2DEUxCI76ArnQH2DiY2hgHu0,6522
-dashscope-1.13.6.dist-info/LICENSE,sha256=Izp5L1DF1Mbza6qojkqNNWlE_mYLnr4rmzx2EBF8YFw,11413
-dashscope-1.13.6.dist-info/METADATA,sha256=Jy5x-xQoVYSzUjXvxxKHQv-wuDbdcVZIdB-_iwYnMlg,7227
-dashscope-1.13.6.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-dashscope-1.13.6.dist-info/entry_points.txt,sha256=raEp5dOuj8whJ7yqZlDM8WQ5p2RfnGrGNo0QLQEnatY,50
-dashscope-1.13.6.dist-info/top_level.txt,sha256=woqavFJK9zas5xTqynmALqOtlafghjsk63Xk86powTU,10
-dashscope-1.13.6.dist-info/RECORD,,
+dashscope-1.14.1.dist-info/LICENSE,sha256=Izp5L1DF1Mbza6qojkqNNWlE_mYLnr4rmzx2EBF8YFw,11413
+dashscope-1.14.1.dist-info/METADATA,sha256=Hlg9nly8P6C_7fj2RwM8HEDuP8dt5unBCUmwGEy-FoY,6659
+dashscope-1.14.1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+dashscope-1.14.1.dist-info/entry_points.txt,sha256=raEp5dOuj8whJ7yqZlDM8WQ5p2RfnGrGNo0QLQEnatY,50
+dashscope-1.14.1.dist-info/top_level.txt,sha256=woqavFJK9zas5xTqynmALqOtlafghjsk63Xk86powTU,10
+dashscope-1.14.1.dist-info/RECORD,,