tokencostauto 0.1.25__py3-none-any.whl → 0.1.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencostauto/costs.py +10 -5
- {tokencostauto-0.1.25.dist-info → tokencostauto-0.1.26.dist-info}/METADATA +1 -1
- tokencostauto-0.1.26.dist-info/RECORD +9 -0
- tokencostauto-0.1.25.dist-info/RECORD +0 -9
- {tokencostauto-0.1.25.dist-info → tokencostauto-0.1.26.dist-info}/WHEEL +0 -0
- {tokencostauto-0.1.25.dist-info → tokencostauto-0.1.26.dist-info}/licenses/LICENSE +0 -0
- {tokencostauto-0.1.25.dist-info → tokencostauto-0.1.26.dist-info}/top_level.txt +0 -0
tokencostauto/costs.py
CHANGED
@@ -79,7 +79,7 @@ def count_message_tokens(messages: List[Dict[str, str]], model: str) -> int:
|
|
79
79
|
try:
|
80
80
|
encoding = tiktoken.encoding_for_model(model)
|
81
81
|
except KeyError:
|
82
|
-
logger.
|
82
|
+
logger.info("Model not found. Using cl100k_base encoding.")
|
83
83
|
encoding = tiktoken.get_encoding("cl100k_base")
|
84
84
|
if model in {
|
85
85
|
"gpt-3.5-turbo-0613",
|
@@ -100,17 +100,22 @@ def count_message_tokens(messages: List[Dict[str, str]], model: str) -> int:
|
|
100
100
|
tokens_per_message = 4
|
101
101
|
tokens_per_name = -1 # if there's a name, the role is omitted
|
102
102
|
elif "gpt-3.5-turbo" in model:
|
103
|
-
logger.
|
103
|
+
logger.info(
|
104
104
|
"gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
|
105
105
|
)
|
106
106
|
return count_message_tokens(messages, model="gpt-3.5-turbo-0613")
|
107
107
|
elif "gpt-4o" in model:
|
108
|
-
logger.
|
108
|
+
logger.info(
|
109
109
|
"Warning: gpt-4o may update over time. Returning num tokens assuming gpt-4o-2024-05-13."
|
110
110
|
)
|
111
111
|
return count_message_tokens(messages, model="gpt-4o-2024-05-13")
|
112
|
+
elif "gpt-4.1" in model:
|
113
|
+
logger.info(
|
114
|
+
"Warning: gpt-4.1 may update over time. Returning num tokens assuming gpt-4o-2024-05-13."
|
115
|
+
)
|
116
|
+
return count_message_tokens(messages, model="gpt-4o-2024-05-13")
|
112
117
|
elif "gpt-4" in model:
|
113
|
-
logger.
|
118
|
+
logger.info(
|
114
119
|
"gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
|
115
120
|
)
|
116
121
|
return count_message_tokens(messages, model="gpt-4-0613")
|
@@ -154,7 +159,7 @@ def count_string_tokens(prompt: str, model: str) -> int:
|
|
154
159
|
try:
|
155
160
|
encoding = tiktoken.encoding_for_model(model)
|
156
161
|
except KeyError:
|
157
|
-
logger.
|
162
|
+
logger.info("Warning: model not found. Using cl100k_base encoding.")
|
158
163
|
encoding = tiktoken.get_encoding("cl100k_base")
|
159
164
|
|
160
165
|
return len(encoding.encode(prompt))
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: tokencostauto
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.26
|
4
4
|
Summary: To calculate token and translated USD cost of string and message calls to OpenAI, for example when used by AI agents
|
5
5
|
Author-email: Trisha Pan <trishaepan@gmail.com>, Alex Reibman <areibman@gmail.com>, Pratyush Shukla <ps4534@nyu.edu>, Thiago MadPin <madpin@gmail.com>
|
6
6
|
Project-URL: Homepage, https://github.com/madpin/tokencostaudo
|
@@ -0,0 +1,9 @@
|
|
1
|
+
tokencostauto/__init__.py,sha256=-4d_ryFH62SgNXPXA8vGPFZoAKtOBjnsg37EB_RkZG8,289
|
2
|
+
tokencostauto/constants.py,sha256=_82MlTkTrdrwzyRosQD7d3JdgNP9KAUM-cZo8DE00P0,3395
|
3
|
+
tokencostauto/costs.py,sha256=tXsgrTypq-dCHaHtoXcg2XepezWsAvZpl9gEsv_53iE,10679
|
4
|
+
tokencostauto/model_prices.json,sha256=pjU4wUl3QQHwEvJFywqGIFYGcIb_Q3IxL4NSEL87uNg,468528
|
5
|
+
tokencostauto-0.1.26.dist-info/licenses/LICENSE,sha256=4PLv_CD6Ughnsvg_nM2XeTqGwVK6lQVR77kVWbPq-0U,1065
|
6
|
+
tokencostauto-0.1.26.dist-info/METADATA,sha256=gMp2jWQc0d2vCamN5pQwQiJ3eqAOEqzsW3Ju5vrCjHo,204076
|
7
|
+
tokencostauto-0.1.26.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
|
8
|
+
tokencostauto-0.1.26.dist-info/top_level.txt,sha256=szZQTUJRotfIaeZCDsOgvofIkLt2ak88RP13oI51-TU,14
|
9
|
+
tokencostauto-0.1.26.dist-info/RECORD,,
|
@@ -1,9 +0,0 @@
|
|
1
|
-
tokencostauto/__init__.py,sha256=-4d_ryFH62SgNXPXA8vGPFZoAKtOBjnsg37EB_RkZG8,289
|
2
|
-
tokencostauto/constants.py,sha256=_82MlTkTrdrwzyRosQD7d3JdgNP9KAUM-cZo8DE00P0,3395
|
3
|
-
tokencostauto/costs.py,sha256=vCtP8QEnIYpbDs0ls4DzK6I6SR1AJiX3hmbooyQC1aM,10459
|
4
|
-
tokencostauto/model_prices.json,sha256=pjU4wUl3QQHwEvJFywqGIFYGcIb_Q3IxL4NSEL87uNg,468528
|
5
|
-
tokencostauto-0.1.25.dist-info/licenses/LICENSE,sha256=4PLv_CD6Ughnsvg_nM2XeTqGwVK6lQVR77kVWbPq-0U,1065
|
6
|
-
tokencostauto-0.1.25.dist-info/METADATA,sha256=BCVJyt2HGDmvRnuztA1LURMlkajYoCuRoxjL5anN_Mc,204076
|
7
|
-
tokencostauto-0.1.25.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
|
8
|
-
tokencostauto-0.1.25.dist-info/top_level.txt,sha256=szZQTUJRotfIaeZCDsOgvofIkLt2ak88RP13oI51-TU,14
|
9
|
-
tokencostauto-0.1.25.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|