validmind 2.4.0__py3-none-any.whl → 2.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validmind/__version__.py +1 -1
- validmind/api_client.py +17 -15
- validmind/datasets/nlp/cnn_dailymail.py +10 -1
- validmind/datasets/nlp/datasets/cnn_dailymail_100_with_predictions.csv +33 -209
- validmind/datasets/nlp/datasets/cnn_dailymail_500_with_predictions.csv +156 -1051
- validmind/models/huggingface.py +0 -1
- validmind/template.py +2 -0
- validmind/tests/data_validation/TabularDescriptionTables.py +96 -148
- validmind/tests/model_validation/embeddings/ClusterDistribution.py +1 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +1 -1
- validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +1 -1
- validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +1 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysis.py +3 -2
- validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +9 -2
- validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +0 -1
- validmind/tests/model_validation/sklearn/HyperParametersTuning.py +2 -1
- validmind/tests/run.py +1 -1
- validmind/vm_models/dataset/dataset.py +18 -6
- validmind/vm_models/test_suite/summary.py +2 -2
- {validmind-2.4.0.dist-info → validmind-2.4.4.dist-info}/METADATA +3 -2
- {validmind-2.4.0.dist-info → validmind-2.4.4.dist-info}/RECORD +24 -24
- {validmind-2.4.0.dist-info → validmind-2.4.4.dist-info}/LICENSE +0 -0
- {validmind-2.4.0.dist-info → validmind-2.4.4.dist-info}/WHEEL +0 -0
- {validmind-2.4.0.dist-info → validmind-2.4.4.dist-info}/entry_points.txt +0 -0
validmind/__version__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "2.4.
|
1
|
+
__version__ = "2.4.4"
|
validmind/api_client.py
CHANGED
@@ -32,8 +32,7 @@ logger = get_logger(__name__)
|
|
32
32
|
_api_key = os.getenv("VM_API_KEY")
|
33
33
|
_api_secret = os.getenv("VM_API_SECRET")
|
34
34
|
_api_host = os.getenv("VM_API_HOST")
|
35
|
-
|
36
|
-
_project = os.getenv("VM_API_PROJECT")
|
35
|
+
_model_cuid = os.getenv("VM_API_MODEL")
|
37
36
|
_run_cuid = os.getenv("VM_RUN_CUID")
|
38
37
|
|
39
38
|
__api_session: aiohttp.ClientSession = None
|
@@ -56,7 +55,7 @@ def get_api_config() -> Dict[str, Optional[str]]:
|
|
56
55
|
"VM_API_KEY": _api_key,
|
57
56
|
"VM_API_SECRET": _api_secret,
|
58
57
|
"VM_API_HOST": _api_host,
|
59
|
-
"
|
58
|
+
"VM_API_MODEL": _model_cuid,
|
60
59
|
"VM_RUN_CUID": _run_cuid,
|
61
60
|
}
|
62
61
|
|
@@ -65,15 +64,15 @@ def get_api_host() -> Optional[str]:
|
|
65
64
|
return _api_host
|
66
65
|
|
67
66
|
|
68
|
-
def
|
69
|
-
return
|
67
|
+
def get_api_model() -> Optional[str]:
|
68
|
+
return _model_cuid
|
70
69
|
|
71
70
|
|
72
71
|
def get_api_headers() -> Dict[str, str]:
|
73
72
|
return {
|
74
73
|
"X-API-KEY": _api_key,
|
75
74
|
"X-API-SECRET": _api_secret,
|
76
|
-
"X-PROJECT-CUID":
|
75
|
+
"X-PROJECT-CUID": _model_cuid,
|
77
76
|
}
|
78
77
|
|
79
78
|
|
@@ -82,6 +81,7 @@ def init(
|
|
82
81
|
api_key: Optional[str] = None,
|
83
82
|
api_secret: Optional[str] = None,
|
84
83
|
api_host: Optional[str] = None,
|
84
|
+
model: Optional[str] = None,
|
85
85
|
):
|
86
86
|
"""
|
87
87
|
Initializes the API client instances and calls the /ping endpoint to ensure
|
@@ -91,7 +91,8 @@ def init(
|
|
91
91
|
retrieve them from the environment variables `VM_API_KEY` and `VM_API_SECRET`.
|
92
92
|
|
93
93
|
Args:
|
94
|
-
project (str): The project CUID
|
94
|
+
project (str, optional): The project CUID. Alias for model. Defaults to None.
|
95
|
+
model (str, optional): The model CUID. Defaults to None.
|
95
96
|
api_key (str, optional): The API key. Defaults to None.
|
96
97
|
api_secret (str, optional): The API secret. Defaults to None.
|
97
98
|
api_host (str, optional): The API host. Defaults to None.
|
@@ -99,7 +100,7 @@ def init(
|
|
99
100
|
Raises:
|
100
101
|
ValueError: If the API key and secret are not provided
|
101
102
|
"""
|
102
|
-
global _api_key, _api_secret, _api_host, _run_cuid,
|
103
|
+
global _api_key, _api_secret, _api_host, _run_cuid, _model_cuid
|
103
104
|
|
104
105
|
if api_key == "...":
|
105
106
|
# special case to detect when running a notebook with the standard init snippet
|
@@ -107,9 +108,9 @@ def init(
|
|
107
108
|
# the notebook
|
108
109
|
api_host = api_key = api_secret = project = None
|
109
110
|
|
110
|
-
|
111
|
+
_model_cuid = project or model or os.getenv("VM_API_MODEL")
|
111
112
|
|
112
|
-
if
|
113
|
+
if _model_cuid is None:
|
113
114
|
raise MissingProjectIdError()
|
114
115
|
|
115
116
|
_api_key = api_key or os.getenv("VM_API_KEY")
|
@@ -143,7 +144,7 @@ def _get_session() -> aiohttp.ClientSession:
|
|
143
144
|
{
|
144
145
|
"X-API-KEY": _api_key,
|
145
146
|
"X-API-SECRET": _api_secret,
|
146
|
-
"X-PROJECT-CUID":
|
147
|
+
"X-PROJECT-CUID": _model_cuid,
|
147
148
|
}
|
148
149
|
)
|
149
150
|
|
@@ -157,7 +158,7 @@ def __ping() -> Dict[str, Any]:
|
|
157
158
|
headers={
|
158
159
|
"X-API-KEY": _api_key,
|
159
160
|
"X-API-SECRET": _api_secret,
|
160
|
-
"X-PROJECT-CUID":
|
161
|
+
"X-PROJECT-CUID": _model_cuid,
|
161
162
|
},
|
162
163
|
)
|
163
164
|
if r.status_code != 200:
|
@@ -172,13 +173,14 @@ def __ping() -> Dict[str, Any]:
|
|
172
173
|
if client_config.project is None:
|
173
174
|
ack_connected = True
|
174
175
|
|
176
|
+
# TODO: use model object when backend moves away from project
|
175
177
|
client_config.project = client_info["project"]
|
176
178
|
client_config.documentation_template = client_info.get("documentation_template", {})
|
177
179
|
client_config.feature_flags = client_info.get("feature_flags", {})
|
178
180
|
|
179
181
|
if ack_connected:
|
180
182
|
logger.info(
|
181
|
-
f"Connected to ValidMind
|
183
|
+
f"Connected to ValidMind... Current Model: {client_config.project['name']}"
|
182
184
|
f" ({client_config.project['cuid']})"
|
183
185
|
)
|
184
186
|
|
@@ -563,7 +565,7 @@ def start_run() -> str:
|
|
563
565
|
headers={
|
564
566
|
"X-API-KEY": _api_key,
|
565
567
|
"X-API-SECRET": _api_secret,
|
566
|
-
"X-PROJECT-CUID":
|
568
|
+
"X-PROJECT-CUID": _model_cuid,
|
567
569
|
},
|
568
570
|
)
|
569
571
|
|
@@ -584,7 +586,7 @@ def get_ai_key() -> str:
|
|
584
586
|
headers={
|
585
587
|
"X-API-KEY": _api_key,
|
586
588
|
"X-API-SECRET": _api_secret,
|
587
|
-
"X-PROJECT-CUID":
|
589
|
+
"X-PROJECT-CUID": _model_cuid,
|
588
590
|
},
|
589
591
|
)
|
590
592
|
|
@@ -15,6 +15,7 @@ text_column = "article"
|
|
15
15
|
target_column = "highlights"
|
16
16
|
gpt_35_prediction_column = "gpt_35_prediction"
|
17
17
|
t5_prediction = "t5_prediction"
|
18
|
+
bert_embedding_prediction_column = "bert_embedding_model_prediction"
|
18
19
|
|
19
20
|
# Define the path to the dataset directory
|
20
21
|
current_path = os.path.dirname(os.path.abspath(__file__))
|
@@ -55,7 +56,15 @@ def load_data(source="online", dataset_size=None):
|
|
55
56
|
|
56
57
|
# Load the dataset
|
57
58
|
df = pd.read_csv(data_file)
|
58
|
-
df = df[
|
59
|
+
df = df[
|
60
|
+
[
|
61
|
+
"article",
|
62
|
+
"highlights",
|
63
|
+
"gpt_35_prediction",
|
64
|
+
"t5_prediction",
|
65
|
+
"bert_embedding_model_prediction",
|
66
|
+
]
|
67
|
+
]
|
59
68
|
|
60
69
|
train_df = df.sample(frac=0.7, random_state=42)
|
61
70
|
test_df = df.drop(train_df.index)
|