instmodel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- instmodel/__init__.py +4 -0
- instmodel/embeddings.py +359 -0
- instmodel/instruction_model.py +377 -0
- instmodel/model.py +872 -0
- instmodel/training_utils.py +21 -0
- instmodel-0.1.0.dist-info/METADATA +17 -0
- instmodel-0.1.0.dist-info/RECORD +9 -0
- instmodel-0.1.0.dist-info/WHEEL +5 -0
- instmodel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load_model_from_label(model_location):
|
|
8
|
+
if os.path.exists(model_location):
|
|
9
|
+
with open(model_location, "r") as f:
|
|
10
|
+
model_json = json.load(f)
|
|
11
|
+
else:
|
|
12
|
+
raise ValueError(f"Model not found at {model_location}")
|
|
13
|
+
return model_json
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def generate_features(model_label, dataset, output_features):
|
|
17
|
+
model_json = load_model_from_label(model_label)
|
|
18
|
+
|
|
19
|
+
return generate_features_IO(
|
|
20
|
+
model_label, dataset, model_json["features"], output_features
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def generate_features_IO(model_label, dataset, input_features, output_features):
|
|
25
|
+
# before going to s3, check if the model is already in the local cache in folder models
|
|
26
|
+
|
|
27
|
+
model_json = load_model_from_label(model_label)
|
|
28
|
+
|
|
29
|
+
*_, output = instruction_model_inference(
|
|
30
|
+
model_json, dataset[input_features].to_numpy()
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
dataset_new = dataset.copy()
|
|
34
|
+
dataset_new[output_features] = output
|
|
35
|
+
|
|
36
|
+
return dataset_new
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def apply_activation(instruction, buffers, output_index):
|
|
40
|
+
"""
|
|
41
|
+
Applies the activation function specified in the instruction to the buffer at output_index.
|
|
42
|
+
|
|
43
|
+
Parameters:
|
|
44
|
+
instruction (dict): A dictionary that may contain an "activation" key.
|
|
45
|
+
buffers (dict or list): A container of NumPy arrays.
|
|
46
|
+
output_index: The key or index for the output buffer within `buffers`.
|
|
47
|
+
"""
|
|
48
|
+
act = instruction.get("activation")
|
|
49
|
+
if act is None:
|
|
50
|
+
# No activation specified
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
x = buffers[output_index]
|
|
54
|
+
|
|
55
|
+
if act == "RELU":
|
|
56
|
+
buffers[output_index] = np.maximum(0, x)
|
|
57
|
+
|
|
58
|
+
elif act == "SIGMOID":
|
|
59
|
+
# Using a numerically stable sigmoid computation
|
|
60
|
+
buffers[output_index] = np.where(
|
|
61
|
+
x >= 0, 1 / (1 + np.exp(-x)), np.exp(x) / (1 + np.exp(x))
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
elif act == "SOFTMAX":
|
|
65
|
+
# Determine axis based on array dimensions
|
|
66
|
+
if x.ndim == 1:
|
|
67
|
+
# For a 1D vector, subtract the max for numerical stability
|
|
68
|
+
shifted_x = x - np.max(x)
|
|
69
|
+
exp_x = np.exp(shifted_x)
|
|
70
|
+
buffers[output_index] = exp_x / np.sum(exp_x)
|
|
71
|
+
else:
|
|
72
|
+
# For 2D (or higher) arrays, assume softmax is computed along axis=1
|
|
73
|
+
shifted_x = x - np.max(x, axis=1, keepdims=True)
|
|
74
|
+
exp_x = np.exp(shifted_x)
|
|
75
|
+
buffers[output_index] = exp_x / np.sum(exp_x, axis=1, keepdims=True)
|
|
76
|
+
|
|
77
|
+
elif act == "TANH":
|
|
78
|
+
buffers[output_index] = np.tanh(x)
|
|
79
|
+
|
|
80
|
+
elif act == "SQRT":
|
|
81
|
+
# Compute sqrt(x) for x > 0, else 0
|
|
82
|
+
buffers[output_index] = np.where(x > 0, np.sqrt(x), 0)
|
|
83
|
+
|
|
84
|
+
elif act == "LOG":
|
|
85
|
+
# Compute log(x+1) for x > 0, else 0
|
|
86
|
+
result = np.zeros_like(x)
|
|
87
|
+
mask = x > 0
|
|
88
|
+
result[mask] = np.log(x[mask] + 1)
|
|
89
|
+
buffers[output_index] = result
|
|
90
|
+
|
|
91
|
+
elif act == "LOG10":
|
|
92
|
+
# Compute log10(x+1) for x > 0, else 0
|
|
93
|
+
result = np.zeros_like(x)
|
|
94
|
+
mask = x > 0
|
|
95
|
+
result[mask] = np.log10(x[mask] + 1)
|
|
96
|
+
buffers[output_index] = result
|
|
97
|
+
|
|
98
|
+
elif act == "INVERSE":
|
|
99
|
+
# Compute 1 - x
|
|
100
|
+
buffers[output_index] = 1 - x
|
|
101
|
+
|
|
102
|
+
else:
|
|
103
|
+
raise ValueError(f"Unexpected activation: {act}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def instruction_model_inference(model, input_data):
|
|
107
|
+
import numpy as np
|
|
108
|
+
|
|
109
|
+
# Convert weights, bias, and parameters into float32 numpy arrays.
|
|
110
|
+
weights = [np.array(w, dtype=np.float32) for w in model["weights"]]
|
|
111
|
+
bias = [np.array(b, dtype=np.float32) for b in model["bias"]]
|
|
112
|
+
parameters = [np.array(p, dtype=np.float32) for p in model["parameters"]]
|
|
113
|
+
buffer_sizes = model["buffer_sizes"]
|
|
114
|
+
instructions = model["instructions"]
|
|
115
|
+
buffers = [None] * len(buffer_sizes)
|
|
116
|
+
|
|
117
|
+
# Initialize the first buffers from input_data (can be a list or a single array).
|
|
118
|
+
if isinstance(input_data, list):
|
|
119
|
+
for i, data in enumerate(input_data):
|
|
120
|
+
buffers[i] = data.copy()
|
|
121
|
+
if buffers[i].shape[1] != buffer_sizes[i]:
|
|
122
|
+
raise ValueError(
|
|
123
|
+
f"Input data shape does not match the expected shape. Expected {buffer_sizes[i]} but got {buffers[i].shape[1]}"
|
|
124
|
+
)
|
|
125
|
+
else:
|
|
126
|
+
buffers[0] = input_data.copy()
|
|
127
|
+
if buffers[0].shape[1] != buffer_sizes[0]:
|
|
128
|
+
raise ValueError(
|
|
129
|
+
f"Input data shape does not match the expected shape. Expected {buffer_sizes[0]} but got {buffers[0].shape[1]}"
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# Process each instruction.
|
|
133
|
+
for instruction in instructions:
|
|
134
|
+
if instruction["type"] == "ACTIVATION":
|
|
135
|
+
output_index = instruction["input"]
|
|
136
|
+
apply_activation(instruction, buffers, output_index)
|
|
137
|
+
continue
|
|
138
|
+
# --- New instructions that expect a list of input indices ---
|
|
139
|
+
elif instruction["type"] == "ADD_ELEMENTWISE_BUFFERS":
|
|
140
|
+
# "input" is a list of buffer indices
|
|
141
|
+
input_indices = instruction["input"]
|
|
142
|
+
# Get each buffer and stack along a new axis (assumes all buffers have equal shape)
|
|
143
|
+
stacked = np.stack([buffers[i] for i in input_indices], axis=0)
|
|
144
|
+
result = np.sum(stacked, axis=0)
|
|
145
|
+
output_index = instruction["output"]
|
|
146
|
+
# Compute elementwise addition across the stacked buffers # This ensures "deploy" only runs on valid version tags in the form release-x.x.x
|
|
147
|
+
buffers[output_index] = np.array(result, dtype=np.float32)
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
elif instruction["type"] == "MULTIPLY_ELEMENTWISE_BUFFERS":
|
|
151
|
+
# "input" is a list of buffer indices
|
|
152
|
+
input_indices = instruction["input"]
|
|
153
|
+
stacked = np.stack([buffers[i] for i in input_indices], axis=0)
|
|
154
|
+
# Compute elementwise multiplication across the stacked buffers.
|
|
155
|
+
result = np.prod(stacked, axis=0)
|
|
156
|
+
output_index = instruction["output"]
|
|
157
|
+
buffers[output_index] = np.array(result, dtype=np.float32)
|
|
158
|
+
continue
|
|
159
|
+
|
|
160
|
+
# --- Existing instructions (where "input" is a single index) --
|
|
161
|
+
# In these cases we assume "input" is an integer.
|
|
162
|
+
input_index = instruction["input"]
|
|
163
|
+
# Get the buffer corresponding to the single input for this instruction.
|
|
164
|
+
input_data_buffer = buffers[input_index]
|
|
165
|
+
|
|
166
|
+
if instruction["type"] == "COPY_MASKED":
|
|
167
|
+
output_index = instruction["output"]
|
|
168
|
+
indexes = instruction["indexes"]
|
|
169
|
+
if buffers[output_index] is None:
|
|
170
|
+
buffers[output_index] = np.zeros(
|
|
171
|
+
(input_data_buffer.shape[0], buffer_sizes[output_index]),
|
|
172
|
+
dtype=np.float32,
|
|
173
|
+
)
|
|
174
|
+
# Copy only the selected indices.
|
|
175
|
+
buffers[output_index][:, list(range(len(indexes)))] = input_data_buffer[
|
|
176
|
+
:, indexes
|
|
177
|
+
]
|
|
178
|
+
|
|
179
|
+
elif instruction["type"] == "COPY":
|
|
180
|
+
output_index = instruction["output"]
|
|
181
|
+
internal_index = instruction["internal_index"]
|
|
182
|
+
size = buffer_sizes[input_index]
|
|
183
|
+
if buffers[output_index] is None:
|
|
184
|
+
buffers[output_index] = np.zeros(
|
|
185
|
+
(input_data_buffer.shape[0], buffer_sizes[output_index]),
|
|
186
|
+
dtype=np.float32,
|
|
187
|
+
)
|
|
188
|
+
buffers[output_index][:, internal_index : internal_index + size] = (
|
|
189
|
+
input_data_buffer[:, :]
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
elif instruction["type"] == "MAP_TRANSFORM":
|
|
193
|
+
output_index = instruction["output"]
|
|
194
|
+
internal_input_index = instruction["internal_input_index"]
|
|
195
|
+
internal_output_index = instruction["internal_output_index"]
|
|
196
|
+
map_index = instruction["map"]
|
|
197
|
+
default = instruction["default"]
|
|
198
|
+
size = instruction["size"]
|
|
199
|
+
map_data = {int(k): v for k, v in model["maps"][map_index].items()}
|
|
200
|
+
buffers[output_index][
|
|
201
|
+
:, internal_output_index : internal_output_index + size
|
|
202
|
+
] = np.array(
|
|
203
|
+
[
|
|
204
|
+
np.array(map_data.get(i.item(), default))
|
|
205
|
+
for i in input_data_buffer[
|
|
206
|
+
:, internal_input_index : internal_input_index + 1
|
|
207
|
+
].flatten()
|
|
208
|
+
]
|
|
209
|
+
).reshape(-1, size)
|
|
210
|
+
|
|
211
|
+
elif instruction["type"] == "DOT":
|
|
212
|
+
w_index = instruction["weights"]
|
|
213
|
+
output_index = instruction["output"]
|
|
214
|
+
buffers[output_index] = (
|
|
215
|
+
input_data_buffer @ weights[w_index].T + bias[w_index]
|
|
216
|
+
)
|
|
217
|
+
apply_activation(instruction, buffers, output_index)
|
|
218
|
+
|
|
219
|
+
elif instruction["type"] == "ATTENTION":
|
|
220
|
+
w_index = instruction["weights"]
|
|
221
|
+
output_index = instruction["output"]
|
|
222
|
+
key_index = instruction["key"]
|
|
223
|
+
key_data = buffers[key_index]
|
|
224
|
+
buffers[output_index] = key_data @ weights[w_index].T + bias[w_index]
|
|
225
|
+
x = buffers[output_index]
|
|
226
|
+
x = x - np.max(x, axis=1, keepdims=True)
|
|
227
|
+
exp_x = np.exp(x)
|
|
228
|
+
buffers[output_index] = exp_x / np.sum(exp_x, axis=1, keepdims=True)
|
|
229
|
+
buffers[output_index] = buffers[output_index] * input_data_buffer
|
|
230
|
+
|
|
231
|
+
elif instruction["type"] == "ADD_ELEMENTWISE":
|
|
232
|
+
param = parameters[instruction["parameters"]]
|
|
233
|
+
buffers[input_index] = input_data_buffer + param
|
|
234
|
+
|
|
235
|
+
elif instruction["type"] == "MUL_ELEMENTWISE":
|
|
236
|
+
param = parameters[instruction["parameters"]]
|
|
237
|
+
buffers[input_index] = input_data_buffer * param
|
|
238
|
+
|
|
239
|
+
# Ensure the source buffer (still unassigned to an output) is float32.
|
|
240
|
+
buffers[input_index] = np.array(buffers[input_index], dtype=np.float32)
|
|
241
|
+
|
|
242
|
+
return buffers
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def validate_instruction_model(model):
|
|
246
|
+
input_data_all = np.array(model["validation_data"]["inputs"])
|
|
247
|
+
output_data_all = np.array(model["validation_data"]["expected_outputs"])
|
|
248
|
+
buffers = instruction_model_inference(model, input_data_all)
|
|
249
|
+
output_inference = buffers[-1]
|
|
250
|
+
if not np.allclose(output_data_all, output_inference, atol=1e-6):
|
|
251
|
+
print("output_data_all shape:", output_data_all.shape)
|
|
252
|
+
print("output_inference shape:", output_inference.shape)
|
|
253
|
+
print("MAX ERROR: ", np.max(np.abs(output_data_all - output_inference)))
|
|
254
|
+
print("MEAN ERROR: ", np.mean(np.abs(output_data_all - output_inference)))
|
|
255
|
+
print("All ERROR: ", np.abs(output_data_all - output_inference))
|
|
256
|
+
|
|
257
|
+
assert np.allclose(output_data_all, output_inference, atol=1e-6)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def concatenate_instruction_models(models, start_from_first=False):
|
|
261
|
+
features = models[0]["features"]
|
|
262
|
+
# Start with the first model's buffer_sizes. (Assume at least one model.)
|
|
263
|
+
concatenated_buffer_sizes = models[0]["buffer_sizes"][:]
|
|
264
|
+
for model in models[1:]:
|
|
265
|
+
# Ensure that the last buffer in the previous model matches the first buffer of the next.
|
|
266
|
+
assert concatenated_buffer_sizes[-1] == model["buffer_sizes"][0]
|
|
267
|
+
concatenated_buffer_sizes += model["buffer_sizes"][1:]
|
|
268
|
+
|
|
269
|
+
if start_from_first:
|
|
270
|
+
new_model = models[0]
|
|
271
|
+
current_buffer_offset = len(new_model["buffer_sizes"]) - 1
|
|
272
|
+
new_model["buffer_sizes"] = concatenated_buffer_sizes
|
|
273
|
+
else:
|
|
274
|
+
new_model = {
|
|
275
|
+
"features": features,
|
|
276
|
+
"buffer_sizes": concatenated_buffer_sizes,
|
|
277
|
+
"instructions": [],
|
|
278
|
+
"weights": [],
|
|
279
|
+
"bias": [],
|
|
280
|
+
"parameters": [],
|
|
281
|
+
"maps": [],
|
|
282
|
+
}
|
|
283
|
+
current_buffer_offset = 0
|
|
284
|
+
|
|
285
|
+
for model in models[int(start_from_first) :]:
|
|
286
|
+
w_offset = len(new_model["weights"])
|
|
287
|
+
p_offset = len(new_model["parameters"])
|
|
288
|
+
m_offset = len(new_model["maps"])
|
|
289
|
+
for instruction in model["instructions"]:
|
|
290
|
+
new_instruction = instruction.copy()
|
|
291
|
+
if "weights" in new_instruction:
|
|
292
|
+
new_instruction["weights"] += w_offset
|
|
293
|
+
if "parameters" in new_instruction:
|
|
294
|
+
new_instruction["parameters"] += p_offset
|
|
295
|
+
if "map" in new_instruction:
|
|
296
|
+
new_instruction["map"] += m_offset
|
|
297
|
+
if "input" in new_instruction:
|
|
298
|
+
if isinstance(new_instruction["input"], list):
|
|
299
|
+
new_instruction["input"] = [
|
|
300
|
+
x + current_buffer_offset for x in new_instruction["input"]
|
|
301
|
+
]
|
|
302
|
+
else:
|
|
303
|
+
new_instruction["input"] += current_buffer_offset
|
|
304
|
+
if "output" in new_instruction:
|
|
305
|
+
new_instruction["output"] += current_buffer_offset
|
|
306
|
+
if "key" in new_instruction:
|
|
307
|
+
new_instruction["key"] += current_buffer_offset
|
|
308
|
+
new_model["instructions"].append(new_instruction)
|
|
309
|
+
|
|
310
|
+
new_model["weights"] += model["weights"]
|
|
311
|
+
new_model["bias"] += model["bias"]
|
|
312
|
+
new_model["parameters"] += model["parameters"]
|
|
313
|
+
new_model["maps"] += model["maps"]
|
|
314
|
+
|
|
315
|
+
current_buffer_offset += len(model["buffer_sizes"]) - 1
|
|
316
|
+
return new_model
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def create_instructions_model_from_transformation_list(features, transformation_list):
|
|
320
|
+
new_size = (
|
|
321
|
+
len(features)
|
|
322
|
+
+ sum(i["size"] for i in transformation_list if "to" in i)
|
|
323
|
+
- sum(1 for i in transformation_list if "delete" in i)
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
transformation_list = sorted(
|
|
327
|
+
transformation_list, key=lambda x: "delete" in x, reverse=True
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
instructions = []
|
|
331
|
+
new_features = features.copy()
|
|
332
|
+
keep_indexes = [*range(len(features))]
|
|
333
|
+
for transformation in transformation_list:
|
|
334
|
+
if "delete" in transformation:
|
|
335
|
+
new_features.remove(transformation["delete"])
|
|
336
|
+
keep_indexes.remove(features.index(transformation["delete"]))
|
|
337
|
+
elif "to" in transformation:
|
|
338
|
+
new_features.extend(transformation["to"])
|
|
339
|
+
|
|
340
|
+
instructions.append(
|
|
341
|
+
{
|
|
342
|
+
"type": "COPY_MASKED",
|
|
343
|
+
"input": 0,
|
|
344
|
+
"output": 1,
|
|
345
|
+
"indexes": keep_indexes,
|
|
346
|
+
}
|
|
347
|
+
)
|
|
348
|
+
maps = []
|
|
349
|
+
for transformation in transformation_list:
|
|
350
|
+
if "to" in transformation:
|
|
351
|
+
from_idx = features.index(transformation["from"])
|
|
352
|
+
|
|
353
|
+
instructions.append(
|
|
354
|
+
{
|
|
355
|
+
"type": "MAP_TRANSFORM",
|
|
356
|
+
"input": 0,
|
|
357
|
+
"output": 1,
|
|
358
|
+
"internal_input_index": from_idx,
|
|
359
|
+
"internal_output_index": new_features.index(
|
|
360
|
+
transformation["to"][0]
|
|
361
|
+
),
|
|
362
|
+
"map": len(maps),
|
|
363
|
+
"size": transformation["size"],
|
|
364
|
+
"default": transformation["default"],
|
|
365
|
+
}
|
|
366
|
+
)
|
|
367
|
+
maps.append(transformation["map"])
|
|
368
|
+
|
|
369
|
+
return {
|
|
370
|
+
"features": features,
|
|
371
|
+
"buffer_sizes": [len(features), new_size],
|
|
372
|
+
"instructions": instructions,
|
|
373
|
+
"weights": [],
|
|
374
|
+
"bias": [],
|
|
375
|
+
"parameters": [],
|
|
376
|
+
"maps": maps,
|
|
377
|
+
}, new_features
|