corvic-engine 0.3.0rc44__cp38-abi3-win_amd64.whl → 0.3.0rc45__cp38-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corvic/engine/_native.pyd +0 -0
- corvic/model/__init__.py +7 -1
- corvic/model/_base_model.py +35 -42
- corvic/model/_feature_view.py +43 -43
- corvic/model/_pipeline.py +2 -14
- corvic/model/_proto_orm_convert.py +242 -239
- corvic/model/_resource.py +2 -7
- corvic/model/_source.py +14 -16
- corvic/model/_space.py +330 -362
- corvic/orm/__init__.py +38 -138
- {corvic_engine-0.3.0rc44.dist-info → corvic_engine-0.3.0rc45.dist-info}/METADATA +1 -1
- {corvic_engine-0.3.0rc44.dist-info → corvic_engine-0.3.0rc45.dist-info}/RECORD +16 -16
- corvic_generated/model/v1alpha/models_pb2.py +28 -28
- corvic_generated/model/v1alpha/models_pb2.pyi +8 -12
- {corvic_engine-0.3.0rc44.dist-info → corvic_engine-0.3.0rc45.dist-info}/WHEEL +0 -0
- {corvic_engine-0.3.0rc44.dist-info → corvic_engine-0.3.0rc45.dist-info}/licenses/LICENSE +0 -0
@@ -1,7 +1,6 @@
|
|
1
|
-
import dataclasses
|
2
1
|
import datetime
|
3
|
-
from collections.abc import Sequence
|
4
|
-
from typing import
|
2
|
+
from collections.abc import Callable, Sequence
|
3
|
+
from typing import Any, Protocol, TypeVar, cast
|
5
4
|
|
6
5
|
import sqlalchemy as sa
|
7
6
|
import sqlalchemy.orm as sa_orm
|
@@ -9,54 +8,64 @@ from google.protobuf import timestamp_pb2
|
|
9
8
|
from more_itertools import flatten
|
10
9
|
|
11
10
|
from corvic import orm
|
12
|
-
from corvic.result import InvalidArgumentError, Ok
|
11
|
+
from corvic.result import InternalError, InvalidArgumentError, Ok
|
13
12
|
from corvic_generated.model.v1alpha import models_pb2
|
13
|
+
from corvic_generated.orm.v1 import feature_view_pb2
|
14
14
|
|
15
15
|
UNCOMMITTED_ID_PREFIX = "__uncommitted_object-"
|
16
16
|
|
17
|
-
_Proto = (
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
_ID = TypeVar(
|
29
|
-
"_ID",
|
30
|
-
orm.ResourceID,
|
31
|
-
orm.SourceID,
|
32
|
-
orm.FeatureViewID,
|
33
|
-
orm.SpaceID,
|
34
|
-
orm.FeatureViewSourceID,
|
35
|
-
orm.AgentID,
|
36
|
-
orm.PipelineID,
|
37
|
-
orm.RoomID,
|
38
|
-
orm.CompletionModelID,
|
17
|
+
_Proto = TypeVar(
|
18
|
+
"_Proto",
|
19
|
+
models_pb2.Resource,
|
20
|
+
models_pb2.Source,
|
21
|
+
models_pb2.FeatureView,
|
22
|
+
models_pb2.Space,
|
23
|
+
models_pb2.FeatureViewSource,
|
24
|
+
models_pb2.Agent,
|
25
|
+
models_pb2.Pipeline,
|
26
|
+
models_pb2.Room,
|
27
|
+
models_pb2.CompletionModel,
|
39
28
|
)
|
29
|
+
ID = TypeVar("ID", bound=orm.BaseID[Any])
|
30
|
+
|
31
|
+
|
32
|
+
class _OrmModel(Protocol[ID]):
|
33
|
+
id: sa_orm.Mapped[ID | None]
|
34
|
+
|
35
|
+
org_id: sa_orm.Mapped[orm.OrgID | None]
|
36
|
+
|
37
|
+
@sa.ext.hybrid.hybrid_property
|
38
|
+
def created_at(self) -> datetime.datetime | None: ...
|
39
|
+
|
40
|
+
@created_at.inplace.expression
|
41
|
+
@classmethod
|
42
|
+
def _created_at_expression(cls): ...
|
43
|
+
|
44
|
+
|
45
|
+
OrmObj = TypeVar("OrmObj", bound=_OrmModel[Any])
|
40
46
|
|
41
47
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
48
|
+
def _translate_orm_id(
|
49
|
+
obj_id: str, id_class: type[ID]
|
50
|
+
) -> Ok[ID | None] | orm.InvalidORMIdentifierError:
|
51
|
+
if obj_id.startswith(UNCOMMITTED_ID_PREFIX):
|
52
|
+
return Ok(None)
|
53
|
+
parsed_obj_id = id_class(obj_id)
|
54
|
+
match parsed_obj_id.to_db():
|
55
|
+
case orm.InvalidORMIdentifierError() as err:
|
56
|
+
return err
|
57
|
+
case Ok():
|
58
|
+
return Ok(parsed_obj_id)
|
46
59
|
|
47
60
|
|
48
61
|
def _translate_orm_ids(
|
49
|
-
proto_obj: _Proto, obj_id_class: type[
|
50
|
-
) -> Ok[
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
case orm.InvalidORMIdentifierError() as err:
|
57
|
-
return err
|
58
|
-
case Ok():
|
59
|
-
pass
|
62
|
+
proto_obj: _Proto, obj_id_class: type[ID]
|
63
|
+
) -> Ok[tuple[ID | None, orm.RoomID | None]] | orm.InvalidORMIdentifierError:
|
64
|
+
match _translate_orm_id(proto_obj.id, obj_id_class):
|
65
|
+
case orm.InvalidORMIdentifierError() as err:
|
66
|
+
return err
|
67
|
+
case Ok(obj_id):
|
68
|
+
pass
|
60
69
|
|
61
70
|
match proto_obj:
|
62
71
|
case (
|
@@ -79,7 +88,7 @@ def _translate_orm_ids(
|
|
79
88
|
case models_pb2.Room():
|
80
89
|
room_id = cast(orm.RoomID, obj_id)
|
81
90
|
|
82
|
-
return Ok(
|
91
|
+
return Ok((obj_id, room_id))
|
83
92
|
|
84
93
|
|
85
94
|
def timestamp_orm_to_proto(
|
@@ -104,12 +113,11 @@ def resource_orm_to_proto(resource_orm: orm.Resource) -> models_pb2.Resource:
|
|
104
113
|
size=resource_orm.size,
|
105
114
|
original_path=resource_orm.original_path,
|
106
115
|
room_id=str(resource_orm.room_id),
|
107
|
-
source_ids=[str(val.source_id) for val in resource_orm.source_associations],
|
108
116
|
org_id=str(resource_orm.org_id),
|
109
117
|
recent_events=[resource_orm.latest_event] if resource_orm.latest_event else [],
|
110
|
-
pipeline_id=str(resource_orm.
|
111
|
-
if resource_orm.
|
112
|
-
else
|
118
|
+
pipeline_id=str(resource_orm.pipeline_ref.pipeline_id)
|
119
|
+
if resource_orm.pipeline_ref
|
120
|
+
else "",
|
113
121
|
created_at=timestamp_orm_to_proto(resource_orm.created_at),
|
114
122
|
)
|
115
123
|
|
@@ -120,13 +128,10 @@ def source_orm_to_proto(source_orm: orm.Source) -> models_pb2.Source:
|
|
120
128
|
name=source_orm.name,
|
121
129
|
table_op_graph=source_orm.table_op_graph,
|
122
130
|
room_id=str(source_orm.room_id),
|
123
|
-
resource_id=str(source_orm.resource_associations[0].resource_id)
|
124
|
-
if source_orm.resource_associations
|
125
|
-
else "",
|
126
131
|
org_id=str(source_orm.org_id),
|
127
|
-
pipeline_id=str(source_orm.
|
128
|
-
if source_orm.
|
129
|
-
else
|
132
|
+
pipeline_id=str(source_orm.pipeline_ref.pipeline_id)
|
133
|
+
if source_orm.pipeline_ref
|
134
|
+
else "",
|
130
135
|
created_at=timestamp_orm_to_proto(source_orm.created_at),
|
131
136
|
)
|
132
137
|
|
@@ -145,11 +150,17 @@ def agent_orm_to_proto(agent_orm: orm.Agent) -> models_pb2.Agent:
|
|
145
150
|
def feature_view_source_orm_to_proto(
|
146
151
|
feature_view_source_orm: orm.FeatureViewSource,
|
147
152
|
) -> models_pb2.FeatureViewSource:
|
153
|
+
if feature_view_source_orm.table_op_graph.WhichOneof("op") is not None:
|
154
|
+
op = feature_view_source_orm.table_op_graph
|
155
|
+
else:
|
156
|
+
# some legacy feature views were stored without op graphs fill those
|
157
|
+
# with the source's opgraph
|
158
|
+
op = feature_view_source_orm.source.table_op_graph
|
148
159
|
return models_pb2.FeatureViewSource(
|
149
160
|
id=str(feature_view_source_orm.id),
|
150
161
|
room_id=str(feature_view_source_orm.room_id),
|
151
162
|
source=source_orm_to_proto(feature_view_source_orm.source),
|
152
|
-
table_op_graph=
|
163
|
+
table_op_graph=op,
|
153
164
|
drop_disconnected=feature_view_source_orm.drop_disconnected,
|
154
165
|
org_id=str(feature_view_source_orm.org_id),
|
155
166
|
created_at=timestamp_orm_to_proto(feature_view_source_orm.created_at),
|
@@ -203,7 +214,7 @@ def space_orm_to_proto(space_orm: orm.Space) -> models_pb2.Space:
|
|
203
214
|
description=space_orm.description,
|
204
215
|
room_id=str(space_orm.room_id),
|
205
216
|
space_parameters=space_orm.parameters,
|
206
|
-
|
217
|
+
feature_view=feature_view_orm_to_proto(space_orm.feature_view),
|
207
218
|
auto_sync=space_orm.auto_sync if space_orm.auto_sync is not None else False,
|
208
219
|
org_id=str(space_orm.org_id),
|
209
220
|
created_at=timestamp_orm_to_proto(space_orm.created_at),
|
@@ -233,27 +244,31 @@ def completion_model_orm_to_proto(
|
|
233
244
|
)
|
234
245
|
|
235
246
|
|
247
|
+
def _add_orm_to_session(
|
248
|
+
orm_obj: OrmObj, org_id: str, session: sa_orm.Session
|
249
|
+
) -> OrmObj:
|
250
|
+
if org_id:
|
251
|
+
orm_obj.org_id = orm.OrgID(org_id)
|
252
|
+
if not orm_obj.id:
|
253
|
+
session.add(orm_obj)
|
254
|
+
else:
|
255
|
+
orm_obj = session.merge(orm_obj)
|
256
|
+
return orm_obj
|
257
|
+
|
258
|
+
|
236
259
|
def resource_proto_to_orm(
|
237
260
|
proto_obj: models_pb2.Resource, session: sa_orm.Session
|
238
261
|
) -> Ok[orm.Resource] | InvalidArgumentError:
|
239
262
|
match _translate_orm_ids(proto_obj, orm.ResourceID):
|
240
263
|
case orm.InvalidORMIdentifierError() as err:
|
241
264
|
return err
|
242
|
-
case Ok(
|
265
|
+
case Ok((obj_id, room_id)):
|
243
266
|
pass
|
244
|
-
if not
|
245
|
-
return InvalidArgumentError("room id required to commit
|
246
|
-
|
247
|
-
source_ids = list[orm.SourceID]()
|
248
|
-
for source_id in proto_obj.source_ids:
|
249
|
-
orm_id = orm.SourceID(source_id)
|
250
|
-
match orm_id.to_db():
|
251
|
-
case orm.InvalidORMIdentifierError() as err:
|
252
|
-
return err
|
253
|
-
case Ok():
|
254
|
-
source_ids.append(orm_id)
|
267
|
+
if not room_id:
|
268
|
+
return InvalidArgumentError("room id required to commit")
|
269
|
+
|
255
270
|
orm_obj = orm.Resource(
|
256
|
-
id=
|
271
|
+
id=obj_id,
|
257
272
|
name=proto_obj.name,
|
258
273
|
description=proto_obj.description,
|
259
274
|
mime_type=proto_obj.mime_type,
|
@@ -262,25 +277,31 @@ def resource_proto_to_orm(
|
|
262
277
|
size=proto_obj.size,
|
263
278
|
original_path=proto_obj.original_path,
|
264
279
|
latest_event=proto_obj.recent_events[-1] if proto_obj.recent_events else None,
|
265
|
-
room_id=
|
266
|
-
source_associations=[
|
267
|
-
orm.SourceResourceAssociation(
|
268
|
-
room_id=ids.room_id, source_id=src_id, resource_id=ids.obj_id
|
269
|
-
)
|
270
|
-
for src_id in source_ids
|
271
|
-
],
|
280
|
+
room_id=room_id,
|
272
281
|
)
|
273
|
-
|
274
|
-
orm_obj.org_id = orm.OrgID(proto_obj.org_id)
|
275
|
-
for assn in orm_obj.source_associations:
|
276
|
-
assn.org_id = orm_obj.org_id
|
282
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
277
283
|
|
278
|
-
if ids.obj_id:
|
279
|
-
orm_obj = session.merge(orm_obj)
|
280
|
-
else:
|
281
|
-
session.add(orm_obj)
|
282
284
|
|
283
|
-
|
285
|
+
def _ensure_id(
|
286
|
+
proto_obj: _Proto,
|
287
|
+
proto_to_orm: Callable[[_Proto, sa_orm.Session], Ok[OrmObj] | InvalidArgumentError],
|
288
|
+
id_type: type[ID],
|
289
|
+
session: sa_orm.Session,
|
290
|
+
) -> Ok[ID] | orm.InvalidORMIdentifierError | InvalidArgumentError:
|
291
|
+
match _translate_orm_id(proto_obj.id, id_type):
|
292
|
+
case orm.InvalidORMIdentifierError() as err:
|
293
|
+
return err
|
294
|
+
case Ok(orm_id):
|
295
|
+
if orm_id:
|
296
|
+
return Ok(orm_id)
|
297
|
+
match proto_to_orm(proto_obj, session):
|
298
|
+
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
299
|
+
return err
|
300
|
+
case Ok(orm_obj):
|
301
|
+
session.flush()
|
302
|
+
if not orm_obj.id:
|
303
|
+
raise InternalError("internal assertion did not hold")
|
304
|
+
return Ok(orm_obj.id)
|
284
305
|
|
285
306
|
|
286
307
|
def pipeline_proto_to_orm( # noqa: C901
|
@@ -289,68 +310,59 @@ def pipeline_proto_to_orm( # noqa: C901
|
|
289
310
|
match _translate_orm_ids(proto_obj, orm.PipelineID):
|
290
311
|
case orm.InvalidORMIdentifierError() as err:
|
291
312
|
return err
|
292
|
-
case Ok(
|
313
|
+
case Ok((obj_id, room_id)):
|
293
314
|
pass
|
294
|
-
if not
|
295
|
-
return InvalidArgumentError("room id required to commit
|
315
|
+
if not room_id:
|
316
|
+
return InvalidArgumentError("room id required to commit")
|
296
317
|
|
318
|
+
inputs = list[orm.PipelineInput]()
|
297
319
|
orm_obj = orm.Pipeline(
|
298
|
-
id=
|
320
|
+
id=obj_id,
|
299
321
|
name=proto_obj.name,
|
300
|
-
room_id=
|
322
|
+
room_id=room_id,
|
301
323
|
transformation=proto_obj.pipeline_transformation,
|
302
324
|
description=proto_obj.description,
|
303
325
|
)
|
326
|
+
orm_obj = _add_orm_to_session(orm_obj, proto_obj.org_id, session)
|
327
|
+
session.flush()
|
304
328
|
|
305
|
-
if
|
306
|
-
|
307
|
-
else:
|
308
|
-
session.add(orm_obj)
|
329
|
+
if not orm_obj.id:
|
330
|
+
raise InternalError("internal assertion did not hold")
|
309
331
|
|
310
332
|
for name, val in proto_obj.resource_inputs.items():
|
311
|
-
|
312
|
-
continue
|
313
|
-
match resource_proto_to_orm(val, session):
|
333
|
+
match _ensure_id(val, resource_proto_to_orm, orm.ResourceID, session):
|
314
334
|
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
315
335
|
return err
|
316
|
-
case Ok(
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
name=name,
|
327
|
-
)
|
328
|
-
)
|
336
|
+
case Ok(resource_id):
|
337
|
+
pass
|
338
|
+
inputs.append(
|
339
|
+
orm.PipelineInput(
|
340
|
+
resource_id=resource_id,
|
341
|
+
name=name,
|
342
|
+
pipeline_id=orm_obj.id,
|
343
|
+
room_id=room_id,
|
344
|
+
)
|
345
|
+
)
|
329
346
|
|
347
|
+
outputs = list[orm.PipelineOutput]()
|
330
348
|
for name, val in proto_obj.source_outputs.items():
|
331
|
-
|
332
|
-
continue
|
333
|
-
match source_proto_to_orm(val, session):
|
349
|
+
match _ensure_id(val, source_proto_to_orm, orm.SourceID, session):
|
334
350
|
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
335
351
|
return err
|
336
|
-
case Ok(
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
pipeline=orm_obj,
|
345
|
-
source=source_orm,
|
346
|
-
name=name,
|
347
|
-
)
|
348
|
-
)
|
352
|
+
case Ok(source_id):
|
353
|
+
pass
|
354
|
+
outputs.append(
|
355
|
+
orm.PipelineOutput(
|
356
|
+
source_id=source_id, name=name, pipeline_id=orm_obj.id, room_id=room_id
|
357
|
+
)
|
358
|
+
)
|
359
|
+
|
349
360
|
if proto_obj.org_id:
|
350
361
|
org_id = orm.OrgID(proto_obj.org_id)
|
351
|
-
|
352
|
-
|
353
|
-
|
362
|
+
for obj in flatten((inputs, outputs)):
|
363
|
+
obj.org_id = org_id
|
364
|
+
for obj in flatten((inputs, outputs)):
|
365
|
+
session.merge(obj)
|
354
366
|
return Ok(orm_obj)
|
355
367
|
|
356
368
|
|
@@ -360,37 +372,18 @@ def source_proto_to_orm(
|
|
360
372
|
match _translate_orm_ids(proto_obj, orm.SourceID):
|
361
373
|
case orm.InvalidORMIdentifierError() as err:
|
362
374
|
return err
|
363
|
-
case Ok(
|
375
|
+
case Ok((obj_id, room_id)):
|
364
376
|
pass
|
365
|
-
if not
|
366
|
-
return InvalidArgumentError("room id required to commit
|
367
|
-
resource_id = orm.ResourceID(proto_obj.resource_id)
|
368
|
-
if resource_id:
|
369
|
-
associations = [
|
370
|
-
orm.SourceResourceAssociation(
|
371
|
-
room_id=ids.room_id, source_id=ids.obj_id, resource_id=resource_id
|
372
|
-
)
|
373
|
-
]
|
374
|
-
else:
|
375
|
-
associations = list[orm.SourceResourceAssociation]()
|
377
|
+
if not room_id:
|
378
|
+
return InvalidArgumentError("room id required to commit")
|
376
379
|
|
377
380
|
orm_obj = orm.Source(
|
378
|
-
id=
|
381
|
+
id=obj_id,
|
379
382
|
name=proto_obj.name,
|
380
383
|
table_op_graph=proto_obj.table_op_graph,
|
381
|
-
room_id=
|
382
|
-
resource_associations=associations,
|
384
|
+
room_id=room_id,
|
383
385
|
)
|
384
|
-
|
385
|
-
orm_obj.org_id = orm.OrgID(proto_obj.org_id)
|
386
|
-
for assn in orm_obj.resource_associations:
|
387
|
-
assn.org_id = orm.OrgID(proto_obj.org_id)
|
388
|
-
if ids.obj_id:
|
389
|
-
orm_obj = session.merge(orm_obj)
|
390
|
-
else:
|
391
|
-
session.add(orm_obj)
|
392
|
-
|
393
|
-
return Ok(orm_obj)
|
386
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
394
387
|
|
395
388
|
|
396
389
|
def agent_proto_to_orm(
|
@@ -399,26 +392,18 @@ def agent_proto_to_orm(
|
|
399
392
|
match _translate_orm_ids(proto_obj, orm.AgentID):
|
400
393
|
case orm.InvalidORMIdentifierError() as err:
|
401
394
|
return err
|
402
|
-
case Ok(
|
395
|
+
case Ok((obj_id, room_id)):
|
403
396
|
pass
|
404
|
-
if not
|
405
|
-
return InvalidArgumentError("room id required to commit
|
397
|
+
if not room_id:
|
398
|
+
return InvalidArgumentError("room id required to commit")
|
406
399
|
|
407
400
|
orm_obj = orm.Agent(
|
408
|
-
id=
|
401
|
+
id=obj_id,
|
409
402
|
name=proto_obj.name,
|
410
403
|
parameters=proto_obj.agent_parameters,
|
411
|
-
room_id=
|
404
|
+
room_id=room_id,
|
412
405
|
)
|
413
|
-
|
414
|
-
if proto_obj.org_id:
|
415
|
-
orm_obj.org_id = orm.OrgID(proto_obj.org_id)
|
416
|
-
if ids.obj_id:
|
417
|
-
orm_obj = session.merge(orm_obj)
|
418
|
-
else:
|
419
|
-
session.add(orm_obj)
|
420
|
-
|
421
|
-
return Ok(orm_obj)
|
406
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
422
407
|
|
423
408
|
|
424
409
|
def space_proto_to_orm(
|
@@ -427,99 +412,129 @@ def space_proto_to_orm(
|
|
427
412
|
match _translate_orm_ids(proto_obj, orm.SpaceID):
|
428
413
|
case orm.InvalidORMIdentifierError() as err:
|
429
414
|
return err
|
430
|
-
case Ok(
|
415
|
+
case Ok((obj_id, room_id)):
|
416
|
+
pass
|
417
|
+
if not room_id:
|
418
|
+
return InvalidArgumentError("room id required to commit")
|
419
|
+
|
420
|
+
match _ensure_id(
|
421
|
+
proto_obj.feature_view, feature_view_proto_to_orm, orm.FeatureViewID, session
|
422
|
+
):
|
423
|
+
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
424
|
+
return err
|
425
|
+
case Ok(feature_view_id):
|
431
426
|
pass
|
432
|
-
if not ids.room_id:
|
433
|
-
return InvalidArgumentError("room id required to commit resource")
|
434
427
|
|
435
|
-
|
436
|
-
|
437
|
-
if not feature_view:
|
438
|
-
return InvalidArgumentError("feature view required to commit resource")
|
428
|
+
if not feature_view_id:
|
429
|
+
raise InternalError("internal assertion did not hold")
|
439
430
|
|
440
431
|
orm_obj = orm.Space(
|
441
|
-
id=
|
432
|
+
id=obj_id,
|
442
433
|
name=proto_obj.name,
|
443
434
|
description=proto_obj.description,
|
444
|
-
room_id=
|
435
|
+
room_id=room_id,
|
445
436
|
feature_view_id=feature_view_id,
|
446
437
|
parameters=proto_obj.space_parameters,
|
447
438
|
auto_sync=proto_obj.auto_sync,
|
448
|
-
feature_view=feature_view,
|
449
439
|
)
|
450
|
-
|
451
|
-
orm_obj.org_id = orm.OrgID(proto_obj.org_id)
|
452
|
-
|
453
|
-
if ids.obj_id:
|
454
|
-
orm_obj = session.merge(orm_obj)
|
455
|
-
else:
|
456
|
-
session.add(orm_obj)
|
457
|
-
return Ok(orm_obj)
|
440
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
458
441
|
|
459
442
|
|
460
|
-
def feature_view_proto_to_orm(
|
443
|
+
def feature_view_proto_to_orm( # noqa: C901
|
461
444
|
proto_obj: models_pb2.FeatureView, session: sa_orm.Session
|
462
445
|
) -> Ok[orm.FeatureView] | orm.InvalidORMIdentifierError | InvalidArgumentError:
|
463
446
|
match _translate_orm_ids(proto_obj, orm.FeatureViewID):
|
464
447
|
case orm.InvalidORMIdentifierError() as err:
|
465
448
|
return err
|
466
|
-
case Ok(
|
449
|
+
case Ok((obj_id, room_id)):
|
467
450
|
pass
|
468
|
-
if not
|
469
|
-
return InvalidArgumentError("room id required to commit
|
470
|
-
|
471
|
-
feature_view_sources = list[orm.FeatureViewSource]()
|
472
|
-
for fvs in proto_obj.feature_view_sources:
|
473
|
-
match feature_view_source_proto_to_orm(fvs, session):
|
474
|
-
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
475
|
-
return err
|
476
|
-
case Ok(orm_fvs):
|
477
|
-
if orm_fvs.id:
|
478
|
-
orm_fvs = session.merge(orm_fvs)
|
479
|
-
else:
|
480
|
-
session.add(orm_fvs)
|
481
|
-
feature_view_sources.append(orm_fvs)
|
451
|
+
if not room_id:
|
452
|
+
return InvalidArgumentError("room id required to commit")
|
482
453
|
|
483
454
|
orm_obj = orm.FeatureView(
|
484
|
-
id=
|
455
|
+
id=obj_id,
|
485
456
|
name=proto_obj.name,
|
486
457
|
description=proto_obj.description,
|
487
|
-
room_id=
|
488
|
-
feature_view_output=proto_obj.feature_view_output,
|
489
|
-
feature_view_sources=feature_view_sources,
|
458
|
+
room_id=room_id,
|
490
459
|
)
|
491
460
|
if proto_obj.org_id:
|
492
461
|
orm_obj.org_id = orm.OrgID(proto_obj.org_id)
|
493
462
|
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
463
|
+
orm_obj = _add_orm_to_session(orm_obj, proto_obj.org_id, session)
|
464
|
+
session.flush()
|
465
|
+
|
466
|
+
if not orm_obj.id:
|
467
|
+
raise InternalError("internal assertion did not hold")
|
468
|
+
|
469
|
+
new_fv_sources = list[orm.FeatureViewSource]()
|
470
|
+
for fvs in proto_obj.feature_view_sources:
|
471
|
+
match _feature_view_source_proto_to_orm(fvs, orm_obj.id, session):
|
472
|
+
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
473
|
+
return err
|
474
|
+
case Ok(fvs_orm):
|
475
|
+
new_fv_sources.append(fvs_orm)
|
476
|
+
|
477
|
+
old_to_new_source_id = dict[str, str]()
|
478
|
+
for old_fvs, new_fvs in zip(
|
479
|
+
proto_obj.feature_view_sources, new_fv_sources, strict=True
|
480
|
+
):
|
481
|
+
if old_fvs.source.id != str(new_fvs.source_id):
|
482
|
+
old_to_new_source_id[old_fvs.source.id] = str(new_fvs.source_id)
|
483
|
+
|
484
|
+
orm_obj.feature_view_output = (
|
485
|
+
feature_view_pb2.FeatureViewOutput(
|
486
|
+
relationships=[
|
487
|
+
feature_view_pb2.FeatureViewRelationship(
|
488
|
+
start_source_id=old_to_new_source_id.get(
|
489
|
+
old_rel.start_source_id, old_rel.start_source_id
|
490
|
+
),
|
491
|
+
end_source_id=old_to_new_source_id.get(
|
492
|
+
old_rel.end_source_id, old_rel.end_source_id
|
493
|
+
),
|
494
|
+
)
|
495
|
+
for old_rel in proto_obj.feature_view_output.relationships
|
496
|
+
],
|
497
|
+
output_sources=[
|
498
|
+
feature_view_pb2.OutputSource(
|
499
|
+
source_id=old_to_new_source_id.get(
|
500
|
+
old_output_source.source_id, old_output_source.source_id
|
501
|
+
)
|
502
|
+
)
|
503
|
+
for old_output_source in proto_obj.feature_view_output.output_sources
|
504
|
+
],
|
505
|
+
)
|
506
|
+
if old_to_new_source_id
|
507
|
+
else proto_obj.feature_view_output
|
508
|
+
)
|
509
|
+
orm_obj = session.merge(orm_obj)
|
498
510
|
return Ok(orm_obj)
|
499
511
|
|
500
512
|
|
501
|
-
def
|
502
|
-
proto_obj: models_pb2.FeatureViewSource,
|
513
|
+
def _feature_view_source_proto_to_orm(
|
514
|
+
proto_obj: models_pb2.FeatureViewSource,
|
515
|
+
feature_view_id: orm.FeatureViewID,
|
516
|
+
session: sa_orm.Session,
|
503
517
|
) -> Ok[orm.FeatureViewSource] | orm.InvalidORMIdentifierError | InvalidArgumentError:
|
504
|
-
match
|
518
|
+
match _ensure_id(proto_obj.source, source_proto_to_orm, orm.SourceID, session):
|
505
519
|
case orm.InvalidORMIdentifierError() | InvalidArgumentError() as err:
|
506
520
|
return err
|
507
|
-
case Ok(
|
521
|
+
case Ok(source_id):
|
522
|
+
pass
|
523
|
+
match _translate_orm_id(proto_obj.id, orm.FeatureViewSourceID):
|
524
|
+
case orm.InvalidORMIdentifierError() as err:
|
525
|
+
return err
|
526
|
+
case Ok(obj_id):
|
508
527
|
pass
|
509
528
|
|
510
|
-
if source.id:
|
511
|
-
source = session.merge(source)
|
512
|
-
else:
|
513
|
-
session.add(source)
|
514
529
|
orm_obj = orm.FeatureViewSource(
|
515
|
-
room_id=
|
530
|
+
room_id=orm.RoomID(proto_obj.room_id),
|
531
|
+
id=obj_id,
|
516
532
|
table_op_graph=proto_obj.table_op_graph,
|
517
533
|
drop_disconnected=proto_obj.drop_disconnected,
|
518
|
-
|
534
|
+
source_id=source_id,
|
535
|
+
feature_view_id=feature_view_id,
|
519
536
|
)
|
520
|
-
|
521
|
-
orm_obj.org_id = orm.OrgID(proto_obj.org_id)
|
522
|
-
return Ok(orm_obj)
|
537
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
523
538
|
|
524
539
|
|
525
540
|
def room_proto_to_orm(
|
@@ -528,20 +543,14 @@ def room_proto_to_orm(
|
|
528
543
|
match _translate_orm_ids(proto_obj, orm.RoomID):
|
529
544
|
case orm.InvalidORMIdentifierError() as err:
|
530
545
|
return err
|
531
|
-
case Ok(
|
546
|
+
case Ok((obj_id, _)):
|
532
547
|
pass
|
533
548
|
|
534
549
|
orm_obj = orm.Room(
|
535
|
-
id=
|
550
|
+
id=obj_id,
|
536
551
|
name=proto_obj.name,
|
537
552
|
)
|
538
|
-
|
539
|
-
if ids.obj_id:
|
540
|
-
orm_obj = session.merge(orm_obj)
|
541
|
-
else:
|
542
|
-
session.add(orm_obj)
|
543
|
-
|
544
|
-
return Ok(orm_obj)
|
553
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
545
554
|
|
546
555
|
|
547
556
|
def completion_model_proto_to_orm(
|
@@ -550,23 +559,17 @@ def completion_model_proto_to_orm(
|
|
550
559
|
match _translate_orm_ids(proto_obj, orm.CompletionModelID):
|
551
560
|
case orm.InvalidORMIdentifierError() as err:
|
552
561
|
return err
|
553
|
-
case Ok(
|
562
|
+
case Ok((obj_id, _)):
|
554
563
|
pass
|
555
564
|
|
556
565
|
orm_obj = orm.CompletionModel(
|
557
|
-
id=
|
566
|
+
id=obj_id,
|
558
567
|
name=proto_obj.name,
|
559
568
|
description=proto_obj.description,
|
560
569
|
parameters=proto_obj.parameters,
|
561
570
|
secret_api_key=proto_obj.secret_api_key,
|
562
571
|
)
|
563
|
-
|
564
|
-
if ids.obj_id:
|
565
|
-
orm_obj = session.merge(orm_obj)
|
566
|
-
else:
|
567
|
-
session.add(orm_obj)
|
568
|
-
|
569
|
-
return Ok(orm_obj)
|
572
|
+
return Ok(_add_orm_to_session(orm_obj, proto_obj.org_id, session))
|
570
573
|
|
571
574
|
|
572
575
|
def source_delete_orms(
|