corvic-engine 0.3.0rc94__tar.gz → 0.3.0rc96__tar.gz

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/Cargo.lock

@@ -16,7 +16,7 @@ checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.94"
+version = "0.3.0-rc.96"
 dependencies = [
  "numpy",
  "pyo3",
@@ -57,7 +57,7 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.94"
+version = "0.3.0-rc.96"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.94"
+version = "0.3.0-rc.96"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc94
+Version: 0.3.0rc96
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/pyproject.toml

@@ -18,7 +18,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.94" # sync this with tool.poetry.version below
+version = "0.3.0-rc.96" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -73,7 +73,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.94"
+version = "0.3.0-rc.96"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic/emodel/_proto_orm_convert.py

@@ -9,7 +9,7 @@ from google.protobuf import timestamp_pb2
 from corvic import eorm, orm, transfer
 from corvic.result import InternalError, InvalidArgumentError, Ok
 from corvic_generated.model.v1alpha import models_pb2
-from corvic_generated.orm.v1 import feature_view_pb2
+from corvic_generated.orm.v1 import data_connector_pb2, feature_view_pb2
 
 
 class _ModelBelongsToOrgProto(transfer.ProtoModel, Protocol):
@@ -104,6 +104,9 @@ def resource_orm_to_proto(resource_orm: eorm.Resource) -> models_pb2.Resource:
         md5=resource_orm.md5,
         size=resource_orm.size,
         etag=resource_orm.etag,
+        origin_type=resource_orm.origin_info.type
+        if resource_orm.origin_info
+        else data_connector_pb2.DATA_CONNECTION_TYPE_UNSPECIFIED,
         original_modified_at_time=timestamp_orm_to_proto(
             resource_orm.original_modified_at_time
         ),
@@ -328,6 +331,7 @@ def resource_proto_to_orm(
         url=proto_obj.url,
         size=proto_obj.size,
         etag=proto_obj.etag,
+        origin_info=data_connector_pb2.DataConnectionInfo(type=proto_obj.origin_type),
         original_modified_at_time=proto_obj.original_modified_at_time.ToDatetime(
             datetime.UTC
         ),

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic/emodel/_resource.py

@@ -24,6 +24,7 @@ from corvic.emodel._proto_orm_convert import (
 )
 from corvic.result import InternalError, InvalidArgumentError, NotFoundError, Ok
 from corvic_generated.model.v1alpha import models_pb2
+from corvic_generated.orm.v1 import data_connector_pb2
 from corvic_generated.status.v1 import event_pb2
 
 SourceID: TypeAlias = eorm.SourceID
@@ -101,6 +102,10 @@ class Resource(StandardModel[ResourceID, models_pb2.Resource, eorm.Resource]):
     def etag(self) -> str:
         return self.proto_self.etag
 
+    @property
+    def origin_type(self) -> data_connector_pb2.DataConnectionType:
+        return self.proto_self.origin_type
+
     @property
     def original_modified_at_time(self) -> datetime.datetime | None:
         return self.proto_self.original_modified_at_time.ToDatetime(datetime.UTC)
@@ -136,6 +141,16 @@ class Resource(StandardModel[ResourceID, models_pb2.Resource, eorm.Resource]):
             client=self.client,
         )
 
+    def with_origin_type(
+        self, origin_type: data_connector_pb2.DataConnectionType
+    ) -> Resource:
+        proto_self = copy.copy(self.proto_self)
+        proto_self.origin_type = origin_type
+        return Resource(
+            proto_self=proto_self,
+            client=self.client,
+        )
+
     def with_original_modified_at_time(
         self, original_modified_at_time: datetime.datetime
     ) -> Resource:

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic/eorm/__init__.py

@@ -79,6 +79,9 @@ orm.Base.registry.update_type_annotation_map(
         data_connector_pb2.DataConnectorParameters: orm.ProtoMessageDecorator(
             data_connector_pb2.DataConnectorParameters()
         ),
+        data_connector_pb2.DataConnectionInfo: orm.ProtoMessageDecorator(
+            data_connector_pb2.DataConnectionInfo()
+        ),
         # ID types
         RoomID: orm.IntIDDecorator(RoomID()),
         ResourceID: orm.IntIDDecorator(ResourceID()),
@@ -183,6 +186,9 @@ class Resource(orm.BelongsToOrgMixin, BelongsToRoomMixin, orm.Base, kw_only=True
         server_default=None,
         default=None,
     )
+    origin_info: sa_orm.Mapped[data_connector_pb2.DataConnectionInfo | None] = (
+        sa_orm.mapped_column(default=None, nullable=True)
+    )
     id: sa_orm.Mapped[ResourceID | None] = orm.primary_key_identity_column()
     latest_event: sa_orm.Mapped[event_pb2.Event | None] = sa_orm.mapped_column(
         default=None, nullable=True

corvic_engine-0.3.0rc96/python/corvic_generated/ingest/v2/resource_pb2.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: corvic/ingest/v2/resource.proto
+# Protobuf Python Version: 4.25.8
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from buf.validate import validate_pb2 as buf_dot_validate_dot_validate__pb2
+from corvic_generated.status.v1 import event_pb2 as corvic_dot_status_dot_v1_dot_event__pb2
+from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x63orvic/ingest/v2/resource.proto\x12\x10\x63orvic.ingest.v2\x1a\x1b\x62uf/validate/validate.proto\x1a\x1c\x63orvic/status/v1/event.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x99\x04\n\x10ResourceMetadata\x12\x1e\n\x04name\x18\x01 \x01(\tB\n\xbaH\x07r\x05\x10\x01\x18\xc8\x01R\x04name\x12%\n\tmime_type\x18\x02 \x01(\tB\x08\xbaH\x05r\x03\x18\xc8\x01R\x08mimeType\x12\x7f\n\x07room_id\x18\x03 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x06roomId\x12-\n\roriginal_path\x18\x05 \x01(\tB\x08\xbaH\x05r\x03\x18\xe8\x07R\x0coriginalPath\x12*\n\x0b\x64\x65scription\x18\x06 \x01(\tB\x08\xbaH\x05r\x03\x18\xe8\x07R\x0b\x64\x65scription\x12\x87\x01\n\x0bpipeline_id\x18\x08 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\npipelineId\x12;\n\ttype_hint\x18\x04 \x01(\x0e\x32\x1e.corvic.ingest.v2.ResourceTypeR\x08typeHint\x12\x1b\n\x04size\x18\x07 \x01(\x03\x42\x07\xbaH\x04\"\x02 \x00R\x04size\"\xef\x03\n\rResourceEntry\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tmime_type\x18\x03 \x01(\tR\x08mimeType\x12\x10\n\x03md5\x18\x05 \x01(\tR\x03md5\x12\x17\n\x07room_id\x18\x06 \x01(\tR\x06roomId\x12\x12\n\x04size\x18\x07 \x01(\x04R\x04size\x12#\n\roriginal_path\x18\t \x01(\tR\x0coriginalPath\x12 \n\x0b\x64\x65scription\x18\x0b \x01(\tR\x0b\x64\x65scription\x12\x45\n\x0borigin_type\x18\x0f \x01(\x0e\x32$.corvic.ingest.v2.ResourceOriginTypeR\noriginType\x12\x1f\n\x0bpipeline_id\x18\x0c \x01(\tR\npipelineId\x12\x1b\n\turl_depth\x18\r \x01(\x04R\x08urlDepth\x12\x1b\n\turl_width\x18\x0e \x01(\x04R\x08urlWidth\x12\x37\n\x18referenced_by_source_ids\x18\n \x03(\tR\x15referencedBySourceIds\x12<\n\rrecent_events\x18\x08 \x03(\x0b\x32\x17.corvic.status.v1.EventR\x0crecentEvents\"p\n\x16\x43reateUploadURLRequest\x12>\n\x08metadata\x18\x01 \x01(\x0b\x32\".corvic.ingest.v2.ResourceMetadataR\x08metadata\x12\x16\n\x06origin\x18\x02 \x01(\tR\x06origin\"A\n\x17\x43reateUploadURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"s\n\x15UploadURLTokenPayload\x12>\n\x08metadata\x18\x01 \x01(\x0b\x32\".corvic.ingest.v2.ResourceMetadataR\x08metadata\x12\x1a\n\x08\x66ilename\x18\x02 \x01(\tR\x08\x66ilename\"0\n\x18\x46inalizeUploadURLRequest\x12\x14\n\x05token\x18\x01 \x01(\tR\x05token\"R\n\x19\x46inalizeUploadURLResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\xaa\x03\n\"FetchAndFinalizeExternalURLRequest\x12\x7f\n\x07room_id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x06roomId\x12\x87\x01\n\x0bpipeline_id\x18\x02 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\npipelineId\x12-\n\roriginal_path\x18\x03 \x01(\tB\x08\xbaH\x05r\x03\x18\x80\x10R\x0coriginalPath\x12$\n\turl_depth\x18\x04 \x01(\x04\x42\x07\xbaH\x04\x32\x02\x18\nR\x08urlDepth\x12$\n\turl_width\x18\x05 \x01(\x04\x42\x07\xbaH\x04\x32\x02 \x00R\x08urlWidth\"\\\n#FetchAndFinalizeExternalURLResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x8f\x01\n\x15\x44\x65leteResourceRequest\x12v\n\x02id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"\x18\n\x16\x44\x65leteResourceResponse\"\x8c\x01\n\x12GetResourceRequest\x12v\n\x02id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"L\n\x13GetResourceResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x97\x01\n\x14ListResourcesRequest\x12\x7f\n\x07room_id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x06roomId\"N\n\x15ListResourcesResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x8b\x01\n\x0cResourceList\x12{\n\x02id\x18\x01 \x03(\tBk\xbaHh\x92\x01\x65\"cr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"\xdb\x01\n\x15WatchResourcesRequest\x12\x81\x01\n\x07room_id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')H\x00R\x06roomId\x12\x32\n\x03ids\x18\x02 \x01(\x0b\x32\x1e.corvic.ingest.v2.ResourceListH\x00R\x03idsB\n\n\x08selector\"d\n\x16WatchResourcesResponse\x12J\n\x10updated_resource\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x0fupdatedResource\"\xb5\x01\n#ListResourcesPaginatedCursorPayload\x12T\n\x19\x63reate_time_of_last_entry\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x15\x63reateTimeOfLastEntry\x12\x17\n\x07room_id\x18\x02 \x01(\tR\x06roomId\x12\x1f\n\x0bpipeline_id\x18\x03 \x01(\tR\npipelineId\"\xef\x02\n\x1dListResourcesPaginatedRequest\x12\x80\x01\n\x07room_id\x18\x01 \x01(\tBg\xbaHdr\x02\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd8\x01\x01R\x06roomId\x12\x88\x01\n\x0bpipeline_id\x18\x04 \x01(\tBg\xbaHdr\x02\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd8\x01\x01R\npipelineId\x12(\n\x10\x65ntries_per_page\x18\x02 \x01(\rR\x0e\x65ntriesPerPage\x12\x16\n\x06\x63ursor\x18\x03 \x01(\tR\x06\x63ursor\"\x91\x01\n\x1eListResourcesPaginatedResponse\x12J\n\x10resource_entries\x18\x03 \x03(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x0fresourceEntries\x12\x16\n\x06\x63ursor\x18\x02 \x01(\tR\x06\x63ursorJ\x04\x08\x01\x10\x02R\x05\x65ntry\"\x9a\x01\n CreateResourceDownloadURLRequest\x12v\n\x02id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"5\n!CreateResourceDownloadURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url\"\x99\x01\n\x1f\x43reateResourcePreviewURLRequest\x12v\n\x02id\x18\x01 \x01(\tBf\xbaHcr\x04\x10\x01\x18\x14\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"4\n CreateResourcePreviewURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url*\xab\x01\n\x0cResourceType\x12\x1d\n\x19RESOURCE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x13RESOURCE_TYPE_TABLE\x10\x01\x1a\x02\x08\x01\x12!\n\x1dRESOURCE_TYPE_DIMENSION_TABLE\x10\x02\x12\x1c\n\x18RESOURCE_TYPE_FACT_TABLE\x10\x03\x12\x1e\n\x1aRESOURCE_TYPE_PDF_DOCUMENT\x10\x04*\xfd\x01\n\x12ResourceOriginType\x12$\n RESOURCE_ORIGIN_TYPE_UNSPECIFIED\x10\x00\x12$\n RESOURCE_ORIGIN_TYPE_FILE_UPLOAD\x10\x01\x12\"\n\x1eRESOURCE_ORIGIN_TYPE_WEB_CRAWL\x10\x02\x12%\n!RESOURCE_ORIGIN_TYPE_S3_CONNECTOR\x10\x03\x12&\n\"RESOURCE_ORIGIN_TYPE_GCS_CONNECTOR\x10\x04\x12(\n$RESOURCE_ORIGIN_TYPE_AZURE_CONNECTOR\x10\x05\x32\xac\t\n\x0fResourceService\x12h\n\x0f\x43reateUploadURL\x12(.corvic.ingest.v2.CreateUploadURLRequest\x1a).corvic.ingest.v2.CreateUploadURLResponse\"\x00\x12q\n\x11\x46inalizeUploadURL\x12*.corvic.ingest.v2.FinalizeUploadURLRequest\x1a+.corvic.ingest.v2.FinalizeUploadURLResponse\"\x03\x90\x02\x02\x12\x8c\x01\n\x1b\x46\x65tchAndFinalizeExternalURL\x12\x34.corvic.ingest.v2.FetchAndFinalizeExternalURLRequest\x1a\x35.corvic.ingest.v2.FetchAndFinalizeExternalURLResponse\"\x00\x12\x65\n\x0e\x44\x65leteResource\x12\'.corvic.ingest.v2.DeleteResourceRequest\x1a(.corvic.ingest.v2.DeleteResourceResponse\"\x00\x12_\n\x0bGetResource\x12$.corvic.ingest.v2.GetResourceRequest\x1a%.corvic.ingest.v2.GetResourceResponse\"\x03\x90\x02\x01\x12g\n\rListResources\x12&.corvic.ingest.v2.ListResourcesRequest\x1a\'.corvic.ingest.v2.ListResourcesResponse\"\x03\x90\x02\x01\x30\x01\x12\x80\x01\n\x16ListResourcesPaginated\x12/.corvic.ingest.v2.ListResourcesPaginatedRequest\x1a\x30.corvic.ingest.v2.ListResourcesPaginatedResponse\"\x03\x90\x02\x01\x12j\n\x0eWatchResources\x12\'.corvic.ingest.v2.WatchResourcesRequest\x1a(.corvic.ingest.v2.WatchResourcesResponse\"\x03\x90\x02\x01\x30\x01\x12\x86\x01\n\x19\x43reateResourceDownloadURL\x12\x32.corvic.ingest.v2.CreateResourceDownloadURLRequest\x1a\x33.corvic.ingest.v2.CreateResourceDownloadURLResponse\"\x00\x12\x83\x01\n\x18\x43reateResourcePreviewURL\x12\x31.corvic.ingest.v2.CreateResourcePreviewURLRequest\x1a\x32.corvic.ingest.v2.CreateResourcePreviewURLResponse\"\x00\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'corvic.ingest.v2.resource_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  _globals['_RESOURCETYPE'].values_by_name["RESOURCE_TYPE_TABLE"]._options = None
+  _globals['_RESOURCETYPE'].values_by_name["RESOURCE_TYPE_TABLE"]._serialized_options = b'\010\001'
+  _globals['_RESOURCEMETADATA'].fields_by_name['name']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['name']._serialized_options = b'\272H\007r\005\020\001\030\310\001'
+  _globals['_RESOURCEMETADATA'].fields_by_name['mime_type']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['mime_type']._serialized_options = b'\272H\005r\003\030\310\001'
+  _globals['_RESOURCEMETADATA'].fields_by_name['room_id']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['room_id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_RESOURCEMETADATA'].fields_by_name['original_path']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['original_path']._serialized_options = b'\272H\005r\003\030\350\007'
+  _globals['_RESOURCEMETADATA'].fields_by_name['description']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['description']._serialized_options = b'\272H\005r\003\030\350\007'
+  _globals['_RESOURCEMETADATA'].fields_by_name['pipeline_id']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['pipeline_id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_RESOURCEMETADATA'].fields_by_name['size']._options = None
+  _globals['_RESOURCEMETADATA'].fields_by_name['size']._serialized_options = b'\272H\004\"\002 \000'
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['room_id']._options = None
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['room_id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['pipeline_id']._options = None
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['pipeline_id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['original_path']._options = None
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['original_path']._serialized_options = b'\272H\005r\003\030\200\020'
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['url_depth']._options = None
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['url_depth']._serialized_options = b'\272H\0042\002\030\n'
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['url_width']._options = None
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST'].fields_by_name['url_width']._serialized_options = b'\272H\0042\002 \000'
+  _globals['_DELETERESOURCEREQUEST'].fields_by_name['id']._options = None
+  _globals['_DELETERESOURCEREQUEST'].fields_by_name['id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_GETRESOURCEREQUEST'].fields_by_name['id']._options = None
+  _globals['_GETRESOURCEREQUEST'].fields_by_name['id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_LISTRESOURCESREQUEST'].fields_by_name['room_id']._options = None
+  _globals['_LISTRESOURCESREQUEST'].fields_by_name['room_id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_RESOURCELIST'].fields_by_name['id']._options = None
+  _globals['_RESOURCELIST'].fields_by_name['id']._serialized_options = b'\272Hh\222\001e\"cr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_WATCHRESOURCESREQUEST'].fields_by_name['room_id']._options = None
+  _globals['_WATCHRESOURCESREQUEST'].fields_by_name['room_id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['room_id']._options = None
+  _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['room_id']._serialized_options = b'\272Hdr\002\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\330\001\001'
+  _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['pipeline_id']._options = None
+  _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['pipeline_id']._serialized_options = b'\272Hdr\002\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\330\001\001'
+  _globals['_CREATERESOURCEDOWNLOADURLREQUEST'].fields_by_name['id']._options = None
+  _globals['_CREATERESOURCEDOWNLOADURLREQUEST'].fields_by_name['id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_CREATERESOURCEPREVIEWURLREQUEST'].fields_by_name['id']._options = None
+  _globals['_CREATERESOURCEPREVIEWURLREQUEST'].fields_by_name['id']._serialized_options = b'\272Hcr\004\020\001\030\024\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
+  _globals['_RESOURCESERVICE'].methods_by_name['FinalizeUploadURL']._options = None
+  _globals['_RESOURCESERVICE'].methods_by_name['FinalizeUploadURL']._serialized_options = b'\220\002\002'
+  _globals['_RESOURCESERVICE'].methods_by_name['GetResource']._options = None
+  _globals['_RESOURCESERVICE'].methods_by_name['GetResource']._serialized_options = b'\220\002\001'
+  _globals['_RESOURCESERVICE'].methods_by_name['ListResources']._options = None
+  _globals['_RESOURCESERVICE'].methods_by_name['ListResources']._serialized_options = b'\220\002\001'
+  _globals['_RESOURCESERVICE'].methods_by_name['ListResourcesPaginated']._options = None
+  _globals['_RESOURCESERVICE'].methods_by_name['ListResourcesPaginated']._serialized_options = b'\220\002\001'
+  _globals['_RESOURCESERVICE'].methods_by_name['WatchResources']._options = None
+  _globals['_RESOURCESERVICE'].methods_by_name['WatchResources']._serialized_options = b'\220\002\001'
+  _globals['_RESOURCETYPE']._serialized_start=4390
+  _globals['_RESOURCETYPE']._serialized_end=4561
+  _globals['_RESOURCEORIGINTYPE']._serialized_start=4564
+  _globals['_RESOURCEORIGINTYPE']._serialized_end=4817
+  _globals['_RESOURCEMETADATA']._serialized_start=180
+  _globals['_RESOURCEMETADATA']._serialized_end=717
+  _globals['_RESOURCEENTRY']._serialized_start=720
+  _globals['_RESOURCEENTRY']._serialized_end=1215
+  _globals['_CREATEUPLOADURLREQUEST']._serialized_start=1217
+  _globals['_CREATEUPLOADURLREQUEST']._serialized_end=1329
+  _globals['_CREATEUPLOADURLRESPONSE']._serialized_start=1331
+  _globals['_CREATEUPLOADURLRESPONSE']._serialized_end=1396
+  _globals['_UPLOADURLTOKENPAYLOAD']._serialized_start=1398
+  _globals['_UPLOADURLTOKENPAYLOAD']._serialized_end=1513
+  _globals['_FINALIZEUPLOADURLREQUEST']._serialized_start=1515
+  _globals['_FINALIZEUPLOADURLREQUEST']._serialized_end=1563
+  _globals['_FINALIZEUPLOADURLRESPONSE']._serialized_start=1565
+  _globals['_FINALIZEUPLOADURLRESPONSE']._serialized_end=1647
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST']._serialized_start=1650
+  _globals['_FETCHANDFINALIZEEXTERNALURLREQUEST']._serialized_end=2076
+  _globals['_FETCHANDFINALIZEEXTERNALURLRESPONSE']._serialized_start=2078
+  _globals['_FETCHANDFINALIZEEXTERNALURLRESPONSE']._serialized_end=2170
+  _globals['_DELETERESOURCEREQUEST']._serialized_start=2173
+  _globals['_DELETERESOURCEREQUEST']._serialized_end=2316
+  _globals['_DELETERESOURCERESPONSE']._serialized_start=2318
+  _globals['_DELETERESOURCERESPONSE']._serialized_end=2342
+  _globals['_GETRESOURCEREQUEST']._serialized_start=2345
+  _globals['_GETRESOURCEREQUEST']._serialized_end=2485
+  _globals['_GETRESOURCERESPONSE']._serialized_start=2487
+  _globals['_GETRESOURCERESPONSE']._serialized_end=2563
+  _globals['_LISTRESOURCESREQUEST']._serialized_start=2566
+  _globals['_LISTRESOURCESREQUEST']._serialized_end=2717
+  _globals['_LISTRESOURCESRESPONSE']._serialized_start=2719
+  _globals['_LISTRESOURCESRESPONSE']._serialized_end=2797
+  _globals['_RESOURCELIST']._serialized_start=2800
+  _globals['_RESOURCELIST']._serialized_end=2939
+  _globals['_WATCHRESOURCESREQUEST']._serialized_start=2942
+  _globals['_WATCHRESOURCESREQUEST']._serialized_end=3161
+  _globals['_WATCHRESOURCESRESPONSE']._serialized_start=3163
+  _globals['_WATCHRESOURCESRESPONSE']._serialized_end=3263
+  _globals['_LISTRESOURCESPAGINATEDCURSORPAYLOAD']._serialized_start=3266
+  _globals['_LISTRESOURCESPAGINATEDCURSORPAYLOAD']._serialized_end=3447
+  _globals['_LISTRESOURCESPAGINATEDREQUEST']._serialized_start=3450
+  _globals['_LISTRESOURCESPAGINATEDREQUEST']._serialized_end=3817
+  _globals['_LISTRESOURCESPAGINATEDRESPONSE']._serialized_start=3820
+  _globals['_LISTRESOURCESPAGINATEDRESPONSE']._serialized_end=3965
+  _globals['_CREATERESOURCEDOWNLOADURLREQUEST']._serialized_start=3968
+  _globals['_CREATERESOURCEDOWNLOADURLREQUEST']._serialized_end=4122
+  _globals['_CREATERESOURCEDOWNLOADURLRESPONSE']._serialized_start=4124
+  _globals['_CREATERESOURCEDOWNLOADURLRESPONSE']._serialized_end=4177
+  _globals['_CREATERESOURCEPREVIEWURLREQUEST']._serialized_start=4180
+  _globals['_CREATERESOURCEPREVIEWURLREQUEST']._serialized_end=4333
+  _globals['_CREATERESOURCEPREVIEWURLRESPONSE']._serialized_start=4335
+  _globals['_CREATERESOURCEPREVIEWURLRESPONSE']._serialized_end=4387
+  _globals['_RESOURCESERVICE']._serialized_start=4820
+  _globals['_RESOURCESERVICE']._serialized_end=6016
+# @@protoc_insertion_point(module_scope)

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic_generated/ingest/v2/resource_pb2.pyi

@@ -17,11 +17,26 @@ class ResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     RESOURCE_TYPE_DIMENSION_TABLE: _ClassVar[ResourceType]
     RESOURCE_TYPE_FACT_TABLE: _ClassVar[ResourceType]
     RESOURCE_TYPE_PDF_DOCUMENT: _ClassVar[ResourceType]
+
+class ResourceOriginType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    RESOURCE_ORIGIN_TYPE_UNSPECIFIED: _ClassVar[ResourceOriginType]
+    RESOURCE_ORIGIN_TYPE_FILE_UPLOAD: _ClassVar[ResourceOriginType]
+    RESOURCE_ORIGIN_TYPE_WEB_CRAWL: _ClassVar[ResourceOriginType]
+    RESOURCE_ORIGIN_TYPE_S3_CONNECTOR: _ClassVar[ResourceOriginType]
+    RESOURCE_ORIGIN_TYPE_GCS_CONNECTOR: _ClassVar[ResourceOriginType]
+    RESOURCE_ORIGIN_TYPE_AZURE_CONNECTOR: _ClassVar[ResourceOriginType]
 RESOURCE_TYPE_UNSPECIFIED: ResourceType
 RESOURCE_TYPE_TABLE: ResourceType
 RESOURCE_TYPE_DIMENSION_TABLE: ResourceType
 RESOURCE_TYPE_FACT_TABLE: ResourceType
 RESOURCE_TYPE_PDF_DOCUMENT: ResourceType
+RESOURCE_ORIGIN_TYPE_UNSPECIFIED: ResourceOriginType
+RESOURCE_ORIGIN_TYPE_FILE_UPLOAD: ResourceOriginType
+RESOURCE_ORIGIN_TYPE_WEB_CRAWL: ResourceOriginType
+RESOURCE_ORIGIN_TYPE_S3_CONNECTOR: ResourceOriginType
+RESOURCE_ORIGIN_TYPE_GCS_CONNECTOR: ResourceOriginType
+RESOURCE_ORIGIN_TYPE_AZURE_CONNECTOR: ResourceOriginType
 
 class ResourceMetadata(_message.Message):
     __slots__ = ("name", "mime_type", "room_id", "original_path", "description", "pipeline_id", "type_hint", "size")
@@ -44,7 +59,7 @@ class ResourceMetadata(_message.Message):
     def __init__(self, name: _Optional[str] = ..., mime_type: _Optional[str] = ..., room_id: _Optional[str] = ..., original_path: _Optional[str] = ..., description: _Optional[str] = ..., pipeline_id: _Optional[str] = ..., type_hint: _Optional[_Union[ResourceType, str]] = ..., size: _Optional[int] = ...) -> None: ...
 
 class ResourceEntry(_message.Message):
-    __slots__ = ("id", "name", "mime_type", "md5", "room_id", "size", "original_path", "description", "pipeline_id", "referenced_by_source_ids", "recent_events")
+    __slots__ = ("id", "name", "mime_type", "md5", "room_id", "size", "original_path", "description", "origin_type", "pipeline_id", "url_depth", "url_width", "referenced_by_source_ids", "recent_events")
     ID_FIELD_NUMBER: _ClassVar[int]
     NAME_FIELD_NUMBER: _ClassVar[int]
     MIME_TYPE_FIELD_NUMBER: _ClassVar[int]
@@ -53,7 +68,10 @@ class ResourceEntry(_message.Message):
     SIZE_FIELD_NUMBER: _ClassVar[int]
     ORIGINAL_PATH_FIELD_NUMBER: _ClassVar[int]
     DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
+    ORIGIN_TYPE_FIELD_NUMBER: _ClassVar[int]
     PIPELINE_ID_FIELD_NUMBER: _ClassVar[int]
+    URL_DEPTH_FIELD_NUMBER: _ClassVar[int]
+    URL_WIDTH_FIELD_NUMBER: _ClassVar[int]
     REFERENCED_BY_SOURCE_IDS_FIELD_NUMBER: _ClassVar[int]
     RECENT_EVENTS_FIELD_NUMBER: _ClassVar[int]
     id: str
@@ -64,10 +82,13 @@ class ResourceEntry(_message.Message):
     size: int
     original_path: str
     description: str
+    origin_type: ResourceOriginType
     pipeline_id: str
+    url_depth: int
+    url_width: int
     referenced_by_source_ids: _containers.RepeatedScalarFieldContainer[str]
     recent_events: _containers.RepeatedCompositeFieldContainer[_event_pb2.Event]
-    def __init__(self, id: _Optional[str] = ..., name: _Optional[str] = ..., mime_type: _Optional[str] = ..., md5: _Optional[str] = ..., room_id: _Optional[str] = ..., size: _Optional[int] = ..., original_path: _Optional[str] = ..., description: _Optional[str] = ..., pipeline_id: _Optional[str] = ..., referenced_by_source_ids: _Optional[_Iterable[str]] = ..., recent_events: _Optional[_Iterable[_Union[_event_pb2.Event, _Mapping]]] = ...) -> None: ...
+    def __init__(self, id: _Optional[str] = ..., name: _Optional[str] = ..., mime_type: _Optional[str] = ..., md5: _Optional[str] = ..., room_id: _Optional[str] = ..., size: _Optional[int] = ..., original_path: _Optional[str] = ..., description: _Optional[str] = ..., origin_type: _Optional[_Union[ResourceOriginType, str]] = ..., pipeline_id: _Optional[str] = ..., url_depth: _Optional[int] = ..., url_width: _Optional[int] = ..., referenced_by_source_ids: _Optional[_Iterable[str]] = ..., recent_events: _Optional[_Iterable[_Union[_event_pb2.Event, _Mapping]]] = ...) -> None: ...
 
 class CreateUploadURLRequest(_message.Message):
     __slots__ = ("metadata", "origin")
@@ -105,6 +126,26 @@ class FinalizeUploadURLResponse(_message.Message):
     entry: ResourceEntry
     def __init__(self, entry: _Optional[_Union[ResourceEntry, _Mapping]] = ...) -> None: ...
 
+class FetchAndFinalizeExternalURLRequest(_message.Message):
+    __slots__ = ("room_id", "pipeline_id", "original_path", "url_depth", "url_width")
+    ROOM_ID_FIELD_NUMBER: _ClassVar[int]
+    PIPELINE_ID_FIELD_NUMBER: _ClassVar[int]
+    ORIGINAL_PATH_FIELD_NUMBER: _ClassVar[int]
+    URL_DEPTH_FIELD_NUMBER: _ClassVar[int]
+    URL_WIDTH_FIELD_NUMBER: _ClassVar[int]
+    room_id: str
+    pipeline_id: str
+    original_path: str
+    url_depth: int
+    url_width: int
+    def __init__(self, room_id: _Optional[str] = ..., pipeline_id: _Optional[str] = ..., original_path: _Optional[str] = ..., url_depth: _Optional[int] = ..., url_width: _Optional[int] = ...) -> None: ...
+
+class FetchAndFinalizeExternalURLResponse(_message.Message):
+    __slots__ = ("entry",)
+    ENTRY_FIELD_NUMBER: _ClassVar[int]
+    entry: ResourceEntry
+    def __init__(self, entry: _Optional[_Union[ResourceEntry, _Mapping]] = ...) -> None: ...
+
 class DeleteResourceRequest(_message.Message):
     __slots__ = ("id",)
     ID_FIELD_NUMBER: _ClassVar[int]

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic_generated/ingest/v2/resource_pb2_grpc.py

@@ -24,6 +24,11 @@ class ResourceServiceStub(object):
                 request_serializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FinalizeUploadURLRequest.SerializeToString,
                 response_deserializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FinalizeUploadURLResponse.FromString,
                 )
+        self.FetchAndFinalizeExternalURL = channel.unary_unary(
+                '/corvic.ingest.v2.ResourceService/FetchAndFinalizeExternalURL',
+                request_serializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FetchAndFinalizeExternalURLRequest.SerializeToString,
+                response_deserializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FetchAndFinalizeExternalURLResponse.FromString,
+                )
         self.DeleteResource = channel.unary_unary(
                 '/corvic.ingest.v2.ResourceService/DeleteResource',
                 request_serializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.DeleteResourceRequest.SerializeToString,
@@ -94,6 +99,12 @@ class ResourceServiceServicer(object):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def FetchAndFinalizeExternalURL(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
     def DeleteResource(self, request, context):
         """Missing associated documentation comment in .proto file."""
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
@@ -159,6 +170,11 @@ def add_ResourceServiceServicer_to_server(servicer, server):
                     request_deserializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FinalizeUploadURLRequest.FromString,
                     response_serializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FinalizeUploadURLResponse.SerializeToString,
             ),
+            'FetchAndFinalizeExternalURL': grpc.unary_unary_rpc_method_handler(
+                    servicer.FetchAndFinalizeExternalURL,
+                    request_deserializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FetchAndFinalizeExternalURLRequest.FromString,
+                    response_serializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.FetchAndFinalizeExternalURLResponse.SerializeToString,
+            ),
             'DeleteResource': grpc.unary_unary_rpc_method_handler(
                     servicer.DeleteResource,
                     request_deserializer=corvic_dot_ingest_dot_v2_dot_resource__pb2.DeleteResourceRequest.FromString,
@@ -238,6 +254,23 @@ class ResourceService(object):
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
+    @staticmethod
+    def FetchAndFinalizeExternalURL(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/corvic.ingest.v2.ResourceService/FetchAndFinalizeExternalURL',
+            corvic_dot_ingest_dot_v2_dot_resource__pb2.FetchAndFinalizeExternalURLRequest.SerializeToString,
+            corvic_dot_ingest_dot_v2_dot_resource__pb2.FetchAndFinalizeExternalURLResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
     @staticmethod
     def DeleteResource(request,
             target,

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic_generated/ingest/v2/resource_pb2_grpc.pyi

@@ -47,6 +47,10 @@ class ResourceServiceStub:
     This is an alternative to UploadResource when uploads are large as clients
     directly write to the bucket room location.
     """
+    FetchAndFinalizeExternalURL: grpc.UnaryUnaryMultiCallable[
+        corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLRequest,
+        corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLResponse,
+    ]
     DeleteResource: grpc.UnaryUnaryMultiCallable[
         corvic_generated.ingest.v2.resource_pb2.DeleteResourceRequest,
         corvic_generated.ingest.v2.resource_pb2.DeleteResourceResponse,
@@ -116,6 +120,10 @@ class ResourceServiceAsyncStub:
     This is an alternative to UploadResource when uploads are large as clients
     directly write to the bucket room location.
     """
+    FetchAndFinalizeExternalURL: grpc.aio.UnaryUnaryMultiCallable[
+        corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLRequest,
+        corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLResponse,
+    ]
     DeleteResource: grpc.aio.UnaryUnaryMultiCallable[
         corvic_generated.ingest.v2.resource_pb2.DeleteResourceRequest,
         corvic_generated.ingest.v2.resource_pb2.DeleteResourceResponse,
@@ -190,6 +198,12 @@ class ResourceServiceServicer(metaclass=abc.ABCMeta):
         directly write to the bucket room location.
         """
     @abc.abstractmethod
+    def FetchAndFinalizeExternalURL(
+        self,
+        request: corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLRequest,
+        context: _ServicerContext,
+    ) -> typing.Union[corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLResponse, collections.abc.Awaitable[corvic_generated.ingest.v2.resource_pb2.FetchAndFinalizeExternalURLResponse]]: ...
+    @abc.abstractmethod
     def DeleteResource(
         self,
         request: corvic_generated.ingest.v2.resource_pb2.DeleteResourceRequest,

corvic_engine-0.3.0rc96/python/corvic_generated/model/v1alpha/models_pb2.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: corvic/model/v1alpha/models.proto
+# Protobuf Python Version: 4.25.8
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from corvic_generated.orm.v1 import data_connector_pb2 as corvic_dot_orm_dot_v1_dot_data__connector__pb2
+from corvic_generated.orm.v1 import feature_view_pb2 as corvic_dot_orm_dot_v1_dot_feature__view__pb2
+from corvic_generated.orm.v1 import pipeline_pb2 as corvic_dot_orm_dot_v1_dot_pipeline__pb2
+from corvic_generated.orm.v1 import space_pb2 as corvic_dot_orm_dot_v1_dot_space__pb2
+from corvic_generated.orm.v1 import table_pb2 as corvic_dot_orm_dot_v1_dot_table__pb2
+from corvic_generated.status.v1 import event_pb2 as corvic_dot_status_dot_v1_dot_event__pb2
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n!corvic/model/v1alpha/models.proto\x12\x14\x63orvic.model.v1alpha\x1a\"corvic/orm/v1/data_connector.proto\x1a corvic/orm/v1/feature_view.proto\x1a\x1c\x63orvic/orm/v1/pipeline.proto\x1a\x19\x63orvic/orm/v1/space.proto\x1a\x19\x63orvic/orm/v1/table.proto\x1a\x1c\x63orvic/status/v1/event.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x90\x01\n\x04Room\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x15\n\x06org_id\x18\x03 \x01(\tR\x05orgId\x12>\n\ncreated_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x42\r\n\x0b_created_at\"\xcb\x05\n\x08Resource\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12\x1b\n\tmime_type\x18\x04 \x01(\tR\x08mimeType\x12\x10\n\x03url\x18\x05 \x01(\tR\x03url\x12\x12\n\x04size\x18\x06 \x01(\x04R\x04size\x12\x10\n\x03md5\x18\x07 \x01(\tR\x03md5\x12#\n\roriginal_path\x18\x08 \x01(\tR\x0coriginalPath\x12\x12\n\x04\x65tag\x18\x11 \x01(\tR\x04\x65tag\x12\x42\n\x0borigin_type\x18\x13 \x01(\x0e\x32!.corvic.orm.v1.DataConnectionTypeR\noriginType\x12Z\n\x19original_modified_at_time\x18\x12 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x16originalModifiedAtTime\x88\x01\x01\x12\x17\n\x07room_id\x18\t \x01(\tR\x06roomId\x12\x15\n\x06org_id\x18\x0b \x01(\tR\x05orgId\x12\x1f\n\x0bpipeline_id\x18\r \x01(\tR\npipelineId\x12.\n\x13pipeline_input_name\x18\x0f \x01(\tR\x11pipelineInputName\x12<\n\rrecent_events\x18\x0e \x03(\x0b\x32\x17.corvic.status.v1.EventR\x0crecentEvents\x12\x1f\n\x0bis_terminal\x18\x10 \x01(\x08R\nisTerminal\x12>\n\ncreated_at\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x01R\tcreatedAt\x88\x01\x01\x42\x1c\n\x1a_original_modified_at_timeB\r\n\x0b_created_at\"\xfc\x02\n\x06Source\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x43\n\x0etable_op_graph\x18\x03 \x01(\x0b\x32\x1d.corvic.orm.v1.TableComputeOpR\x0ctableOpGraph\x12\x17\n\x07room_id\x18\x04 \x01(\tR\x06roomId\x12\x15\n\x06org_id\x18\x06 \x01(\tR\x05orgId\x12\x1f\n\x0bpipeline_id\x18\x08 \x01(\tR\npipelineId\x12>\n\ncreated_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x12Q\n\x13prop_table_op_graph\x18\t \x01(\x0b\x32\x1d.corvic.orm.v1.TableComputeOpH\x01R\x10propTableOpGraph\x88\x01\x01\x42\r\n\x0b_created_atB\x16\n\x14_prop_table_op_graph\"\xe9\x03\n\x08Pipeline\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\t \x01(\tR\x0b\x64\x65scription\x12\x17\n\x07room_id\x18\x03 \x01(\tR\x06roomId\x12X\n\x0esource_outputs\x18\x05 \x03(\x0b\x32\x31.corvic.model.v1alpha.Pipeline.SourceOutputsEntryR\rsourceOutputs\x12^\n\x17pipeline_transformation\x18\x06 \x01(\x0b\x32%.corvic.orm.v1.PipelineTransformationR\x16pipelineTransformation\x12\x15\n\x06org_id\x18\x07 \x01(\tR\x05orgId\x12>\n\ncreated_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x1a^\n\x12SourceOutputsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32\x1c.corvic.model.v1alpha.SourceR\x05value:\x02\x38\x01\x42\r\n\x0b_created_at\"\xca\x02\n\x11\x46\x65\x61tureViewSource\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x34\n\x06source\x18\x02 \x01(\x0b\x32\x1c.corvic.model.v1alpha.SourceR\x06source\x12\x43\n\x0etable_op_graph\x18\x03 \x01(\x0b\x32\x1d.corvic.orm.v1.TableComputeOpR\x0ctableOpGraph\x12+\n\x11\x64rop_disconnected\x18\x04 \x01(\x08R\x10\x64ropDisconnected\x12\x15\n\x06org_id\x18\x05 \x01(\tR\x05orgId\x12>\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x12\x17\n\x07room_id\x18\x07 \x01(\tR\x06roomIdB\r\n\x0b_created_at\"\x9c\x03\n\x0b\x46\x65\x61tureView\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12\x17\n\x07room_id\x18\x04 \x01(\tR\x06roomId\x12P\n\x13\x66\x65\x61ture_view_output\x18\x05 \x01(\x0b\x32 .corvic.orm.v1.FeatureViewOutputR\x11\x66\x65\x61tureViewOutput\x12Y\n\x14\x66\x65\x61ture_view_sources\x18\x06 \x03(\x0b\x32\'.corvic.model.v1alpha.FeatureViewSourceR\x12\x66\x65\x61tureViewSources\x12\x1b\n\tspace_ids\x18\x07 \x03(\tR\x08spaceIds\x12\x15\n\x06org_id\x18\x08 \x01(\tR\x05orgId\x12>\n\ncreated_at\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x42\r\n\x0b_created_at\"\xfa\x02\n\x05Space\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12\x17\n\x07room_id\x18\x04 \x01(\tR\x06roomId\x12I\n\x10space_parameters\x18\x05 \x01(\x0b\x32\x1e.corvic.orm.v1.SpaceParametersR\x0fspaceParameters\x12\x44\n\x0c\x66\x65\x61ture_view\x18\x06 \x01(\x0b\x32!.corvic.model.v1alpha.FeatureViewR\x0b\x66\x65\x61tureView\x12\x1b\n\tauto_sync\x18\t \x01(\x08R\x08\x61utoSync\x12\x15\n\x06org_id\x18\x07 \x01(\tR\x05orgId\x12>\n\ncreated_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x42\r\n\x0b_created_at\"\x87\x04\n\rDataConnector\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x15\n\x06org_id\x18\x03 \x01(\tR\x05orgId\x12\x46\n\nparameters\x18\x04 \x01(\x0b\x32&.corvic.orm.v1.DataConnectorParametersR\nparameters\x12\x1b\n\tbase_path\x18\x05 \x01(\tR\x08\x62\x61sePath\x12\x1d\n\nsecret_key\x18\x06 \x01(\tR\tsecretKey\x12Q\n\x14last_validation_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x12lastValidationTime\x88\x01\x01\x12]\n\x1alast_successful_validation\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x01R\x18lastSuccessfulValidation\x88\x01\x01\x12>\n\ncreated_at\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02R\tcreatedAt\x88\x01\x01\x42\x17\n\x15_last_validation_timeB\x1d\n\x1b_last_successful_validationB\r\n\x0b_created_at\"\xdd\x03\n\x16PipelineDataConnection\x12\x1f\n\x0bpipeline_id\x18\x01 \x01(\tR\npipelineId\x12J\n\x0e\x64\x61ta_connector\x18\x02 \x01(\x0b\x32#.corvic.model.v1alpha.DataConnectorR\rdataConnector\x12\x17\n\x07room_id\x18\x03 \x01(\tR\x06roomId\x12\x15\n\x06org_id\x18\x04 \x01(\tR\x05orgId\x12\x16\n\x06prefix\x18\x05 \x01(\tR\x06prefix\x12\x12\n\x04glob\x18\x06 \x01(\tR\x04glob\x12J\n\x13last_ingestion_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x11lastIngestionTime\x12_\n\x1elast_successful_ingestion_time\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x1blastSuccessfulIngestionTime\x12>\n\ncreated_at\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\tcreatedAt\x88\x01\x01\x42\r\n\x0b_created_atb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'corvic.model.v1alpha.models_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  _globals['_PIPELINE_SOURCEOUTPUTSENTRY']._options = None
+  _globals['_PIPELINE_SOURCEOUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_ROOM']._serialized_start=277
+  _globals['_ROOM']._serialized_end=421
+  _globals['_RESOURCE']._serialized_start=424
+  _globals['_RESOURCE']._serialized_end=1139
+  _globals['_SOURCE']._serialized_start=1142
+  _globals['_SOURCE']._serialized_end=1522
+  _globals['_PIPELINE']._serialized_start=1525
+  _globals['_PIPELINE']._serialized_end=2014
+  _globals['_PIPELINE_SOURCEOUTPUTSENTRY']._serialized_start=1905
+  _globals['_PIPELINE_SOURCEOUTPUTSENTRY']._serialized_end=1999
+  _globals['_FEATUREVIEWSOURCE']._serialized_start=2017
+  _globals['_FEATUREVIEWSOURCE']._serialized_end=2347
+  _globals['_FEATUREVIEW']._serialized_start=2350
+  _globals['_FEATUREVIEW']._serialized_end=2762
+  _globals['_SPACE']._serialized_start=2765
+  _globals['_SPACE']._serialized_end=3143
+  _globals['_DATACONNECTOR']._serialized_start=3146
+  _globals['_DATACONNECTOR']._serialized_end=3665
+  _globals['_PIPELINEDATACONNECTION']._serialized_start=3668
+  _globals['_PIPELINEDATACONNECTION']._serialized_end=4145
+# @@protoc_insertion_point(module_scope)

{corvic_engine-0.3.0rc94 → corvic_engine-0.3.0rc96}/python/corvic_generated/model/v1alpha/models_pb2.pyi

@@ -25,7 +25,7 @@ class Room(_message.Message):
     def __init__(self, id: _Optional[str] = ..., name: _Optional[str] = ..., org_id: _Optional[str] = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
 
 class Resource(_message.Message):
-    __slots__ = ("id", "name", "description", "mime_type", "url", "size", "md5", "original_path", "etag", "original_modified_at_time", "room_id", "org_id", "pipeline_id", "pipeline_input_name", "recent_events", "is_terminal", "created_at")
+    __slots__ = ("id", "name", "description", "mime_type", "url", "size", "md5", "original_path", "etag", "origin_type", "original_modified_at_time", "room_id", "org_id", "pipeline_id", "pipeline_input_name", "recent_events", "is_terminal", "created_at")
     ID_FIELD_NUMBER: _ClassVar[int]
     NAME_FIELD_NUMBER: _ClassVar[int]
     DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
@@ -35,6 +35,7 @@ class Resource(_message.Message):
     MD5_FIELD_NUMBER: _ClassVar[int]
     ORIGINAL_PATH_FIELD_NUMBER: _ClassVar[int]
     ETAG_FIELD_NUMBER: _ClassVar[int]
+    ORIGIN_TYPE_FIELD_NUMBER: _ClassVar[int]
     ORIGINAL_MODIFIED_AT_TIME_FIELD_NUMBER: _ClassVar[int]
     ROOM_ID_FIELD_NUMBER: _ClassVar[int]
     ORG_ID_FIELD_NUMBER: _ClassVar[int]
@@ -52,6 +53,7 @@ class Resource(_message.Message):
     md5: str
     original_path: str
     etag: str
+    origin_type: _data_connector_pb2.DataConnectionType
     original_modified_at_time: _timestamp_pb2.Timestamp
     room_id: str
     org_id: str
@@ -60,7 +62,7 @@ class Resource(_message.Message):
     recent_events: _containers.RepeatedCompositeFieldContainer[_event_pb2.Event]
     is_terminal: bool
     created_at: _timestamp_pb2.Timestamp
-    def __init__(self, id: _Optional[str] = ..., name: _Optional[str] = ..., description: _Optional[str] = ..., mime_type: _Optional[str] = ..., url: _Optional[str] = ..., size: _Optional[int] = ..., md5: _Optional[str] = ..., original_path: _Optional[str] = ..., etag: _Optional[str] = ..., original_modified_at_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., room_id: _Optional[str] = ..., org_id: _Optional[str] = ..., pipeline_id: _Optional[str] = ..., pipeline_input_name: _Optional[str] = ..., recent_events: _Optional[_Iterable[_Union[_event_pb2.Event, _Mapping]]] = ..., is_terminal: bool = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+    def __init__(self, id: _Optional[str] = ..., name: _Optional[str] = ..., description: _Optional[str] = ..., mime_type: _Optional[str] = ..., url: _Optional[str] = ..., size: _Optional[int] = ..., md5: _Optional[str] = ..., original_path: _Optional[str] = ..., etag: _Optional[str] = ..., origin_type: _Optional[_Union[_data_connector_pb2.DataConnectionType, str]] = ..., original_modified_at_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., room_id: _Optional[str] = ..., org_id: _Optional[str] = ..., pipeline_id: _Optional[str] = ..., pipeline_input_name: _Optional[str] = ..., recent_events: _Optional[_Iterable[_Union[_event_pb2.Event, _Mapping]]] = ..., is_terminal: bool = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
 
 class Source(_message.Message):
     __slots__ = ("id", "name", "table_op_graph", "room_id", "org_id", "pipeline_id", "created_at", "prop_table_op_graph")

corvic_engine-0.3.0rc96/python/corvic_generated/orm/v1/data_connector_pb2.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: corvic/orm/v1/data_connector.proto
+# Protobuf Python Version: 4.25.8
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\"corvic/orm/v1/data_connector.proto\x12\rcorvic.orm.v1\"K\n\rS3Credentials\x12\"\n\raccess_key_id\x18\x01 \x01(\tR\x0b\x61\x63\x63\x65ssKeyId\x12\x16\n\x06region\x18\x02 \x01(\tR\x06region\"B\n\x0eGCSCredentials\x12\x30\n\x14service_account_json\x18\x01 \x01(\tR\x12serviceAccountJson\"x\n\x14\x41zureBlobCredentials\x12\x1f\n\x0b\x61\x63\x63ount_url\x18\x01 \x01(\tR\naccountUrl\x12?\n\x0bsecret_type\x18\x02 \x01(\x0e\x32\x1e.corvic.orm.v1.AzureSecretTypeR\nsecretType\"\x96\x02\n\x17\x44\x61taConnectorParameters\x12\x45\n\x0es3_credentials\x18\x01 \x01(\x0b\x32\x1c.corvic.orm.v1.S3CredentialsH\x00R\rs3Credentials\x12H\n\x0fgcs_credentials\x18\x02 \x01(\x0b\x32\x1d.corvic.orm.v1.GCSCredentialsH\x00R\x0egcsCredentials\x12[\n\x16\x61zure_blob_credentials\x18\x03 \x01(\x0b\x32#.corvic.orm.v1.AzureBlobCredentialsH\x00R\x14\x61zureBlobCredentialsB\r\n\x0b\x63redentials\"K\n\x12\x44\x61taConnectionInfo\x12\x35\n\x04type\x18\x01 \x01(\x0e\x32!.corvic.orm.v1.DataConnectionTypeR\x04type*\x8d\x01\n\x0f\x41zureSecretType\x12!\n\x1d\x41ZURE_SECRET_TYPE_UNSPECIFIED\x10\x00\x12(\n$AZURE_SECRET_TYPE_ACCOUNT_ACCESS_KEY\x10\x01\x12-\n)AZURE_SECRET_TYPE_SHARED_ACCESS_SIGNATURE\x10\x02*\xfd\x01\n\x12\x44\x61taConnectionType\x12$\n DATA_CONNECTION_TYPE_UNSPECIFIED\x10\x00\x12$\n DATA_CONNECTION_TYPE_FILE_UPLOAD\x10\x01\x12\"\n\x1e\x44\x41TA_CONNECTION_TYPE_WEB_CRAWL\x10\x02\x12%\n!DATA_CONNECTION_TYPE_S3_CONNECTOR\x10\x03\x12&\n\"DATA_CONNECTION_TYPE_GCS_CONNECTOR\x10\x04\x12(\n$DATA_CONNECTION_TYPE_AZURE_CONNECTOR\x10\x05\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'corvic.orm.v1.data_connector_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  _globals['_AZURESECRETTYPE']._serialized_start=679
+  _globals['_AZURESECRETTYPE']._serialized_end=820
+  _globals['_DATACONNECTIONTYPE']._serialized_start=823
+  _globals['_DATACONNECTIONTYPE']._serialized_end=1076
+  _globals['_S3CREDENTIALS']._serialized_start=53
+  _globals['_S3CREDENTIALS']._serialized_end=128
+  _globals['_GCSCREDENTIALS']._serialized_start=130
+  _globals['_GCSCREDENTIALS']._serialized_end=196
+  _globals['_AZUREBLOBCREDENTIALS']._serialized_start=198
+  _globals['_AZUREBLOBCREDENTIALS']._serialized_end=318
+  _globals['_DATACONNECTORPARAMETERS']._serialized_start=321
+  _globals['_DATACONNECTORPARAMETERS']._serialized_end=599
+  _globals['_DATACONNECTIONINFO']._serialized_start=601
+  _globals['_DATACONNECTIONINFO']._serialized_end=676
+# @@protoc_insertion_point(module_scope)