corvic-engine 0.3.0rc49__tar.gz → 0.3.0rc52__tar.gz

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/Cargo.lock

@@ -22,7 +22,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "corvic-engine"
-version = "0.3.0-rc.49"
+version = "0.3.0-rc.52"
 dependencies = [
  "numpy",
  "pyo3",
@@ -63,7 +63,7 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "engine-walk"
-version = "0.3.0-rc.49"
+version = "0.3.0-rc.52"
 dependencies = [
  "version_check",
 ]

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["python"]
 
 [workspace.package]
-version = "0.3.0-rc.49"
+version = "0.3.0-rc.52"
 edition = "2021"
 publish = false
 

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corvic-engine
-Version: 0.3.0rc49
+Version: 0.3.0rc52
 Classifier: Environment :: Console
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: Implementation :: CPython

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/pyproject.toml

@@ -20,7 +20,7 @@ authors = [
 ]
 readme = "README.md"
 description = "Seamless embedding generation and retrieval."
-version = "0.3.0-rc.49" # sync this with tool.poetry.version below
+version = "0.3.0-rc.52" # sync this with tool.poetry.version below
 
 # TODO(ddn): Pull dependencies from tool.poetry.dependencies. We use maturin as
 # the build system but poetry for the development environment.
@@ -74,7 +74,7 @@ packages = [
 ]
 description = ""
 authors = []
-version = "0.3.0-rc.49"
+version = "0.3.0-rc.52"
 
 [tool.poetry.scripts]
 check = "corvic_check.cli:main"
@@ -167,7 +167,7 @@ torch = [
 ]
 torch-geometric = "^2.5.3"
 deepgnn-torch = "^0.1.60"
-transformers = "^4.45.0"
+transformers = "^4.48.0"
 umap-learn = "^0.5.5"
 pillow = "10.4.0"
 

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/python/corvic/model/_resource.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import copy
 import datetime
+import uuid
 from collections.abc import Iterable, Sequence
 from typing import TypeAlias
 
@@ -127,6 +128,7 @@ class Resource(BaseModel[ResourceID, models_pb2.Resource, orm.Resource]):
         cls,
         *,
         room_id: RoomID | None = None,
+        pipeline_id: PipelineID | None = None,
         limit: int | None = None,
         created_before: datetime.datetime | None = None,
         client: system.Client | None = None,
@@ -136,13 +138,19 @@ class Resource(BaseModel[ResourceID, models_pb2.Resource, orm.Resource]):
     ) -> Ok[list[Resource]] | NotFoundError | InvalidArgumentError:
         """List resources."""
         client = client or Defaults.get_default_client()
-        additional_query_transform = None
 
-        def url_filter(query: sa.Select[tuple[orm.Resource]]):
-            return query.filter_by(url=url)
-
-        if url is not None:
-            additional_query_transform = url_filter
+        def query_transform(query: sa.Select[tuple[orm.Resource]]):
+            if url:
+                query = query.where(orm.Resource.url == url)
+            if pipeline_id:
+                query = query.where(
+                    orm.Resource.id.in_(
+                        sa.select(orm.PipelineInput.resource_id).where(
+                            orm.PipelineInput.pipeline_id == pipeline_id
+                        )
+                    )
+                )
+            return query
 
         match cls.list_as_proto(
             client,
@@ -151,7 +159,7 @@ class Resource(BaseModel[ResourceID, models_pb2.Resource, orm.Resource]):
             created_before=created_before,
             ids=ids,
             existing_session=existing_session,
-            additional_query_transform=additional_query_transform,
+            additional_query_transform=query_transform,
         ):
             case NotFoundError() | InvalidArgumentError() as err:
                 return err
@@ -219,7 +227,9 @@ class Resource(BaseModel[ResourceID, models_pb2.Resource, orm.Resource]):
         client = client or Defaults.get_default_client()
         room_id = room_id or Defaults.get_default_room_id(client)
 
-        blob = client.storage_manager.make_tabular_blob(room_id, "anonymous_tables")
+        blob = client.storage_manager.make_tabular_blob(
+            room_id, f"polars_dataframe/{uuid.uuid4()}"
+        )
         with blob.open(mode="wb") as stream:
             data_frame.write_parquet(stream)
 

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/python/corvic/system/storage.py

@@ -144,7 +144,7 @@ class StorageManager:
 
     def make_tabular_blob(self, room_id: orm.RoomID, suffix: str | None = None) -> Blob:
         if suffix:
-            name = f"{self._render_room_id(room_id)}/{uuid.uuid4()}/{suffix}"
+            name = f"{self._render_room_id(room_id)}/{suffix}"
         else:
             name = f"{self._render_room_id(room_id)}/{uuid.uuid4()}"
         return self.bucket.blob(f"{self.tabular_prefix}/{name}")
@@ -153,14 +153,14 @@ class StorageManager:
         self, room_id: orm.RoomID, suffix: str | None = None
     ) -> Blob:
         if suffix:
-            name = f"{self._render_room_id(room_id)}/{uuid.uuid4()}/{suffix}"
+            name = f"{self._render_room_id(room_id)}/{suffix}"
         else:
             name = f"{self._render_room_id(room_id)}/{uuid.uuid4()}"
         return self.bucket.blob(f"{self.unstructured_prefix}/{name}")
 
     def make_vector_blob(self, room_id: orm.RoomID, suffix: str | None = None) -> Blob:
         if suffix:
-            name = f"{self._render_room_id(room_id)}/{uuid.uuid4()}/{suffix}"
+            name = f"{self._render_room_id(room_id)}/{suffix}"
         else:
             name = f"{self._render_room_id(room_id)}/{uuid.uuid4()}"
         return self.bucket.blob(f"{self.vector_prefix}/{name}")
@@ -173,3 +173,12 @@ class StorageManager:
 
     def get_vector_blob_from_blob_name(self, blob_name: str) -> Blob:
         return self.bucket.blob(f"{self.vector_prefix}/{blob_name}")
+
+    def tabular_prefix_for_room_id(self, room_id: orm.RoomID) -> str:
+        return f"{self.tabular_prefix}/{self._render_room_id(room_id)}"
+
+    def unstructured_prefix_for_room_id(self, room_id: orm.RoomID) -> str:
+        return f"{self.unstructured_prefix}/{self._render_room_id(room_id)}"
+
+    def vector_prefix_for_room_id(self, room_id: orm.RoomID) -> str:
+        return f"{self.vector_prefix}/{self._render_room_id(room_id)}"

corvic_engine-0.3.0rc52/python/corvic/system_sqlite/__init__.py

@@ -0,0 +1,19 @@
+"""Single-machine, sqlite-backed implementation of corvic.system."""
+
+from corvic.system_sqlite.client import (
+    TABULAR_PREFIX,
+    UNSTRUCTURED_PREFIX,
+    VECTOR_PREFIX,
+    Client,
+    FSBlobClient,
+    RDBMSBlobClient,
+)
+
+__all__ = [
+    "Client",
+    "FSBlobClient",
+    "RDBMSBlobClient",
+    "TABULAR_PREFIX",
+    "UNSTRUCTURED_PREFIX",
+    "VECTOR_PREFIX",
+]

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/python/corvic/system_sqlite/client.py

@@ -20,6 +20,10 @@ from corvic.system_sqlite.fs_blob_store import FSBlobClient
 from corvic.system_sqlite.rdbms_blob_store import RDBMSBlobClient
 from corvic.system_sqlite.staging import DuckDBStaging
 
+UNSTRUCTURED_PREFIX = "unstructured_data"
+TABULAR_PREFIX = "tabular_data"
+VECTOR_PREFIX = "vectors"
+
 
 @contextlib.contextmanager
 def _context_requester_org_id_is(value: str | corvic.orm.OrgID) -> Iterator[None]:
@@ -133,9 +137,9 @@ class Client(corvic.system.Client):
         self._storage_manager = corvic.system.StorageManager(
             self._blob_client,
             bucket_name=bucket.name,
-            unstructured_prefix="unstructured_data",
-            tabular_prefix="tabular_data",
-            vector_prefix="vectors",
+            unstructured_prefix=UNSTRUCTURED_PREFIX,
+            tabular_prefix=TABULAR_PREFIX,
+            vector_prefix=VECTOR_PREFIX,
         )
         corvic.orm.Base.metadata.create_all(self._sa_engine)
         duck_db_conn = duckdb.connect(":memory:")

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/python/corvic/table/table.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import dataclasses
 import functools
+import uuid
 from collections.abc import Iterable, Mapping, Sequence
 from typing import (
     Any,
@@ -453,7 +454,8 @@ class Table:
                 TableComputeContext(
                     self.op_graph,
                     output_url_prefix=self.client.storage_manager.make_tabular_blob(
-                        room_id=room_id, suffix="anonymous_tables"
+                        room_id=room_id,
+                        suffix=f"anonymous_tables-{uuid.uuid4()}.parquet",
                     ).url,
                 )
             ],

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/python/corvic_generated/ingest/v2/resource_pb2.py

@@ -18,7 +18,7 @@ from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor
 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x63orvic/ingest/v2/resource.proto\x12\x10\x63orvic.ingest.v2\x1a\x1b\x62uf/validate/validate.proto\x1a\x1c\x63orvic/status/v1/event.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x89\x03\n\x10ResourceMetadata\x12\x1b\n\x04name\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\x12\x1b\n\tmime_type\x18\x02 \x01(\tR\x08mimeType\x12y\n\x07room_id\x18\x03 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x06roomId\x12#\n\roriginal_path\x18\x05 \x01(\tR\x0coriginalPath\x12 \n\x0b\x64\x65scription\x18\x06 \x01(\tR\x0b\x64\x65scription\x12\x1f\n\x0bpipeline_id\x18\x08 \x01(\tR\npipelineId\x12;\n\ttype_hint\x18\x04 \x01(\x0e\x32\x1e.corvic.ingest.v2.ResourceTypeR\x08typeHint\x12\x1b\n\x04size\x18\x07 \x01(\x03\x42\x07\xbaH\x04\"\x02 \x00R\x04size\"\xf7\x02\n\rResourceEntry\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x1b\n\x04name\x18\x02 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\x12\x1b\n\tmime_type\x18\x03 \x01(\tR\x08mimeType\x12\x10\n\x03md5\x18\x05 \x01(\tR\x03md5\x12\x17\n\x07room_id\x18\x06 \x01(\tR\x06roomId\x12\x12\n\x04size\x18\x07 \x01(\x04R\x04size\x12#\n\roriginal_path\x18\t \x01(\tR\x0coriginalPath\x12 \n\x0b\x64\x65scription\x18\x0b \x01(\tR\x0b\x64\x65scription\x12\x1f\n\x0bpipeline_id\x18\x0c \x01(\tR\npipelineId\x12\x37\n\x18referenced_by_source_ids\x18\n \x03(\tR\x15referencedBySourceIds\x12<\n\rrecent_events\x18\x08 \x03(\x0b\x32\x17.corvic.status.v1.EventR\x0crecentEvents\"p\n\x16\x43reateUploadURLRequest\x12>\n\x08metadata\x18\x01 \x01(\x0b\x32\".corvic.ingest.v2.ResourceMetadataR\x08metadata\x12\x16\n\x06origin\x18\x02 \x01(\tR\x06origin\"A\n\x17\x43reateUploadURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"s\n\x15UploadURLTokenPayload\x12>\n\x08metadata\x18\x01 \x01(\x0b\x32\".corvic.ingest.v2.ResourceMetadataR\x08metadata\x12\x1a\n\x08\x66ilename\x18\x02 \x01(\tR\x08\x66ilename\"0\n\x18\x46inalizeUploadURLRequest\x12\x14\n\x05token\x18\x01 \x01(\tR\x05token\"R\n\x19\x46inalizeUploadURLResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x89\x01\n\x15\x44\x65leteResourceRequest\x12p\n\x02id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"\x18\n\x16\x44\x65leteResourceResponse\"\x86\x01\n\x12GetResourceRequest\x12p\n\x02id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"L\n\x13GetResourceResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x94\x01\n\x14ListResourcesRequest\x12|\n\x07room_id\x18\x01 \x01(\tBc\xbaH`\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd0\x01\x01R\x06roomId\"N\n\x15ListResourcesResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x85\x01\n\x0cResourceList\x12u\n\x02id\x18\x01 \x03(\tBe\xbaHb\x92\x01_\"]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"\xd4\x01\n\x15WatchResourcesRequest\x12{\n\x07room_id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')H\x00R\x06roomId\x12\x32\n\x03ids\x18\x02 \x01(\x0b\x32\x1e.corvic.ingest.v2.ResourceListH\x00R\x03idsB\n\n\x08selector\"d\n\x16WatchResourcesResponse\x12J\n\x10updated_resource\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x0fupdatedResource\"\x94\x01\n#ListResourcesPaginatedCursorPayload\x12T\n\x19\x63reate_time_of_last_entry\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x15\x63reateTimeOfLastEntry\x12\x17\n\x07room_id\x18\x02 \x01(\tR\x06roomId\"\xdf\x01\n\x1dListResourcesPaginatedRequest\x12|\n\x07room_id\x18\x01 \x01(\tBc\xbaH`\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd0\x01\x01R\x06roomId\x12(\n\x10\x65ntries_per_page\x18\x02 \x01(\rR\x0e\x65ntriesPerPage\x12\x16\n\x06\x63ursor\x18\x03 \x01(\tR\x06\x63ursor\"o\n\x1eListResourcesPaginatedResponse\x12\x35\n\x05\x65ntry\x18\x01 \x03(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\x12\x16\n\x06\x63ursor\x18\x02 \x01(\tR\x06\x63ursor\"\x94\x01\n CreateResourceDownloadURLRequest\x12p\n\x02id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"5\n!CreateResourceDownloadURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url*\xab\x01\n\x0cResourceType\x12\x1d\n\x19RESOURCE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x13RESOURCE_TYPE_TABLE\x10\x01\x1a\x02\x08\x01\x12!\n\x1dRESOURCE_TYPE_DIMENSION_TABLE\x10\x02\x12\x1c\n\x18RESOURCE_TYPE_FACT_TABLE\x10\x03\x12\x1e\n\x1aRESOURCE_TYPE_PDF_DOCUMENT\x10\x04\x32\x97\x07\n\x0fResourceService\x12h\n\x0f\x43reateUploadURL\x12(.corvic.ingest.v2.CreateUploadURLRequest\x1a).corvic.ingest.v2.CreateUploadURLResponse\"\x00\x12q\n\x11\x46inalizeUploadURL\x12*.corvic.ingest.v2.FinalizeUploadURLRequest\x1a+.corvic.ingest.v2.FinalizeUploadURLResponse\"\x03\x90\x02\x02\x12\x65\n\x0e\x44\x65leteResource\x12\'.corvic.ingest.v2.DeleteResourceRequest\x1a(.corvic.ingest.v2.DeleteResourceResponse\"\x00\x12_\n\x0bGetResource\x12$.corvic.ingest.v2.GetResourceRequest\x1a%.corvic.ingest.v2.GetResourceResponse\"\x03\x90\x02\x01\x12g\n\rListResources\x12&.corvic.ingest.v2.ListResourcesRequest\x1a\'.corvic.ingest.v2.ListResourcesResponse\"\x03\x90\x02\x01\x30\x01\x12\x80\x01\n\x16ListResourcesPaginated\x12/.corvic.ingest.v2.ListResourcesPaginatedRequest\x1a\x30.corvic.ingest.v2.ListResourcesPaginatedResponse\"\x03\x90\x02\x01\x12j\n\x0eWatchResources\x12\'.corvic.ingest.v2.WatchResourcesRequest\x1a(.corvic.ingest.v2.WatchResourcesResponse\"\x03\x90\x02\x01\x30\x01\x12\x86\x01\n\x19\x43reateResourceDownloadURL\x12\x32.corvic.ingest.v2.CreateResourceDownloadURLRequest\x1a\x33.corvic.ingest.v2.CreateResourceDownloadURLResponse\"\x00\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x63orvic/ingest/v2/resource.proto\x12\x10\x63orvic.ingest.v2\x1a\x1b\x62uf/validate/validate.proto\x1a\x1c\x63orvic/status/v1/event.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x89\x03\n\x10ResourceMetadata\x12\x1b\n\x04name\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\x12\x1b\n\tmime_type\x18\x02 \x01(\tR\x08mimeType\x12y\n\x07room_id\x18\x03 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x06roomId\x12#\n\roriginal_path\x18\x05 \x01(\tR\x0coriginalPath\x12 \n\x0b\x64\x65scription\x18\x06 \x01(\tR\x0b\x64\x65scription\x12\x1f\n\x0bpipeline_id\x18\x08 \x01(\tR\npipelineId\x12;\n\ttype_hint\x18\x04 \x01(\x0e\x32\x1e.corvic.ingest.v2.ResourceTypeR\x08typeHint\x12\x1b\n\x04size\x18\x07 \x01(\x03\x42\x07\xbaH\x04\"\x02 \x00R\x04size\"\xf7\x02\n\rResourceEntry\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x1b\n\x04name\x18\x02 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\x12\x1b\n\tmime_type\x18\x03 \x01(\tR\x08mimeType\x12\x10\n\x03md5\x18\x05 \x01(\tR\x03md5\x12\x17\n\x07room_id\x18\x06 \x01(\tR\x06roomId\x12\x12\n\x04size\x18\x07 \x01(\x04R\x04size\x12#\n\roriginal_path\x18\t \x01(\tR\x0coriginalPath\x12 \n\x0b\x64\x65scription\x18\x0b \x01(\tR\x0b\x64\x65scription\x12\x1f\n\x0bpipeline_id\x18\x0c \x01(\tR\npipelineId\x12\x37\n\x18referenced_by_source_ids\x18\n \x03(\tR\x15referencedBySourceIds\x12<\n\rrecent_events\x18\x08 \x03(\x0b\x32\x17.corvic.status.v1.EventR\x0crecentEvents\"p\n\x16\x43reateUploadURLRequest\x12>\n\x08metadata\x18\x01 \x01(\x0b\x32\".corvic.ingest.v2.ResourceMetadataR\x08metadata\x12\x16\n\x06origin\x18\x02 \x01(\tR\x06origin\"A\n\x17\x43reateUploadURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"s\n\x15UploadURLTokenPayload\x12>\n\x08metadata\x18\x01 \x01(\x0b\x32\".corvic.ingest.v2.ResourceMetadataR\x08metadata\x12\x1a\n\x08\x66ilename\x18\x02 \x01(\tR\x08\x66ilename\"0\n\x18\x46inalizeUploadURLRequest\x12\x14\n\x05token\x18\x01 \x01(\tR\x05token\"R\n\x19\x46inalizeUploadURLResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x89\x01\n\x15\x44\x65leteResourceRequest\x12p\n\x02id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"\x18\n\x16\x44\x65leteResourceResponse\"\x86\x01\n\x12GetResourceRequest\x12p\n\x02id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"L\n\x13GetResourceResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x94\x01\n\x14ListResourcesRequest\x12|\n\x07room_id\x18\x01 \x01(\tBc\xbaH`\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd0\x01\x01R\x06roomId\"N\n\x15ListResourcesResponse\x12\x35\n\x05\x65ntry\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x05\x65ntry\"\x85\x01\n\x0cResourceList\x12u\n\x02id\x18\x01 \x03(\tBe\xbaHb\x92\x01_\"]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"\xd4\x01\n\x15WatchResourcesRequest\x12{\n\x07room_id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')H\x00R\x06roomId\x12\x32\n\x03ids\x18\x02 \x01(\x0b\x32\x1e.corvic.ingest.v2.ResourceListH\x00R\x03idsB\n\n\x08selector\"d\n\x16WatchResourcesResponse\x12J\n\x10updated_resource\x18\x01 \x01(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x0fupdatedResource\"\xb5\x01\n#ListResourcesPaginatedCursorPayload\x12T\n\x19\x63reate_time_of_last_entry\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x15\x63reateTimeOfLastEntry\x12\x17\n\x07room_id\x18\x02 \x01(\tR\x06roomId\x12\x1f\n\x0bpipeline_id\x18\x03 \x01(\tR\npipelineId\"\xe6\x02\n\x1dListResourcesPaginatedRequest\x12|\n\x07room_id\x18\x01 \x01(\tBc\xbaH`\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd0\x01\x01R\x06roomId\x12\x84\x01\n\x0bpipeline_id\x18\x04 \x01(\tBc\xbaH`\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\xd0\x01\x01R\npipelineId\x12(\n\x10\x65ntries_per_page\x18\x02 \x01(\rR\x0e\x65ntriesPerPage\x12\x16\n\x06\x63ursor\x18\x03 \x01(\tR\x06\x63ursor\"\x91\x01\n\x1eListResourcesPaginatedResponse\x12J\n\x10resource_entries\x18\x03 \x03(\x0b\x32\x1f.corvic.ingest.v2.ResourceEntryR\x0fresourceEntries\x12\x16\n\x06\x63ursor\x18\x02 \x01(\tR\x06\x63ursorJ\x04\x08\x01\x10\x02R\x05\x65ntry\"\x94\x01\n CreateResourceDownloadURLRequest\x12p\n\x02id\x18\x01 \x01(\tB`\xbaH]\xba\x01Z\n\x0estring.pattern\x12\x16value must be a number\x1a\x30this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')R\x02id\"5\n!CreateResourceDownloadURLResponse\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url*\xab\x01\n\x0cResourceType\x12\x1d\n\x19RESOURCE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x13RESOURCE_TYPE_TABLE\x10\x01\x1a\x02\x08\x01\x12!\n\x1dRESOURCE_TYPE_DIMENSION_TABLE\x10\x02\x12\x1c\n\x18RESOURCE_TYPE_FACT_TABLE\x10\x03\x12\x1e\n\x1aRESOURCE_TYPE_PDF_DOCUMENT\x10\x04\x32\x97\x07\n\x0fResourceService\x12h\n\x0f\x43reateUploadURL\x12(.corvic.ingest.v2.CreateUploadURLRequest\x1a).corvic.ingest.v2.CreateUploadURLResponse\"\x00\x12q\n\x11\x46inalizeUploadURL\x12*.corvic.ingest.v2.FinalizeUploadURLRequest\x1a+.corvic.ingest.v2.FinalizeUploadURLResponse\"\x03\x90\x02\x02\x12\x65\n\x0e\x44\x65leteResource\x12\'.corvic.ingest.v2.DeleteResourceRequest\x1a(.corvic.ingest.v2.DeleteResourceResponse\"\x00\x12_\n\x0bGetResource\x12$.corvic.ingest.v2.GetResourceRequest\x1a%.corvic.ingest.v2.GetResourceResponse\"\x03\x90\x02\x01\x12g\n\rListResources\x12&.corvic.ingest.v2.ListResourcesRequest\x1a\'.corvic.ingest.v2.ListResourcesResponse\"\x03\x90\x02\x01\x30\x01\x12\x80\x01\n\x16ListResourcesPaginated\x12/.corvic.ingest.v2.ListResourcesPaginatedRequest\x1a\x30.corvic.ingest.v2.ListResourcesPaginatedResponse\"\x03\x90\x02\x01\x12j\n\x0eWatchResources\x12\'.corvic.ingest.v2.WatchResourcesRequest\x1a(.corvic.ingest.v2.WatchResourcesResponse\"\x03\x90\x02\x01\x30\x01\x12\x86\x01\n\x19\x43reateResourceDownloadURL\x12\x32.corvic.ingest.v2.CreateResourceDownloadURLRequest\x1a\x33.corvic.ingest.v2.CreateResourceDownloadURLResponse\"\x00\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -47,6 +47,8 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_WATCHRESOURCESREQUEST'].fields_by_name['room_id']._serialized_options = b'\272H]\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
   _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['room_id']._options = None
   _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['room_id']._serialized_options = b'\272H`\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\320\001\001'
+  _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['pipeline_id']._options = None
+  _globals['_LISTRESOURCESPAGINATEDREQUEST'].fields_by_name['pipeline_id']._serialized_options = b'\272H`\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')\320\001\001'
   _globals['_CREATERESOURCEDOWNLOADURLREQUEST'].fields_by_name['id']._options = None
   _globals['_CREATERESOURCEDOWNLOADURLREQUEST'].fields_by_name['id']._serialized_options = b'\272H]\272\001Z\n\016string.pattern\022\026value must be a number\0320this.matches(\'^[0-9]+$\') && !this.endsWith(\'\\n\')'
   _globals['_RESOURCESERVICE'].methods_by_name['FinalizeUploadURL']._options = None
@@ -59,8 +61,8 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_RESOURCESERVICE'].methods_by_name['ListResourcesPaginated']._serialized_options = b'\220\002\001'
   _globals['_RESOURCESERVICE'].methods_by_name['WatchResources']._options = None
   _globals['_RESOURCESERVICE'].methods_by_name['WatchResources']._serialized_options = b'\220\002\001'
-  _globals['_RESOURCETYPE']._serialized_start=3147
-  _globals['_RESOURCETYPE']._serialized_end=3318
+  _globals['_RESOURCETYPE']._serialized_start=3350
+  _globals['_RESOURCETYPE']._serialized_end=3521
   _globals['_RESOURCEMETADATA']._serialized_start=180
   _globals['_RESOURCEMETADATA']._serialized_end=573
   _globals['_RESOURCEENTRY']._serialized_start=576
@@ -94,15 +96,15 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_WATCHRESOURCESRESPONSE']._serialized_start=2348
   _globals['_WATCHRESOURCESRESPONSE']._serialized_end=2448
   _globals['_LISTRESOURCESPAGINATEDCURSORPAYLOAD']._serialized_start=2451
-  _globals['_LISTRESOURCESPAGINATEDCURSORPAYLOAD']._serialized_end=2599
-  _globals['_LISTRESOURCESPAGINATEDREQUEST']._serialized_start=2602
-  _globals['_LISTRESOURCESPAGINATEDREQUEST']._serialized_end=2825
-  _globals['_LISTRESOURCESPAGINATEDRESPONSE']._serialized_start=2827
-  _globals['_LISTRESOURCESPAGINATEDRESPONSE']._serialized_end=2938
-  _globals['_CREATERESOURCEDOWNLOADURLREQUEST']._serialized_start=2941
-  _globals['_CREATERESOURCEDOWNLOADURLREQUEST']._serialized_end=3089
-  _globals['_CREATERESOURCEDOWNLOADURLRESPONSE']._serialized_start=3091
-  _globals['_CREATERESOURCEDOWNLOADURLRESPONSE']._serialized_end=3144
-  _globals['_RESOURCESERVICE']._serialized_start=3321
-  _globals['_RESOURCESERVICE']._serialized_end=4240
+  _globals['_LISTRESOURCESPAGINATEDCURSORPAYLOAD']._serialized_end=2632
+  _globals['_LISTRESOURCESPAGINATEDREQUEST']._serialized_start=2635
+  _globals['_LISTRESOURCESPAGINATEDREQUEST']._serialized_end=2993
+  _globals['_LISTRESOURCESPAGINATEDRESPONSE']._serialized_start=2996
+  _globals['_LISTRESOURCESPAGINATEDRESPONSE']._serialized_end=3141
+  _globals['_CREATERESOURCEDOWNLOADURLREQUEST']._serialized_start=3144
+  _globals['_CREATERESOURCEDOWNLOADURLREQUEST']._serialized_end=3292
+  _globals['_CREATERESOURCEDOWNLOADURLRESPONSE']._serialized_start=3294
+  _globals['_CREATERESOURCEDOWNLOADURLRESPONSE']._serialized_end=3347
+  _globals['_RESOURCESERVICE']._serialized_start=3524
+  _globals['_RESOURCESERVICE']._serialized_end=4443
 # @@protoc_insertion_point(module_scope)

{corvic_engine-0.3.0rc49 → corvic_engine-0.3.0rc52}/python/corvic_generated/ingest/v2/resource_pb2.pyi

@@ -160,30 +160,34 @@ class WatchResourcesResponse(_message.Message):
     def __init__(self, updated_resource: _Optional[_Union[ResourceEntry, _Mapping]] = ...) -> None: ...
 
 class ListResourcesPaginatedCursorPayload(_message.Message):
-    __slots__ = ("create_time_of_last_entry", "room_id")
+    __slots__ = ("create_time_of_last_entry", "room_id", "pipeline_id")
     CREATE_TIME_OF_LAST_ENTRY_FIELD_NUMBER: _ClassVar[int]
     ROOM_ID_FIELD_NUMBER: _ClassVar[int]
+    PIPELINE_ID_FIELD_NUMBER: _ClassVar[int]
     create_time_of_last_entry: _timestamp_pb2.Timestamp
     room_id: str
-    def __init__(self, create_time_of_last_entry: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., room_id: _Optional[str] = ...) -> None: ...
+    pipeline_id: str
+    def __init__(self, create_time_of_last_entry: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., room_id: _Optional[str] = ..., pipeline_id: _Optional[str] = ...) -> None: ...
 
 class ListResourcesPaginatedRequest(_message.Message):
-    __slots__ = ("room_id", "entries_per_page", "cursor")
+    __slots__ = ("room_id", "pipeline_id", "entries_per_page", "cursor")
     ROOM_ID_FIELD_NUMBER: _ClassVar[int]
+    PIPELINE_ID_FIELD_NUMBER: _ClassVar[int]
     ENTRIES_PER_PAGE_FIELD_NUMBER: _ClassVar[int]
     CURSOR_FIELD_NUMBER: _ClassVar[int]
     room_id: str
+    pipeline_id: str
     entries_per_page: int
     cursor: str
-    def __init__(self, room_id: _Optional[str] = ..., entries_per_page: _Optional[int] = ..., cursor: _Optional[str] = ...) -> None: ...
+    def __init__(self, room_id: _Optional[str] = ..., pipeline_id: _Optional[str] = ..., entries_per_page: _Optional[int] = ..., cursor: _Optional[str] = ...) -> None: ...
 
 class ListResourcesPaginatedResponse(_message.Message):
-    __slots__ = ("entry", "cursor")
-    ENTRY_FIELD_NUMBER: _ClassVar[int]
+    __slots__ = ("resource_entries", "cursor")
+    RESOURCE_ENTRIES_FIELD_NUMBER: _ClassVar[int]
     CURSOR_FIELD_NUMBER: _ClassVar[int]
-    entry: _containers.RepeatedCompositeFieldContainer[ResourceEntry]
+    resource_entries: _containers.RepeatedCompositeFieldContainer[ResourceEntry]
     cursor: str
-    def __init__(self, entry: _Optional[_Iterable[_Union[ResourceEntry, _Mapping]]] = ..., cursor: _Optional[str] = ...) -> None: ...
+    def __init__(self, resource_entries: _Optional[_Iterable[_Union[ResourceEntry, _Mapping]]] = ..., cursor: _Optional[str] = ...) -> None: ...
 
 class CreateResourceDownloadURLRequest(_message.Message):
     __slots__ = ("id",)

corvic_engine-0.3.0rc49/python/corvic/system_sqlite/__init__.py

@@ -1,9 +0,0 @@
-"""Single-machine, sqlite-backed implementation of corvic.system."""
-
-from corvic.system_sqlite.client import Client, FSBlobClient, RDBMSBlobClient
-
-__all__ = [
-    "Client",
-    "FSBlobClient",
-    "RDBMSBlobClient",
-]