pixeltable 0.4.20__py3-none-any.whl → 0.4.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/_version.py +1 -1
- pixeltable/catalog/catalog.py +24 -0
- pixeltable/catalog/table.py +20 -24
- pixeltable/catalog/table_version.py +0 -15
- pixeltable/globals.py +3 -0
- pixeltable/metadata/__init__.py +1 -1
- pixeltable/metadata/converters/convert_30.py +32 -14
- pixeltable/metadata/converters/convert_41.py +12 -0
- pixeltable/metadata/converters/util.py +7 -13
- pixeltable/metadata/notes.py +1 -0
- pixeltable/metadata/schema.py +12 -4
- pixeltable/share/packager.py +6 -2
- pixeltable/share/publish.py +14 -5
- {pixeltable-0.4.20.dist-info → pixeltable-0.4.22.dist-info}/METADATA +1 -1
- {pixeltable-0.4.20.dist-info → pixeltable-0.4.22.dist-info}/RECORD +18 -17
- {pixeltable-0.4.20.dist-info → pixeltable-0.4.22.dist-info}/WHEEL +0 -0
- {pixeltable-0.4.20.dist-info → pixeltable-0.4.22.dist-info}/entry_points.txt +0 -0
- {pixeltable-0.4.20.dist-info → pixeltable-0.4.22.dist-info}/licenses/LICENSE +0 -0
pixeltable/_version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__: str = '0.4.
|
|
1
|
+
__version__: str = '0.4.22'
|
pixeltable/catalog/catalog.py
CHANGED
|
@@ -1294,6 +1294,30 @@ class Catalog:
|
|
|
1294
1294
|
# It's a new version of a table that has a physical store, so we need to create a TableVersion instance.
|
|
1295
1295
|
TableVersion.create_replica(md)
|
|
1296
1296
|
|
|
1297
|
+
def get_additional_md(self, tbl_id: UUID) -> dict[str, Any]:
|
|
1298
|
+
"""Return the additional_md field of the given table."""
|
|
1299
|
+
assert Env.get().in_xact
|
|
1300
|
+
conn = Env.get().conn
|
|
1301
|
+
q = sql.select(schema.Table.additional_md).where(schema.Table.id == str(tbl_id))
|
|
1302
|
+
row = conn.execute(q).one()
|
|
1303
|
+
assert isinstance(row[0], dict)
|
|
1304
|
+
return row[0]
|
|
1305
|
+
|
|
1306
|
+
def update_additional_md(self, tbl_id: UUID, additional_md: dict[str, Any]) -> None:
|
|
1307
|
+
"""
|
|
1308
|
+
Update the additional_md field of the given table. The new additional_md is merged with the
|
|
1309
|
+
existing one via a JSON dictionary merge, giving preference to the new values.
|
|
1310
|
+
"""
|
|
1311
|
+
assert self._in_write_xact
|
|
1312
|
+
conn = Env.get().conn
|
|
1313
|
+
q = (
|
|
1314
|
+
sql.update(schema.Table)
|
|
1315
|
+
.where(schema.Table.id == str(tbl_id))
|
|
1316
|
+
.values({schema.Table.additional_md: schema.Table.additional_md.op('||')(additional_md)})
|
|
1317
|
+
)
|
|
1318
|
+
result = conn.execute(q)
|
|
1319
|
+
assert result.rowcount == 1, result.rowcount
|
|
1320
|
+
|
|
1297
1321
|
@retry_loop(for_write=False)
|
|
1298
1322
|
def get_table(self, path: Path, if_not_exists: IfNotExistsParam) -> Table | None:
|
|
1299
1323
|
obj = Catalog.get()._get_schema_object(
|
pixeltable/catalog/table.py
CHANGED
|
@@ -171,6 +171,10 @@ class Table(SchemaObject):
|
|
|
171
171
|
"""Return the version of this table. Used by tests to ascertain version changes."""
|
|
172
172
|
return self._tbl_version_path.version()
|
|
173
173
|
|
|
174
|
+
def _get_pxt_uri(self) -> str | None:
|
|
175
|
+
with catalog.Catalog.get().begin_xact(tbl_id=self._id):
|
|
176
|
+
return catalog.Catalog.get().get_additional_md(self._id).get('pxt_uri')
|
|
177
|
+
|
|
174
178
|
def __hash__(self) -> int:
|
|
175
179
|
return hash(self._tbl_version_path.tbl_id)
|
|
176
180
|
|
|
@@ -1616,61 +1620,53 @@ class Table(SchemaObject):
|
|
|
1616
1620
|
.. warning::
|
|
1617
1621
|
This operation is irreversible.
|
|
1618
1622
|
"""
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
|
|
1623
|
+
with catalog.Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
|
|
1622
1624
|
self.__check_mutable('revert')
|
|
1623
1625
|
self._tbl_version.get().revert()
|
|
1624
1626
|
# remove cached md in order to force a reload on the next operation
|
|
1625
1627
|
self._tbl_version_path.clear_cached_md()
|
|
1626
1628
|
|
|
1627
|
-
def push(self
|
|
1629
|
+
def push(self) -> None:
|
|
1628
1630
|
from pixeltable.share import push_replica
|
|
1629
1631
|
from pixeltable.share.protocol import PxtUri
|
|
1630
1632
|
|
|
1631
|
-
|
|
1632
|
-
pxt_uri = tbl_version.pxt_uri
|
|
1633
|
-
|
|
1634
|
-
if tbl_version.is_replica:
|
|
1633
|
+
if self._tbl_version_path.tbl_version.get().is_replica:
|
|
1635
1634
|
raise excs.Error(f'push(): Cannot push replica table {self._name!r}. (Did you mean `pull()`?)')
|
|
1635
|
+
|
|
1636
|
+
pxt_uri = self._get_pxt_uri()
|
|
1636
1637
|
if pxt_uri is None:
|
|
1637
1638
|
raise excs.Error(
|
|
1638
1639
|
f'push(): Table {self._name!r} has not yet been published to Pixeltable Cloud. '
|
|
1639
1640
|
'To publish it, use `pxt.publish()` instead.'
|
|
1640
1641
|
)
|
|
1641
1642
|
|
|
1643
|
+
if self._tbl_version is None:
|
|
1644
|
+
# Named snapshots never have new versions to push.
|
|
1645
|
+
env.Env.get().console_logger.info('push(): Everything up to date.')
|
|
1646
|
+
return
|
|
1647
|
+
|
|
1642
1648
|
# Parse the pxt URI to extract org/db and create a UUID-based URI for pushing
|
|
1643
1649
|
parsed_uri = PxtUri(uri=pxt_uri)
|
|
1644
1650
|
uuid_uri_obj = PxtUri.from_components(org=parsed_uri.org, id=self._id, db=parsed_uri.db)
|
|
1645
1651
|
uuid_uri = str(uuid_uri_obj)
|
|
1646
1652
|
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
else:
|
|
1651
|
-
versioned_path = catalog.Path.parse(self._path())._replace(version=version)
|
|
1652
|
-
versioned_tbl = catalog.Catalog.get().get_table(versioned_path, IfNotExistsParam.IGNORE)
|
|
1653
|
-
if versioned_tbl is None:
|
|
1654
|
-
raise excs.Error(f'Table {self._name!r} has no known version {version}')
|
|
1655
|
-
assert versioned_tbl._id == self._id
|
|
1656
|
-
push_replica(uuid_uri, versioned_tbl)
|
|
1657
|
-
|
|
1658
|
-
def pull(self, *, version: int | None = None) -> None:
|
|
1653
|
+
push_replica(uuid_uri, self)
|
|
1654
|
+
|
|
1655
|
+
def pull(self) -> None:
|
|
1659
1656
|
from pixeltable.share import pull_replica
|
|
1660
1657
|
from pixeltable.share.protocol import PxtUri
|
|
1661
1658
|
|
|
1659
|
+
pxt_uri = self._get_pxt_uri()
|
|
1662
1660
|
tbl_version = self._tbl_version_path.tbl_version.get()
|
|
1663
|
-
pxt_uri = tbl_version.pxt_uri
|
|
1664
1661
|
|
|
1665
|
-
if not tbl_version.is_replica:
|
|
1662
|
+
if not tbl_version.is_replica or pxt_uri is None:
|
|
1666
1663
|
raise excs.Error(
|
|
1667
1664
|
f'pull(): Table {self._name!r} is not a replica of a Pixeltable Cloud table (nothing to `pull()`).'
|
|
1668
1665
|
)
|
|
1669
|
-
assert pxt_uri is not None
|
|
1670
1666
|
|
|
1671
1667
|
# Parse the pxt URI to extract org/db and create a UUID-based URI for pulling
|
|
1672
1668
|
parsed_uri = PxtUri(uri=pxt_uri)
|
|
1673
|
-
uuid_uri_obj = PxtUri.from_components(org=parsed_uri.org, id=self._id, db=parsed_uri.db
|
|
1669
|
+
uuid_uri_obj = PxtUri.from_components(org=parsed_uri.org, id=self._id, db=parsed_uri.db)
|
|
1674
1670
|
uuid_uri = str(uuid_uri_obj)
|
|
1675
1671
|
|
|
1676
1672
|
pull_replica(self._path(), uuid_uri)
|
|
@@ -1431,21 +1431,6 @@ class TableVersion:
|
|
|
1431
1431
|
self._tbl_md.external_stores.pop(idx)
|
|
1432
1432
|
self._write_md(new_version=True, new_schema_version=True)
|
|
1433
1433
|
|
|
1434
|
-
@property
|
|
1435
|
-
def pxt_uri(self) -> str | None:
|
|
1436
|
-
return self._tbl_md.additional_md.get('pxt_uri')
|
|
1437
|
-
|
|
1438
|
-
def update_pxt_uri(self, pxt_uri: str | None) -> None:
|
|
1439
|
-
if self._tbl_md.additional_md.get('pxt_uri') == pxt_uri:
|
|
1440
|
-
return # Nothing to do
|
|
1441
|
-
|
|
1442
|
-
if pxt_uri is None:
|
|
1443
|
-
del self._tbl_md.additional_md['pxt_uri'] # must be present due to preceding check
|
|
1444
|
-
else:
|
|
1445
|
-
self._tbl_md.additional_md['pxt_uri'] = pxt_uri
|
|
1446
|
-
|
|
1447
|
-
self._write_md(new_version=False, new_schema_version=False)
|
|
1448
|
-
|
|
1449
1434
|
@property
|
|
1450
1435
|
def tbl_md(self) -> schema.TableMd:
|
|
1451
1436
|
return self._tbl_md
|
pixeltable/globals.py
CHANGED
|
@@ -306,6 +306,9 @@ def create_view(
|
|
|
306
306
|
raise excs.Error('`base` must be an instance of `Table` or `DataFrame`')
|
|
307
307
|
assert isinstance(base, (catalog.Table, DataFrame))
|
|
308
308
|
|
|
309
|
+
if tbl_version_path.is_replica():
|
|
310
|
+
raise excs.Error('Cannot create a view or snapshot on top of a replica')
|
|
311
|
+
|
|
309
312
|
path_obj = catalog.Path.parse(path)
|
|
310
313
|
if_exists_ = catalog.IfExistsParam.validated(if_exists, 'if_exists')
|
|
311
314
|
media_validation_ = catalog.MediaValidation.validated(media_validation, 'media_validation')
|
pixeltable/metadata/__init__.py
CHANGED
|
@@ -18,7 +18,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
|
|
|
18
18
|
_logger = logging.getLogger('pixeltable')
|
|
19
19
|
|
|
20
20
|
# current version of the metadata; this is incremented whenever the metadata schema changes
|
|
21
|
-
VERSION =
|
|
21
|
+
VERSION = 42
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def create_system_info(engine: sql.engine.Engine) -> None:
|
|
@@ -1,31 +1,49 @@
|
|
|
1
1
|
import copy
|
|
2
|
-
|
|
2
|
+
import uuid
|
|
3
|
+
from typing import Any
|
|
3
4
|
|
|
4
5
|
import sqlalchemy as sql
|
|
6
|
+
from sqlalchemy import orm
|
|
7
|
+
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
|
5
8
|
|
|
6
9
|
from pixeltable.metadata import register_converter
|
|
7
|
-
from pixeltable.metadata.converters.util import
|
|
8
|
-
convert_table_md,
|
|
9
|
-
convert_table_schema_version_record,
|
|
10
|
-
convert_table_version_record,
|
|
11
|
-
)
|
|
12
|
-
from pixeltable.metadata.schema import TableSchemaVersion, TableVersion
|
|
10
|
+
from pixeltable.metadata.converters.util import convert_sql_table_record, convert_table_md
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
@register_converter(version=30)
|
|
16
14
|
def _(engine: sql.engine.Engine) -> None:
|
|
17
15
|
convert_table_md(engine, table_md_updater=__update_table_md)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
engine, table_schema_version_record_updater=__update_table_schema_version_record
|
|
21
|
-
)
|
|
16
|
+
convert_sql_table_record(TableVersionAtV30, engine, record_updater=__update_table_version_record)
|
|
17
|
+
convert_sql_table_record(TableSchemaVersionAtV30, engine, record_updater=__update_table_schema_version_record)
|
|
22
18
|
|
|
23
19
|
|
|
24
|
-
def __update_table_md(md: dict, tbl_id: UUID) -> None:
|
|
20
|
+
def __update_table_md(md: dict, tbl_id: uuid.UUID) -> None:
|
|
25
21
|
md['tbl_id'] = str(tbl_id)
|
|
26
22
|
|
|
27
23
|
|
|
28
|
-
|
|
24
|
+
# We can't use the ORM declarations from pixeltable.metadata.schema, because those might have changed since the
|
|
25
|
+
# version being converted. So we include static declarations here of the tables as they existed at version 30.
|
|
26
|
+
|
|
27
|
+
Base: type = orm.declarative_base()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TableVersionAtV30(Base):
|
|
31
|
+
__tablename__ = 'tableversions'
|
|
32
|
+
|
|
33
|
+
tbl_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), primary_key=True, nullable=False)
|
|
34
|
+
version: orm.Mapped[int] = orm.mapped_column(sql.BigInteger, primary_key=True, nullable=False)
|
|
35
|
+
md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class TableSchemaVersionAtV30(Base):
|
|
39
|
+
__tablename__ = 'tableschemaversions'
|
|
40
|
+
|
|
41
|
+
tbl_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), primary_key=True, nullable=False)
|
|
42
|
+
schema_version: orm.Mapped[int] = orm.mapped_column(sql.BigInteger, primary_key=True, nullable=False)
|
|
43
|
+
md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False) # TableSchemaVersionMd
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def __update_table_version_record(record: TableVersionAtV30) -> None:
|
|
29
47
|
"""
|
|
30
48
|
Update TableVersion with table_id.
|
|
31
49
|
"""
|
|
@@ -35,7 +53,7 @@ def __update_table_version_record(record: TableVersion) -> None:
|
|
|
35
53
|
record.md = md
|
|
36
54
|
|
|
37
55
|
|
|
38
|
-
def __update_table_schema_version_record(record:
|
|
56
|
+
def __update_table_schema_version_record(record: TableSchemaVersionAtV30) -> None:
|
|
39
57
|
"""
|
|
40
58
|
Update TableSchemaVersion with table_id.
|
|
41
59
|
"""
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import sqlalchemy as sql
|
|
2
|
+
|
|
3
|
+
from pixeltable.metadata import register_converter
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@register_converter(version=41)
|
|
7
|
+
def _(engine: sql.engine.Engine) -> None:
|
|
8
|
+
with engine.begin() as conn:
|
|
9
|
+
conn.execute(sql.text("ALTER TABLE dirs ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
|
|
10
|
+
conn.execute(sql.text("ALTER TABLE tables ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
|
|
11
|
+
conn.execute(sql.text("ALTER TABLE tableversions ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
|
|
12
|
+
conn.execute(sql.text("ALTER TABLE tableschemaversions ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import logging
|
|
3
|
-
from typing import Any, Callable
|
|
3
|
+
from typing import Any, Callable, TypeVar
|
|
4
4
|
from uuid import UUID
|
|
5
5
|
|
|
6
6
|
import sqlalchemy as sql
|
|
7
7
|
|
|
8
|
-
from pixeltable.metadata.schema import Function, Table, TableSchemaVersion
|
|
8
|
+
from pixeltable.metadata.schema import Function, Table, TableSchemaVersion
|
|
9
9
|
|
|
10
10
|
__logger = logging.getLogger('pixeltable')
|
|
11
11
|
|
|
@@ -147,19 +147,13 @@ def __update_schema_column(table_schema_version_md: dict, schema_column_updater:
|
|
|
147
147
|
schema_column_updater(schema_col)
|
|
148
148
|
|
|
149
149
|
|
|
150
|
-
|
|
151
|
-
engine: sql.engine.Engine, table_version_record_updater: Callable[[TableVersion], None] | None
|
|
152
|
-
) -> None:
|
|
153
|
-
with sql.orm.Session(engine, future=True) as session:
|
|
154
|
-
for record in session.query(TableVersion).all():
|
|
155
|
-
table_version_record_updater(record)
|
|
156
|
-
session.commit()
|
|
150
|
+
T = TypeVar('T')
|
|
157
151
|
|
|
158
152
|
|
|
159
|
-
def
|
|
160
|
-
engine: sql.engine.Engine,
|
|
153
|
+
def convert_sql_table_record(
|
|
154
|
+
schema: type[T], engine: sql.engine.Engine, record_updater: Callable[[T], None] | None
|
|
161
155
|
) -> None:
|
|
162
156
|
with sql.orm.Session(engine, future=True) as session:
|
|
163
|
-
for record in session.query(
|
|
164
|
-
|
|
157
|
+
for record in session.query(schema).all():
|
|
158
|
+
record_updater(record)
|
|
165
159
|
session.commit()
|
pixeltable/metadata/notes.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# rather than as a comment, so that the existence of a description can be enforced by
|
|
3
3
|
# the unit tests when new versions are added.
|
|
4
4
|
VERSION_NOTES = {
|
|
5
|
+
42: 'Add additional_md columns to metadata tables',
|
|
5
6
|
41: 'Cellmd columns for array and json columns',
|
|
6
7
|
40: 'Convert error property columns to cellmd columns',
|
|
7
8
|
39: 'ColumnHandles in external stores',
|
pixeltable/metadata/schema.py
CHANGED
|
@@ -68,6 +68,7 @@ class SystemInfo(Base):
|
|
|
68
68
|
"""A single-row table that contains system-wide metadata."""
|
|
69
69
|
|
|
70
70
|
__tablename__ = 'systeminfo'
|
|
71
|
+
|
|
71
72
|
dummy = sql.Column(Integer, primary_key=True, default=0, nullable=False)
|
|
72
73
|
md = sql.Column(JSONB, nullable=False) # SystemInfoMd
|
|
73
74
|
|
|
@@ -76,7 +77,7 @@ class SystemInfo(Base):
|
|
|
76
77
|
class DirMd:
|
|
77
78
|
name: str
|
|
78
79
|
user: str | None
|
|
79
|
-
additional_md: dict[str, Any]
|
|
80
|
+
additional_md: dict[str, Any] # deprecated
|
|
80
81
|
|
|
81
82
|
|
|
82
83
|
class Dir(Base):
|
|
@@ -87,6 +88,7 @@ class Dir(Base):
|
|
|
87
88
|
)
|
|
88
89
|
parent_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=True)
|
|
89
90
|
md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False) # DirMd
|
|
91
|
+
additional_md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False, default=dict)
|
|
90
92
|
|
|
91
93
|
# used to force acquisition of an X-lock via an Update stmt
|
|
92
94
|
lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
|
|
@@ -196,7 +198,9 @@ class TableMd:
|
|
|
196
198
|
column_md: dict[int, ColumnMd] # col_id -> ColumnMd
|
|
197
199
|
index_md: dict[int, IndexMd] # index_id -> IndexMd
|
|
198
200
|
view_md: ViewMd | None
|
|
199
|
-
|
|
201
|
+
# TODO: Remove additional_md from this and other Md dataclasses (and switch to using the separate additional_md
|
|
202
|
+
# column in all cases)
|
|
203
|
+
additional_md: dict[str, Any] # deprecated
|
|
200
204
|
|
|
201
205
|
has_pending_ops: bool = False
|
|
202
206
|
|
|
@@ -241,6 +245,7 @@ class Table(Base):
|
|
|
241
245
|
id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), primary_key=True, nullable=False)
|
|
242
246
|
dir_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=False)
|
|
243
247
|
md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False) # TableMd
|
|
248
|
+
additional_md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False, default=dict)
|
|
244
249
|
|
|
245
250
|
# used to force acquisition of an X-lock via an Update stmt
|
|
246
251
|
lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
|
|
@@ -257,16 +262,18 @@ class TableVersionMd:
|
|
|
257
262
|
# A version fragment cannot be queried or instantiated via get_table(). A fragment represents a version of a
|
|
258
263
|
# replica table that has incomplete data, and exists only to provide base table support for a dependent view.
|
|
259
264
|
is_fragment: bool = False
|
|
260
|
-
additional_md: dict[str, Any] = dataclasses.field(default_factory=dict)
|
|
265
|
+
additional_md: dict[str, Any] = dataclasses.field(default_factory=dict) # deprecated
|
|
261
266
|
|
|
262
267
|
|
|
263
268
|
class TableVersion(Base):
|
|
264
269
|
__tablename__ = 'tableversions'
|
|
270
|
+
|
|
265
271
|
tbl_id: orm.Mapped[uuid.UUID] = orm.mapped_column(
|
|
266
272
|
UUID(as_uuid=True), ForeignKey('tables.id'), primary_key=True, nullable=False
|
|
267
273
|
)
|
|
268
274
|
version: orm.Mapped[int] = orm.mapped_column(BigInteger, primary_key=True, nullable=False)
|
|
269
275
|
md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)
|
|
276
|
+
additional_md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False, default=dict)
|
|
270
277
|
|
|
271
278
|
|
|
272
279
|
@dataclasses.dataclass
|
|
@@ -299,7 +306,7 @@ class TableSchemaVersionMd:
|
|
|
299
306
|
# default validation strategy for any media column of this table
|
|
300
307
|
# stores column.MediaValiation.name.lower()
|
|
301
308
|
media_validation: str
|
|
302
|
-
additional_md: dict[str, Any]
|
|
309
|
+
additional_md: dict[str, Any] # deprecated
|
|
303
310
|
|
|
304
311
|
|
|
305
312
|
# versioning: each table schema change results in a new record
|
|
@@ -311,6 +318,7 @@ class TableSchemaVersion(Base):
|
|
|
311
318
|
)
|
|
312
319
|
schema_version: orm.Mapped[int] = orm.mapped_column(BigInteger, primary_key=True, nullable=False)
|
|
313
320
|
md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False) # TableSchemaVersionMd
|
|
321
|
+
additional_md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False, default=dict)
|
|
314
322
|
|
|
315
323
|
|
|
316
324
|
class PendingTableOp(Base):
|
pixeltable/share/packager.py
CHANGED
|
@@ -377,7 +377,7 @@ class TableRestorer:
|
|
|
377
377
|
self.tmp_dir = TempStore.create_path()
|
|
378
378
|
self.media_files = {}
|
|
379
379
|
|
|
380
|
-
def restore(self, bundle_path: Path) -> pxt.Table:
|
|
380
|
+
def restore(self, bundle_path: Path, pxt_uri: str | None = None) -> pxt.Table:
|
|
381
381
|
# Extract tarball
|
|
382
382
|
print(f'Extracting table data into: {self.tmp_dir}')
|
|
383
383
|
with tarfile.open(bundle_path, 'r:bz2') as tf:
|
|
@@ -423,7 +423,11 @@ class TableRestorer:
|
|
|
423
423
|
_logger.info(f'Importing table {tv.name!r}.')
|
|
424
424
|
self.__import_table(self.tmp_dir, tv, md)
|
|
425
425
|
|
|
426
|
-
|
|
426
|
+
tbl = cat.get_table_by_id(UUID(tbl_md[0].tbl_md.tbl_id))
|
|
427
|
+
if pxt_uri is not None:
|
|
428
|
+
# Set pxt_uri for the newly created table
|
|
429
|
+
cat.update_additional_md(tbl._id, {'pxt_uri': pxt_uri})
|
|
430
|
+
return tbl
|
|
427
431
|
|
|
428
432
|
def __import_table(self, bundle_path: Path, tv: catalog.TableVersion, tbl_md: TableVersionCompleteMd) -> None:
|
|
429
433
|
"""
|
pixeltable/share/publish.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import dataclasses
|
|
2
2
|
import json
|
|
3
|
+
import logging
|
|
3
4
|
import os
|
|
4
5
|
import sys
|
|
5
6
|
import urllib.parse
|
|
@@ -32,6 +33,8 @@ from .protocol.replica import (
|
|
|
32
33
|
ReplicateResponse,
|
|
33
34
|
)
|
|
34
35
|
|
|
36
|
+
_logger = logging.getLogger('pixeltable')
|
|
37
|
+
|
|
35
38
|
# These URLs are abstracted out for now, but will be replaced with actual (hard-coded) URLs once the
|
|
36
39
|
# pixeltable.com URLs are available.
|
|
37
40
|
|
|
@@ -41,6 +44,8 @@ PIXELTABLE_API_URL = os.environ.get('PIXELTABLE_API_URL', 'https://internal-api.
|
|
|
41
44
|
def push_replica(
|
|
42
45
|
dest_tbl_uri: str, src_tbl: pxt.Table, bucket: str | None = None, access: Literal['public', 'private'] = 'private'
|
|
43
46
|
) -> str:
|
|
47
|
+
_logger.info(f'Publishing replica for {src_tbl._name!r} to: {dest_tbl_uri}')
|
|
48
|
+
|
|
44
49
|
packager = TablePackager(src_tbl)
|
|
45
50
|
|
|
46
51
|
# Create the publish request using packager's bundle_md
|
|
@@ -53,11 +58,15 @@ def push_replica(
|
|
|
53
58
|
is_public=access == 'public',
|
|
54
59
|
)
|
|
55
60
|
|
|
61
|
+
_logger.debug(f'Sending PublishRequest: {publish_request}')
|
|
62
|
+
|
|
56
63
|
response = requests.post(PIXELTABLE_API_URL, data=publish_request.model_dump_json(), headers=_api_headers())
|
|
57
64
|
if response.status_code != 200:
|
|
58
65
|
raise excs.Error(f'Error publishing {src_tbl._display_name()}: {response.text}')
|
|
59
66
|
publish_response = PublishResponse.model_validate(response.json())
|
|
60
67
|
|
|
68
|
+
_logger.debug(f'Received PublishResponse: {publish_response}')
|
|
69
|
+
|
|
61
70
|
upload_id = publish_response.upload_id
|
|
62
71
|
destination_uri = publish_response.destination_uri
|
|
63
72
|
|
|
@@ -95,8 +104,8 @@ def push_replica(
|
|
|
95
104
|
confirmed_tbl_uri = finalize_response.confirmed_table_uri
|
|
96
105
|
Env.get().console_logger.info(f'The published table is now available at: {confirmed_tbl_uri}')
|
|
97
106
|
|
|
98
|
-
with Catalog.get().begin_xact(tbl_id=src_tbl.
|
|
99
|
-
|
|
107
|
+
with Catalog.get().begin_xact(tbl_id=src_tbl._id, for_write=True):
|
|
108
|
+
Catalog.get().update_additional_md(src_tbl._id, {'pxt_uri': str(confirmed_tbl_uri)})
|
|
100
109
|
|
|
101
110
|
return str(confirmed_tbl_uri)
|
|
102
111
|
|
|
@@ -144,14 +153,14 @@ def pull_replica(dest_path: str, src_tbl_uri: str) -> pxt.Table:
|
|
|
144
153
|
_download_from_presigned_url(url=parsed_location.geturl(), output_path=bundle_path)
|
|
145
154
|
else:
|
|
146
155
|
raise excs.Error(f'Unexpected response from server: unsupported bundle uri: {bundle_uri}')
|
|
147
|
-
|
|
148
|
-
|
|
156
|
+
|
|
157
|
+
pxt_uri = str(clone_response.table_uri)
|
|
149
158
|
md_list = [dataclasses.asdict(md) for md in clone_response.md]
|
|
150
159
|
restorer = TableRestorer(
|
|
151
160
|
dest_path, {'pxt_version': pxt.__version__, 'pxt_md_version': clone_response.pxt_md_version, 'md': md_list}
|
|
152
161
|
)
|
|
153
162
|
|
|
154
|
-
tbl = restorer.restore(bundle_path)
|
|
163
|
+
tbl = restorer.restore(bundle_path, pxt_uri)
|
|
155
164
|
Env.get().console_logger.info(f'Created local replica {tbl._path()!r} from URI: {src_tbl_uri}')
|
|
156
165
|
return tbl
|
|
157
166
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pixeltable
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.22
|
|
4
4
|
Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
|
|
5
5
|
Project-URL: homepage, https://pixeltable.com/
|
|
6
6
|
Project-URL: repository, https://github.com/pixeltable/pixeltable
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
pixeltable/__init__.py,sha256=_glawKwHDw5Jjzfe2ef-SRS1EQFacpPMnyqvQdh06PU,1694
|
|
2
|
-
pixeltable/_version.py,sha256=
|
|
2
|
+
pixeltable/_version.py,sha256=3LUYumasSAz5moNiWNci9TlBBzYLzP157hY2nHkRLU4,28
|
|
3
3
|
pixeltable/config.py,sha256=aVYorLvk1Z4zUBPOlmWfXsubkxubIueL9R_klngtFw0,9413
|
|
4
4
|
pixeltable/dataframe.py,sha256=CuyO0XydUqAIBu8vht79gMfZCGLNr5ClQUJKV9IPAFQ,64779
|
|
5
5
|
pixeltable/env.py,sha256=BdKRoy58cUl2zC7foFqLmIpUVmPZhkC2ibork-irMTs,47570
|
|
6
6
|
pixeltable/exceptions.py,sha256=Gm8d3TL2iiv6Pj2DLd29wp_j41qNBhxXL9iTQnL4Nk4,1116
|
|
7
|
-
pixeltable/globals.py,sha256=
|
|
7
|
+
pixeltable/globals.py,sha256=KMtlXPV92_PpjqfKSQy-wh2DM6EjAHU7OtEVX0VjMcY,42810
|
|
8
8
|
pixeltable/plan.py,sha256=SuVT6cY7nsJXYOwcyyCfWpcorWSU6vv56dBs_03QJ3w,54078
|
|
9
9
|
pixeltable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
10
|
pixeltable/store.py,sha256=Dz9N85DISVZlX1gLf3x_sWVdP-j9DBaUURMv55kuEmI,27552
|
|
11
11
|
pixeltable/type_system.py,sha256=mhks0nbY4JtBHuLssGr5JCPRVXWyZUy6nmsOpqBX4cI,56143
|
|
12
12
|
pixeltable/catalog/__init__.py,sha256=GL0MLxqCBHlhKWqhC3e9B4kwTazagTOiqBHHRjyWbTg,726
|
|
13
|
-
pixeltable/catalog/catalog.py,sha256=
|
|
13
|
+
pixeltable/catalog/catalog.py,sha256=6RdmeZX23NLSQwgBipAdUAfrm-wt29Ge7XIrIDNtxb4,106829
|
|
14
14
|
pixeltable/catalog/column.py,sha256=ttNGZbmbnDmWiIOIbpESPRWYFomtVqKxwQhHyaNFasA,13955
|
|
15
15
|
pixeltable/catalog/dir.py,sha256=VYTscPlKR6XhupPTXlJ8txAHxS5GSpPJ3LIleDJagVQ,2047
|
|
16
16
|
pixeltable/catalog/globals.py,sha256=EGlqpgmw5rhzfQC4MTKVYpdxXStFSz0J4LKxAXXXS3k,2617
|
|
@@ -18,9 +18,9 @@ pixeltable/catalog/insertable_table.py,sha256=W83qtX8dN3NWLr9Nm75QadM7tZ83wBbIcz
|
|
|
18
18
|
pixeltable/catalog/named_function.py,sha256=vZ-j7P4HugWh9OmUzBMwyRYvO3tQn9jWyJz_1stPavU,1210
|
|
19
19
|
pixeltable/catalog/path.py,sha256=gPq_ezwDEU-7dxGE3-kEiKTwvZz0JwzoLy9rSgtHouc,3558
|
|
20
20
|
pixeltable/catalog/schema_object.py,sha256=Xzax_sGpvHw7mPi-srOBYFrcOv-4MT_KXrN2y8_qwGc,1766
|
|
21
|
-
pixeltable/catalog/table.py,sha256=
|
|
21
|
+
pixeltable/catalog/table.py,sha256=QQ06oT8lS9Yu_JSD3JXihZyzVfC5IjlBmTPVFbxIOXo,83754
|
|
22
22
|
pixeltable/catalog/table_metadata.py,sha256=MV79FWMbZy238XsjR8pY0ROsSSPqXQHERry7mf-ds9E,3737
|
|
23
|
-
pixeltable/catalog/table_version.py,sha256=
|
|
23
|
+
pixeltable/catalog/table_version.py,sha256=11s0HAmMw0OmWPnXLlOrB9ju02Y1YzB6gDxTfaP_6v4,71312
|
|
24
24
|
pixeltable/catalog/table_version_handle.py,sha256=KP3-AnHP9LANpMzaP7P2E-HY7E33sMJfkOAz36S5kkg,3710
|
|
25
25
|
pixeltable/catalog/table_version_path.py,sha256=_imZieXYDrHR1VcDdjm6Bu2u5DFRgb85AegBeVXeiiY,9209
|
|
26
26
|
pixeltable/catalog/tbl_ops.py,sha256=2B8D6nwPAr6wEcMbDaEiMOjbvsAOXIGMcZ0qKRGrS2E,1240
|
|
@@ -140,9 +140,9 @@ pixeltable/iterators/document.py,sha256=qOOGSkuBKGh2dKUDvwJ0PA_Du1S32xleM6McZuKv
|
|
|
140
140
|
pixeltable/iterators/image.py,sha256=RrFdf5cnFIQzWKJk4uYi1m1p2qAiz909THYhRQ27DbY,3603
|
|
141
141
|
pixeltable/iterators/string.py,sha256=URj5edWp-CsorjN_8nnfWGvtIFs_Zh4VPm6htlJbFkU,1257
|
|
142
142
|
pixeltable/iterators/video.py,sha256=62_smE_YlMY6FSC5xhYnUVXLl7JUCKlk9UptrvMuz_o,21551
|
|
143
|
-
pixeltable/metadata/__init__.py,sha256=
|
|
144
|
-
pixeltable/metadata/notes.py,sha256=
|
|
145
|
-
pixeltable/metadata/schema.py,sha256=
|
|
143
|
+
pixeltable/metadata/__init__.py,sha256=uW8KCLWIe1-7sOVpNDGYF0HpqrrDrFVm1aKoL-2bxd8,3289
|
|
144
|
+
pixeltable/metadata/notes.py,sha256=MOuNOLPOnaDBDUkkGfnodOk-QYSDRNJzU3XgihH9USQ,1654
|
|
145
|
+
pixeltable/metadata/schema.py,sha256=lepxO_ZrtKd1r4_wr6uRozYZAJmzKISDjGv6evWD7Ek,13472
|
|
146
146
|
pixeltable/metadata/utils.py,sha256=ocun76zdpgoj1BHROQNZZXxhQ_aQwmuKfXRM2iSmTxY,2630
|
|
147
147
|
pixeltable/metadata/converters/convert_10.py,sha256=myYIo1DyccnsQUxDKG6mafnU5ge_EhZpHg_pesKBoK4,708
|
|
148
148
|
pixeltable/metadata/converters/convert_12.py,sha256=Ci-qyZW1gqci-8wnjeOB5afdq7KTuN-hVSV9OqSPx8g,162
|
|
@@ -163,7 +163,7 @@ pixeltable/metadata/converters/convert_26.py,sha256=a24OXiHUKklYQdDNkzYLllESNDzm
|
|
|
163
163
|
pixeltable/metadata/converters/convert_27.py,sha256=1P7DSAXPwHqkXvNxFqjIOcdI8dlsL8ikfaaveBvQHyM,904
|
|
164
164
|
pixeltable/metadata/converters/convert_28.py,sha256=v2CqpXAbCTQ-3gbn_ZtSl0fXueK1Q7VDECbIc7Ci1VQ,694
|
|
165
165
|
pixeltable/metadata/converters/convert_29.py,sha256=rHZp1tnvnbkIK08IYiT1RgP0-Y-Jw-G2L6VMWvEHUs4,4887
|
|
166
|
-
pixeltable/metadata/converters/convert_30.py,sha256=
|
|
166
|
+
pixeltable/metadata/converters/convert_30.py,sha256=OCrlv56d_-TWbAiquo6WwWKs5aHhcc6uNsdW2ue7EiE,2289
|
|
167
167
|
pixeltable/metadata/converters/convert_31.py,sha256=pFjZGpuFlGRPxKYYidiSCIMNaEEXVuLTnELi3y_pSQo,424
|
|
168
168
|
pixeltable/metadata/converters/convert_32.py,sha256=YENfuQ_mqhnZWrvHJffaGEqd1IwS2RjTGPkEvOs5sUs,406
|
|
169
169
|
pixeltable/metadata/converters/convert_33.py,sha256=ZZV3FTyyouBM1eNymXxfHV-Oqmgu2s0KNP6AG3zc5eM,574
|
|
@@ -174,12 +174,13 @@ pixeltable/metadata/converters/convert_37.py,sha256=IVZGtKFaaYMGBs39V_H_okWvpxxa
|
|
|
174
174
|
pixeltable/metadata/converters/convert_38.py,sha256=bH4ofK9wGMFOX00Go-ni6s041P9B5tzHooZ32396h3c,1150
|
|
175
175
|
pixeltable/metadata/converters/convert_39.py,sha256=ycZAmH2KhJC28_ao-3Ka71jAPldmjUjoQYhb7C-rla4,4627
|
|
176
176
|
pixeltable/metadata/converters/convert_40.py,sha256=qtsbIGv_YcMKswsSam2PBsspIl0mDsCOSiaRW8yN1wM,2892
|
|
177
|
-
pixeltable/metadata/converters/
|
|
177
|
+
pixeltable/metadata/converters/convert_41.py,sha256=VbNqjCzkhdKnkhopiolP3PppkNRx6613zu_cR7XmA60,620
|
|
178
|
+
pixeltable/metadata/converters/util.py,sha256=YOF_QTle9RWPEe-QJY8IfRWat6RWziDUsSZdwcOvlSM,7285
|
|
178
179
|
pixeltable/mypy/__init__.py,sha256=cD_oHXClR_bDM8qVNIfaOAgRhQjPfcWvLcinz79ua6o,54
|
|
179
180
|
pixeltable/mypy/mypy_plugin.py,sha256=_eV2PtFDLUztb-QlysdEmF4qzVDALEQkH8aapbLYqy4,5438
|
|
180
181
|
pixeltable/share/__init__.py,sha256=5NafGEUosRUQW61lj2dq4O3NAj8EehT9iK81mvh7Kis,105
|
|
181
|
-
pixeltable/share/packager.py,sha256=
|
|
182
|
-
pixeltable/share/publish.py,sha256=
|
|
182
|
+
pixeltable/share/packager.py,sha256=7a-bDFwR7DuwplzTfBWjjWn4pe00n_rUCpgeNBGOPAs,39053
|
|
183
|
+
pixeltable/share/publish.py,sha256=7p923XlgMgwHF8sDZjSpMPe83nyhnMjCME3-ydkGJ8c,12044
|
|
183
184
|
pixeltable/share/protocol/__init__.py,sha256=IS51n0-RYDy5ET2d2Y17Agevr8TYbKN8xRWiV_TJ1MQ,786
|
|
184
185
|
pixeltable/share/protocol/common.py,sha256=qJjlk9NJKgb85JKCdTloPvSlBVFNFs5xRY5slT90T3s,6082
|
|
185
186
|
pixeltable/share/protocol/operation_types.py,sha256=yr0id4nsihUaGnb2gS9HQ0sG9mIGSmllCzqgdreyGeg,959
|
|
@@ -211,8 +212,8 @@ pixeltable/utils/pytorch.py,sha256=77x2g4N6dkfYvqbxjYi_rBRiNnMMZAb2cNtBw0hOCHg,3
|
|
|
211
212
|
pixeltable/utils/s3_store.py,sha256=rpUrgMbrI06AE2TnQTODjkspo9OutOX6knCeTrShhXA,16657
|
|
212
213
|
pixeltable/utils/sql.py,sha256=Sa4Lh-VGe8GToU5W7DRiWf2lMl9B6saPqemiT0ZdHEc,806
|
|
213
214
|
pixeltable/utils/transactional_directory.py,sha256=OFKmu90oP7KwBAljwjnzP_w8euGdAXob3y4Nx9SCNHA,1357
|
|
214
|
-
pixeltable-0.4.
|
|
215
|
-
pixeltable-0.4.
|
|
216
|
-
pixeltable-0.4.
|
|
217
|
-
pixeltable-0.4.
|
|
218
|
-
pixeltable-0.4.
|
|
215
|
+
pixeltable-0.4.22.dist-info/METADATA,sha256=OPHiW8MGtH4E4ePQxed2hDIDr8XBx_OyDiRD8ST3c7w,25549
|
|
216
|
+
pixeltable-0.4.22.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
217
|
+
pixeltable-0.4.22.dist-info/entry_points.txt,sha256=rrKugZmxDtGnXCnEQ5UJMaaSYY7-g1cLjUZ4W1moIhM,98
|
|
218
|
+
pixeltable-0.4.22.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
219
|
+
pixeltable-0.4.22.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|