pixeltable 0.4.14__py3-none-any.whl → 0.4.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/__init__.py +2 -1
- pixeltable/catalog/catalog.py +8 -0
- pixeltable/catalog/table_metadata.py +1 -0
- pixeltable/catalog/table_version.py +1 -28
- pixeltable/catalog/update_status.py +12 -0
- pixeltable/dataframe.py +1 -0
- pixeltable/env.py +4 -0
- pixeltable/exec/expr_eval/expr_eval_node.py +11 -0
- pixeltable/functions/video.py +123 -9
- pixeltable/functions/whisperx.py +2 -0
- pixeltable/functions/yolox.py +2 -0
- pixeltable/globals.py +46 -27
- pixeltable/io/__init__.py +1 -0
- pixeltable/iterators/__init__.py +1 -0
- pixeltable/share/publish.py +0 -1
- pixeltable/store.py +1 -1
- {pixeltable-0.4.14.dist-info → pixeltable-0.4.15.dist-info}/METADATA +1 -1
- {pixeltable-0.4.14.dist-info → pixeltable-0.4.15.dist-info}/RECORD +21 -21
- {pixeltable-0.4.14.dist-info → pixeltable-0.4.15.dist-info}/WHEEL +0 -0
- {pixeltable-0.4.14.dist-info → pixeltable-0.4.15.dist-info}/entry_points.txt +0 -0
- {pixeltable-0.4.14.dist-info → pixeltable-0.4.15.dist-info}/licenses/LICENSE +0 -0
pixeltable/__init__.py
CHANGED
|
@@ -20,7 +20,6 @@ from .globals import (
|
|
|
20
20
|
array,
|
|
21
21
|
configure_logging,
|
|
22
22
|
create_dir,
|
|
23
|
-
create_replica,
|
|
24
23
|
create_snapshot,
|
|
25
24
|
create_table,
|
|
26
25
|
create_view,
|
|
@@ -34,6 +33,8 @@ from .globals import (
|
|
|
34
33
|
list_tables,
|
|
35
34
|
ls,
|
|
36
35
|
move,
|
|
36
|
+
publish,
|
|
37
|
+
replicate,
|
|
37
38
|
tool,
|
|
38
39
|
tools,
|
|
39
40
|
)
|
pixeltable/catalog/catalog.py
CHANGED
|
@@ -407,6 +407,11 @@ class Catalog:
|
|
|
407
407
|
else:
|
|
408
408
|
raise
|
|
409
409
|
|
|
410
|
+
except KeyboardInterrupt:
|
|
411
|
+
has_exc = True
|
|
412
|
+
_logger.debug('Caught KeyboardInterrupt')
|
|
413
|
+
raise
|
|
414
|
+
|
|
410
415
|
except:
|
|
411
416
|
has_exc = True
|
|
412
417
|
raise
|
|
@@ -427,6 +432,9 @@ class Catalog:
|
|
|
427
432
|
# stored metadata
|
|
428
433
|
for handle in self._modified_tvs:
|
|
429
434
|
self._clear_tv_cache(handle.id, handle.effective_version)
|
|
435
|
+
# Clear potentially corrupted cached metadata after error
|
|
436
|
+
if tbl is not None:
|
|
437
|
+
tbl.clear_cached_md()
|
|
430
438
|
self._modified_tvs = set()
|
|
431
439
|
|
|
432
440
|
@property
|
|
@@ -38,6 +38,7 @@ class IndexMetadata(TypedDict):
|
|
|
38
38
|
index_type: Literal['embedding']
|
|
39
39
|
"""The type of index (currently only `'embedding'` is supported, but others will be added in the future)."""
|
|
40
40
|
parameters: EmbeddingIndexParams
|
|
41
|
+
"""Parameters specific to the index type."""
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
class TableMetadata(TypedDict):
|
|
@@ -18,7 +18,6 @@ from pixeltable import exprs, index
|
|
|
18
18
|
from pixeltable.env import Env
|
|
19
19
|
from pixeltable.iterators import ComponentIterator
|
|
20
20
|
from pixeltable.metadata import schema
|
|
21
|
-
from pixeltable.utils.exception_handler import run_cleanup_on_exception
|
|
22
21
|
from pixeltable.utils.filecache import FileCache
|
|
23
22
|
from pixeltable.utils.object_stores import ObjectOps
|
|
24
23
|
|
|
@@ -604,18 +603,7 @@ class TableVersion:
|
|
|
604
603
|
idx_info = self.IndexInfo(id=idx_id, name=idx_name, idx=idx, col=col, val_col=val_col, undo_col=undo_col)
|
|
605
604
|
self._tbl_md.index_md[idx_id] = idx_md
|
|
606
605
|
self.idxs_by_name[idx_name] = idx_info
|
|
607
|
-
|
|
608
|
-
idx.create_index(self._store_idx_name(idx_id), val_col)
|
|
609
|
-
finally:
|
|
610
|
-
|
|
611
|
-
def cleanup_index() -> None:
|
|
612
|
-
"""Delete the newly added in-memory index structure"""
|
|
613
|
-
del self.idxs_by_name[idx_name]
|
|
614
|
-
del self._tbl_md.index_md[idx_id]
|
|
615
|
-
self.next_idx_id = idx_id
|
|
616
|
-
|
|
617
|
-
# Run cleanup only if there has been an exception; otherwise, skip cleanup.
|
|
618
|
-
run_cleanup_on_exception(cleanup_index)
|
|
606
|
+
idx.create_index(self._store_idx_name(idx_id), val_col)
|
|
619
607
|
|
|
620
608
|
def _add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
|
|
621
609
|
val_col, undo_vol = self._create_index_columns(idx)
|
|
@@ -750,21 +738,6 @@ class TableVersion:
|
|
|
750
738
|
num_excs += excs_per_col
|
|
751
739
|
computed_values += plan.ctx.num_computed_exprs * row_count
|
|
752
740
|
finally:
|
|
753
|
-
# Ensure cleanup occurs if an exception or keyboard interruption happens during `load_column()`.
|
|
754
|
-
def cleanup_on_error() -> None:
|
|
755
|
-
"""Delete columns that are added as part of current add_columns operation and re-initialize
|
|
756
|
-
the sqlalchemy schema"""
|
|
757
|
-
self.cols = [col for col in self.cols if col not in cols_to_add]
|
|
758
|
-
for col in cols_to_add:
|
|
759
|
-
# remove columns that we already added
|
|
760
|
-
if col.id in self.cols_by_id:
|
|
761
|
-
del self.cols_by_id[col.id]
|
|
762
|
-
if col.name is not None and col.name in self.cols_by_name:
|
|
763
|
-
del self.cols_by_name[col.name]
|
|
764
|
-
self.store_tbl.create_sa_tbl()
|
|
765
|
-
|
|
766
|
-
# Run cleanup only if there has been an exception; otherwise, skip cleanup.
|
|
767
|
-
run_cleanup_on_exception(cleanup_on_error)
|
|
768
741
|
plan.close()
|
|
769
742
|
|
|
770
743
|
pxt.catalog.Catalog.get().record_column_dependencies(self)
|
|
@@ -57,27 +57,35 @@ class UpdateStatus:
|
|
|
57
57
|
"""
|
|
58
58
|
|
|
59
59
|
updated_cols: list[str] = field(default_factory=list)
|
|
60
|
+
"""Columns that were updated."""
|
|
60
61
|
cols_with_excs: list[str] = field(default_factory=list)
|
|
62
|
+
"""Columns that encountered exceptions."""
|
|
61
63
|
|
|
62
64
|
# stats for the rows affected by the operation
|
|
63
65
|
row_count_stats: RowCountStats = field(default_factory=RowCountStats)
|
|
66
|
+
"""Row count statistics for rows affected by this operation."""
|
|
64
67
|
|
|
65
68
|
# stats for changes cascaded to other tables
|
|
66
69
|
cascade_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
|
|
70
|
+
"""Row count statistics for changes cascaded to other tables."""
|
|
67
71
|
|
|
68
72
|
# stats for the rows affected by the operation in an external store
|
|
69
73
|
ext_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
|
|
74
|
+
"""Row count statistics for rows affected in an external store."""
|
|
70
75
|
|
|
71
76
|
@property
|
|
72
77
|
def num_rows(self) -> int:
|
|
78
|
+
"""Total number of rows affected (including cascaded changes)."""
|
|
73
79
|
return self.row_count_stats.num_rows + self.cascade_row_count_stats.num_rows
|
|
74
80
|
|
|
75
81
|
@property
|
|
76
82
|
def num_excs(self) -> int:
|
|
83
|
+
"""Total number of exceptions encountered (including cascaded changes)."""
|
|
77
84
|
return self.row_count_stats.num_excs + self.cascade_row_count_stats.num_excs
|
|
78
85
|
|
|
79
86
|
@property
|
|
80
87
|
def num_computed_values(self) -> int:
|
|
88
|
+
"""Total number of computed values affected (including cascaded changes)."""
|
|
81
89
|
return self.row_count_stats.computed_values + self.cascade_row_count_stats.computed_values
|
|
82
90
|
|
|
83
91
|
def insert_to_update(self) -> 'UpdateStatus':
|
|
@@ -164,16 +172,20 @@ class UpdateStatus:
|
|
|
164
172
|
|
|
165
173
|
@property
|
|
166
174
|
def external_rows_updated(self) -> int:
|
|
175
|
+
"""Number of rows updated in an external store."""
|
|
167
176
|
return self.ext_row_count_stats.upd_rows
|
|
168
177
|
|
|
169
178
|
@property
|
|
170
179
|
def external_rows_created(self) -> int:
|
|
180
|
+
"""Number of rows created in an external store."""
|
|
171
181
|
return self.ext_row_count_stats.ins_rows
|
|
172
182
|
|
|
173
183
|
@property
|
|
174
184
|
def external_rows_deleted(self) -> int:
|
|
185
|
+
"""Number of rows deleted from an external store."""
|
|
175
186
|
return self.ext_row_count_stats.del_rows
|
|
176
187
|
|
|
177
188
|
@property
|
|
178
189
|
def ext_num_rows(self) -> int:
|
|
190
|
+
"""Total number of rows affected in an external store."""
|
|
179
191
|
return self.ext_row_count_stats.num_rows
|
pixeltable/dataframe.py
CHANGED
pixeltable/env.py
CHANGED
|
@@ -770,6 +770,10 @@ class Env:
|
|
|
770
770
|
library_name=library_name or package_name, # defaults to package_name unless specified otherwise
|
|
771
771
|
)
|
|
772
772
|
|
|
773
|
+
def require_binary(self, binary_name: str) -> None:
|
|
774
|
+
if not shutil.which(binary_name):
|
|
775
|
+
raise excs.Error(f'{binary_name} is not installed or not in PATH. Please install it to use this feature.')
|
|
776
|
+
|
|
773
777
|
def require_package(self, package_name: str, min_version: Optional[list[int]] = None) -> None:
|
|
774
778
|
"""
|
|
775
779
|
Checks whether the specified optional package is available. If not, raises an exception
|
|
@@ -390,6 +390,17 @@ class ExprEvalNode(ExecNode):
|
|
|
390
390
|
# end the main loop if we had an unhandled exception
|
|
391
391
|
try:
|
|
392
392
|
t.result()
|
|
393
|
+
except KeyboardInterrupt:
|
|
394
|
+
# ExprEvalNode instances are long-running and reused across multiple operations.
|
|
395
|
+
# When a user interrupts an operation (Ctrl+C), the main evaluation loop properly
|
|
396
|
+
# handles the KeyboardInterrupt and terminates the current operation. However,
|
|
397
|
+
# background tasks spawned by evaluators may complete asynchronously after the
|
|
398
|
+
# operation has ended, and their done callbacks will fire during subsequent
|
|
399
|
+
# operations. These "phantom" KeyboardInterrupt exceptions from previous
|
|
400
|
+
# operations' background tasks should not interfere with new operations, so we
|
|
401
|
+
# absorb them here rather than propagating them via self.error/self.exc_event.
|
|
402
|
+
_logger.debug('Task completed with KeyboardInterrupt (user cancellation)')
|
|
403
|
+
pass
|
|
393
404
|
except asyncio.CancelledError:
|
|
394
405
|
pass
|
|
395
406
|
except Exception as exc:
|
pixeltable/functions/video.py
CHANGED
|
@@ -4,7 +4,6 @@ Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs)
|
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
6
|
import pathlib
|
|
7
|
-
import shutil
|
|
8
7
|
import subprocess
|
|
9
8
|
from typing import Literal, NoReturn
|
|
10
9
|
|
|
@@ -327,6 +326,7 @@ def clip(
|
|
|
327
326
|
Returns:
|
|
328
327
|
New video containing only the specified time range or None if start_time is beyond the end of the video.
|
|
329
328
|
"""
|
|
329
|
+
Env.get().require_binary('ffmpeg')
|
|
330
330
|
if start_time < 0:
|
|
331
331
|
raise pxt.Error(f'start_time must be non-negative, got {start_time}')
|
|
332
332
|
if end_time is not None and end_time <= start_time:
|
|
@@ -335,8 +335,6 @@ def clip(
|
|
|
335
335
|
raise pxt.Error(f'duration must be positive, got {duration}')
|
|
336
336
|
if end_time is not None and duration is not None:
|
|
337
337
|
raise pxt.Error('end_time and duration cannot both be specified')
|
|
338
|
-
if not shutil.which('ffmpeg'):
|
|
339
|
-
raise pxt.Error('ffmpeg is not installed or not in PATH. Please install ffmpeg to use get_clip().')
|
|
340
338
|
|
|
341
339
|
video_duration = av_utils.get_video_duration(video)
|
|
342
340
|
if video_duration is not None and start_time > video_duration:
|
|
@@ -388,10 +386,9 @@ def segment_video(video: pxt.Video, *, duration: float) -> list[str]:
|
|
|
388
386
|
>>> duration = tbl.video.get_duration()
|
|
389
387
|
>>> tbl.select(segment_paths=tbl.video.segment_video(duration=duration / 2 + 1)).collect()
|
|
390
388
|
"""
|
|
389
|
+
Env.get().require_binary('ffmpeg')
|
|
391
390
|
if duration <= 0:
|
|
392
391
|
raise pxt.Error(f'duration must be positive, got {duration}')
|
|
393
|
-
if not shutil.which('ffmpeg'):
|
|
394
|
-
raise pxt.Error('ffmpeg is not installed or not in PATH. Please install ffmpeg to use segment_video().')
|
|
395
392
|
|
|
396
393
|
base_path = TempStore.create_path(extension='')
|
|
397
394
|
|
|
@@ -436,10 +433,9 @@ def concat_videos(videos: list[pxt.Video]) -> pxt.Video:
|
|
|
436
433
|
Returns:
|
|
437
434
|
A new video containing the merged videos.
|
|
438
435
|
"""
|
|
436
|
+
Env.get().require_binary('ffmpeg')
|
|
439
437
|
if len(videos) == 0:
|
|
440
438
|
raise pxt.Error('concat_videos(): empty argument list')
|
|
441
|
-
if not shutil.which('ffmpeg'):
|
|
442
|
-
raise pxt.Error('ffmpeg is not installed or not in PATH. Please install ffmpeg to use concat_videos().')
|
|
443
439
|
|
|
444
440
|
# Check that all videos have the same resolution
|
|
445
441
|
resolutions: list[tuple[int, int]] = []
|
|
@@ -528,6 +524,125 @@ def concat_videos(videos: list[pxt.Video]) -> pxt.Video:
|
|
|
528
524
|
filelist_path.unlink()
|
|
529
525
|
|
|
530
526
|
|
|
527
|
+
@pxt.udf
|
|
528
|
+
def with_audio(
|
|
529
|
+
video: pxt.Video,
|
|
530
|
+
audio: pxt.Audio,
|
|
531
|
+
*,
|
|
532
|
+
video_start_time: float = 0.0,
|
|
533
|
+
video_duration: float | None = None,
|
|
534
|
+
audio_start_time: float = 0.0,
|
|
535
|
+
audio_duration: float | None = None,
|
|
536
|
+
) -> pxt.Video:
|
|
537
|
+
"""
|
|
538
|
+
Creates a new video that combines the video stream from `video` and the audio stream from `audio`.
|
|
539
|
+
The `start_time` and `duration` parameters can be used to select a specific time range from each input.
|
|
540
|
+
If the audio input (or selected time range) is longer than the video, the audio will be truncated.
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
__Requirements:__
|
|
544
|
+
|
|
545
|
+
- `ffmpeg` needs to be installed and in PATH
|
|
546
|
+
|
|
547
|
+
Args:
|
|
548
|
+
video: Input video.
|
|
549
|
+
audio: Input audio.
|
|
550
|
+
video_start_time: Start time in the video input (in seconds).
|
|
551
|
+
video_duration: Duration of video segment (in seconds). If None, uses the remainder of the video after
|
|
552
|
+
`video_start_time`. `video_duration` determines the duration of the output video.
|
|
553
|
+
audio_start_time: Start time in the audio input (in seconds).
|
|
554
|
+
audio_duration: Duration of audio segment (in seconds). If None, uses the remainder of the audio after
|
|
555
|
+
`audio_start_time`. If the audio is longer than the output video, it will be truncated.
|
|
556
|
+
|
|
557
|
+
Returns:
|
|
558
|
+
A new video file with the audio track added.
|
|
559
|
+
|
|
560
|
+
Examples:
|
|
561
|
+
Add background music to a video:
|
|
562
|
+
|
|
563
|
+
>>> tbl.select(tbl.video.with_audio(tbl.music_track)).collect()
|
|
564
|
+
|
|
565
|
+
Add audio starting 5 seconds into both files:
|
|
566
|
+
|
|
567
|
+
>>> tbl.select(
|
|
568
|
+
... tbl.video.with_audio(
|
|
569
|
+
... tbl.music_track,
|
|
570
|
+
... video_start_time=5.0,
|
|
571
|
+
... audio_start_time=5.0
|
|
572
|
+
... )
|
|
573
|
+
... ).collect()
|
|
574
|
+
|
|
575
|
+
Use a 10-second clip from the middle of both files:
|
|
576
|
+
|
|
577
|
+
>>> tbl.select(
|
|
578
|
+
... tbl.video.with_audio(
|
|
579
|
+
... tbl.music_track,
|
|
580
|
+
... video_start_time=30.0,
|
|
581
|
+
... video_duration=10.0,
|
|
582
|
+
... audio_start_time=15.0,
|
|
583
|
+
... audio_duration=10.0
|
|
584
|
+
... )
|
|
585
|
+
... ).collect()
|
|
586
|
+
"""
|
|
587
|
+
Env.get().require_binary('ffmpeg')
|
|
588
|
+
if video_start_time < 0:
|
|
589
|
+
raise pxt.Error(f'video_offset must be non-negative, got {video_start_time}')
|
|
590
|
+
if audio_start_time < 0:
|
|
591
|
+
raise pxt.Error(f'audio_offset must be non-negative, got {audio_start_time}')
|
|
592
|
+
if video_duration is not None and video_duration <= 0:
|
|
593
|
+
raise pxt.Error(f'video_duration must be positive, got {video_duration}')
|
|
594
|
+
if audio_duration is not None and audio_duration <= 0:
|
|
595
|
+
raise pxt.Error(f'audio_duration must be positive, got {audio_duration}')
|
|
596
|
+
|
|
597
|
+
output_path = str(TempStore.create_path(extension='.mp4'))
|
|
598
|
+
|
|
599
|
+
cmd = ['ffmpeg']
|
|
600
|
+
if video_start_time > 0:
|
|
601
|
+
# fast seek, must precede -i
|
|
602
|
+
cmd.extend(['-ss', str(video_start_time)])
|
|
603
|
+
if video_duration is not None:
|
|
604
|
+
cmd.extend(['-t', str(video_duration)])
|
|
605
|
+
else:
|
|
606
|
+
video_duration = av_utils.get_video_duration(video)
|
|
607
|
+
cmd.extend(['-i', str(video)])
|
|
608
|
+
|
|
609
|
+
if audio_start_time > 0:
|
|
610
|
+
cmd.extend(['-ss', str(audio_start_time)])
|
|
611
|
+
if audio_duration is not None:
|
|
612
|
+
cmd.extend(['-t', str(audio_duration)])
|
|
613
|
+
cmd.extend(['-i', str(audio)])
|
|
614
|
+
|
|
615
|
+
cmd.extend(
|
|
616
|
+
[
|
|
617
|
+
'-map',
|
|
618
|
+
'0:v:0', # video from first input
|
|
619
|
+
'-map',
|
|
620
|
+
'1:a:0', # audio from second input
|
|
621
|
+
'-c:v',
|
|
622
|
+
'copy', # avoid re-encoding
|
|
623
|
+
'-c:a',
|
|
624
|
+
'copy', # avoid re-encoding
|
|
625
|
+
'-t',
|
|
626
|
+
str(video_duration), # limit output duration to video duration
|
|
627
|
+
'-loglevel',
|
|
628
|
+
'error', # only show errors
|
|
629
|
+
output_path,
|
|
630
|
+
]
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
_logger.debug(f'with_audio(): {" ".join(cmd)}')
|
|
634
|
+
|
|
635
|
+
try:
|
|
636
|
+
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
|
637
|
+
output_file = pathlib.Path(output_path)
|
|
638
|
+
if not output_file.exists() or output_file.stat().st_size == 0:
|
|
639
|
+
stderr_output = result.stderr.strip() if result.stderr is not None else ''
|
|
640
|
+
raise pxt.Error(f'ffmpeg failed to create output file for commandline: {" ".join(cmd)}\n{stderr_output}')
|
|
641
|
+
return output_path
|
|
642
|
+
except subprocess.CalledProcessError as e:
|
|
643
|
+
_handle_ffmpeg_error(e)
|
|
644
|
+
|
|
645
|
+
|
|
531
646
|
@pxt.udf(is_method=True)
|
|
532
647
|
def overlay_text(
|
|
533
648
|
video: pxt.Video,
|
|
@@ -614,8 +729,7 @@ def overlay_text(
|
|
|
614
729
|
... )
|
|
615
730
|
... ).collect()
|
|
616
731
|
"""
|
|
617
|
-
|
|
618
|
-
raise pxt.Error('ffmpeg is not installed or not in PATH. Please install ffmpeg to use overlay_text().')
|
|
732
|
+
Env.get().require_binary('ffmpeg')
|
|
619
733
|
if font_size <= 0:
|
|
620
734
|
raise pxt.Error(f'font_size must be positive, got {font_size}')
|
|
621
735
|
if opacity < 0.0 or opacity > 1.0:
|
pixeltable/functions/whisperx.py
CHANGED
pixeltable/functions/yolox.py
CHANGED
pixeltable/globals.py
CHANGED
|
@@ -397,40 +397,54 @@ def create_snapshot(
|
|
|
397
397
|
)
|
|
398
398
|
|
|
399
399
|
|
|
400
|
-
def
|
|
401
|
-
destination: str,
|
|
400
|
+
def publish(
|
|
402
401
|
source: str | catalog.Table,
|
|
402
|
+
destination_uri: str,
|
|
403
403
|
bucket_name: str | None = None,
|
|
404
404
|
access: Literal['public', 'private'] = 'private',
|
|
405
|
-
) ->
|
|
405
|
+
) -> None:
|
|
406
406
|
"""
|
|
407
|
-
|
|
408
|
-
|
|
407
|
+
Publishes a replica of a local Pixeltable table to Pixeltable cloud. A given table can be published to at most one
|
|
408
|
+
URI per Pixeltable cloud database.
|
|
409
409
|
|
|
410
410
|
Args:
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
411
|
+
source: Path or table handle of the local table to be published.
|
|
412
|
+
destination_uri: Remote URI where the replica will be published, such as `'pxt://org_name/my_dir/my_table'`.
|
|
413
|
+
bucket_name: The name of the bucket to use to store replica's data. The bucket must be registered with
|
|
414
|
+
Pixeltable cloud. If no `bucket_name` is provided, the default storage bucket for the destination
|
|
415
|
+
database will be used.
|
|
416
416
|
access: Access control for the replica.
|
|
417
417
|
|
|
418
418
|
- `'public'`: Anyone can access this replica.
|
|
419
|
-
- `'private'`: Only the
|
|
419
|
+
- `'private'`: Only the host organization can access.
|
|
420
420
|
"""
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
421
|
+
if not destination_uri.startswith('pxt://'):
|
|
422
|
+
raise excs.Error("`destination_uri` must be a remote Pixeltable URI with the prefix 'pxt://'")
|
|
423
|
+
|
|
424
|
+
if isinstance(source, str):
|
|
425
|
+
source = get_table(source)
|
|
426
|
+
|
|
427
|
+
share.push_replica(destination_uri, source, bucket_name, access)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def replicate(remote_uri: str, local_path: str) -> catalog.Table:
|
|
431
|
+
"""
|
|
432
|
+
Retrieve a replica from Pixeltable cloud as a local table. This will create a full local copy of the replica in a
|
|
433
|
+
way that preserves the table structure of the original source data. Once replicated, the local table can be
|
|
434
|
+
queried offline just as any other Pixeltable table.
|
|
435
|
+
|
|
436
|
+
Args:
|
|
437
|
+
remote_uri: Remote URI of the table to be replicated, such as `'pxt://org_name/my_dir/my_table'`.
|
|
438
|
+
local_path: Local table path where the replica will be created, such as `'my_new_dir.my_new_tbl'`. It can be
|
|
439
|
+
the same or different from the cloud table name.
|
|
440
|
+
|
|
441
|
+
Returns:
|
|
442
|
+
A handle to the newly created local replica table.
|
|
443
|
+
"""
|
|
444
|
+
if not remote_uri.startswith('pxt://'):
|
|
445
|
+
raise excs.Error("`remote_uri` must be a remote Pixeltable URI with the prefix 'pxt://'")
|
|
446
|
+
|
|
447
|
+
return share.pull_replica(local_path, remote_uri)
|
|
434
448
|
|
|
435
449
|
|
|
436
450
|
def get_table(path: str) -> catalog.Table:
|
|
@@ -498,10 +512,11 @@ def move(path: str, new_path: str) -> None:
|
|
|
498
512
|
def drop_table(
|
|
499
513
|
table: str | catalog.Table, force: bool = False, if_not_exists: Literal['error', 'ignore'] = 'error'
|
|
500
514
|
) -> None:
|
|
501
|
-
"""Drop a table, view, or
|
|
515
|
+
"""Drop a table, view, snapshot, or replica.
|
|
502
516
|
|
|
503
517
|
Args:
|
|
504
|
-
table: Fully qualified name
|
|
518
|
+
table: Fully qualified name or table handle of the table to be dropped; or a remote URI of a cloud replica to
|
|
519
|
+
be deleted.
|
|
505
520
|
force: If `True`, will also drop all views and sub-views of this table.
|
|
506
521
|
if_not_exists: Directive regarding how to handle if the path does not exist.
|
|
507
522
|
Must be one of the following:
|
|
@@ -541,13 +556,17 @@ def drop_table(
|
|
|
541
556
|
assert isinstance(table, str)
|
|
542
557
|
tbl_path = table
|
|
543
558
|
|
|
559
|
+
if_not_exists_ = catalog.IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
|
|
560
|
+
|
|
544
561
|
if tbl_path.startswith('pxt://'):
|
|
545
562
|
# Remote table
|
|
563
|
+
if force:
|
|
564
|
+
raise excs.Error('Cannot use `force=True` with a cloud replica URI.')
|
|
565
|
+
# TODO: Handle if_not_exists properly
|
|
546
566
|
share.delete_replica(tbl_path)
|
|
547
567
|
else:
|
|
548
568
|
# Local table
|
|
549
569
|
path_obj = catalog.Path.parse(tbl_path)
|
|
550
|
-
if_not_exists_ = catalog.IfNotExistsParam.validated(if_not_exists, 'if_not_exists')
|
|
551
570
|
Catalog.get().drop_table(path_obj, force=force, if_not_exists=if_not_exists_)
|
|
552
571
|
|
|
553
572
|
|
pixeltable/io/__init__.py
CHANGED
pixeltable/iterators/__init__.py
CHANGED
pixeltable/share/publish.py
CHANGED
|
@@ -254,7 +254,6 @@ def _download_from_presigned_url(
|
|
|
254
254
|
session.close()
|
|
255
255
|
|
|
256
256
|
|
|
257
|
-
# TODO: This will be replaced by drop_table with cloud table uri
|
|
258
257
|
def delete_replica(dest_path: str) -> None:
|
|
259
258
|
"""Delete cloud replica"""
|
|
260
259
|
delete_request_json = {'operation_type': 'delete_snapshot', 'table_uri': dest_path}
|
pixeltable/store.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pixeltable
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.15
|
|
4
4
|
Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
|
|
5
5
|
Project-URL: homepage, https://pixeltable.com/
|
|
6
6
|
Project-URL: repository, https://github.com/pixeltable/pixeltable
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
pixeltable/__init__.py,sha256=
|
|
1
|
+
pixeltable/__init__.py,sha256=NJIh6nHictV4OeKGRFJ8QQ-FU5doqPT7PFGD7qFkTuM,1628
|
|
2
2
|
pixeltable/__version__.py,sha256=LnMIuAxx6nAQDMev_jnZyUdgsaiE3F8lulfXQBRl9qQ,112
|
|
3
3
|
pixeltable/config.py,sha256=GEHneFtLb47thJZ6nTuJpzcpDG39lNzKPGFdKKfOvkQ,8540
|
|
4
|
-
pixeltable/dataframe.py,sha256=
|
|
5
|
-
pixeltable/env.py,sha256=
|
|
4
|
+
pixeltable/dataframe.py,sha256=Qu1nCwbif9QEtBGyETnGXlGgnznMa_nQjmDnc6DcqOc,64356
|
|
5
|
+
pixeltable/env.py,sha256=_v0Ny62RMwJkMgiXoflnXmWbvLRx0eJ4I8CRuZVcT4M,45598
|
|
6
6
|
pixeltable/exceptions.py,sha256=Gm8d3TL2iiv6Pj2DLd29wp_j41qNBhxXL9iTQnL4Nk4,1116
|
|
7
|
-
pixeltable/globals.py,sha256=
|
|
7
|
+
pixeltable/globals.py,sha256=mLQk77dsj4QfjGL0jC-aTHAVO327hG1mjmHeOBfiuvw,39950
|
|
8
8
|
pixeltable/plan.py,sha256=lG_H9mkixP_Dqcy14DqDNpGBs-JXAJa9kVy6rTUsuc4,49258
|
|
9
9
|
pixeltable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
pixeltable/store.py,sha256=
|
|
10
|
+
pixeltable/store.py,sha256=zgzn-28gvRxRm29hc7pFX50Ls_ca-pJOHSDdzFZzeug,22886
|
|
11
11
|
pixeltable/type_system.py,sha256=4xnPN684UKC3lhzOoOqVAgR1lOPlHeEHt5Njpc6cGQQ,56131
|
|
12
12
|
pixeltable/catalog/__init__.py,sha256=GL0MLxqCBHlhKWqhC3e9B4kwTazagTOiqBHHRjyWbTg,726
|
|
13
|
-
pixeltable/catalog/catalog.py,sha256=
|
|
13
|
+
pixeltable/catalog/catalog.py,sha256=x_stqOC6CFdklpefFuVflrptGmOqxH4Myo4XzYB3xmU,100345
|
|
14
14
|
pixeltable/catalog/column.py,sha256=qEO1eyyJMaWvC_YXnyjs8f2dz_D52jxVYDN1_iVzEPE,13737
|
|
15
15
|
pixeltable/catalog/dir.py,sha256=VYTscPlKR6XhupPTXlJ8txAHxS5GSpPJ3LIleDJagVQ,2047
|
|
16
16
|
pixeltable/catalog/globals.py,sha256=uMIDsbeDzFxZbcgKDTOiT5plC1gAKgz1oxxdh1odIPw,2648
|
|
@@ -19,12 +19,12 @@ pixeltable/catalog/named_function.py,sha256=vZ-j7P4HugWh9OmUzBMwyRYvO3tQn9jWyJz_
|
|
|
19
19
|
pixeltable/catalog/path.py,sha256=O3FfxrvyX2crijBhp_2k4-3mG3BFxwba-tlPB74QtJQ,3780
|
|
20
20
|
pixeltable/catalog/schema_object.py,sha256=rQ6-3rzqnOHyEEHi97kai2S7BO3D9AkH7rirnfbGc14,1785
|
|
21
21
|
pixeltable/catalog/table.py,sha256=bPG9u-UnktClTQmqQ6cB0MSzp2uxfNkpzDUngJhG2ew,81010
|
|
22
|
-
pixeltable/catalog/table_metadata.py,sha256=
|
|
23
|
-
pixeltable/catalog/table_version.py,sha256=
|
|
22
|
+
pixeltable/catalog/table_metadata.py,sha256=cLqEgpRKmJOaJnKdJWMz3NLzZ59n41UIzfa_EloDZNU,3567
|
|
23
|
+
pixeltable/catalog/table_version.py,sha256=Wqm_10XivTJAT5MGVF76fNGrCEW1jiUD182RnexWuRQ,64176
|
|
24
24
|
pixeltable/catalog/table_version_handle.py,sha256=FTPRqcGY-h-POcWyZbd9b8P2D5zIw5OSUvwF_dbyCGo,3608
|
|
25
25
|
pixeltable/catalog/table_version_path.py,sha256=IaFVDH06_6ZMuBv5eLNCRTlWizpvz95jgAzqp4OVx_o,9713
|
|
26
26
|
pixeltable/catalog/tbl_ops.py,sha256=Vdcz4Nzvdw09zcQaCEaOr9Uufk2rQHgG0vBvMbQp9R8,1145
|
|
27
|
-
pixeltable/catalog/update_status.py,sha256=
|
|
27
|
+
pixeltable/catalog/update_status.py,sha256=jeRy7WUZ80l4gPY1c5vYcxT1F82nL5KphNrSKKfOdk8,7667
|
|
28
28
|
pixeltable/catalog/view.py,sha256=Snki7QNlKThba9MSeEYCaBIwaqfiR0v_FMdIO9SJkFE,15420
|
|
29
29
|
pixeltable/exec/__init__.py,sha256=9B31dWrTJ9sdQiVd1OFtHgxa8A2YmAPBDPd6gnFjEqQ,580
|
|
30
30
|
pixeltable/exec/aggregation_node.py,sha256=54YO1tddnIGOlWUwVff9xcghrZr60UNAgBekF7OuxWA,4401
|
|
@@ -39,7 +39,7 @@ pixeltable/exec/row_update_node.py,sha256=zU0eSyn81-vRrjAMOadRqU8luTshnPUtIbS7np
|
|
|
39
39
|
pixeltable/exec/sql_node.py,sha256=99GW3729YzYT0vYxhACte8XQ-NxIBn1cgfRXzPbz7O4,27233
|
|
40
40
|
pixeltable/exec/expr_eval/__init__.py,sha256=sQThSEByK_DLfB-_-18RFhpARx49cSXYEkpCDyi0vQI,61
|
|
41
41
|
pixeltable/exec/expr_eval/evaluators.py,sha256=-6s_y29Wh8p35SVKkXtnA0NkzcHVw1Z8PgHGiFrMsqs,17135
|
|
42
|
-
pixeltable/exec/expr_eval/expr_eval_node.py,sha256=
|
|
42
|
+
pixeltable/exec/expr_eval/expr_eval_node.py,sha256=xi0H3vlhf00gcPI_Zojt7nhpVsKqV5GaWip8TfIz_4k,19781
|
|
43
43
|
pixeltable/exec/expr_eval/globals.py,sha256=FL9DwFub1fc8ppzzTdrQytBwylyDkH4mIwciWnIGBcc,7816
|
|
44
44
|
pixeltable/exec/expr_eval/row_buffer.py,sha256=YY0thdlMNNReEOTyPp36xKPeMeXSZ0VrI9bJsXgo7sU,2744
|
|
45
45
|
pixeltable/exec/expr_eval/schedulers.py,sha256=pNTHiwL8omHa_kodv61gynBHFbJnSg5CktvezrO_yc8,24337
|
|
@@ -106,16 +106,16 @@ pixeltable/functions/string.py,sha256=LdBNOna5PUSPmM5VlJ_qhmwzyFhumW0k6Dvx2rXSZt
|
|
|
106
106
|
pixeltable/functions/timestamp.py,sha256=3GVCVIWdry96Qk5XXuvbJ58Tp30iY5snBibzl2CHjQc,9143
|
|
107
107
|
pixeltable/functions/together.py,sha256=A8J19BXywyWQ6a2_n05-8uIG5jquOBGqPmW3mb-NrIc,8842
|
|
108
108
|
pixeltable/functions/util.py,sha256=uQNkyBSkTVMe1wbUI2Q0nz-mM3qPVTF86yK8c9OFIcE,954
|
|
109
|
-
pixeltable/functions/video.py,sha256=
|
|
109
|
+
pixeltable/functions/video.py,sha256=LQ-1KHoU6DjgCWlj0koBSVO6vH4JKvfl1lGIntWoSSI,30854
|
|
110
110
|
pixeltable/functions/vision.py,sha256=17h9bOm3NJyQzFMBwXDHMqnkcuCspyQJgHdBOXV1Ip8,15380
|
|
111
111
|
pixeltable/functions/whisper.py,sha256=u2QcDU7JdtgLIImCkFPkzjWEjLTJIrlSkAWqeITyIJw,3103
|
|
112
|
-
pixeltable/functions/whisperx.py,sha256=
|
|
113
|
-
pixeltable/functions/yolox.py,sha256=
|
|
112
|
+
pixeltable/functions/whisperx.py,sha256=NapvBQlIR6FfYeUP328Kjv7a9zgFZPghiFtsvgzhKZk,7724
|
|
113
|
+
pixeltable/functions/yolox.py,sha256=IXTE-1NDY7cQp94UDFu0wZjMMcTociUZc4GgEOcNU3I,3559
|
|
114
114
|
pixeltable/index/__init__.py,sha256=97aFuxiP_oz1ldn5iq8IWApkOV8XG6ZIBW5-9rkS0vM,122
|
|
115
115
|
pixeltable/index/base.py,sha256=200s7v3Zy810bRlbSAYzxxaEjVssl6r8esTHiSvWRwQ,1704
|
|
116
116
|
pixeltable/index/btree.py,sha256=8B06D67ay0DFUtEBC5q4bLjxMq7ILpKyyoLAiSaamzA,2503
|
|
117
117
|
pixeltable/index/embedding_index.py,sha256=rXAh3ZPrsUoaqNB8JSYlceDkcjs82hIkqSDwRi70Pio,11638
|
|
118
|
-
pixeltable/io/__init__.py,sha256=
|
|
118
|
+
pixeltable/io/__init__.py,sha256=k8jlxi2IB_yA8yYTk9lf7qyuayxUbvs9xDuZcO77sFY,716
|
|
119
119
|
pixeltable/io/datarows.py,sha256=s2fDQTttGxq7cS5JwKFEJRSKn6WsXTaGdmm9VJSl_2M,6154
|
|
120
120
|
pixeltable/io/external_store.py,sha256=rOYBwTqcZZVU2toWxJ_9Iy2w2YO0DhuABrM2xGmqHSo,14787
|
|
121
121
|
pixeltable/io/fiftyone.py,sha256=6KyOqJbRXJzF96Ob4B3yZzyU0yveLbvuf6thjfrZZwE,6891
|
|
@@ -127,7 +127,7 @@ pixeltable/io/pandas.py,sha256=xQmkwbqE9_fjbbPUgeG5yNICrbVVK73UHxDL-cgrQw0,9007
|
|
|
127
127
|
pixeltable/io/parquet.py,sha256=qVvg9nixJnK9gXYxZocD8HE13SznyLrgW9IsehtT4j4,4101
|
|
128
128
|
pixeltable/io/table_data_conduit.py,sha256=8jwQ3IOoOBS-8j2TEfgiqsFUD85kEP5IjoC0dg2uPEk,22058
|
|
129
129
|
pixeltable/io/utils.py,sha256=qzBTmqdIawXMt2bfXQOraYnEstL69eC2Z33nl8RrwJk,4244
|
|
130
|
-
pixeltable/iterators/__init__.py,sha256=
|
|
130
|
+
pixeltable/iterators/__init__.py,sha256=hbQhrz0LShLGOPcT1SnWl1s1nBUzAjWino64CfFT_UA,537
|
|
131
131
|
pixeltable/iterators/audio.py,sha256=khW4M4P_J5twz1Eu3nuDP_73C9iWeCYZ5K_zQysvYxI,9671
|
|
132
132
|
pixeltable/iterators/base.py,sha256=ZC0ZvXL4iw6AmT8cu-Mdx-T2UG9nmJYV1C6LK4efAfw,1669
|
|
133
133
|
pixeltable/iterators/document.py,sha256=UMmLeLQxyNfbhI8p3rBrE-sYrnJhpEUVaMLM50Iq-e8,19972
|
|
@@ -172,7 +172,7 @@ pixeltable/mypy/__init__.py,sha256=cD_oHXClR_bDM8qVNIfaOAgRhQjPfcWvLcinz79ua6o,5
|
|
|
172
172
|
pixeltable/mypy/mypy_plugin.py,sha256=KCjzKOeKW5CBqJOq9Ch7ZJ25ICPc4nlTB49DxtC6oDM,5460
|
|
173
173
|
pixeltable/share/__init__.py,sha256=PTX1mw61Ss4acEOI-sUlu0HaoVsosLqwDfh0ldn8Hkg,84
|
|
174
174
|
pixeltable/share/packager.py,sha256=UjpEv7wU_R4BKstMYw7PPULrSa4szN9KP8wwM4NU1-g,31563
|
|
175
|
-
pixeltable/share/publish.py,sha256=
|
|
175
|
+
pixeltable/share/publish.py,sha256=Cpbn10duCXoe5k0RaB2WpJjbeMhXd5DXEV7OLQpmtJA,10899
|
|
176
176
|
pixeltable/utils/__init__.py,sha256=45qEM20L2VuIe-Cc3BTKWFqQb-S7A8qDtmmgl77zYK0,1728
|
|
177
177
|
pixeltable/utils/arrow.py,sha256=U7vb_ffPCR7zv-phyBMPMDosPdKN6LK4IVMpfm2mRy8,10424
|
|
178
178
|
pixeltable/utils/av.py,sha256=omJufz62dzaTTwlR7quKfcT7apf8KkBLJ9cQ9240dt0,4016
|
|
@@ -197,8 +197,8 @@ pixeltable/utils/pytorch.py,sha256=77x2g4N6dkfYvqbxjYi_rBRiNnMMZAb2cNtBw0hOCHg,3
|
|
|
197
197
|
pixeltable/utils/s3_store.py,sha256=l6X6vakjxtryUBJMsFYk7PUJLsMdmrVcUkF7LwZXdk0,14714
|
|
198
198
|
pixeltable/utils/sql.py,sha256=Sa4Lh-VGe8GToU5W7DRiWf2lMl9B6saPqemiT0ZdHEc,806
|
|
199
199
|
pixeltable/utils/transactional_directory.py,sha256=OFKmu90oP7KwBAljwjnzP_w8euGdAXob3y4Nx9SCNHA,1357
|
|
200
|
-
pixeltable-0.4.
|
|
201
|
-
pixeltable-0.4.
|
|
202
|
-
pixeltable-0.4.
|
|
203
|
-
pixeltable-0.4.
|
|
204
|
-
pixeltable-0.4.
|
|
200
|
+
pixeltable-0.4.15.dist-info/METADATA,sha256=yaHrz0jidyCrfvXfZ1ANwI5nGvTAGkaroXZRrpMUZpU,25631
|
|
201
|
+
pixeltable-0.4.15.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
202
|
+
pixeltable-0.4.15.dist-info/entry_points.txt,sha256=rrKugZmxDtGnXCnEQ5UJMaaSYY7-g1cLjUZ4W1moIhM,98
|
|
203
|
+
pixeltable-0.4.15.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
204
|
+
pixeltable-0.4.15.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|