pixeltable 0.4.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- {pixeltable-0.4.1 → pixeltable-0.4.2}/PKG-INFO +1 -1
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/__version__.py +2 -2
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/__init__.py +2 -1
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/catalog.py +75 -21
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/column.py +10 -0
- pixeltable-0.4.2/pixeltable/catalog/globals.py +234 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/insertable_table.py +2 -1
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/table.py +135 -4
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/table_version.py +106 -66
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/table_version_handle.py +26 -1
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/view.py +4 -2
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/column_property_ref.py +2 -11
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/column_ref.py +19 -17
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/data_row.py +9 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/row_builder.py +44 -13
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/external_store.py +79 -52
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/globals.py +1 -1
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/label_studio.py +45 -41
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/__init__.py +1 -1
- pixeltable-0.4.2/pixeltable/metadata/converters/convert_38.py +39 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/notes.py +1 -0
- pixeltable-0.4.2/pixeltable/metadata/utils.py +78 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/plan.py +22 -18
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/store.py +114 -103
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pyproject.toml +5 -4
- pixeltable-0.4.1/pixeltable/catalog/globals.py +0 -131
- {pixeltable-0.4.1 → pixeltable-0.4.2}/LICENSE +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/README.md +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/dir.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/named_function.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/path.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/schema_object.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/catalog/table_version_path.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/config.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/dataframe.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/env.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exceptions.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/aggregation_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/cache_prefetch_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/component_iteration_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/data_row_batch.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/exec_context.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/exec_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/expr_eval/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/expr_eval/evaluators.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/expr_eval/expr_eval_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/expr_eval/globals.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/expr_eval/row_buffer.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/expr_eval/schedulers.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/in_memory_data_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/row_update_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exec/sql_node.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/arithmetic_expr.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/array_slice.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/comparison.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/compound_predicate.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/expr.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/expr_dict.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/expr_set.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/function_call.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/globals.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/in_predicate.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/inline_expr.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/is_null.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/json_mapper.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/json_path.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/literal.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/method_ref.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/object_ref.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/rowid_ref.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/similarity_expr.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/sql_element_cache.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/string_op.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/type_cast.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/exprs/variable.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/ext/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/ext/functions/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/ext/functions/whisperx.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/ext/functions/yolox.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/aggregate_function.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/callable_function.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/expr_template_function.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/function.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/function_registry.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/globals.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/mcp.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/query_template_function.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/signature.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/tools.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/func/udf.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/anthropic.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/audio.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/bedrock.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/date.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/deepseek.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/fireworks.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/gemini.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/globals.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/groq.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/huggingface.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/image.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/json.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/llama_cpp.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/math.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/mistralai.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/ollama.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/openai.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/replicate.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/string.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/timestamp.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/together.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/util.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/video.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/vision.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/functions/whisper.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/globals.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/index/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/index/base.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/index/btree.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/index/embedding_index.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/datarows.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/fiftyone.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/hf_datasets.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/pandas.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/parquet.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/table_data_conduit.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/io/utils.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/audio.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/base.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/document.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/image.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/string.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/iterators/video.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_10.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_12.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_13.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_14.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_15.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_16.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_17.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_18.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_19.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_20.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_21.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_22.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_23.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_24.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_25.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_26.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_27.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_28.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_29.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_30.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_31.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_32.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_33.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_34.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_35.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_36.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/convert_37.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/converters/util.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/metadata/schema.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/py.typed +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/share/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/share/packager.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/share/publish.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/type_system.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/__init__.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/arrow.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/coco.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/code.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/console_output.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/coroutine.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/dbms.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/description_helper.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/documents.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/exception_handler.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/filecache.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/formatter.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/http_server.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/iceberg.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/media_store.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/pytorch.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/s3.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/sql.py +0 -0
- {pixeltable-0.4.1 → pixeltable-0.4.2}/pixeltable/utils/transactional_directory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: pixeltable
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
# These version placeholders will be replaced during build.
|
|
2
|
-
__version__ = '0.4.
|
|
3
|
-
__version_tuple__ = (0, 4,
|
|
2
|
+
__version__ = '0.4.2'
|
|
3
|
+
__version_tuple__ = (0, 4, 2)
|
|
@@ -8,6 +8,7 @@ from .globals import (
|
|
|
8
8
|
IfNotExistsParam,
|
|
9
9
|
MediaValidation,
|
|
10
10
|
QColumnId,
|
|
11
|
+
RowCountStats,
|
|
11
12
|
UpdateStatus,
|
|
12
13
|
is_valid_identifier,
|
|
13
14
|
is_valid_path,
|
|
@@ -18,6 +19,6 @@ from .path import Path
|
|
|
18
19
|
from .schema_object import SchemaObject
|
|
19
20
|
from .table import Table
|
|
20
21
|
from .table_version import TableVersion
|
|
21
|
-
from .table_version_handle import TableVersionHandle
|
|
22
|
+
from .table_version_handle import ColumnHandle, TableVersionHandle
|
|
22
23
|
from .table_version_path import TableVersionPath
|
|
23
24
|
from .view import View
|
|
@@ -308,7 +308,11 @@ class Catalog:
|
|
|
308
308
|
# we still got a serialization error, despite getting x-locks at the beginning
|
|
309
309
|
msg = f'{tbl.tbl_name()} ({tbl.tbl_id})' if tbl is not None else ''
|
|
310
310
|
_logger.debug(f'Exception: serialization failure: {msg} ({e})')
|
|
311
|
-
raise excs.Error(
|
|
311
|
+
raise excs.Error(
|
|
312
|
+
'That Pixeltable operation could not be completed because it conflicted with another '
|
|
313
|
+
'operation that was run on a different process.\n'
|
|
314
|
+
'Please re-run the operation.'
|
|
315
|
+
) from None
|
|
312
316
|
else:
|
|
313
317
|
raise
|
|
314
318
|
|
|
@@ -911,7 +915,7 @@ class Catalog:
|
|
|
911
915
|
'This is likely due to data corruption in the replicated table.'
|
|
912
916
|
)
|
|
913
917
|
|
|
914
|
-
self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
|
|
918
|
+
self.store_tbl_md(UUID(tbl_id), None, new_tbl_md, new_version_md, new_schema_version_md)
|
|
915
919
|
|
|
916
920
|
@_retry_loop(for_write=False)
|
|
917
921
|
def get_table(self, path: Path) -> Table:
|
|
@@ -1228,6 +1232,43 @@ class Catalog:
|
|
|
1228
1232
|
self._tbls[tbl_id] = view
|
|
1229
1233
|
return view
|
|
1230
1234
|
|
|
1235
|
+
@_retry_loop(for_write=False)
|
|
1236
|
+
def collect_tbl_history(self, tbl_id: UUID, n: Optional[int]) -> list[schema.FullTableMd]:
|
|
1237
|
+
"""
|
|
1238
|
+
Returns the history of up to n versions of the table with the given UUID.
|
|
1239
|
+
|
|
1240
|
+
Args:
|
|
1241
|
+
tbl_id: the UUID of the table to collect history for.
|
|
1242
|
+
n: Optional limit on the maximum number of versions returned.
|
|
1243
|
+
|
|
1244
|
+
Returns:
|
|
1245
|
+
A sequence of rows, ordered by version number
|
|
1246
|
+
Each row contains a TableVersion and a TableSchemaVersion object.
|
|
1247
|
+
"""
|
|
1248
|
+
q = (
|
|
1249
|
+
sql.select(schema.TableVersion, schema.TableSchemaVersion)
|
|
1250
|
+
.select_from(schema.TableVersion)
|
|
1251
|
+
.join(
|
|
1252
|
+
schema.TableSchemaVersion,
|
|
1253
|
+
sql.cast(schema.TableVersion.md['schema_version'], sql.Integer)
|
|
1254
|
+
== schema.TableSchemaVersion.schema_version,
|
|
1255
|
+
)
|
|
1256
|
+
.where(schema.TableVersion.tbl_id == tbl_id)
|
|
1257
|
+
.where(schema.TableSchemaVersion.tbl_id == tbl_id)
|
|
1258
|
+
.order_by(schema.TableVersion.version.desc())
|
|
1259
|
+
)
|
|
1260
|
+
if n is not None:
|
|
1261
|
+
q = q.limit(n)
|
|
1262
|
+
src_rows = Env.get().session.execute(q).fetchall()
|
|
1263
|
+
return [
|
|
1264
|
+
schema.FullTableMd(
|
|
1265
|
+
None,
|
|
1266
|
+
schema.md_from_dict(schema.TableVersionMd, row.TableVersion.md),
|
|
1267
|
+
schema.md_from_dict(schema.TableSchemaVersionMd, row.TableSchemaVersion.md),
|
|
1268
|
+
)
|
|
1269
|
+
for row in src_rows
|
|
1270
|
+
]
|
|
1271
|
+
|
|
1231
1272
|
def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
|
|
1232
1273
|
"""
|
|
1233
1274
|
Loads metadata from the store for a given table UUID and version.
|
|
@@ -1297,19 +1338,27 @@ class Catalog:
|
|
|
1297
1338
|
def store_tbl_md(
|
|
1298
1339
|
self,
|
|
1299
1340
|
tbl_id: UUID,
|
|
1341
|
+
dir_id: Optional[UUID],
|
|
1300
1342
|
tbl_md: Optional[schema.TableMd],
|
|
1301
1343
|
version_md: Optional[schema.TableVersionMd],
|
|
1302
1344
|
schema_version_md: Optional[schema.TableSchemaVersionMd],
|
|
1303
1345
|
) -> None:
|
|
1304
1346
|
"""
|
|
1305
|
-
Stores metadata to the DB.
|
|
1306
|
-
|
|
1347
|
+
Stores metadata to the DB.
|
|
1348
|
+
|
|
1349
|
+
Args:
|
|
1350
|
+
tbl_id: UUID of the table to store metadata for.
|
|
1351
|
+
dir_id: If specified, the tbl_md will be added to the given directory; if None, the table must already exist
|
|
1352
|
+
tbl_md: If specified, `tbl_md` will be inserted, or updated (only one such record can exist per UUID)
|
|
1353
|
+
version_md: inserted as a new record if present
|
|
1354
|
+
schema_version_md: will be inserted as a new record if present
|
|
1307
1355
|
|
|
1308
1356
|
If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
|
|
1309
1357
|
"""
|
|
1310
|
-
conn = Env.get().conn
|
|
1311
1358
|
assert self._in_write_xact
|
|
1359
|
+
session = Env.get().session
|
|
1312
1360
|
|
|
1361
|
+
# Construct and insert or update table record if requested.
|
|
1313
1362
|
if tbl_md is not None:
|
|
1314
1363
|
assert tbl_md.tbl_id == str(tbl_id)
|
|
1315
1364
|
if version_md is not None:
|
|
@@ -1317,32 +1366,37 @@ class Catalog:
|
|
|
1317
1366
|
assert tbl_md.current_schema_version == version_md.schema_version
|
|
1318
1367
|
if schema_version_md is not None:
|
|
1319
1368
|
assert tbl_md.current_schema_version == schema_version_md.schema_version
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
.
|
|
1324
|
-
|
|
1325
|
-
|
|
1369
|
+
if dir_id is not None:
|
|
1370
|
+
# We are inserting a record while creating a new table.
|
|
1371
|
+
tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_md))
|
|
1372
|
+
session.add(tbl_record)
|
|
1373
|
+
else:
|
|
1374
|
+
# Update the existing table record.
|
|
1375
|
+
result = session.execute(
|
|
1376
|
+
sql.update(schema.Table.__table__)
|
|
1377
|
+
.values({schema.Table.md: dataclasses.asdict(tbl_md)})
|
|
1378
|
+
.where(schema.Table.id == tbl_id)
|
|
1379
|
+
)
|
|
1380
|
+
assert result.rowcount == 1, result.rowcount
|
|
1326
1381
|
|
|
1382
|
+
# Construct and insert new table version record if requested.
|
|
1327
1383
|
if version_md is not None:
|
|
1328
1384
|
assert version_md.tbl_id == str(tbl_id)
|
|
1329
1385
|
if schema_version_md is not None:
|
|
1330
1386
|
assert version_md.schema_version == schema_version_md.schema_version
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
|
|
1334
|
-
)
|
|
1387
|
+
tbl_version_record = schema.TableVersion(
|
|
1388
|
+
tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
|
|
1335
1389
|
)
|
|
1390
|
+
session.add(tbl_version_record)
|
|
1336
1391
|
|
|
1392
|
+
# Construct and insert a new schema version record if requested.
|
|
1337
1393
|
if schema_version_md is not None:
|
|
1338
1394
|
assert schema_version_md.tbl_id == str(tbl_id)
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
tbl_id=tbl_id,
|
|
1342
|
-
schema_version=schema_version_md.schema_version,
|
|
1343
|
-
md=dataclasses.asdict(schema_version_md),
|
|
1344
|
-
)
|
|
1395
|
+
schema_version_record = schema.TableSchemaVersion(
|
|
1396
|
+
tbl_id=tbl_id, schema_version=schema_version_md.schema_version, md=dataclasses.asdict(schema_version_md)
|
|
1345
1397
|
)
|
|
1398
|
+
session.add(schema_version_record)
|
|
1399
|
+
session.flush() # Inform SQLAlchemy that we want to write these changes to the DB.
|
|
1346
1400
|
|
|
1347
1401
|
def delete_tbl_md(self, tbl_id: UUID) -> None:
|
|
1348
1402
|
"""
|
|
@@ -15,6 +15,7 @@ from .globals import MediaValidation, is_valid_identifier
|
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
17
|
from .table_version import TableVersion
|
|
18
|
+
from .table_version_handle import ColumnHandle
|
|
18
19
|
from .table_version_path import TableVersionPath
|
|
19
20
|
|
|
20
21
|
_logger = logging.getLogger('pixeltable')
|
|
@@ -148,6 +149,15 @@ class Column:
|
|
|
148
149
|
)
|
|
149
150
|
warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
|
|
150
151
|
|
|
152
|
+
@property
|
|
153
|
+
def handle(self) -> 'ColumnHandle':
|
|
154
|
+
"""Returns a ColumnHandle for this Column."""
|
|
155
|
+
from .table_version_handle import ColumnHandle
|
|
156
|
+
|
|
157
|
+
assert self.tbl is not None
|
|
158
|
+
assert self.id is not None
|
|
159
|
+
return ColumnHandle(self.tbl.handle, self.id)
|
|
160
|
+
|
|
151
161
|
@property
|
|
152
162
|
def value_expr(self) -> Optional[exprs.Expr]:
|
|
153
163
|
assert self.value_expr_dict is None or self._value_expr is not None
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import enum
|
|
5
|
+
import itertools
|
|
6
|
+
import logging
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import TYPE_CHECKING, Optional
|
|
9
|
+
from uuid import UUID
|
|
10
|
+
|
|
11
|
+
import pixeltable.exceptions as excs
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from IPython.lib.pretty import RepresentationPrinter
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_logger = logging.getLogger('pixeltable')
|
|
18
|
+
|
|
19
|
+
# name of the position column in a component view
|
|
20
|
+
_POS_COLUMN_NAME = 'pos'
|
|
21
|
+
_ROWID_COLUMN_NAME = '_rowid'
|
|
22
|
+
|
|
23
|
+
# Set of symbols that are predefined in the `InsertableTable` class (and are therefore not allowed as column names).
|
|
24
|
+
# This will be populated lazily to avoid circular imports.
|
|
25
|
+
_PREDEF_SYMBOLS: Optional[set[str]] = None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class QColumnId:
|
|
30
|
+
"""Qualified column id"""
|
|
31
|
+
|
|
32
|
+
tbl_id: UUID
|
|
33
|
+
col_id: int
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class RowCountStats:
|
|
38
|
+
"""
|
|
39
|
+
Statistics about the counts of rows affected by a table operation.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
ins_rows: int = 0 # rows inserted
|
|
43
|
+
del_rows: int = 0 # rows deleted
|
|
44
|
+
upd_rows: int = 0 # rows updated
|
|
45
|
+
num_excs: int = 0 # total number of exceptions
|
|
46
|
+
# TODO: disambiguate what this means: # of slots computed or # of columns computed?
|
|
47
|
+
computed_values: int = 0 # number of computed values (e.g., computed columns) affected by the operation
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def num_rows(self) -> int:
|
|
51
|
+
return self.ins_rows + self.del_rows + self.upd_rows
|
|
52
|
+
|
|
53
|
+
def insert_to_update(self) -> 'RowCountStats':
|
|
54
|
+
"""
|
|
55
|
+
Convert insert row count stats to update row count stats.
|
|
56
|
+
This is used when an insert operation is treated as an update.
|
|
57
|
+
"""
|
|
58
|
+
return RowCountStats(
|
|
59
|
+
ins_rows=0,
|
|
60
|
+
del_rows=self.del_rows,
|
|
61
|
+
upd_rows=self.upd_rows + self.ins_rows,
|
|
62
|
+
num_excs=self.num_excs,
|
|
63
|
+
computed_values=self.computed_values,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def __add__(self, other: 'RowCountStats') -> 'RowCountStats':
|
|
67
|
+
"""
|
|
68
|
+
Add the stats from two RowCountStats objects together.
|
|
69
|
+
"""
|
|
70
|
+
return RowCountStats(
|
|
71
|
+
ins_rows=self.ins_rows + other.ins_rows,
|
|
72
|
+
del_rows=self.del_rows + other.del_rows,
|
|
73
|
+
upd_rows=self.upd_rows + other.upd_rows,
|
|
74
|
+
num_excs=self.num_excs + other.num_excs,
|
|
75
|
+
computed_values=self.computed_values + other.computed_values,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class UpdateStatus:
|
|
81
|
+
"""
|
|
82
|
+
Information about updates that resulted from a table operation.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
updated_cols: list[str] = dataclasses.field(default_factory=list)
|
|
86
|
+
cols_with_excs: list[str] = dataclasses.field(default_factory=list)
|
|
87
|
+
|
|
88
|
+
# stats for the rows affected by the operation
|
|
89
|
+
row_count_stats: RowCountStats = field(default_factory=RowCountStats)
|
|
90
|
+
|
|
91
|
+
# stats for changes cascaded to other tables
|
|
92
|
+
cascade_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def num_rows(self) -> int:
|
|
96
|
+
return self.row_count_stats.num_rows + self.cascade_row_count_stats.num_rows
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def num_excs(self) -> int:
|
|
100
|
+
return self.row_count_stats.num_excs + self.cascade_row_count_stats.num_excs
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def num_computed_values(self) -> int:
|
|
104
|
+
return self.row_count_stats.computed_values + self.cascade_row_count_stats.computed_values
|
|
105
|
+
|
|
106
|
+
def insert_to_update(self) -> 'UpdateStatus':
|
|
107
|
+
"""
|
|
108
|
+
Convert the update status from an insert operation to an update operation.
|
|
109
|
+
This is used when an insert operation is treated as an update.
|
|
110
|
+
"""
|
|
111
|
+
return UpdateStatus(
|
|
112
|
+
updated_cols=self.updated_cols,
|
|
113
|
+
cols_with_excs=self.cols_with_excs,
|
|
114
|
+
row_count_stats=self.row_count_stats.insert_to_update(),
|
|
115
|
+
cascade_row_count_stats=self.cascade_row_count_stats.insert_to_update(),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def to_cascade(self) -> 'UpdateStatus':
|
|
119
|
+
"""
|
|
120
|
+
Convert the update status to a cascade update status.
|
|
121
|
+
This is used when an operation cascades changes to other tables.
|
|
122
|
+
"""
|
|
123
|
+
return UpdateStatus(
|
|
124
|
+
updated_cols=self.updated_cols,
|
|
125
|
+
cols_with_excs=self.cols_with_excs,
|
|
126
|
+
row_count_stats=RowCountStats(),
|
|
127
|
+
cascade_row_count_stats=self.cascade_row_count_stats + self.row_count_stats,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def __add__(self, other: 'UpdateStatus') -> UpdateStatus:
|
|
131
|
+
"""
|
|
132
|
+
Add the update status from two UpdateStatus objects together.
|
|
133
|
+
"""
|
|
134
|
+
return UpdateStatus(
|
|
135
|
+
updated_cols=list(dict.fromkeys(self.updated_cols + other.updated_cols)),
|
|
136
|
+
cols_with_excs=list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs)),
|
|
137
|
+
row_count_stats=self.row_count_stats + other.row_count_stats,
|
|
138
|
+
cascade_row_count_stats=self.cascade_row_count_stats + other.cascade_row_count_stats,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def insert_msg(self) -> str:
|
|
143
|
+
"""Return a message describing the results of an insert operation."""
|
|
144
|
+
if self.num_excs == 0:
|
|
145
|
+
cols_with_excs_str = ''
|
|
146
|
+
else:
|
|
147
|
+
cols_with_excs_str = (
|
|
148
|
+
f' across {len(self.cols_with_excs)} column{"" if len(self.cols_with_excs) == 1 else "s"}'
|
|
149
|
+
)
|
|
150
|
+
cols_with_excs_str += f' ({", ".join(self.cols_with_excs)})'
|
|
151
|
+
msg = (
|
|
152
|
+
f'Inserted {self.num_rows} row{"" if self.num_rows == 1 else "s"} '
|
|
153
|
+
f'with {self.num_excs} error{"" if self.num_excs == 1 else "s"}{cols_with_excs_str}.'
|
|
154
|
+
)
|
|
155
|
+
return msg
|
|
156
|
+
|
|
157
|
+
@classmethod
|
|
158
|
+
def __cnt_str(cls, cnt: int, item: str) -> str:
|
|
159
|
+
assert cnt > 0
|
|
160
|
+
return f'{cnt} {item}{"" if cnt == 1 else "s"}'
|
|
161
|
+
|
|
162
|
+
def _repr_pretty_(self, p: 'RepresentationPrinter', cycle: bool) -> None:
|
|
163
|
+
messages = []
|
|
164
|
+
if self.row_count_stats.ins_rows > 0:
|
|
165
|
+
messages.append(f'{self.__cnt_str(self.row_count_stats.ins_rows, "row")} inserted')
|
|
166
|
+
if self.row_count_stats.del_rows > 0:
|
|
167
|
+
messages.append(f'{self.__cnt_str(self.row_count_stats.del_rows, "row")} deleted')
|
|
168
|
+
if self.row_count_stats.upd_rows > 0:
|
|
169
|
+
messages.append(f'{self.__cnt_str(self.row_count_stats.upd_rows, "row")} updated')
|
|
170
|
+
if self.num_computed_values > 0:
|
|
171
|
+
messages.append(f'{self.__cnt_str(self.num_computed_values, "value")} computed')
|
|
172
|
+
if self.row_count_stats.num_excs > 0:
|
|
173
|
+
messages.append(self.__cnt_str(self.row_count_stats.num_excs, 'exception'))
|
|
174
|
+
p.text(', '.join(messages) + '.' if len(messages) > 0 else 'No rows affected.')
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class MediaValidation(enum.Enum):
|
|
178
|
+
ON_READ = 0
|
|
179
|
+
ON_WRITE = 1
|
|
180
|
+
|
|
181
|
+
@classmethod
|
|
182
|
+
def validated(cls, name: str, error_prefix: str) -> MediaValidation:
|
|
183
|
+
try:
|
|
184
|
+
return cls[name.upper()]
|
|
185
|
+
except KeyError:
|
|
186
|
+
val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__)
|
|
187
|
+
raise excs.Error(f'{error_prefix} must be one of: [{val_strs}]') from None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class IfExistsParam(enum.Enum):
|
|
191
|
+
ERROR = 0
|
|
192
|
+
IGNORE = 1
|
|
193
|
+
REPLACE = 2
|
|
194
|
+
REPLACE_FORCE = 3
|
|
195
|
+
|
|
196
|
+
@classmethod
|
|
197
|
+
def validated(cls, param_val: str, param_name: str) -> IfExistsParam:
|
|
198
|
+
try:
|
|
199
|
+
return cls[param_val.upper()]
|
|
200
|
+
except KeyError:
|
|
201
|
+
val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__)
|
|
202
|
+
raise excs.Error(f'{param_name} must be one of: [{val_strs}]') from None
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class IfNotExistsParam(enum.Enum):
|
|
206
|
+
ERROR = 0
|
|
207
|
+
IGNORE = 1
|
|
208
|
+
|
|
209
|
+
@classmethod
|
|
210
|
+
def validated(cls, param_val: str, param_name: str) -> IfNotExistsParam:
|
|
211
|
+
try:
|
|
212
|
+
return cls[param_val.upper()]
|
|
213
|
+
except KeyError:
|
|
214
|
+
val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__)
|
|
215
|
+
raise excs.Error(f'{param_name} must be one of: [{val_strs}]') from None
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def is_valid_identifier(name: str, allow_system_identifiers: bool = False) -> bool:
|
|
219
|
+
return name.isidentifier() and (allow_system_identifiers or not name.startswith('_'))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def is_valid_path(path: str, empty_is_valid: bool, allow_system_paths: bool = False) -> bool:
|
|
223
|
+
if path == '':
|
|
224
|
+
return empty_is_valid
|
|
225
|
+
return all(is_valid_identifier(part, allow_system_paths) for part in path.split('.'))
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def is_system_column_name(name: str) -> bool:
|
|
229
|
+
from pixeltable.catalog import InsertableTable, View
|
|
230
|
+
|
|
231
|
+
global _PREDEF_SYMBOLS # noqa: PLW0603
|
|
232
|
+
if _PREDEF_SYMBOLS is None:
|
|
233
|
+
_PREDEF_SYMBOLS = set(itertools.chain(dir(InsertableTable), dir(View)))
|
|
234
|
+
return name == _POS_COLUMN_NAME or name in _PREDEF_SYMBOLS
|
|
@@ -174,10 +174,11 @@ class InsertableTable(Table):
|
|
|
174
174
|
status = pxt.UpdateStatus()
|
|
175
175
|
with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
|
|
176
176
|
if isinstance(data_source, DFTableDataConduit):
|
|
177
|
-
status
|
|
177
|
+
status = self._tbl_version.get().insert(
|
|
178
178
|
rows=None, df=data_source.pxt_df, print_stats=print_stats, fail_on_exception=fail_on_exception
|
|
179
179
|
)
|
|
180
180
|
else:
|
|
181
|
+
status = UpdateStatus()
|
|
181
182
|
for row_batch in data_source.valid_row_batch():
|
|
182
183
|
status += self._tbl_version.get().insert(
|
|
183
184
|
rows=row_batch, df=None, print_stats=print_stats, fail_on_exception=fail_on_exception
|
|
@@ -9,6 +9,7 @@ from pathlib import Path
|
|
|
9
9
|
from typing import TYPE_CHECKING, Any, Iterable, Literal, Optional, Union, overload
|
|
10
10
|
|
|
11
11
|
from typing import _GenericAlias # type: ignore[attr-defined] # isort: skip
|
|
12
|
+
import datetime
|
|
12
13
|
from uuid import UUID
|
|
13
14
|
|
|
14
15
|
import pandas as pd
|
|
@@ -17,6 +18,7 @@ import sqlalchemy as sql
|
|
|
17
18
|
import pixeltable as pxt
|
|
18
19
|
from pixeltable import catalog, env, exceptions as excs, exprs, index, type_system as ts
|
|
19
20
|
from pixeltable.metadata import schema
|
|
21
|
+
from pixeltable.metadata.utils import MetadataUtils
|
|
20
22
|
|
|
21
23
|
from ..exprs import ColumnRef
|
|
22
24
|
from ..utils.description_helper import DescriptionHelper
|
|
@@ -840,11 +842,12 @@ class Table(SchemaObject):
|
|
|
840
842
|
_ = self._get_views(recursive=True, include_snapshots=False)
|
|
841
843
|
# See if this column has a dependent store. We need to look through all stores in all
|
|
842
844
|
# (transitive) views of this table.
|
|
845
|
+
col_handle = col.handle
|
|
843
846
|
dependent_stores = [
|
|
844
847
|
(view, store)
|
|
845
848
|
for view in (self, *self._get_views(recursive=True, include_snapshots=False))
|
|
846
849
|
for store in view._tbl_version.get().external_stores.values()
|
|
847
|
-
if
|
|
850
|
+
if col_handle in store.get_local_columns()
|
|
848
851
|
]
|
|
849
852
|
if len(dependent_stores) > 0:
|
|
850
853
|
dependent_store_names = [
|
|
@@ -1321,6 +1324,9 @@ class Table(SchemaObject):
|
|
|
1321
1324
|
where: a predicate to filter rows to update.
|
|
1322
1325
|
cascade: if True, also update all computed columns that transitively depend on the updated columns.
|
|
1323
1326
|
|
|
1327
|
+
Returns:
|
|
1328
|
+
An [`UpdateStatus`][pixeltable.UpdateStatus] object containing information about the update.
|
|
1329
|
+
|
|
1324
1330
|
Examples:
|
|
1325
1331
|
Set column `int_col` to 1 for all rows:
|
|
1326
1332
|
|
|
@@ -1419,6 +1425,69 @@ class Table(SchemaObject):
|
|
|
1419
1425
|
FileCache.get().emit_eviction_warnings()
|
|
1420
1426
|
return status
|
|
1421
1427
|
|
|
1428
|
+
def recompute_columns(
|
|
1429
|
+
self, *columns: Union[str, ColumnRef], errors_only: bool = False, cascade: bool = True
|
|
1430
|
+
) -> UpdateStatus:
|
|
1431
|
+
"""Recompute the values in one or more computed columns of this table.
|
|
1432
|
+
|
|
1433
|
+
Args:
|
|
1434
|
+
columns: The names or references of the computed columns to recompute.
|
|
1435
|
+
errors_only: If True, only run the recomputation for rows that have errors in the column (ie, the column's
|
|
1436
|
+
`errortype` property is non-None). Only allowed for recomputing a single column.
|
|
1437
|
+
cascade: if True, also update all computed columns that transitively depend on the recomputed columns.
|
|
1438
|
+
|
|
1439
|
+
Examples:
|
|
1440
|
+
Recompute computed columns `c1` and `c2` for all rows in this table, and everything that transitively
|
|
1441
|
+
depends on them:
|
|
1442
|
+
|
|
1443
|
+
>>> tbl.recompute_columns('c1', 'c2')
|
|
1444
|
+
|
|
1445
|
+
Recompute computed column `c1` for all rows in this table, but don't recompute other columns that depend on
|
|
1446
|
+
it:
|
|
1447
|
+
|
|
1448
|
+
>>> tbl.recompute_columns(tbl.c1, tbl.c2, cascade=False)
|
|
1449
|
+
|
|
1450
|
+
Recompute column `c1` and its dependents, but only for rows that have errors in it:
|
|
1451
|
+
|
|
1452
|
+
>>> tbl.recompute_columns('c1', errors_only=True)
|
|
1453
|
+
"""
|
|
1454
|
+
from pixeltable.catalog import Catalog
|
|
1455
|
+
|
|
1456
|
+
cat = Catalog.get()
|
|
1457
|
+
# lock_mutable_tree=True: we need to be able to see whether any transitive view has column dependents
|
|
1458
|
+
with cat.begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
|
|
1459
|
+
if self._tbl_version_path.is_snapshot():
|
|
1460
|
+
raise excs.Error('Cannot recompute columns of a snapshot.')
|
|
1461
|
+
if len(columns) == 0:
|
|
1462
|
+
raise excs.Error('At least one column must be specified to recompute')
|
|
1463
|
+
if errors_only and len(columns) > 1:
|
|
1464
|
+
raise excs.Error('Cannot use errors_only=True with multiple columns')
|
|
1465
|
+
|
|
1466
|
+
col_names: list[str] = []
|
|
1467
|
+
for column in columns:
|
|
1468
|
+
col_name: str
|
|
1469
|
+
col: Column
|
|
1470
|
+
if isinstance(column, str):
|
|
1471
|
+
col = self._tbl_version_path.get_column(column, include_bases=True)
|
|
1472
|
+
if col is None:
|
|
1473
|
+
raise excs.Error(f'Unknown column: {column!r}')
|
|
1474
|
+
col_name = column
|
|
1475
|
+
else:
|
|
1476
|
+
assert isinstance(column, ColumnRef)
|
|
1477
|
+
col = column.col
|
|
1478
|
+
if not self._tbl_version_path.has_column(col, include_bases=True):
|
|
1479
|
+
raise excs.Error(f'Unknown column: {col.name!r}')
|
|
1480
|
+
col_name = col.name
|
|
1481
|
+
if not col.is_computed:
|
|
1482
|
+
raise excs.Error(f'Column {col_name!r} is not a computed column')
|
|
1483
|
+
if col.tbl.id != self._tbl_version_path.tbl_id:
|
|
1484
|
+
raise excs.Error(f'Cannot recompute column of a base: {col_name!r}')
|
|
1485
|
+
col_names.append(col_name)
|
|
1486
|
+
|
|
1487
|
+
status = self._tbl_version.get().recompute_columns(col_names, errors_only=errors_only, cascade=cascade)
|
|
1488
|
+
FileCache.get().emit_eviction_warnings()
|
|
1489
|
+
return status
|
|
1490
|
+
|
|
1422
1491
|
def delete(self, where: Optional['exprs.Expr'] = None) -> UpdateStatus:
|
|
1423
1492
|
"""Delete rows in this table.
|
|
1424
1493
|
|
|
@@ -1532,7 +1601,7 @@ class Table(SchemaObject):
|
|
|
1532
1601
|
from pixeltable.catalog import Catalog
|
|
1533
1602
|
|
|
1534
1603
|
if self._tbl_version_path.is_snapshot():
|
|
1535
|
-
return pxt.io.SyncStatus
|
|
1604
|
+
return pxt.io.SyncStatus()
|
|
1536
1605
|
# we lock the entire tree starting at the root base table in order to ensure that all synced columns can
|
|
1537
1606
|
# have their updates propagated down the tree
|
|
1538
1607
|
base_tv = self._tbl_version_path.get_tbl_versions()[-1]
|
|
@@ -1548,11 +1617,11 @@ class Table(SchemaObject):
|
|
|
1548
1617
|
if store not in all_stores:
|
|
1549
1618
|
raise excs.Error(f'Table `{self._name}` has no external store with that name: {store}')
|
|
1550
1619
|
|
|
1551
|
-
sync_status = pxt.io.SyncStatus
|
|
1620
|
+
sync_status = pxt.io.SyncStatus()
|
|
1552
1621
|
for store in stores:
|
|
1553
1622
|
store_obj = self._tbl_version.get().external_stores[store]
|
|
1554
1623
|
store_sync_status = store_obj.sync(self, export_data=export_data, import_data=import_data)
|
|
1555
|
-
sync_status
|
|
1624
|
+
sync_status += store_sync_status
|
|
1556
1625
|
|
|
1557
1626
|
return sync_status
|
|
1558
1627
|
|
|
@@ -1561,3 +1630,65 @@ class Table(SchemaObject):
|
|
|
1561
1630
|
|
|
1562
1631
|
def _ipython_key_completions_(self) -> list[str]:
|
|
1563
1632
|
return list(self._get_schema().keys())
|
|
1633
|
+
|
|
1634
|
+
def history(self, n: Optional[int] = None) -> pixeltable.dataframe.DataFrameResultSet:
|
|
1635
|
+
"""Returns rows of information about the versions of this table, most recent first.
|
|
1636
|
+
|
|
1637
|
+
Args:
|
|
1638
|
+
n: a limit to the number of versions listed
|
|
1639
|
+
|
|
1640
|
+
Examples:
|
|
1641
|
+
Report history:
|
|
1642
|
+
|
|
1643
|
+
>>> tbl.history()
|
|
1644
|
+
|
|
1645
|
+
Report only the most recent 5 changes to the table:
|
|
1646
|
+
|
|
1647
|
+
>>> tbl.history(n=5)
|
|
1648
|
+
|
|
1649
|
+
Returns:
|
|
1650
|
+
A list of information about each version, ordered from most recent to oldest version.
|
|
1651
|
+
"""
|
|
1652
|
+
from pixeltable.catalog import Catalog
|
|
1653
|
+
|
|
1654
|
+
if n is None:
|
|
1655
|
+
n = 1000_000_000
|
|
1656
|
+
if not isinstance(n, int) or n < 1:
|
|
1657
|
+
raise excs.Error(f'Invalid value for n: {n}')
|
|
1658
|
+
|
|
1659
|
+
# Retrieve the table history components from the catalog
|
|
1660
|
+
tbl_id = self._id
|
|
1661
|
+
# Collect an extra version, if available, to allow for computation of the first version's schema change
|
|
1662
|
+
vers_list = Catalog.get().collect_tbl_history(tbl_id, n + 1)
|
|
1663
|
+
|
|
1664
|
+
# Construct the metadata change description dictionary
|
|
1665
|
+
md_list = [(vers_md.version_md.version, vers_md.schema_version_md.columns) for vers_md in vers_list]
|
|
1666
|
+
md_dict = MetadataUtils._create_md_change_dict(md_list)
|
|
1667
|
+
|
|
1668
|
+
# Construct report lines
|
|
1669
|
+
if len(vers_list) > n:
|
|
1670
|
+
assert len(vers_list) == n + 1
|
|
1671
|
+
over_count = 1
|
|
1672
|
+
else:
|
|
1673
|
+
over_count = 0
|
|
1674
|
+
|
|
1675
|
+
report_lines: list[list[Any]] = []
|
|
1676
|
+
for vers_md in vers_list[0 : len(vers_list) - over_count]:
|
|
1677
|
+
version = vers_md.version_md.version
|
|
1678
|
+
schema_change = md_dict.get(version, '')
|
|
1679
|
+
change_type = 'schema' if schema_change != '' else 'data'
|
|
1680
|
+
report_line = [
|
|
1681
|
+
version,
|
|
1682
|
+
datetime.datetime.fromtimestamp(vers_md.version_md.created_at),
|
|
1683
|
+
change_type,
|
|
1684
|
+
schema_change,
|
|
1685
|
+
]
|
|
1686
|
+
report_lines.append(report_line)
|
|
1687
|
+
|
|
1688
|
+
report_schema = {
|
|
1689
|
+
'version': ts.IntType(),
|
|
1690
|
+
'created_at': ts.TimestampType(),
|
|
1691
|
+
'change': ts.StringType(),
|
|
1692
|
+
'schema_change': ts.StringType(),
|
|
1693
|
+
}
|
|
1694
|
+
return pxt.dataframe.DataFrameResultSet(report_lines, report_schema)
|