pixeltable 0.2.8__py3-none-any.whl → 0.2.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pixeltable might be problematic. Click here for more details.

Files changed (77) hide show
  1. pixeltable/__init__.py +15 -33
  2. pixeltable/__version__.py +2 -2
  3. pixeltable/catalog/catalog.py +1 -1
  4. pixeltable/catalog/column.py +29 -11
  5. pixeltable/catalog/dir.py +2 -2
  6. pixeltable/catalog/insertable_table.py +5 -55
  7. pixeltable/catalog/named_function.py +2 -2
  8. pixeltable/catalog/schema_object.py +2 -7
  9. pixeltable/catalog/table.py +307 -186
  10. pixeltable/catalog/table_version.py +109 -63
  11. pixeltable/catalog/table_version_path.py +28 -5
  12. pixeltable/catalog/view.py +20 -10
  13. pixeltable/dataframe.py +129 -26
  14. pixeltable/env.py +29 -18
  15. pixeltable/exec/exec_context.py +5 -0
  16. pixeltable/exec/exec_node.py +1 -0
  17. pixeltable/exec/in_memory_data_node.py +29 -24
  18. pixeltable/exec/sql_scan_node.py +1 -1
  19. pixeltable/exprs/column_ref.py +13 -8
  20. pixeltable/exprs/data_row.py +4 -0
  21. pixeltable/exprs/expr.py +16 -1
  22. pixeltable/exprs/function_call.py +4 -4
  23. pixeltable/exprs/row_builder.py +29 -20
  24. pixeltable/exprs/similarity_expr.py +4 -3
  25. pixeltable/ext/functions/yolox.py +2 -1
  26. pixeltable/func/__init__.py +1 -0
  27. pixeltable/func/aggregate_function.py +14 -12
  28. pixeltable/func/callable_function.py +8 -6
  29. pixeltable/func/expr_template_function.py +13 -19
  30. pixeltable/func/function.py +3 -6
  31. pixeltable/func/query_template_function.py +84 -0
  32. pixeltable/func/signature.py +68 -23
  33. pixeltable/func/udf.py +13 -10
  34. pixeltable/functions/__init__.py +6 -91
  35. pixeltable/functions/eval.py +26 -14
  36. pixeltable/functions/fireworks.py +25 -23
  37. pixeltable/functions/globals.py +62 -0
  38. pixeltable/functions/huggingface.py +20 -16
  39. pixeltable/functions/image.py +170 -1
  40. pixeltable/functions/openai.py +95 -128
  41. pixeltable/functions/string.py +10 -2
  42. pixeltable/functions/together.py +95 -84
  43. pixeltable/functions/util.py +16 -0
  44. pixeltable/functions/video.py +94 -16
  45. pixeltable/functions/whisper.py +74 -0
  46. pixeltable/globals.py +1 -1
  47. pixeltable/io/__init__.py +10 -0
  48. pixeltable/io/external_store.py +370 -0
  49. pixeltable/io/globals.py +51 -22
  50. pixeltable/io/label_studio.py +639 -0
  51. pixeltable/io/parquet.py +1 -1
  52. pixeltable/iterators/__init__.py +9 -0
  53. pixeltable/iterators/string.py +40 -0
  54. pixeltable/metadata/__init__.py +6 -8
  55. pixeltable/metadata/converters/convert_10.py +2 -4
  56. pixeltable/metadata/converters/convert_12.py +7 -2
  57. pixeltable/metadata/converters/convert_13.py +6 -8
  58. pixeltable/metadata/converters/convert_14.py +2 -4
  59. pixeltable/metadata/converters/convert_15.py +44 -0
  60. pixeltable/metadata/converters/convert_16.py +18 -0
  61. pixeltable/metadata/converters/util.py +66 -0
  62. pixeltable/metadata/schema.py +3 -3
  63. pixeltable/plan.py +8 -7
  64. pixeltable/store.py +1 -1
  65. pixeltable/tool/create_test_db_dump.py +147 -54
  66. pixeltable/tool/embed_udf.py +9 -0
  67. pixeltable/type_system.py +1 -2
  68. pixeltable/utils/code.py +34 -0
  69. {pixeltable-0.2.8.dist-info → pixeltable-0.2.10.dist-info}/METADATA +1 -1
  70. pixeltable-0.2.10.dist-info/RECORD +131 -0
  71. pixeltable/datatransfer/__init__.py +0 -1
  72. pixeltable/datatransfer/label_studio.py +0 -452
  73. pixeltable/datatransfer/remote.py +0 -85
  74. pixeltable/functions/pil/image.py +0 -147
  75. pixeltable-0.2.8.dist-info/RECORD +0 -124
  76. {pixeltable-0.2.8.dist-info → pixeltable-0.2.10.dist-info}/LICENSE +0 -0
  77. {pixeltable-0.2.8.dist-info → pixeltable-0.2.10.dist-info}/WHEEL +0 -0
pixeltable/io/globals.py CHANGED
@@ -1,17 +1,21 @@
1
- from typing import Any, Optional
1
+ from typing import Any, Optional, Literal
2
2
 
3
- import pixeltable as pxt
3
+ import pixeltable.exceptions as excs
4
4
  from pixeltable import Table
5
+ from pixeltable.io.external_store import SyncStatus
5
6
 
6
7
 
7
8
  def create_label_studio_project(
8
9
  t: Table,
9
10
  label_config: str,
10
- col_mapping: Optional[dict[str, str]] = None,
11
+ name: Optional[str] = None,
11
12
  title: Optional[str] = None,
13
+ media_import_method: Literal['post', 'file', 'url'] = 'post',
14
+ col_mapping: Optional[dict[str, str]] = None,
12
15
  sync_immediately: bool = True,
13
16
  **kwargs: Any
14
- ) -> None:
17
+ ) -> SyncStatus:
18
+ # TODO(aaron-siegel): Add link in docstring to a Label Studio howto
15
19
  """
16
20
  Creates a new Label Studio project and links it to the specified `Table`.
17
21
 
@@ -26,33 +30,58 @@ def create_label_studio_project(
26
30
  will always have a JSON-typed column `annotations` representing the output.
27
31
 
28
32
  By default, Pixeltable will link each of these columns to a column of the specified `Table`
29
- with the same name. If any of the data fields are missing, an exception will be thrown. If
33
+ with the same name. If any of the data fields are missing, an exception will be raised. If
30
34
  the `annotations` column is missing, it will be created. The default names can be overridden
31
35
  by specifying an optional `col_mapping`, with Pixeltable column names as keys and Label
32
- Studio field names as values.
36
+ Studio field names as values. In all cases, the Pixeltable columns must have types that are
37
+ consistent with their corresponding Label Studio fields; otherwise, an exception will be raised.
38
+
39
+ The API key and URL for a valid Label Studio server must be specified in Pixeltable config. Either:
40
+
41
+ * Set the `LABEL_STUDIO_API_KEY` and `LABEL_STUDIO_URL` environment variables; or
42
+ * Specify `api_key` and `url` fields in the `label-studio` section of `$PIXELTABLE_HOME/config.yaml`.
33
43
 
34
44
  Args:
35
45
  t: The Table to link to.
36
46
  label_config: The Label Studio project configuration, in XML format.
37
- col_mapping: An optional mapping of local column names to remote column names.
38
- title: An optional title for the Label Studio project. If not specified, the
39
- name of the `Table` will be used as a default.
47
+ name: An optional name for the new project in Pixeltable. If specified, must be a valid
48
+ Pixeltable identifier and must not be the name of any other external data store
49
+ linked to `t`. If not specified, a default name will be used of the form
50
+ `ls_project_0`, `ls_project_1`, etc.
51
+ title: An optional title for the Label Studio project. This is the title that annotators
52
+ will see inside Label Studio. Unlike `name`, it does not need to be an identifier and
53
+ does not need to be unique. If not specified, the table name `t.get_name()` will be used.
54
+ media_import_method: The method to use when transferring media files to Label Studio:
55
+ - `post`: Media will be sent to Label Studio via HTTP post. This should generally only be used for
56
+ prototyping; due to restrictions in Label Studio, it can only be used with projects that have
57
+ just one data field, and does not scale well.
58
+ - `file`: Media will be sent to Label Studio as a file on the local filesystem. This method can be
59
+ used if Pixeltable and Label Studio are running on the same host.
60
+ - `url`: Media will be sent to Label Studio as externally accessible URLs. This method cannot be
61
+ used with local media files or with media generated by computed columns.
62
+ The default is `post`.
63
+ col_mapping: An optional mapping of local column names to Label Studio fields.
40
64
  sync_immediately: If `True`, immediately perform an initial synchronization by
41
- importing all rows of the `Table` as Label Studio tasks.
65
+ exporting all rows of the `Table` as Label Studio tasks.
66
+ kwargs: Additional keyword arguments are passed to the `start_project` method in the Label
67
+ Studio SDK, as described here:
68
+ https://labelstud.io/sdk/project.html#label_studio_sdk.project.Project.start_project
42
69
  """
43
- from pixeltable.datatransfer.label_studio import LabelStudioProject, ANNOTATIONS_COLUMN
70
+ from pixeltable.io.label_studio import LabelStudioProject
44
71
 
45
- ls_project = LabelStudioProject.create(title or t.get_name(), label_config, **kwargs)
46
-
47
- # Create a column to hold the annotations, if one does not yet exist.
48
- if col_mapping is not None and ANNOTATIONS_COLUMN in col_mapping.values():
49
- local_annotations_column = next(k for k, v in col_mapping.items() if v == ANNOTATIONS_COLUMN)
50
- else:
51
- local_annotations_column = ANNOTATIONS_COLUMN
52
- if local_annotations_column not in t.column_names():
53
- t[local_annotations_column] = pxt.JsonType(nullable=True)
72
+ ls_project = LabelStudioProject.create(
73
+ t,
74
+ label_config,
75
+ name,
76
+ title,
77
+ media_import_method,
78
+ col_mapping,
79
+ **kwargs
80
+ )
54
81
 
55
82
  # Link the project to `t`, and sync if appropriate.
56
- t._link(ls_project, col_mapping)
83
+ t._link_external_store(ls_project)
57
84
  if sync_immediately:
58
- t.sync()
85
+ return t.sync()
86
+ else:
87
+ return SyncStatus.empty()