pyspiral 0.5.0__cp310-abi3-macosx_11_0_arm64.whl → 0.6.1__cp310-abi3-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyspiral-0.5.0.dist-info → pyspiral-0.6.1.dist-info}/METADATA +7 -3
- pyspiral-0.6.1.dist-info/RECORD +99 -0
- {pyspiral-0.5.0.dist-info → pyspiral-0.6.1.dist-info}/WHEEL +1 -1
- spiral/__init__.py +11 -3
- spiral/_lib.abi3.so +0 -0
- spiral/adbc.py +6 -6
- spiral/api/__init__.py +8 -2
- spiral/api/client.py +1 -1
- spiral/api/key_space_indexes.py +23 -0
- spiral/api/projects.py +15 -0
- spiral/api/text_indexes.py +1 -1
- spiral/cli/__init__.py +15 -6
- spiral/cli/admin.py +2 -4
- spiral/cli/app.py +4 -2
- spiral/cli/fs.py +5 -6
- spiral/cli/iceberg.py +97 -0
- spiral/cli/key_spaces.py +89 -0
- spiral/cli/login.py +6 -7
- spiral/cli/orgs.py +7 -8
- spiral/cli/printer.py +3 -3
- spiral/cli/projects.py +5 -6
- spiral/cli/tables.py +131 -0
- spiral/cli/telemetry.py +3 -4
- spiral/cli/text.py +115 -0
- spiral/cli/types.py +3 -4
- spiral/cli/workloads.py +7 -8
- spiral/client.py +111 -8
- spiral/core/authn/__init__.pyi +27 -0
- spiral/core/client/__init__.pyi +152 -63
- spiral/core/table/__init__.pyi +17 -27
- spiral/core/table/metastore/__init__.pyi +0 -4
- spiral/core/table/spec/__init__.pyi +0 -2
- spiral/{tables/dataset.py → dataset.py} +13 -7
- spiral/{tables/debug → debug}/manifests.py +15 -6
- spiral/{tables/debug → debug}/scan.py +3 -3
- spiral/expressions/base.py +3 -3
- spiral/expressions/udf.py +1 -1
- spiral/{iceberg/client.py → iceberg.py} +1 -3
- spiral/key_space_index.py +44 -0
- spiral/project.py +171 -18
- spiral/protogen/_/arrow/flight/protocol/sql/__init__.py +1668 -1110
- spiral/protogen/_/google/protobuf/__init__.py +2190 -0
- spiral/protogen/_/message_pool.py +3 -0
- spiral/protogen/_/py.typed +0 -0
- spiral/protogen/_/scandal/__init__.py +138 -126
- spiral/protogen/_/spfs/__init__.py +72 -0
- spiral/protogen/_/spql/__init__.py +61 -0
- spiral/protogen/_/substrait/__init__.py +5256 -2459
- spiral/protogen/_/substrait/extensions/__init__.py +103 -49
- spiral/{tables/scan.py → scan.py} +38 -44
- spiral/settings.py +14 -3
- spiral/snapshot.py +55 -0
- spiral/streaming_/__init__.py +3 -0
- spiral/streaming_/reader.py +131 -0
- spiral/streaming_/stream.py +146 -0
- spiral/substrait_.py +9 -9
- spiral/table.py +259 -0
- spiral/text_index.py +17 -0
- spiral/{tables/transaction.py → transaction.py} +11 -15
- pyspiral-0.5.0.dist-info/RECORD +0 -103
- spiral/cli/iceberg/__init__.py +0 -7
- spiral/cli/iceberg/namespaces.py +0 -47
- spiral/cli/iceberg/tables.py +0 -60
- spiral/cli/indexes/__init__.py +0 -40
- spiral/cli/indexes/args.py +0 -39
- spiral/cli/indexes/workers.py +0 -59
- spiral/cli/tables/__init__.py +0 -88
- spiral/cli/tables/args.py +0 -42
- spiral/core/index/__init__.pyi +0 -7
- spiral/iceberg/__init__.py +0 -3
- spiral/indexes/__init__.py +0 -5
- spiral/indexes/client.py +0 -137
- spiral/indexes/index.py +0 -28
- spiral/indexes/scan.py +0 -22
- spiral/protogen/_/spiral/table/__init__.py +0 -22
- spiral/protogen/substrait/__init__.py +0 -3399
- spiral/protogen/substrait/extensions/__init__.py +0 -115
- spiral/tables/__init__.py +0 -12
- spiral/tables/client.py +0 -133
- spiral/tables/maintenance.py +0 -12
- spiral/tables/snapshot.py +0 -78
- spiral/tables/table.py +0 -145
- {pyspiral-0.5.0.dist-info → pyspiral-0.6.1.dist-info}/entry_points.txt +0 -0
- /spiral/{protogen/_/spiral → debug}/__init__.py +0 -0
- /spiral/{tables/debug → debug}/metrics.py +0 -0
- /spiral/{tables/debug → protogen/_/google}/__init__.py +0 -0
spiral/cli/login.py
CHANGED
@@ -1,22 +1,21 @@
|
|
1
1
|
import jwt
|
2
|
-
from rich import print
|
3
2
|
|
4
|
-
from spiral.cli import state
|
3
|
+
from spiral.cli import CONSOLE, state
|
5
4
|
|
6
5
|
|
7
6
|
def command(org_id: str | None = None, force: bool = False):
|
8
7
|
token = state.settings.device_code_auth.authenticate(force=force, org_id=org_id)
|
9
|
-
print("Successfully logged in.")
|
10
|
-
print(token.expose_secret())
|
8
|
+
CONSOLE.print("Successfully logged in.")
|
9
|
+
CONSOLE.print(token.expose_secret(), soft_wrap=True)
|
11
10
|
|
12
11
|
|
13
12
|
def whoami():
|
14
13
|
"""Display the current user's information."""
|
15
14
|
payload = jwt.decode(state.settings.authn.token().expose_secret(), options={"verify_signature": False})
|
16
|
-
print(f"{payload['org_id']}")
|
17
|
-
print(f"{payload['sub']}")
|
15
|
+
CONSOLE.print(f"{payload['org_id']}")
|
16
|
+
CONSOLE.print(f"{payload['sub']}")
|
18
17
|
|
19
18
|
|
20
19
|
def logout():
|
21
20
|
state.settings.device_code_auth.logout()
|
22
|
-
print("Logged out.")
|
21
|
+
CONSOLE.print("Logged out.")
|
spiral/cli/orgs.py
CHANGED
@@ -2,13 +2,12 @@ import webbrowser
|
|
2
2
|
from typing import Annotated
|
3
3
|
|
4
4
|
import jwt
|
5
|
-
import rich
|
6
5
|
import typer
|
7
6
|
from rich.table import Table
|
8
7
|
from typer import Option
|
9
8
|
|
10
9
|
from spiral.api.organizations import CreateOrgRequest, InviteUserRequest, OrgRole, PortalLinkIntent, PortalLinkRequest
|
11
|
-
from spiral.cli import AsyncTyper, state
|
10
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
|
12
11
|
from spiral.cli.types import OrganizationArg
|
13
12
|
|
14
13
|
app = AsyncTyper(short_help="Org admin.")
|
@@ -17,7 +16,7 @@ app = AsyncTyper(short_help="Org admin.")
|
|
17
16
|
@app.command(help="Switch the active organization.")
|
18
17
|
def switch(org_id: OrganizationArg):
|
19
18
|
state.settings.device_code_auth.authenticate(org_id=org_id)
|
20
|
-
|
19
|
+
CONSOLE.print(f"Switched to organization: {org_id}")
|
21
20
|
|
22
21
|
|
23
22
|
@app.command(help="Create a new organization.")
|
@@ -29,7 +28,7 @@ def create(
|
|
29
28
|
# Authenticate to the new organization
|
30
29
|
state.settings.device_code_auth.authenticate(org_id=res.org.id)
|
31
30
|
|
32
|
-
|
31
|
+
CONSOLE.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
|
33
32
|
|
34
33
|
|
35
34
|
@app.command(help="List organizations.")
|
@@ -40,7 +39,7 @@ def ls():
|
|
40
39
|
for m in state.settings.api.organization.list_memberships():
|
41
40
|
table.add_row("👉" if m.org.id == org_id else "", m.org.id, m.org.name, m.role)
|
42
41
|
|
43
|
-
|
42
|
+
CONSOLE.print(table)
|
44
43
|
|
45
44
|
|
46
45
|
@app.command(help="Invite a user to the organization.")
|
@@ -48,7 +47,7 @@ def invite(email: str, role: OrgRole = OrgRole.MEMBER, expires_in_days: int = 7)
|
|
48
47
|
state.settings.api.organization.invite_user(
|
49
48
|
InviteUserRequest(email=email, role=role, expires_in_days=expires_in_days)
|
50
49
|
)
|
51
|
-
|
50
|
+
CONSOLE.print(f"Invited {email} as a {role.value}.")
|
52
51
|
|
53
52
|
|
54
53
|
@app.command(help="Configure single sign-on for your organization.")
|
@@ -78,7 +77,7 @@ def domains():
|
|
78
77
|
|
79
78
|
def _do_action(intent: PortalLinkIntent):
|
80
79
|
res = state.settings.api.organization.portal_link(PortalLinkRequest(intent=intent))
|
81
|
-
|
80
|
+
CONSOLE.print(f"Opening the configuration portal:\n{res.url}")
|
82
81
|
webbrowser.open(res.url)
|
83
82
|
|
84
83
|
|
@@ -86,5 +85,5 @@ def current_org_id():
|
|
86
85
|
if token := state.settings.authn.token():
|
87
86
|
if org_id := jwt.decode(token.expose_secret(), options={"verify_signature": False}).get("org_id"):
|
88
87
|
return org_id
|
89
|
-
|
88
|
+
ERR_CONSOLE.print("You are not logged in to an organization.")
|
90
89
|
raise typer.Exit(1)
|
spiral/cli/printer.py
CHANGED
@@ -2,14 +2,14 @@ import json
|
|
2
2
|
from collections.abc import Iterable
|
3
3
|
from typing import TypeVar
|
4
4
|
|
5
|
-
import
|
5
|
+
import betterproto2
|
6
6
|
from pydantic import BaseModel
|
7
7
|
from rich.console import ConsoleRenderable, Group
|
8
8
|
from rich.padding import Padding
|
9
9
|
from rich.pretty import Pretty
|
10
10
|
from rich.table import Table
|
11
11
|
|
12
|
-
T = TypeVar("T", bound=
|
12
|
+
T = TypeVar("T", bound=betterproto2.Message)
|
13
13
|
M = TypeVar("M", bound=BaseModel)
|
14
14
|
|
15
15
|
|
@@ -34,7 +34,7 @@ def table_of_protos(cls: type[T], messages: Iterable[T], fields: list[str] = Non
|
|
34
34
|
|
35
35
|
TODO(ngates): add a CLI switch to emit JSON results instead of tables.
|
36
36
|
"""
|
37
|
-
cols = fields or cls().
|
37
|
+
cols = fields or cls()._betterproto2.sorted_field_names
|
38
38
|
table = Table(*cols, title=title or f"{cls.__name__}s")
|
39
39
|
for msg in messages:
|
40
40
|
table.add_row(*[getattr(msg, col, "") for col in cols])
|
spiral/cli/projects.py
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
from typing import Annotated
|
2
2
|
|
3
|
-
import rich
|
4
3
|
import typer
|
5
4
|
from typer import Option
|
6
5
|
|
@@ -19,7 +18,7 @@ from spiral.api.projects import (
|
|
19
18
|
Project,
|
20
19
|
WorkloadPrincipalConditions,
|
21
20
|
)
|
22
|
-
from spiral.cli import AsyncTyper, printer, state
|
21
|
+
from spiral.cli import CONSOLE, AsyncTyper, printer, state
|
23
22
|
from spiral.cli.types import ProjectArg
|
24
23
|
|
25
24
|
app = AsyncTyper(short_help="Projects and grants.")
|
@@ -28,7 +27,7 @@ app = AsyncTyper(short_help="Projects and grants.")
|
|
28
27
|
@app.command(help="List projects.")
|
29
28
|
def ls():
|
30
29
|
projects = list(state.settings.api.project.list())
|
31
|
-
|
30
|
+
CONSOLE.print(printer.table_of_models(Project, projects))
|
32
31
|
|
33
32
|
|
34
33
|
@app.command(help="Create a new project.")
|
@@ -39,7 +38,7 @@ def create(
|
|
39
38
|
name: Annotated[str | None, Option(help="Friendly name for the project.")] = None,
|
40
39
|
):
|
41
40
|
res = state.settings.api.project.create(CreateProjectRequest(id_prefix=id_prefix, name=name))
|
42
|
-
|
41
|
+
CONSOLE.print(f"Created project {res.project.id}")
|
43
42
|
|
44
43
|
|
45
44
|
@app.command(help="Grant a role on a project.")
|
@@ -127,10 +126,10 @@ def grant(
|
|
127
126
|
),
|
128
127
|
)
|
129
128
|
|
130
|
-
|
129
|
+
CONSOLE.print(f"Granted role {role} on project {project}")
|
131
130
|
|
132
131
|
|
133
132
|
@app.command(help="List project grants.")
|
134
133
|
def grants(project: ProjectArg):
|
135
134
|
project_grants = list(state.settings.api.project.list_grants(project))
|
136
|
-
|
135
|
+
CONSOLE.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
|
spiral/cli/tables.py
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
from typing import Annotated
|
2
|
+
|
3
|
+
import questionary
|
4
|
+
import rich
|
5
|
+
import rich.table
|
6
|
+
import typer
|
7
|
+
from questionary import Choice
|
8
|
+
from typer import Argument, Option
|
9
|
+
|
10
|
+
from spiral import Spiral
|
11
|
+
from spiral.api.projects import TableResource
|
12
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
|
13
|
+
from spiral.cli.types import ProjectArg
|
14
|
+
from spiral.debug.manifests import display_manifests
|
15
|
+
from spiral.table import Table
|
16
|
+
|
17
|
+
app = AsyncTyper(short_help="Spiral Tables.")
|
18
|
+
|
19
|
+
|
20
|
+
def ask_table(project_id: str, title: str = "Select a table") -> str:
|
21
|
+
tables: list[TableResource] = list(state.spiral.project(project_id).list_tables())
|
22
|
+
|
23
|
+
if not tables:
|
24
|
+
ERR_CONSOLE.print("No tables found")
|
25
|
+
raise typer.Exit(1)
|
26
|
+
|
27
|
+
return questionary.select( # pyright: ignore[reportAny]
|
28
|
+
title,
|
29
|
+
choices=[
|
30
|
+
Choice(title=f"{table.dataset}.{table.table}", value=f"{table.dataset}.{table.table}")
|
31
|
+
for table in sorted(tables, key=lambda t: (t.dataset, t.table))
|
32
|
+
],
|
33
|
+
).ask()
|
34
|
+
|
35
|
+
|
36
|
+
def get_table(
|
37
|
+
project: ProjectArg,
|
38
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
39
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
40
|
+
) -> tuple[str, Table]:
|
41
|
+
if table is None:
|
42
|
+
identifier = ask_table(project)
|
43
|
+
else:
|
44
|
+
identifier = table
|
45
|
+
if dataset is not None:
|
46
|
+
identifier = f"{dataset}.{table}"
|
47
|
+
return identifier, state.spiral.project(project).table(identifier)
|
48
|
+
|
49
|
+
|
50
|
+
@app.command(help="List tables.")
|
51
|
+
def ls(
|
52
|
+
project: ProjectArg,
|
53
|
+
):
|
54
|
+
tables = Spiral().project(project).list_tables()
|
55
|
+
|
56
|
+
rich_table = rich.table.Table("id", "dataset", "name", title="Spiral tables")
|
57
|
+
for table in tables:
|
58
|
+
rich_table.add_row(table.id, table.dataset, table.table)
|
59
|
+
CONSOLE.print(rich_table)
|
60
|
+
|
61
|
+
|
62
|
+
@app.command(help="Show the table key schema.")
|
63
|
+
def key_schema(
|
64
|
+
project: ProjectArg,
|
65
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
66
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
67
|
+
):
|
68
|
+
_, t = get_table(project, table, dataset)
|
69
|
+
CONSOLE.print(t.key_schema)
|
70
|
+
|
71
|
+
|
72
|
+
@app.command(help="Compute the full table schema.")
|
73
|
+
def schema(
|
74
|
+
project: ProjectArg,
|
75
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
76
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
77
|
+
):
|
78
|
+
_, t = get_table(project, table, dataset)
|
79
|
+
CONSOLE.print(t.schema())
|
80
|
+
|
81
|
+
|
82
|
+
@app.command(help="Flush Write-Ahead-Log.")
|
83
|
+
def flush(
|
84
|
+
project: ProjectArg,
|
85
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
86
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
87
|
+
):
|
88
|
+
identifier, t = get_table(project, table, dataset)
|
89
|
+
state.spiral._ops().flush_wal(t.core) # pyright: ignore[reportPrivateUsage]
|
90
|
+
CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
|
91
|
+
|
92
|
+
|
93
|
+
@app.command(help="Display scan.")
|
94
|
+
def debug(
|
95
|
+
project: ProjectArg,
|
96
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
97
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
98
|
+
column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
|
99
|
+
):
|
100
|
+
_, t = get_table(project, table, dataset)
|
101
|
+
scan = state.spiral.scan(t[column_group] if column_group != "." else t)
|
102
|
+
scan._debug() # pyright: ignore[reportPrivateUsage]
|
103
|
+
|
104
|
+
|
105
|
+
@app.command(help="Display all manifests.")
|
106
|
+
def manifests(
|
107
|
+
project: ProjectArg,
|
108
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
109
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
110
|
+
):
|
111
|
+
_, t = get_table(project, table, dataset)
|
112
|
+
s = t.snapshot()
|
113
|
+
|
114
|
+
key_space_state = state.spiral._ops().key_space_state(s.core) # pyright: ignore[reportPrivateUsage]
|
115
|
+
key_space_manifest = key_space_state.manifest
|
116
|
+
|
117
|
+
column_groups_states = state.spiral._ops().column_groups_states(s.core, key_space_state) # pyright: ignore[reportPrivateUsage]
|
118
|
+
|
119
|
+
display_manifests(key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states])
|
120
|
+
|
121
|
+
|
122
|
+
@app.command(help="Display the manifests which would be read by a scan of the given column group.")
|
123
|
+
def scan_manifests(
|
124
|
+
project: ProjectArg,
|
125
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
126
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
127
|
+
column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
|
128
|
+
):
|
129
|
+
_, t = get_table(project, table, dataset)
|
130
|
+
scan = state.spiral.scan(t[column_group] if column_group != "." else t)
|
131
|
+
scan._dump_manifests() # pyright: ignore[reportPrivateUsage]
|
spiral/cli/telemetry.py
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
import pyperclip
|
2
|
-
import rich
|
3
2
|
|
4
3
|
from spiral.api.telemetry import IssueExportTokenResponse
|
5
|
-
from spiral.cli import AsyncTyper, state
|
4
|
+
from spiral.cli import CONSOLE, AsyncTyper, state
|
6
5
|
|
7
6
|
app = AsyncTyper(short_help="Client-side telemetry.")
|
8
7
|
|
@@ -14,5 +13,5 @@ def export():
|
|
14
13
|
command = f"export SPIRAL_OTEL_TOKEN={res.token}"
|
15
14
|
pyperclip.copy(command)
|
16
15
|
|
17
|
-
|
18
|
-
|
16
|
+
CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
|
17
|
+
CONSOLE.print("[dim]Token is valid for 1h.[/dim]")
|
spiral/cli/text.py
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
from typing import Annotated
|
2
|
+
|
3
|
+
import questionary
|
4
|
+
import rich
|
5
|
+
import typer
|
6
|
+
from questionary import Choice
|
7
|
+
from typer import Option
|
8
|
+
|
9
|
+
from spiral.api.projects import TextIndexResource
|
10
|
+
from spiral.api.text_indexes import CreateWorkerRequest, SyncIndexRequest
|
11
|
+
from spiral.api.types import IndexId
|
12
|
+
from spiral.api.workers import CPU, GcpRegion, Memory, ResourceClass
|
13
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
|
14
|
+
from spiral.cli.types import ProjectArg
|
15
|
+
|
16
|
+
app = AsyncTyper(short_help="Text Indexes.")
|
17
|
+
|
18
|
+
|
19
|
+
def ask_index(project_id, title="Select an index"):
|
20
|
+
indexes: list[TextIndexResource] = list(state.spiral.project(project_id).list_text_indexes())
|
21
|
+
|
22
|
+
if not indexes:
|
23
|
+
ERR_CONSOLE.print("No indexes found")
|
24
|
+
raise typer.Exit(1)
|
25
|
+
|
26
|
+
return questionary.select(
|
27
|
+
title,
|
28
|
+
choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
|
29
|
+
).ask()
|
30
|
+
|
31
|
+
|
32
|
+
def get_index_id(
|
33
|
+
project: ProjectArg,
|
34
|
+
name: Annotated[str | None, Option(help="Index name.")] = None,
|
35
|
+
) -> IndexId:
|
36
|
+
if name is None:
|
37
|
+
return ask_index(project)
|
38
|
+
|
39
|
+
indexes: list[TextIndexResource] = list(state.spiral.project(project).list_text_indexes())
|
40
|
+
for index in indexes:
|
41
|
+
if index.name == name:
|
42
|
+
return index.id
|
43
|
+
raise ValueError(f"Index not found: {name}")
|
44
|
+
|
45
|
+
|
46
|
+
@app.command(help="List indexes.")
|
47
|
+
def ls(
|
48
|
+
project: ProjectArg,
|
49
|
+
):
|
50
|
+
"""List indexes."""
|
51
|
+
indexes = state.spiral.project(project).list_text_indexes()
|
52
|
+
|
53
|
+
rich_table = rich.table.Table("id", "name", title="Text Indexes")
|
54
|
+
for index in indexes:
|
55
|
+
rich_table.add_row(index.id, index.name)
|
56
|
+
CONSOLE.print(rich_table)
|
57
|
+
|
58
|
+
|
59
|
+
@app.command(help="Trigger a sync job for an index.")
|
60
|
+
def sync(
|
61
|
+
project: ProjectArg,
|
62
|
+
name: Annotated[str | None, Option(help="Index name.")] = None,
|
63
|
+
resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
|
64
|
+
):
|
65
|
+
"""Trigger a sync job."""
|
66
|
+
index_id = get_index_id(project, name)
|
67
|
+
response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
|
68
|
+
CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
|
69
|
+
|
70
|
+
|
71
|
+
@app.command(name="serve", help="Spin up a worker to serve an index.")
|
72
|
+
def serve(
|
73
|
+
project: ProjectArg,
|
74
|
+
index: Annotated[str | None, Option(help="Index name.")] = None,
|
75
|
+
region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
|
76
|
+
cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
|
77
|
+
memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
|
78
|
+
):
|
79
|
+
"""Spin up a worker."""
|
80
|
+
index_id = get_index_id(project, index)
|
81
|
+
request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
|
82
|
+
response = state.spiral.api.text_indexes.create_worker(index_id, request)
|
83
|
+
CONSOLE.print(f"Created worker {response.worker_id} for {index_id}.")
|
84
|
+
|
85
|
+
|
86
|
+
@app.command(name="workers", help="List search workers.")
|
87
|
+
def workers(
|
88
|
+
project: ProjectArg,
|
89
|
+
index: Annotated[str | None, Option(help="Index name.")] = None,
|
90
|
+
):
|
91
|
+
"""List text search workers."""
|
92
|
+
index_id = get_index_id(project, index)
|
93
|
+
worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
|
94
|
+
|
95
|
+
rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
|
96
|
+
for worker_id in worker_ids:
|
97
|
+
try:
|
98
|
+
worker = state.spiral.api.text_indexes.get_worker(worker_id)
|
99
|
+
rich_table.add_row(
|
100
|
+
worker_id,
|
101
|
+
worker.url,
|
102
|
+
)
|
103
|
+
except Exception:
|
104
|
+
rich_table.add_row(
|
105
|
+
worker_id,
|
106
|
+
"Unavailable",
|
107
|
+
)
|
108
|
+
CONSOLE.print(rich_table)
|
109
|
+
|
110
|
+
|
111
|
+
@app.command(name="shutdown", help="Shutdown a search worker.")
|
112
|
+
def shutdown(worker_id: str):
|
113
|
+
"""Shutdown a worker."""
|
114
|
+
state.spiral.api.text_indexes.shutdown_worker(worker_id)
|
115
|
+
CONSOLE.print(f"Requested worker {worker_id} to shutdown.")
|
spiral/cli/types.py
CHANGED
@@ -1,20 +1,19 @@
|
|
1
1
|
from typing import Annotated
|
2
2
|
|
3
3
|
import questionary
|
4
|
-
import rich
|
5
4
|
import typer
|
6
5
|
from questionary import Choice
|
7
6
|
from typer import Argument
|
8
7
|
|
9
8
|
from spiral.api.types import OrgId, ProjectId
|
10
|
-
from spiral.cli import state
|
9
|
+
from spiral.cli import ERR_CONSOLE, state
|
11
10
|
|
12
11
|
|
13
12
|
def ask_project(title="Select a project"):
|
14
13
|
projects = list(state.settings.api.project.list())
|
15
14
|
|
16
15
|
if not projects:
|
17
|
-
|
16
|
+
ERR_CONSOLE.print("No projects found")
|
18
17
|
raise typer.Exit(1)
|
19
18
|
|
20
19
|
return questionary.select(
|
@@ -33,7 +32,7 @@ def _org_default():
|
|
33
32
|
memberships = list(state.settings.api.organization.list_memberships())
|
34
33
|
|
35
34
|
if not memberships:
|
36
|
-
|
35
|
+
ERR_CONSOLE.print("No organizations found")
|
37
36
|
raise typer.Exit(1)
|
38
37
|
|
39
38
|
return questionary.select(
|
spiral/cli/workloads.py
CHANGED
@@ -2,12 +2,11 @@ from typing import Annotated
|
|
2
2
|
|
3
3
|
import pyperclip
|
4
4
|
import questionary
|
5
|
-
import rich
|
6
5
|
from questionary import Choice
|
7
6
|
from typer import Argument, Option
|
8
7
|
|
9
8
|
from spiral.api.workloads import CreateWorkloadRequest, IssueWorkloadCredentialsResponse, Workload
|
10
|
-
from spiral.cli import AsyncTyper, printer, state
|
9
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, printer, state
|
11
10
|
from spiral.cli.types import ProjectArg
|
12
11
|
|
13
12
|
app = AsyncTyper()
|
@@ -19,7 +18,7 @@ def create(
|
|
19
18
|
name: Annotated[str | None, Option(help="Friendly name for the workload.")] = None,
|
20
19
|
):
|
21
20
|
res = state.settings.api.workload.create(project, CreateWorkloadRequest(name=name))
|
22
|
-
|
21
|
+
CONSOLE.print(f"Created workload {res.workload.id}")
|
23
22
|
|
24
23
|
|
25
24
|
@app.command(help="List workloads.")
|
@@ -27,7 +26,7 @@ def ls(
|
|
27
26
|
project: ProjectArg,
|
28
27
|
):
|
29
28
|
workloads = list(state.settings.api.workload.list(project))
|
30
|
-
|
29
|
+
CONSOLE.print(printer.table_of_models(Workload, workloads, fields=["id", "project_id", "name"]))
|
31
30
|
|
32
31
|
|
33
32
|
@app.command(help="Issue new workflow credentials.")
|
@@ -46,14 +45,14 @@ def issue_credentials(workload_id: Annotated[str, Argument(help="Workload ID.")]
|
|
46
45
|
|
47
46
|
if choice == 1:
|
48
47
|
pyperclip.copy(res.client_secret)
|
49
|
-
|
48
|
+
CONSOLE.print("[green]Secret copied to clipboard![/green]")
|
50
49
|
break
|
51
50
|
elif choice == 2:
|
52
|
-
|
51
|
+
CONSOLE.print(f"[green]Token Secret:[/green] {res.client_secret}")
|
53
52
|
break
|
54
53
|
elif choice == 3:
|
55
54
|
break
|
56
55
|
else:
|
57
|
-
|
56
|
+
ERR_CONSOLE.print("Invalid choice. Please try again.")
|
58
57
|
|
59
|
-
|
58
|
+
CONSOLE.print(f"[green]Token ID:[/green] {res.client_id}")
|
spiral/client.py
CHANGED
@@ -1,24 +1,29 @@
|
|
1
|
+
from datetime import datetime, timedelta
|
1
2
|
from typing import TYPE_CHECKING
|
2
3
|
|
3
4
|
import jwt
|
5
|
+
import pyarrow as pa
|
4
6
|
|
5
7
|
from spiral.api import SpiralAPI
|
6
8
|
from spiral.api.projects import CreateProjectRequest, CreateProjectResponse
|
9
|
+
from spiral.core.client import Operations
|
7
10
|
from spiral.core.client import Spiral as CoreSpiral
|
11
|
+
from spiral.datetime_ import timestamp_micros
|
12
|
+
from spiral.expressions import ExprLike
|
13
|
+
from spiral.scan import Scan
|
8
14
|
from spiral.settings import Settings, settings
|
9
15
|
|
10
16
|
if TYPE_CHECKING:
|
11
17
|
from spiral.iceberg import Iceberg
|
18
|
+
from spiral.key_space_index import KeySpaceIndex
|
12
19
|
from spiral.project import Project
|
20
|
+
from spiral.table import Table
|
21
|
+
from spiral.text_index import TextIndex
|
13
22
|
|
14
23
|
|
15
24
|
class Spiral:
|
16
25
|
def __init__(self, config: Settings | None = None):
|
17
26
|
self._config = config or settings()
|
18
|
-
self._api = self._config.api
|
19
|
-
self._core = CoreSpiral(
|
20
|
-
api_url=self._config.spiraldb.uri, spfs_url=self._config.spfs.uri, authn=self._config.authn
|
21
|
-
)
|
22
27
|
self._org = None
|
23
28
|
|
24
29
|
@property
|
@@ -27,7 +32,11 @@ class Spiral:
|
|
27
32
|
|
28
33
|
@property
|
29
34
|
def api(self) -> SpiralAPI:
|
30
|
-
return self.
|
35
|
+
return self._config.api
|
36
|
+
|
37
|
+
@property
|
38
|
+
def _core(self) -> CoreSpiral:
|
39
|
+
return self._config.core
|
31
40
|
|
32
41
|
@property
|
33
42
|
def organization(self) -> str:
|
@@ -45,7 +54,7 @@ class Spiral:
|
|
45
54
|
"""List project IDs."""
|
46
55
|
from .project import Project
|
47
56
|
|
48
|
-
return [Project(self,
|
57
|
+
return [Project(self, project_id=p.id, name=p.name) for p in self.api.project.list()]
|
49
58
|
|
50
59
|
def create_project(
|
51
60
|
self,
|
@@ -64,7 +73,102 @@ class Spiral:
|
|
64
73
|
from spiral.project import Project
|
65
74
|
|
66
75
|
# We avoid an API call since we'd just be fetching a human-readable name. Seems a waste in most cases.
|
67
|
-
return Project(self,
|
76
|
+
return Project(self, project_id=project_id, name=project_id)
|
77
|
+
|
78
|
+
def table(self, table_id: str) -> "Table":
|
79
|
+
"""Open a table using an ID."""
|
80
|
+
from spiral.table import Table
|
81
|
+
|
82
|
+
return Table(self, self._core.table(table_id))
|
83
|
+
|
84
|
+
def text_index(self, index_id: str) -> "TextIndex":
|
85
|
+
"""Open a text index using an ID."""
|
86
|
+
from spiral.text_index import TextIndex
|
87
|
+
|
88
|
+
return TextIndex(self._core.text_index(index_id))
|
89
|
+
|
90
|
+
def key_space_index(self, index_id: str) -> "KeySpaceIndex":
|
91
|
+
"""Open a key space index using an ID."""
|
92
|
+
from spiral.key_space_index import KeySpaceIndex
|
93
|
+
|
94
|
+
return KeySpaceIndex(self._core.key_space_index(index_id))
|
95
|
+
|
96
|
+
def scan(
|
97
|
+
self,
|
98
|
+
*projections: ExprLike,
|
99
|
+
where: ExprLike | None = None,
|
100
|
+
asof: datetime | int | None = None,
|
101
|
+
exclude_keys: bool = False,
|
102
|
+
) -> Scan:
|
103
|
+
"""Starts a read transaction on the Spiral.
|
104
|
+
|
105
|
+
Args:
|
106
|
+
projections: a set of expressions that return struct arrays.
|
107
|
+
where: a query expression to apply to the data.
|
108
|
+
asof: only data written before the given timestamp will be returned, caveats around compaction.
|
109
|
+
exclude_keys: whether to exclude the key columns in the scan result, defaults to False.
|
110
|
+
Note that if a projection includes a key column, it will be included in the result.
|
111
|
+
"""
|
112
|
+
from spiral import expressions as se
|
113
|
+
|
114
|
+
if isinstance(asof, datetime):
|
115
|
+
asof = timestamp_micros(asof)
|
116
|
+
|
117
|
+
# Combine all projections into a single struct.
|
118
|
+
projection = se.merge(*projections)
|
119
|
+
if where is not None:
|
120
|
+
where = se.lift(where)
|
121
|
+
|
122
|
+
return Scan(
|
123
|
+
self._core.scan(
|
124
|
+
projection.__expr__,
|
125
|
+
filter=where.__expr__ if where else None,
|
126
|
+
asof=asof,
|
127
|
+
exclude_keys=exclude_keys,
|
128
|
+
),
|
129
|
+
)
|
130
|
+
|
131
|
+
# TODO(marko): This should be query, and search should be query + scan.
|
132
|
+
def search(
|
133
|
+
self,
|
134
|
+
top_k: int,
|
135
|
+
*rank_by: ExprLike,
|
136
|
+
filters: ExprLike | None = None,
|
137
|
+
freshness_window: timedelta | None = None,
|
138
|
+
) -> pa.RecordBatchReader:
|
139
|
+
"""Queries the index with the given rank by and filters clauses. Returns a steam of scored keys.
|
140
|
+
|
141
|
+
Args:
|
142
|
+
top_k: The number of top results to return.
|
143
|
+
rank_by: Rank by expressions are combined for scoring.
|
144
|
+
See `se.text.find` and `se.text.boost` for scoring expressions.
|
145
|
+
filters: The `filters` expression is used to filter the results.
|
146
|
+
It must return a boolean value and use only conjunctions (ANDs). Expressions in filters
|
147
|
+
statement are considered either a `must` or `must_not` clause in search terminology.
|
148
|
+
freshness_window: If provided, the index will not be refreshed if its freshness does not exceed this window.
|
149
|
+
"""
|
150
|
+
from spiral import expressions as se
|
151
|
+
|
152
|
+
if not rank_by:
|
153
|
+
raise ValueError("At least one rank by expression is required.")
|
154
|
+
rank_by = se.or_(*rank_by)
|
155
|
+
if filters is not None:
|
156
|
+
filters = se.lift(filters)
|
157
|
+
|
158
|
+
if freshness_window is None:
|
159
|
+
freshness_window = timedelta(seconds=0)
|
160
|
+
freshness_window_s = int(freshness_window.total_seconds())
|
161
|
+
|
162
|
+
return self._core.search(
|
163
|
+
top_k=top_k,
|
164
|
+
rank_by=rank_by.__expr__,
|
165
|
+
filters=filters.__expr__ if filters else None,
|
166
|
+
freshness_window_s=freshness_window_s,
|
167
|
+
)
|
168
|
+
|
169
|
+
def _ops(self) -> Operations:
|
170
|
+
"""Access maintenance operations."""
|
171
|
+
return self._core._ops(format=settings().file_format)
|
68
172
|
|
69
173
|
@property
|
70
174
|
def iceberg(self) -> "Iceberg":
|
@@ -72,7 +176,6 @@ class Spiral:
|
|
72
176
|
Apache Iceberg is a powerful open-source table format designed for high-performance data lakes.
|
73
177
|
Iceberg brings reliability, scalability, and advanced features like time travel, schema evolution,
|
74
178
|
and ACID transactions to your warehouse.
|
75
|
-
|
76
179
|
"""
|
77
180
|
from spiral.iceberg import Iceberg
|
78
181
|
|