pyspiral 0.7.18__cp312-abi3-manylinux_2_28_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyspiral-0.7.18.dist-info/METADATA +52 -0
- pyspiral-0.7.18.dist-info/RECORD +110 -0
- pyspiral-0.7.18.dist-info/WHEEL +4 -0
- pyspiral-0.7.18.dist-info/entry_points.txt +3 -0
- spiral/__init__.py +55 -0
- spiral/_lib.abi3.so +0 -0
- spiral/adbc.py +411 -0
- spiral/api/__init__.py +78 -0
- spiral/api/admin.py +15 -0
- spiral/api/client.py +164 -0
- spiral/api/filesystems.py +134 -0
- spiral/api/key_space_indexes.py +23 -0
- spiral/api/organizations.py +77 -0
- spiral/api/projects.py +219 -0
- spiral/api/telemetry.py +19 -0
- spiral/api/text_indexes.py +56 -0
- spiral/api/types.py +23 -0
- spiral/api/workers.py +40 -0
- spiral/api/workloads.py +52 -0
- spiral/arrow_.py +216 -0
- spiral/cli/__init__.py +88 -0
- spiral/cli/__main__.py +4 -0
- spiral/cli/admin.py +14 -0
- spiral/cli/app.py +108 -0
- spiral/cli/console.py +95 -0
- spiral/cli/fs.py +76 -0
- spiral/cli/iceberg.py +97 -0
- spiral/cli/key_spaces.py +103 -0
- spiral/cli/login.py +25 -0
- spiral/cli/orgs.py +90 -0
- spiral/cli/printer.py +53 -0
- spiral/cli/projects.py +147 -0
- spiral/cli/state.py +7 -0
- spiral/cli/tables.py +197 -0
- spiral/cli/telemetry.py +17 -0
- spiral/cli/text.py +115 -0
- spiral/cli/types.py +50 -0
- spiral/cli/workloads.py +58 -0
- spiral/client.py +256 -0
- spiral/core/__init__.pyi +0 -0
- spiral/core/_tools/__init__.pyi +5 -0
- spiral/core/authn/__init__.pyi +21 -0
- spiral/core/client/__init__.pyi +285 -0
- spiral/core/config/__init__.pyi +35 -0
- spiral/core/expr/__init__.pyi +15 -0
- spiral/core/expr/images/__init__.pyi +3 -0
- spiral/core/expr/list_/__init__.pyi +4 -0
- spiral/core/expr/refs/__init__.pyi +4 -0
- spiral/core/expr/str_/__init__.pyi +3 -0
- spiral/core/expr/struct_/__init__.pyi +6 -0
- spiral/core/expr/text/__init__.pyi +5 -0
- spiral/core/expr/udf/__init__.pyi +14 -0
- spiral/core/expr/video/__init__.pyi +3 -0
- spiral/core/table/__init__.pyi +141 -0
- spiral/core/table/manifests/__init__.pyi +35 -0
- spiral/core/table/metastore/__init__.pyi +58 -0
- spiral/core/table/spec/__init__.pyi +215 -0
- spiral/dataloader.py +299 -0
- spiral/dataset.py +264 -0
- spiral/datetime_.py +27 -0
- spiral/debug/__init__.py +0 -0
- spiral/debug/manifests.py +87 -0
- spiral/debug/metrics.py +56 -0
- spiral/debug/scan.py +266 -0
- spiral/enrichment.py +306 -0
- spiral/expressions/__init__.py +274 -0
- spiral/expressions/base.py +167 -0
- spiral/expressions/file.py +17 -0
- spiral/expressions/http.py +17 -0
- spiral/expressions/list_.py +68 -0
- spiral/expressions/s3.py +16 -0
- spiral/expressions/str_.py +39 -0
- spiral/expressions/struct.py +59 -0
- spiral/expressions/text.py +62 -0
- spiral/expressions/tiff.py +222 -0
- spiral/expressions/udf.py +60 -0
- spiral/grpc_.py +32 -0
- spiral/iceberg.py +31 -0
- spiral/iterable_dataset.py +106 -0
- spiral/key_space_index.py +44 -0
- spiral/project.py +227 -0
- spiral/protogen/_/__init__.py +0 -0
- spiral/protogen/_/arrow/__init__.py +0 -0
- spiral/protogen/_/arrow/flight/__init__.py +0 -0
- spiral/protogen/_/arrow/flight/protocol/__init__.py +0 -0
- spiral/protogen/_/arrow/flight/protocol/sql/__init__.py +2548 -0
- spiral/protogen/_/google/__init__.py +0 -0
- spiral/protogen/_/google/protobuf/__init__.py +2310 -0
- spiral/protogen/_/message_pool.py +3 -0
- spiral/protogen/_/py.typed +0 -0
- spiral/protogen/_/scandal/__init__.py +190 -0
- spiral/protogen/_/spfs/__init__.py +72 -0
- spiral/protogen/_/spql/__init__.py +61 -0
- spiral/protogen/_/substrait/__init__.py +6196 -0
- spiral/protogen/_/substrait/extensions/__init__.py +169 -0
- spiral/protogen/__init__.py +0 -0
- spiral/protogen/util.py +41 -0
- spiral/py.typed +0 -0
- spiral/scan.py +363 -0
- spiral/server.py +17 -0
- spiral/settings.py +36 -0
- spiral/snapshot.py +56 -0
- spiral/streaming_/__init__.py +3 -0
- spiral/streaming_/reader.py +133 -0
- spiral/streaming_/stream.py +157 -0
- spiral/substrait_.py +274 -0
- spiral/table.py +224 -0
- spiral/text_index.py +17 -0
- spiral/transaction.py +155 -0
- spiral/types_.py +6 -0
spiral/cli/orgs.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import webbrowser
|
|
2
|
+
from typing import Annotated
|
|
3
|
+
|
|
4
|
+
import jwt
|
|
5
|
+
import typer
|
|
6
|
+
from rich.table import Table
|
|
7
|
+
from typer import Option
|
|
8
|
+
|
|
9
|
+
from spiral.api.organizations import CreateOrgRequest, InviteUserRequest, OrgRole, PortalLinkIntent, PortalLinkRequest
|
|
10
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
|
|
11
|
+
from spiral.cli.types import OrganizationArg
|
|
12
|
+
from spiral.core.authn import DeviceCodeAuth
|
|
13
|
+
|
|
14
|
+
app = AsyncTyper(short_help="Org admin.")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@app.command(help="Switch the active organization.")
|
|
18
|
+
def switch(org_id: OrganizationArg):
|
|
19
|
+
DeviceCodeAuth.default().authenticate(org_id=org_id)
|
|
20
|
+
CONSOLE.print(f"Switched to organization: {org_id}")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@app.command(help="Create a new organization.")
|
|
24
|
+
def create(
|
|
25
|
+
name: Annotated[str | None, Option(help="The human-readable name of the organization.")] = None,
|
|
26
|
+
):
|
|
27
|
+
res = state.spiral.api.organization.create(CreateOrgRequest(name=name))
|
|
28
|
+
|
|
29
|
+
# Authenticate to the new organization
|
|
30
|
+
DeviceCodeAuth.default().authenticate(org_id=res.org.id)
|
|
31
|
+
|
|
32
|
+
CONSOLE.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.command(help="List organizations.")
|
|
36
|
+
def ls():
|
|
37
|
+
org_id = current_org_id()
|
|
38
|
+
|
|
39
|
+
table = Table("", "id", "name", "role", title="Organizations")
|
|
40
|
+
for m in state.spiral.api.organization.list_memberships():
|
|
41
|
+
table.add_row("👉" if m.org.id == org_id else "", m.org.id, m.org.name, m.role)
|
|
42
|
+
|
|
43
|
+
CONSOLE.print(table)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@app.command(help="Invite a user to the organization.")
|
|
47
|
+
def invite(email: str, role: OrgRole = OrgRole.MEMBER, expires_in_days: int = 7):
|
|
48
|
+
state.spiral.api.organization.invite_user(
|
|
49
|
+
InviteUserRequest(email=email, role=role, expires_in_days=expires_in_days)
|
|
50
|
+
)
|
|
51
|
+
CONSOLE.print(f"Invited {email} as a {role.value}.")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@app.command(help="Configure single sign-on for your organization.")
|
|
55
|
+
def sso():
|
|
56
|
+
_do_action(PortalLinkIntent.SSO)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.command(help="Configure directory services for your organization.")
|
|
60
|
+
def directory():
|
|
61
|
+
_do_action(PortalLinkIntent.DIRECTORY_SYNC)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command(help="Configure audit logs for your organization.")
|
|
65
|
+
def audit_logs():
|
|
66
|
+
_do_action(PortalLinkIntent.AUDIT_LOGS)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@app.command(help="Configure log streams for your organization.")
|
|
70
|
+
def log_streams():
|
|
71
|
+
_do_action(PortalLinkIntent.LOG_STREAMS)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@app.command(help="Configure domains for your organization.")
|
|
75
|
+
def domains():
|
|
76
|
+
_do_action(PortalLinkIntent.DOMAIN_VERIFICATION)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _do_action(intent: PortalLinkIntent):
|
|
80
|
+
res = state.spiral.api.organization.portal_link(PortalLinkRequest(intent=intent))
|
|
81
|
+
CONSOLE.print(f"Opening the configuration portal:\n{res.url}")
|
|
82
|
+
webbrowser.open(res.url)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def current_org_id():
|
|
86
|
+
if token := state.spiral.authn.token():
|
|
87
|
+
if org_id := jwt.decode(token.expose_secret(), options={"verify_signature": False}).get("org_id"):
|
|
88
|
+
return org_id
|
|
89
|
+
ERR_CONSOLE.print("You are not logged in to an organization.")
|
|
90
|
+
raise typer.Exit(1)
|
spiral/cli/printer.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from collections.abc import Iterable
|
|
3
|
+
from typing import TypeVar
|
|
4
|
+
|
|
5
|
+
import betterproto2
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from rich.console import ConsoleRenderable, Group
|
|
8
|
+
from rich.padding import Padding
|
|
9
|
+
from rich.pretty import Pretty
|
|
10
|
+
from rich.table import Table
|
|
11
|
+
|
|
12
|
+
T = TypeVar("T", bound=betterproto2.Message)
|
|
13
|
+
M = TypeVar("M", bound=BaseModel)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def table_of_models(cls: type[M], messages: Iterable[M], fields: list[str] = None, title: str = None) -> Table:
|
|
17
|
+
"""Centralized logic for printing tables of Pydantic models."""
|
|
18
|
+
cols = fields or cls.model_fields.keys()
|
|
19
|
+
table = Table(*cols, title=title or f"{cls.__name__}s")
|
|
20
|
+
for msg in messages:
|
|
21
|
+
table.add_row(*[_renderable(msg, col) for col in cols])
|
|
22
|
+
return table
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _renderable(msg, col):
|
|
26
|
+
attr = getattr(msg, col, "")
|
|
27
|
+
if isinstance(attr, dict):
|
|
28
|
+
return json.dumps(attr)
|
|
29
|
+
return attr
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def table_of_protos(cls: type[T], messages: Iterable[T], fields: list[str] = None, title: str = None) -> Table:
|
|
33
|
+
"""Centralized logic for printing tables of proto messages.
|
|
34
|
+
|
|
35
|
+
TODO(ngates): add a CLI switch to emit JSON results instead of tables.
|
|
36
|
+
"""
|
|
37
|
+
cols = fields or cls()._betterproto2.sorted_field_names
|
|
38
|
+
table = Table(*cols, title=title or f"{cls.__name__}s")
|
|
39
|
+
for msg in messages:
|
|
40
|
+
table.add_row(*[getattr(msg, col, "") for col in cols])
|
|
41
|
+
return table
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def proto[T](message: T, title: str = None, fields: list[str] = None) -> ConsoleRenderable:
|
|
45
|
+
"""Centralized logic for printing a single proto message."""
|
|
46
|
+
value = Pretty({k: v for k, v in message.to_dict().items() if not fields or k in fields})
|
|
47
|
+
if title:
|
|
48
|
+
return Group(
|
|
49
|
+
f"[bold]{title}[/bold]",
|
|
50
|
+
Padding.indent(value, level=2),
|
|
51
|
+
)
|
|
52
|
+
else:
|
|
53
|
+
return value
|
spiral/cli/projects.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from typing import Annotated, Literal
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
from typer import Option
|
|
5
|
+
|
|
6
|
+
from spiral.api.organizations import OrgRole
|
|
7
|
+
from spiral.api.projects import (
|
|
8
|
+
AwsAssumedRolePrincipalConditions,
|
|
9
|
+
CreateProjectRequest,
|
|
10
|
+
GcpServiceAccountPrincipalConditions,
|
|
11
|
+
GitHubConditions,
|
|
12
|
+
GitHubPrincipalConditions,
|
|
13
|
+
Grant,
|
|
14
|
+
GrantRoleRequest,
|
|
15
|
+
ModalConditions,
|
|
16
|
+
ModalPrincipalConditions,
|
|
17
|
+
OrgRolePrincipalConditions,
|
|
18
|
+
OrgUserPrincipalConditions,
|
|
19
|
+
Project,
|
|
20
|
+
WorkloadPrincipalConditions,
|
|
21
|
+
)
|
|
22
|
+
from spiral.cli import CONSOLE, AsyncTyper, printer, state
|
|
23
|
+
from spiral.cli.types import ProjectArg
|
|
24
|
+
|
|
25
|
+
app = AsyncTyper(short_help="Projects and grants.")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@app.command(help="List projects.")
|
|
29
|
+
def ls():
|
|
30
|
+
projects = list(state.spiral.api.project.list())
|
|
31
|
+
CONSOLE.print(printer.table_of_models(Project, projects))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@app.command(help="Create a new project.")
|
|
35
|
+
def create(
|
|
36
|
+
id_prefix: Annotated[
|
|
37
|
+
str | None, Option(help="An optional ID prefix to which a random number will be appended.")
|
|
38
|
+
] = None,
|
|
39
|
+
name: Annotated[str | None, Option(help="Friendly name for the project.")] = None,
|
|
40
|
+
):
|
|
41
|
+
res = state.spiral.api.project.create(CreateProjectRequest(id_prefix=id_prefix, name=name))
|
|
42
|
+
CONSOLE.print(f"Created project {res.project.id}")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@app.command(help="Grant a role on a project to a principal.")
|
|
46
|
+
def grant(
|
|
47
|
+
project: ProjectArg,
|
|
48
|
+
role: Annotated[Literal["viewer", "editor", "admin"], Option(help="Project role to grant.")],
|
|
49
|
+
org_id: Annotated[
|
|
50
|
+
str | None, Option(help="Pass an organization ID to grant a role to an organization user(s).")
|
|
51
|
+
] = None,
|
|
52
|
+
org_user: Annotated[
|
|
53
|
+
str | None, Option(help="Pass a user ID when using --org-id to grant a role to grant a role to a user.")
|
|
54
|
+
] = None,
|
|
55
|
+
org_role: Annotated[
|
|
56
|
+
Literal["owner", "member", "guest"] | None,
|
|
57
|
+
Option(help="Pass an org role when using --org-id to grant a role to all users with that role."),
|
|
58
|
+
] = None,
|
|
59
|
+
workload_id: Annotated[str | None, Option(help="Pass a workload ID to grant a role to a workload.")] = None,
|
|
60
|
+
github: Annotated[
|
|
61
|
+
str | None, Option(help="Pass an `{org}/{repo}` string to grant a role to a job running in GitHub Actions.")
|
|
62
|
+
] = None,
|
|
63
|
+
modal: Annotated[
|
|
64
|
+
str | None,
|
|
65
|
+
Option(help="Pass a `{workspace_id}/{env_name}` string to grant a role to a job running in Modal environment."),
|
|
66
|
+
] = None,
|
|
67
|
+
gcp_service_account: Annotated[
|
|
68
|
+
str | None,
|
|
69
|
+
Option(help="Pass a `{service_account_email}/{unique_id}` to grant a role to a GCP service account."),
|
|
70
|
+
] = None,
|
|
71
|
+
aws_iam_role: Annotated[
|
|
72
|
+
str | None,
|
|
73
|
+
Option(help="Pass a `{account_id}/{role_name}` to grant a Spiral role to an AWS IAM Role."),
|
|
74
|
+
] = None,
|
|
75
|
+
conditions: list[str] | None = Option(
|
|
76
|
+
default=None,
|
|
77
|
+
help="`{key}={value}` token conditions to apply to the grant",
|
|
78
|
+
),
|
|
79
|
+
):
|
|
80
|
+
# Check mutual exclusion
|
|
81
|
+
if sum(int(bool(opt)) for opt in {org_id, workload_id, github, modal, gcp_service_account, aws_iam_role}) != 1:
|
|
82
|
+
raise typer.BadParameter(
|
|
83
|
+
"Only one of [--org-id, --workload-id, --github, --modal, --gcp-service-account, --aws-iam-role] "
|
|
84
|
+
"may be specified."
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
if github:
|
|
88
|
+
org, repo = github.split("/", 1)
|
|
89
|
+
github_conditions = None
|
|
90
|
+
if conditions is not None:
|
|
91
|
+
github_conditions = GitHubConditions()
|
|
92
|
+
for k, v in dict(c.split("=", 1) for c in conditions).items():
|
|
93
|
+
github_conditions = github_conditions.model_copy(update={k: v})
|
|
94
|
+
principal = GitHubPrincipalConditions(org=org, repo=repo, conditions=github_conditions)
|
|
95
|
+
|
|
96
|
+
elif modal:
|
|
97
|
+
workspace_id, environment_name = modal.split("/", 1)
|
|
98
|
+
modal_conditions = None
|
|
99
|
+
if conditions is not None:
|
|
100
|
+
modal_conditions = ModalConditions()
|
|
101
|
+
for k, v in dict(c.split("=", 1) for c in conditions).items():
|
|
102
|
+
modal_conditions = modal_conditions.model_copy(update={k: v})
|
|
103
|
+
principal = ModalPrincipalConditions(
|
|
104
|
+
workspace_id=workspace_id, environment_name=environment_name, conditions=modal_conditions
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
elif org_id:
|
|
108
|
+
# Check mutual exclusion
|
|
109
|
+
if sum(int(bool(opt)) for opt in {org_user, org_role}) != 1:
|
|
110
|
+
raise typer.BadParameter("Only one of --org-user or --org-role may be specified.")
|
|
111
|
+
|
|
112
|
+
if org_user is not None:
|
|
113
|
+
principal = OrgUserPrincipalConditions(org_id=org_id, user_id=org_user)
|
|
114
|
+
elif org_role is not None:
|
|
115
|
+
principal = OrgRolePrincipalConditions(org_id=org_id, role=OrgRole(org_role))
|
|
116
|
+
else:
|
|
117
|
+
raise typer.BadParameter("One of --org-user or --org-role must be specified with --org-id.")
|
|
118
|
+
|
|
119
|
+
elif workload_id:
|
|
120
|
+
principal = WorkloadPrincipalConditions(workload_id=workload_id)
|
|
121
|
+
|
|
122
|
+
elif gcp_service_account:
|
|
123
|
+
service_account, unique_id = gcp_service_account.split("/", 1)
|
|
124
|
+
principal = GcpServiceAccountPrincipalConditions(service_account=service_account, unique_id=unique_id)
|
|
125
|
+
|
|
126
|
+
elif aws_iam_role:
|
|
127
|
+
account_id, role_name = aws_iam_role.split("/", 1)
|
|
128
|
+
principal = AwsAssumedRolePrincipalConditions(account_id=account_id, role_name=role_name)
|
|
129
|
+
|
|
130
|
+
else:
|
|
131
|
+
raise ValueError("Invalid grant principal")
|
|
132
|
+
|
|
133
|
+
state.spiral.api.project.grant_role(
|
|
134
|
+
project,
|
|
135
|
+
GrantRoleRequest(
|
|
136
|
+
role_id=role,
|
|
137
|
+
principal=principal,
|
|
138
|
+
),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
CONSOLE.print(f"Granted role {role} on project {project}")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@app.command(help="List project grants.")
|
|
145
|
+
def grants(project: ProjectArg):
|
|
146
|
+
project_grants = list(state.spiral.api.project.list_grants(project))
|
|
147
|
+
CONSOLE.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
|
spiral/cli/state.py
ADDED
spiral/cli/tables.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import Annotated
|
|
3
|
+
|
|
4
|
+
import questionary
|
|
5
|
+
import rich
|
|
6
|
+
import rich.table
|
|
7
|
+
import typer
|
|
8
|
+
from questionary import Choice
|
|
9
|
+
from typer import Argument, Option
|
|
10
|
+
|
|
11
|
+
from spiral import Spiral
|
|
12
|
+
from spiral.api.projects import TableResource
|
|
13
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
|
|
14
|
+
from spiral.cli.types import ProjectArg
|
|
15
|
+
from spiral.debug.manifests import display_manifests
|
|
16
|
+
from spiral.table import Table
|
|
17
|
+
|
|
18
|
+
app = AsyncTyper(short_help="Spiral Tables.")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def ask_table(project_id: str, title: str = "Select a table") -> str:
|
|
22
|
+
tables: list[TableResource] = list(state.spiral.project(project_id).list_tables())
|
|
23
|
+
|
|
24
|
+
if not tables:
|
|
25
|
+
ERR_CONSOLE.print("No tables found")
|
|
26
|
+
raise typer.Exit(1)
|
|
27
|
+
|
|
28
|
+
return questionary.select( # pyright: ignore[reportAny]
|
|
29
|
+
title,
|
|
30
|
+
choices=[
|
|
31
|
+
Choice(title=f"{table.dataset}.{table.table}", value=f"{table.dataset}.{table.table}")
|
|
32
|
+
for table in sorted(tables, key=lambda t: (t.dataset, t.table))
|
|
33
|
+
],
|
|
34
|
+
).ask()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def get_table(
|
|
38
|
+
project: ProjectArg,
|
|
39
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
40
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
41
|
+
) -> tuple[str, Table]:
|
|
42
|
+
if table is None:
|
|
43
|
+
identifier = ask_table(project)
|
|
44
|
+
else:
|
|
45
|
+
identifier = table
|
|
46
|
+
if dataset is not None:
|
|
47
|
+
identifier = f"{dataset}.{table}"
|
|
48
|
+
return identifier, state.spiral.project(project).table(identifier)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command(help="List tables.")
|
|
52
|
+
def ls(
|
|
53
|
+
project: ProjectArg,
|
|
54
|
+
):
|
|
55
|
+
tables = Spiral().project(project).list_tables()
|
|
56
|
+
|
|
57
|
+
rich_table = rich.table.Table("id", "dataset", "name", title="Spiral tables")
|
|
58
|
+
for table in tables:
|
|
59
|
+
rich_table.add_row(table.id, table.dataset, table.table)
|
|
60
|
+
CONSOLE.print(rich_table)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@app.command(help="Show the leading rows of the table.")
|
|
64
|
+
def head(
|
|
65
|
+
project: ProjectArg,
|
|
66
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
67
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
68
|
+
n: Annotated[int, Option("-n", help="Maximum number of rows to show. Defaults to 10.")] = 10,
|
|
69
|
+
):
|
|
70
|
+
import polars as pl
|
|
71
|
+
|
|
72
|
+
_, t = get_table(project, table, dataset)
|
|
73
|
+
|
|
74
|
+
with pl.Config(tbl_rows=-1):
|
|
75
|
+
CONSOLE.print(t.to_polars().limit(n).collect())
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def validate_non_empty_str(text: str) -> bool | str:
|
|
79
|
+
if len(text) > 0:
|
|
80
|
+
return True
|
|
81
|
+
|
|
82
|
+
return "Must provide at least one character."
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def get_string(message: str, validate: Callable[[str], bool | str] = validate_non_empty_str) -> str:
|
|
86
|
+
return questionary.text(message, validate=validate).ask()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@app.command(help="Move table to a different dataset.")
|
|
90
|
+
def move(
|
|
91
|
+
project: ProjectArg,
|
|
92
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
93
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
94
|
+
new_dataset: Annotated[str | None, Option(help="New dataset name.")] = None,
|
|
95
|
+
):
|
|
96
|
+
identifier, _ = get_table(project, table, dataset)
|
|
97
|
+
new_dataset = get_string("Provide a new dataset name")
|
|
98
|
+
|
|
99
|
+
state.spiral.project(project).move_table(identifier, new_dataset)
|
|
100
|
+
CONSOLE.print("Success.")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@app.command(help="Rename table.")
|
|
104
|
+
def rename(
|
|
105
|
+
project: ProjectArg,
|
|
106
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
107
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
108
|
+
new_table: Annotated[str | None, Option(help="New table name.")] = None,
|
|
109
|
+
):
|
|
110
|
+
identifier, _ = get_table(project, table, dataset)
|
|
111
|
+
new_table = get_string("Provide a new table name")
|
|
112
|
+
|
|
113
|
+
state.spiral.project(project).rename_table(identifier, new_table)
|
|
114
|
+
CONSOLE.print("Success.")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@app.command(help="Show the table key schema.")
|
|
118
|
+
def key_schema(
|
|
119
|
+
project: ProjectArg,
|
|
120
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
121
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
122
|
+
):
|
|
123
|
+
_, t = get_table(project, table, dataset)
|
|
124
|
+
CONSOLE.print(t.key_schema)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@app.command(help="Compute the full table schema.")
|
|
128
|
+
def schema(
|
|
129
|
+
project: ProjectArg,
|
|
130
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
131
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
132
|
+
):
|
|
133
|
+
_, t = get_table(project, table, dataset)
|
|
134
|
+
CONSOLE.print(t.schema())
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@app.command(help="Fetch Write-Ahead-Log.")
|
|
138
|
+
def wal(
|
|
139
|
+
project: ProjectArg,
|
|
140
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
141
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
142
|
+
):
|
|
143
|
+
_, t = get_table(project, table, dataset)
|
|
144
|
+
wal_ = t.core.get_wal(asof=None)
|
|
145
|
+
# Don't use CONSOLE.print here so that it can be piped.
|
|
146
|
+
print(wal_)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@app.command(help="Flush Write-Ahead-Log.")
|
|
150
|
+
def flush(
|
|
151
|
+
project: ProjectArg,
|
|
152
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
153
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
154
|
+
):
|
|
155
|
+
identifier, t = get_table(project, table, dataset)
|
|
156
|
+
state.spiral.internal.flush_wal(t.core) # pyright: ignore[reportPrivateUsage]
|
|
157
|
+
CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@app.command(help="Display all manifests.")
|
|
161
|
+
def manifests(
|
|
162
|
+
project: ProjectArg,
|
|
163
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
164
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
165
|
+
):
|
|
166
|
+
_, t = get_table(project, table, dataset)
|
|
167
|
+
s = t.snapshot()
|
|
168
|
+
|
|
169
|
+
key_space_state = state.spiral.internal.key_space_state(s.core) # pyright: ignore[reportPrivateUsage]
|
|
170
|
+
key_space_manifest = key_space_state.manifest
|
|
171
|
+
|
|
172
|
+
column_groups_states = state.spiral.internal.column_groups_states(s.core, key_space_state) # pyright: ignore[reportPrivateUsage]
|
|
173
|
+
display_manifests(key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states])
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@app.command(help="Visualize the scan of a given column group.")
|
|
177
|
+
def debug_scan(
|
|
178
|
+
project: ProjectArg,
|
|
179
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
180
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
181
|
+
column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
|
|
182
|
+
):
|
|
183
|
+
_, t = get_table(project, table, dataset)
|
|
184
|
+
scan = state.spiral.scan(t[column_group] if column_group != "." else t)
|
|
185
|
+
scan._debug() # pyright: ignore[reportPrivateUsage]
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@app.command(help="Display the manifests for a scan of a given column group.")
|
|
189
|
+
def dump_scan(
|
|
190
|
+
project: ProjectArg,
|
|
191
|
+
table: Annotated[str | None, Option(help="Table name.")] = None,
|
|
192
|
+
dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
|
|
193
|
+
column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
|
|
194
|
+
):
|
|
195
|
+
_, t = get_table(project, table, dataset)
|
|
196
|
+
scan = state.spiral.scan(t[column_group] if column_group != "." else t)
|
|
197
|
+
scan._dump_manifests() # pyright: ignore[reportPrivateUsage]
|
spiral/cli/telemetry.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import pyperclip
|
|
2
|
+
|
|
3
|
+
from spiral.api.telemetry import IssueExportTokenResponse
|
|
4
|
+
from spiral.cli import CONSOLE, AsyncTyper, state
|
|
5
|
+
|
|
6
|
+
app = AsyncTyper(short_help="Client-side telemetry.")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@app.command(help="Issue new telemetry export token.")
|
|
10
|
+
def export():
|
|
11
|
+
res: IssueExportTokenResponse = state.spiral.api.telemetry.issue_export_token()
|
|
12
|
+
|
|
13
|
+
command = f"export SPIRAL_OTEL_TOKEN={res.token}"
|
|
14
|
+
pyperclip.copy(command)
|
|
15
|
+
|
|
16
|
+
CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
|
|
17
|
+
CONSOLE.print("[dim]Token is valid for 1h.[/dim]")
|
spiral/cli/text.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
3
|
+
import questionary
|
|
4
|
+
import rich
|
|
5
|
+
import typer
|
|
6
|
+
from questionary import Choice
|
|
7
|
+
from typer import Option
|
|
8
|
+
|
|
9
|
+
from spiral.api.projects import TextIndexResource
|
|
10
|
+
from spiral.api.text_indexes import CreateWorkerRequest, SyncIndexRequest
|
|
11
|
+
from spiral.api.types import IndexId
|
|
12
|
+
from spiral.api.workers import CPU, GcpRegion, Memory, ResourceClass
|
|
13
|
+
from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
|
|
14
|
+
from spiral.cli.types import ProjectArg
|
|
15
|
+
|
|
16
|
+
app = AsyncTyper(short_help="Text Indexes.")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def ask_index(project_id, title="Select an index"):
|
|
20
|
+
indexes: list[TextIndexResource] = list(state.spiral.project(project_id).list_text_indexes())
|
|
21
|
+
|
|
22
|
+
if not indexes:
|
|
23
|
+
ERR_CONSOLE.print("No indexes found")
|
|
24
|
+
raise typer.Exit(1)
|
|
25
|
+
|
|
26
|
+
return questionary.select(
|
|
27
|
+
title,
|
|
28
|
+
choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
|
|
29
|
+
).ask()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_index_id(
|
|
33
|
+
project: ProjectArg,
|
|
34
|
+
name: Annotated[str | None, Option(help="Index name.")] = None,
|
|
35
|
+
) -> IndexId:
|
|
36
|
+
if name is None:
|
|
37
|
+
return ask_index(project)
|
|
38
|
+
|
|
39
|
+
indexes: list[TextIndexResource] = list(state.spiral.project(project).list_text_indexes())
|
|
40
|
+
for index in indexes:
|
|
41
|
+
if index.name == name:
|
|
42
|
+
return index.id
|
|
43
|
+
raise ValueError(f"Index not found: {name}")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@app.command(help="List indexes.")
|
|
47
|
+
def ls(
|
|
48
|
+
project: ProjectArg,
|
|
49
|
+
):
|
|
50
|
+
"""List indexes."""
|
|
51
|
+
indexes = state.spiral.project(project).list_text_indexes()
|
|
52
|
+
|
|
53
|
+
rich_table = rich.table.Table("id", "name", title="Text Indexes")
|
|
54
|
+
for index in indexes:
|
|
55
|
+
rich_table.add_row(index.id, index.name)
|
|
56
|
+
CONSOLE.print(rich_table)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.command(help="Trigger a sync job for an index.")
|
|
60
|
+
def sync(
|
|
61
|
+
project: ProjectArg,
|
|
62
|
+
name: Annotated[str | None, Option(help="Index name.")] = None,
|
|
63
|
+
resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
|
|
64
|
+
):
|
|
65
|
+
"""Trigger a sync job."""
|
|
66
|
+
index_id = get_index_id(project, name)
|
|
67
|
+
response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
|
|
68
|
+
CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@app.command(name="serve", help="Spin up a worker to serve an index.")
|
|
72
|
+
def serve(
|
|
73
|
+
project: ProjectArg,
|
|
74
|
+
index: Annotated[str | None, Option(help="Index name.")] = None,
|
|
75
|
+
region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
|
|
76
|
+
cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
|
|
77
|
+
memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
|
|
78
|
+
):
|
|
79
|
+
"""Spin up a worker."""
|
|
80
|
+
index_id = get_index_id(project, index)
|
|
81
|
+
request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
|
|
82
|
+
response = state.spiral.api.text_indexes.create_worker(index_id, request)
|
|
83
|
+
CONSOLE.print(f"Created worker {response.worker_id} for {index_id}.")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@app.command(name="workers", help="List search workers.")
|
|
87
|
+
def workers(
|
|
88
|
+
project: ProjectArg,
|
|
89
|
+
index: Annotated[str | None, Option(help="Index name.")] = None,
|
|
90
|
+
):
|
|
91
|
+
"""List text search workers."""
|
|
92
|
+
index_id = get_index_id(project, index)
|
|
93
|
+
worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
|
|
94
|
+
|
|
95
|
+
rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
|
|
96
|
+
for worker_id in worker_ids:
|
|
97
|
+
try:
|
|
98
|
+
worker = state.spiral.api.text_indexes.get_worker(worker_id)
|
|
99
|
+
rich_table.add_row(
|
|
100
|
+
worker_id,
|
|
101
|
+
worker.url,
|
|
102
|
+
)
|
|
103
|
+
except Exception:
|
|
104
|
+
rich_table.add_row(
|
|
105
|
+
worker_id,
|
|
106
|
+
"Unavailable",
|
|
107
|
+
)
|
|
108
|
+
CONSOLE.print(rich_table)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@app.command(name="shutdown", help="Shutdown a search worker.")
|
|
112
|
+
def shutdown(worker_id: str):
|
|
113
|
+
"""Shutdown a worker."""
|
|
114
|
+
state.spiral.api.text_indexes.shutdown_worker(worker_id)
|
|
115
|
+
CONSOLE.print(f"Requested worker {worker_id} to shutdown.")
|
spiral/cli/types.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
3
|
+
import questionary
|
|
4
|
+
import typer
|
|
5
|
+
from questionary import Choice
|
|
6
|
+
from typer import Argument
|
|
7
|
+
|
|
8
|
+
from spiral.api.types import OrgId, ProjectId
|
|
9
|
+
from spiral.cli import ERR_CONSOLE, state
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def ask_project(title="Select a project"):
|
|
13
|
+
projects = list(state.spiral.api.project.list())
|
|
14
|
+
|
|
15
|
+
if not projects:
|
|
16
|
+
ERR_CONSOLE.print("No projects found")
|
|
17
|
+
raise typer.Exit(1)
|
|
18
|
+
|
|
19
|
+
return questionary.select(
|
|
20
|
+
title,
|
|
21
|
+
choices=[
|
|
22
|
+
Choice(title=f"{project.id} - {project.name}" if project.name else project.id, value=project.id)
|
|
23
|
+
for project in projects
|
|
24
|
+
],
|
|
25
|
+
).ask()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
ProjectArg = Annotated[ProjectId, Argument(help="Project ID", show_default=False, default_factory=ask_project)]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _org_default():
|
|
32
|
+
memberships = list(state.spiral.api.organization.list_memberships())
|
|
33
|
+
|
|
34
|
+
if not memberships:
|
|
35
|
+
ERR_CONSOLE.print("No organizations found")
|
|
36
|
+
raise typer.Exit(1)
|
|
37
|
+
|
|
38
|
+
return questionary.select(
|
|
39
|
+
"Select an organization",
|
|
40
|
+
choices=[
|
|
41
|
+
Choice(
|
|
42
|
+
title=f"{m.org.id} - {m.org.name}" if m.org.name else m.org.id,
|
|
43
|
+
value=m.org.id,
|
|
44
|
+
)
|
|
45
|
+
for m in memberships
|
|
46
|
+
],
|
|
47
|
+
).ask()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
OrganizationArg = Annotated[OrgId, Argument(help="Organization ID", show_default=False, default_factory=_org_default)]
|