pyspiral 0.7.18__cp312-abi3-manylinux_2_28_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. pyspiral-0.7.18.dist-info/METADATA +52 -0
  2. pyspiral-0.7.18.dist-info/RECORD +110 -0
  3. pyspiral-0.7.18.dist-info/WHEEL +4 -0
  4. pyspiral-0.7.18.dist-info/entry_points.txt +3 -0
  5. spiral/__init__.py +55 -0
  6. spiral/_lib.abi3.so +0 -0
  7. spiral/adbc.py +411 -0
  8. spiral/api/__init__.py +78 -0
  9. spiral/api/admin.py +15 -0
  10. spiral/api/client.py +164 -0
  11. spiral/api/filesystems.py +134 -0
  12. spiral/api/key_space_indexes.py +23 -0
  13. spiral/api/organizations.py +77 -0
  14. spiral/api/projects.py +219 -0
  15. spiral/api/telemetry.py +19 -0
  16. spiral/api/text_indexes.py +56 -0
  17. spiral/api/types.py +23 -0
  18. spiral/api/workers.py +40 -0
  19. spiral/api/workloads.py +52 -0
  20. spiral/arrow_.py +216 -0
  21. spiral/cli/__init__.py +88 -0
  22. spiral/cli/__main__.py +4 -0
  23. spiral/cli/admin.py +14 -0
  24. spiral/cli/app.py +108 -0
  25. spiral/cli/console.py +95 -0
  26. spiral/cli/fs.py +76 -0
  27. spiral/cli/iceberg.py +97 -0
  28. spiral/cli/key_spaces.py +103 -0
  29. spiral/cli/login.py +25 -0
  30. spiral/cli/orgs.py +90 -0
  31. spiral/cli/printer.py +53 -0
  32. spiral/cli/projects.py +147 -0
  33. spiral/cli/state.py +7 -0
  34. spiral/cli/tables.py +197 -0
  35. spiral/cli/telemetry.py +17 -0
  36. spiral/cli/text.py +115 -0
  37. spiral/cli/types.py +50 -0
  38. spiral/cli/workloads.py +58 -0
  39. spiral/client.py +256 -0
  40. spiral/core/__init__.pyi +0 -0
  41. spiral/core/_tools/__init__.pyi +5 -0
  42. spiral/core/authn/__init__.pyi +21 -0
  43. spiral/core/client/__init__.pyi +285 -0
  44. spiral/core/config/__init__.pyi +35 -0
  45. spiral/core/expr/__init__.pyi +15 -0
  46. spiral/core/expr/images/__init__.pyi +3 -0
  47. spiral/core/expr/list_/__init__.pyi +4 -0
  48. spiral/core/expr/refs/__init__.pyi +4 -0
  49. spiral/core/expr/str_/__init__.pyi +3 -0
  50. spiral/core/expr/struct_/__init__.pyi +6 -0
  51. spiral/core/expr/text/__init__.pyi +5 -0
  52. spiral/core/expr/udf/__init__.pyi +14 -0
  53. spiral/core/expr/video/__init__.pyi +3 -0
  54. spiral/core/table/__init__.pyi +141 -0
  55. spiral/core/table/manifests/__init__.pyi +35 -0
  56. spiral/core/table/metastore/__init__.pyi +58 -0
  57. spiral/core/table/spec/__init__.pyi +215 -0
  58. spiral/dataloader.py +299 -0
  59. spiral/dataset.py +264 -0
  60. spiral/datetime_.py +27 -0
  61. spiral/debug/__init__.py +0 -0
  62. spiral/debug/manifests.py +87 -0
  63. spiral/debug/metrics.py +56 -0
  64. spiral/debug/scan.py +266 -0
  65. spiral/enrichment.py +306 -0
  66. spiral/expressions/__init__.py +274 -0
  67. spiral/expressions/base.py +167 -0
  68. spiral/expressions/file.py +17 -0
  69. spiral/expressions/http.py +17 -0
  70. spiral/expressions/list_.py +68 -0
  71. spiral/expressions/s3.py +16 -0
  72. spiral/expressions/str_.py +39 -0
  73. spiral/expressions/struct.py +59 -0
  74. spiral/expressions/text.py +62 -0
  75. spiral/expressions/tiff.py +222 -0
  76. spiral/expressions/udf.py +60 -0
  77. spiral/grpc_.py +32 -0
  78. spiral/iceberg.py +31 -0
  79. spiral/iterable_dataset.py +106 -0
  80. spiral/key_space_index.py +44 -0
  81. spiral/project.py +227 -0
  82. spiral/protogen/_/__init__.py +0 -0
  83. spiral/protogen/_/arrow/__init__.py +0 -0
  84. spiral/protogen/_/arrow/flight/__init__.py +0 -0
  85. spiral/protogen/_/arrow/flight/protocol/__init__.py +0 -0
  86. spiral/protogen/_/arrow/flight/protocol/sql/__init__.py +2548 -0
  87. spiral/protogen/_/google/__init__.py +0 -0
  88. spiral/protogen/_/google/protobuf/__init__.py +2310 -0
  89. spiral/protogen/_/message_pool.py +3 -0
  90. spiral/protogen/_/py.typed +0 -0
  91. spiral/protogen/_/scandal/__init__.py +190 -0
  92. spiral/protogen/_/spfs/__init__.py +72 -0
  93. spiral/protogen/_/spql/__init__.py +61 -0
  94. spiral/protogen/_/substrait/__init__.py +6196 -0
  95. spiral/protogen/_/substrait/extensions/__init__.py +169 -0
  96. spiral/protogen/__init__.py +0 -0
  97. spiral/protogen/util.py +41 -0
  98. spiral/py.typed +0 -0
  99. spiral/scan.py +363 -0
  100. spiral/server.py +17 -0
  101. spiral/settings.py +36 -0
  102. spiral/snapshot.py +56 -0
  103. spiral/streaming_/__init__.py +3 -0
  104. spiral/streaming_/reader.py +133 -0
  105. spiral/streaming_/stream.py +157 -0
  106. spiral/substrait_.py +274 -0
  107. spiral/table.py +224 -0
  108. spiral/text_index.py +17 -0
  109. spiral/transaction.py +155 -0
  110. spiral/types_.py +6 -0
spiral/cli/orgs.py ADDED
@@ -0,0 +1,90 @@
1
+ import webbrowser
2
+ from typing import Annotated
3
+
4
+ import jwt
5
+ import typer
6
+ from rich.table import Table
7
+ from typer import Option
8
+
9
+ from spiral.api.organizations import CreateOrgRequest, InviteUserRequest, OrgRole, PortalLinkIntent, PortalLinkRequest
10
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
11
+ from spiral.cli.types import OrganizationArg
12
+ from spiral.core.authn import DeviceCodeAuth
13
+
14
+ app = AsyncTyper(short_help="Org admin.")
15
+
16
+
17
+ @app.command(help="Switch the active organization.")
18
+ def switch(org_id: OrganizationArg):
19
+ DeviceCodeAuth.default().authenticate(org_id=org_id)
20
+ CONSOLE.print(f"Switched to organization: {org_id}")
21
+
22
+
23
+ @app.command(help="Create a new organization.")
24
+ def create(
25
+ name: Annotated[str | None, Option(help="The human-readable name of the organization.")] = None,
26
+ ):
27
+ res = state.spiral.api.organization.create(CreateOrgRequest(name=name))
28
+
29
+ # Authenticate to the new organization
30
+ DeviceCodeAuth.default().authenticate(org_id=res.org.id)
31
+
32
+ CONSOLE.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
33
+
34
+
35
+ @app.command(help="List organizations.")
36
+ def ls():
37
+ org_id = current_org_id()
38
+
39
+ table = Table("", "id", "name", "role", title="Organizations")
40
+ for m in state.spiral.api.organization.list_memberships():
41
+ table.add_row("👉" if m.org.id == org_id else "", m.org.id, m.org.name, m.role)
42
+
43
+ CONSOLE.print(table)
44
+
45
+
46
+ @app.command(help="Invite a user to the organization.")
47
+ def invite(email: str, role: OrgRole = OrgRole.MEMBER, expires_in_days: int = 7):
48
+ state.spiral.api.organization.invite_user(
49
+ InviteUserRequest(email=email, role=role, expires_in_days=expires_in_days)
50
+ )
51
+ CONSOLE.print(f"Invited {email} as a {role.value}.")
52
+
53
+
54
+ @app.command(help="Configure single sign-on for your organization.")
55
+ def sso():
56
+ _do_action(PortalLinkIntent.SSO)
57
+
58
+
59
+ @app.command(help="Configure directory services for your organization.")
60
+ def directory():
61
+ _do_action(PortalLinkIntent.DIRECTORY_SYNC)
62
+
63
+
64
+ @app.command(help="Configure audit logs for your organization.")
65
+ def audit_logs():
66
+ _do_action(PortalLinkIntent.AUDIT_LOGS)
67
+
68
+
69
+ @app.command(help="Configure log streams for your organization.")
70
+ def log_streams():
71
+ _do_action(PortalLinkIntent.LOG_STREAMS)
72
+
73
+
74
+ @app.command(help="Configure domains for your organization.")
75
+ def domains():
76
+ _do_action(PortalLinkIntent.DOMAIN_VERIFICATION)
77
+
78
+
79
+ def _do_action(intent: PortalLinkIntent):
80
+ res = state.spiral.api.organization.portal_link(PortalLinkRequest(intent=intent))
81
+ CONSOLE.print(f"Opening the configuration portal:\n{res.url}")
82
+ webbrowser.open(res.url)
83
+
84
+
85
+ def current_org_id():
86
+ if token := state.spiral.authn.token():
87
+ if org_id := jwt.decode(token.expose_secret(), options={"verify_signature": False}).get("org_id"):
88
+ return org_id
89
+ ERR_CONSOLE.print("You are not logged in to an organization.")
90
+ raise typer.Exit(1)
spiral/cli/printer.py ADDED
@@ -0,0 +1,53 @@
1
+ import json
2
+ from collections.abc import Iterable
3
+ from typing import TypeVar
4
+
5
+ import betterproto2
6
+ from pydantic import BaseModel
7
+ from rich.console import ConsoleRenderable, Group
8
+ from rich.padding import Padding
9
+ from rich.pretty import Pretty
10
+ from rich.table import Table
11
+
12
+ T = TypeVar("T", bound=betterproto2.Message)
13
+ M = TypeVar("M", bound=BaseModel)
14
+
15
+
16
+ def table_of_models(cls: type[M], messages: Iterable[M], fields: list[str] = None, title: str = None) -> Table:
17
+ """Centralized logic for printing tables of Pydantic models."""
18
+ cols = fields or cls.model_fields.keys()
19
+ table = Table(*cols, title=title or f"{cls.__name__}s")
20
+ for msg in messages:
21
+ table.add_row(*[_renderable(msg, col) for col in cols])
22
+ return table
23
+
24
+
25
+ def _renderable(msg, col):
26
+ attr = getattr(msg, col, "")
27
+ if isinstance(attr, dict):
28
+ return json.dumps(attr)
29
+ return attr
30
+
31
+
32
+ def table_of_protos(cls: type[T], messages: Iterable[T], fields: list[str] = None, title: str = None) -> Table:
33
+ """Centralized logic for printing tables of proto messages.
34
+
35
+ TODO(ngates): add a CLI switch to emit JSON results instead of tables.
36
+ """
37
+ cols = fields or cls()._betterproto2.sorted_field_names
38
+ table = Table(*cols, title=title or f"{cls.__name__}s")
39
+ for msg in messages:
40
+ table.add_row(*[getattr(msg, col, "") for col in cols])
41
+ return table
42
+
43
+
44
+ def proto[T](message: T, title: str = None, fields: list[str] = None) -> ConsoleRenderable:
45
+ """Centralized logic for printing a single proto message."""
46
+ value = Pretty({k: v for k, v in message.to_dict().items() if not fields or k in fields})
47
+ if title:
48
+ return Group(
49
+ f"[bold]{title}[/bold]",
50
+ Padding.indent(value, level=2),
51
+ )
52
+ else:
53
+ return value
spiral/cli/projects.py ADDED
@@ -0,0 +1,147 @@
1
+ from typing import Annotated, Literal
2
+
3
+ import typer
4
+ from typer import Option
5
+
6
+ from spiral.api.organizations import OrgRole
7
+ from spiral.api.projects import (
8
+ AwsAssumedRolePrincipalConditions,
9
+ CreateProjectRequest,
10
+ GcpServiceAccountPrincipalConditions,
11
+ GitHubConditions,
12
+ GitHubPrincipalConditions,
13
+ Grant,
14
+ GrantRoleRequest,
15
+ ModalConditions,
16
+ ModalPrincipalConditions,
17
+ OrgRolePrincipalConditions,
18
+ OrgUserPrincipalConditions,
19
+ Project,
20
+ WorkloadPrincipalConditions,
21
+ )
22
+ from spiral.cli import CONSOLE, AsyncTyper, printer, state
23
+ from spiral.cli.types import ProjectArg
24
+
25
+ app = AsyncTyper(short_help="Projects and grants.")
26
+
27
+
28
+ @app.command(help="List projects.")
29
+ def ls():
30
+ projects = list(state.spiral.api.project.list())
31
+ CONSOLE.print(printer.table_of_models(Project, projects))
32
+
33
+
34
+ @app.command(help="Create a new project.")
35
+ def create(
36
+ id_prefix: Annotated[
37
+ str | None, Option(help="An optional ID prefix to which a random number will be appended.")
38
+ ] = None,
39
+ name: Annotated[str | None, Option(help="Friendly name for the project.")] = None,
40
+ ):
41
+ res = state.spiral.api.project.create(CreateProjectRequest(id_prefix=id_prefix, name=name))
42
+ CONSOLE.print(f"Created project {res.project.id}")
43
+
44
+
45
+ @app.command(help="Grant a role on a project to a principal.")
46
+ def grant(
47
+ project: ProjectArg,
48
+ role: Annotated[Literal["viewer", "editor", "admin"], Option(help="Project role to grant.")],
49
+ org_id: Annotated[
50
+ str | None, Option(help="Pass an organization ID to grant a role to an organization user(s).")
51
+ ] = None,
52
+ org_user: Annotated[
53
+ str | None, Option(help="Pass a user ID when using --org-id to grant a role to grant a role to a user.")
54
+ ] = None,
55
+ org_role: Annotated[
56
+ Literal["owner", "member", "guest"] | None,
57
+ Option(help="Pass an org role when using --org-id to grant a role to all users with that role."),
58
+ ] = None,
59
+ workload_id: Annotated[str | None, Option(help="Pass a workload ID to grant a role to a workload.")] = None,
60
+ github: Annotated[
61
+ str | None, Option(help="Pass an `{org}/{repo}` string to grant a role to a job running in GitHub Actions.")
62
+ ] = None,
63
+ modal: Annotated[
64
+ str | None,
65
+ Option(help="Pass a `{workspace_id}/{env_name}` string to grant a role to a job running in Modal environment."),
66
+ ] = None,
67
+ gcp_service_account: Annotated[
68
+ str | None,
69
+ Option(help="Pass a `{service_account_email}/{unique_id}` to grant a role to a GCP service account."),
70
+ ] = None,
71
+ aws_iam_role: Annotated[
72
+ str | None,
73
+ Option(help="Pass a `{account_id}/{role_name}` to grant a Spiral role to an AWS IAM Role."),
74
+ ] = None,
75
+ conditions: list[str] | None = Option(
76
+ default=None,
77
+ help="`{key}={value}` token conditions to apply to the grant",
78
+ ),
79
+ ):
80
+ # Check mutual exclusion
81
+ if sum(int(bool(opt)) for opt in {org_id, workload_id, github, modal, gcp_service_account, aws_iam_role}) != 1:
82
+ raise typer.BadParameter(
83
+ "Only one of [--org-id, --workload-id, --github, --modal, --gcp-service-account, --aws-iam-role] "
84
+ "may be specified."
85
+ )
86
+
87
+ if github:
88
+ org, repo = github.split("/", 1)
89
+ github_conditions = None
90
+ if conditions is not None:
91
+ github_conditions = GitHubConditions()
92
+ for k, v in dict(c.split("=", 1) for c in conditions).items():
93
+ github_conditions = github_conditions.model_copy(update={k: v})
94
+ principal = GitHubPrincipalConditions(org=org, repo=repo, conditions=github_conditions)
95
+
96
+ elif modal:
97
+ workspace_id, environment_name = modal.split("/", 1)
98
+ modal_conditions = None
99
+ if conditions is not None:
100
+ modal_conditions = ModalConditions()
101
+ for k, v in dict(c.split("=", 1) for c in conditions).items():
102
+ modal_conditions = modal_conditions.model_copy(update={k: v})
103
+ principal = ModalPrincipalConditions(
104
+ workspace_id=workspace_id, environment_name=environment_name, conditions=modal_conditions
105
+ )
106
+
107
+ elif org_id:
108
+ # Check mutual exclusion
109
+ if sum(int(bool(opt)) for opt in {org_user, org_role}) != 1:
110
+ raise typer.BadParameter("Only one of --org-user or --org-role may be specified.")
111
+
112
+ if org_user is not None:
113
+ principal = OrgUserPrincipalConditions(org_id=org_id, user_id=org_user)
114
+ elif org_role is not None:
115
+ principal = OrgRolePrincipalConditions(org_id=org_id, role=OrgRole(org_role))
116
+ else:
117
+ raise typer.BadParameter("One of --org-user or --org-role must be specified with --org-id.")
118
+
119
+ elif workload_id:
120
+ principal = WorkloadPrincipalConditions(workload_id=workload_id)
121
+
122
+ elif gcp_service_account:
123
+ service_account, unique_id = gcp_service_account.split("/", 1)
124
+ principal = GcpServiceAccountPrincipalConditions(service_account=service_account, unique_id=unique_id)
125
+
126
+ elif aws_iam_role:
127
+ account_id, role_name = aws_iam_role.split("/", 1)
128
+ principal = AwsAssumedRolePrincipalConditions(account_id=account_id, role_name=role_name)
129
+
130
+ else:
131
+ raise ValueError("Invalid grant principal")
132
+
133
+ state.spiral.api.project.grant_role(
134
+ project,
135
+ GrantRoleRequest(
136
+ role_id=role,
137
+ principal=principal,
138
+ ),
139
+ )
140
+
141
+ CONSOLE.print(f"Granted role {role} on project {project}")
142
+
143
+
144
+ @app.command(help="List project grants.")
145
+ def grants(project: ProjectArg):
146
+ project_grants = list(state.spiral.api.project.list_grants(project))
147
+ CONSOLE.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
spiral/cli/state.py ADDED
@@ -0,0 +1,7 @@
1
+ from spiral import Spiral
2
+ from spiral.settings import settings
3
+
4
+
5
+ @property
6
+ def spiral() -> Spiral:
7
+ return Spiral(settings())
spiral/cli/tables.py ADDED
@@ -0,0 +1,197 @@
1
+ from collections.abc import Callable
2
+ from typing import Annotated
3
+
4
+ import questionary
5
+ import rich
6
+ import rich.table
7
+ import typer
8
+ from questionary import Choice
9
+ from typer import Argument, Option
10
+
11
+ from spiral import Spiral
12
+ from spiral.api.projects import TableResource
13
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
14
+ from spiral.cli.types import ProjectArg
15
+ from spiral.debug.manifests import display_manifests
16
+ from spiral.table import Table
17
+
18
+ app = AsyncTyper(short_help="Spiral Tables.")
19
+
20
+
21
+ def ask_table(project_id: str, title: str = "Select a table") -> str:
22
+ tables: list[TableResource] = list(state.spiral.project(project_id).list_tables())
23
+
24
+ if not tables:
25
+ ERR_CONSOLE.print("No tables found")
26
+ raise typer.Exit(1)
27
+
28
+ return questionary.select( # pyright: ignore[reportAny]
29
+ title,
30
+ choices=[
31
+ Choice(title=f"{table.dataset}.{table.table}", value=f"{table.dataset}.{table.table}")
32
+ for table in sorted(tables, key=lambda t: (t.dataset, t.table))
33
+ ],
34
+ ).ask()
35
+
36
+
37
+ def get_table(
38
+ project: ProjectArg,
39
+ table: Annotated[str | None, Option(help="Table name.")] = None,
40
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
41
+ ) -> tuple[str, Table]:
42
+ if table is None:
43
+ identifier = ask_table(project)
44
+ else:
45
+ identifier = table
46
+ if dataset is not None:
47
+ identifier = f"{dataset}.{table}"
48
+ return identifier, state.spiral.project(project).table(identifier)
49
+
50
+
51
+ @app.command(help="List tables.")
52
+ def ls(
53
+ project: ProjectArg,
54
+ ):
55
+ tables = Spiral().project(project).list_tables()
56
+
57
+ rich_table = rich.table.Table("id", "dataset", "name", title="Spiral tables")
58
+ for table in tables:
59
+ rich_table.add_row(table.id, table.dataset, table.table)
60
+ CONSOLE.print(rich_table)
61
+
62
+
63
+ @app.command(help="Show the leading rows of the table.")
64
+ def head(
65
+ project: ProjectArg,
66
+ table: Annotated[str | None, Option(help="Table name.")] = None,
67
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
68
+ n: Annotated[int, Option("-n", help="Maximum number of rows to show. Defaults to 10.")] = 10,
69
+ ):
70
+ import polars as pl
71
+
72
+ _, t = get_table(project, table, dataset)
73
+
74
+ with pl.Config(tbl_rows=-1):
75
+ CONSOLE.print(t.to_polars().limit(n).collect())
76
+
77
+
78
+ def validate_non_empty_str(text: str) -> bool | str:
79
+ if len(text) > 0:
80
+ return True
81
+
82
+ return "Must provide at least one character."
83
+
84
+
85
+ def get_string(message: str, validate: Callable[[str], bool | str] = validate_non_empty_str) -> str:
86
+ return questionary.text(message, validate=validate).ask()
87
+
88
+
89
+ @app.command(help="Move table to a different dataset.")
90
+ def move(
91
+ project: ProjectArg,
92
+ table: Annotated[str | None, Option(help="Table name.")] = None,
93
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
94
+ new_dataset: Annotated[str | None, Option(help="New dataset name.")] = None,
95
+ ):
96
+ identifier, _ = get_table(project, table, dataset)
97
+ new_dataset = get_string("Provide a new dataset name")
98
+
99
+ state.spiral.project(project).move_table(identifier, new_dataset)
100
+ CONSOLE.print("Success.")
101
+
102
+
103
+ @app.command(help="Rename table.")
104
+ def rename(
105
+ project: ProjectArg,
106
+ table: Annotated[str | None, Option(help="Table name.")] = None,
107
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
108
+ new_table: Annotated[str | None, Option(help="New table name.")] = None,
109
+ ):
110
+ identifier, _ = get_table(project, table, dataset)
111
+ new_table = get_string("Provide a new table name")
112
+
113
+ state.spiral.project(project).rename_table(identifier, new_table)
114
+ CONSOLE.print("Success.")
115
+
116
+
117
+ @app.command(help="Show the table key schema.")
118
+ def key_schema(
119
+ project: ProjectArg,
120
+ table: Annotated[str | None, Option(help="Table name.")] = None,
121
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
122
+ ):
123
+ _, t = get_table(project, table, dataset)
124
+ CONSOLE.print(t.key_schema)
125
+
126
+
127
+ @app.command(help="Compute the full table schema.")
128
+ def schema(
129
+ project: ProjectArg,
130
+ table: Annotated[str | None, Option(help="Table name.")] = None,
131
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
132
+ ):
133
+ _, t = get_table(project, table, dataset)
134
+ CONSOLE.print(t.schema())
135
+
136
+
137
+ @app.command(help="Fetch Write-Ahead-Log.")
138
+ def wal(
139
+ project: ProjectArg,
140
+ table: Annotated[str | None, Option(help="Table name.")] = None,
141
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
142
+ ):
143
+ _, t = get_table(project, table, dataset)
144
+ wal_ = t.core.get_wal(asof=None)
145
+ # Don't use CONSOLE.print here so that it can be piped.
146
+ print(wal_)
147
+
148
+
149
+ @app.command(help="Flush Write-Ahead-Log.")
150
+ def flush(
151
+ project: ProjectArg,
152
+ table: Annotated[str | None, Option(help="Table name.")] = None,
153
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
154
+ ):
155
+ identifier, t = get_table(project, table, dataset)
156
+ state.spiral.internal.flush_wal(t.core) # pyright: ignore[reportPrivateUsage]
157
+ CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
158
+
159
+
160
+ @app.command(help="Display all manifests.")
161
+ def manifests(
162
+ project: ProjectArg,
163
+ table: Annotated[str | None, Option(help="Table name.")] = None,
164
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
165
+ ):
166
+ _, t = get_table(project, table, dataset)
167
+ s = t.snapshot()
168
+
169
+ key_space_state = state.spiral.internal.key_space_state(s.core) # pyright: ignore[reportPrivateUsage]
170
+ key_space_manifest = key_space_state.manifest
171
+
172
+ column_groups_states = state.spiral.internal.column_groups_states(s.core, key_space_state) # pyright: ignore[reportPrivateUsage]
173
+ display_manifests(key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states])
174
+
175
+
176
+ @app.command(help="Visualize the scan of a given column group.")
177
+ def debug_scan(
178
+ project: ProjectArg,
179
+ table: Annotated[str | None, Option(help="Table name.")] = None,
180
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
181
+ column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
182
+ ):
183
+ _, t = get_table(project, table, dataset)
184
+ scan = state.spiral.scan(t[column_group] if column_group != "." else t)
185
+ scan._debug() # pyright: ignore[reportPrivateUsage]
186
+
187
+
188
+ @app.command(help="Display the manifests for a scan of a given column group.")
189
+ def dump_scan(
190
+ project: ProjectArg,
191
+ table: Annotated[str | None, Option(help="Table name.")] = None,
192
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
193
+ column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
194
+ ):
195
+ _, t = get_table(project, table, dataset)
196
+ scan = state.spiral.scan(t[column_group] if column_group != "." else t)
197
+ scan._dump_manifests() # pyright: ignore[reportPrivateUsage]
@@ -0,0 +1,17 @@
1
+ import pyperclip
2
+
3
+ from spiral.api.telemetry import IssueExportTokenResponse
4
+ from spiral.cli import CONSOLE, AsyncTyper, state
5
+
6
+ app = AsyncTyper(short_help="Client-side telemetry.")
7
+
8
+
9
+ @app.command(help="Issue new telemetry export token.")
10
+ def export():
11
+ res: IssueExportTokenResponse = state.spiral.api.telemetry.issue_export_token()
12
+
13
+ command = f"export SPIRAL_OTEL_TOKEN={res.token}"
14
+ pyperclip.copy(command)
15
+
16
+ CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
17
+ CONSOLE.print("[dim]Token is valid for 1h.[/dim]")
spiral/cli/text.py ADDED
@@ -0,0 +1,115 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import rich
5
+ import typer
6
+ from questionary import Choice
7
+ from typer import Option
8
+
9
+ from spiral.api.projects import TextIndexResource
10
+ from spiral.api.text_indexes import CreateWorkerRequest, SyncIndexRequest
11
+ from spiral.api.types import IndexId
12
+ from spiral.api.workers import CPU, GcpRegion, Memory, ResourceClass
13
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
14
+ from spiral.cli.types import ProjectArg
15
+
16
+ app = AsyncTyper(short_help="Text Indexes.")
17
+
18
+
19
+ def ask_index(project_id, title="Select an index"):
20
+ indexes: list[TextIndexResource] = list(state.spiral.project(project_id).list_text_indexes())
21
+
22
+ if not indexes:
23
+ ERR_CONSOLE.print("No indexes found")
24
+ raise typer.Exit(1)
25
+
26
+ return questionary.select(
27
+ title,
28
+ choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
29
+ ).ask()
30
+
31
+
32
+ def get_index_id(
33
+ project: ProjectArg,
34
+ name: Annotated[str | None, Option(help="Index name.")] = None,
35
+ ) -> IndexId:
36
+ if name is None:
37
+ return ask_index(project)
38
+
39
+ indexes: list[TextIndexResource] = list(state.spiral.project(project).list_text_indexes())
40
+ for index in indexes:
41
+ if index.name == name:
42
+ return index.id
43
+ raise ValueError(f"Index not found: {name}")
44
+
45
+
46
+ @app.command(help="List indexes.")
47
+ def ls(
48
+ project: ProjectArg,
49
+ ):
50
+ """List indexes."""
51
+ indexes = state.spiral.project(project).list_text_indexes()
52
+
53
+ rich_table = rich.table.Table("id", "name", title="Text Indexes")
54
+ for index in indexes:
55
+ rich_table.add_row(index.id, index.name)
56
+ CONSOLE.print(rich_table)
57
+
58
+
59
+ @app.command(help="Trigger a sync job for an index.")
60
+ def sync(
61
+ project: ProjectArg,
62
+ name: Annotated[str | None, Option(help="Index name.")] = None,
63
+ resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
64
+ ):
65
+ """Trigger a sync job."""
66
+ index_id = get_index_id(project, name)
67
+ response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
68
+ CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
69
+
70
+
71
+ @app.command(name="serve", help="Spin up a worker to serve an index.")
72
+ def serve(
73
+ project: ProjectArg,
74
+ index: Annotated[str | None, Option(help="Index name.")] = None,
75
+ region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
76
+ cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
77
+ memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
78
+ ):
79
+ """Spin up a worker."""
80
+ index_id = get_index_id(project, index)
81
+ request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
82
+ response = state.spiral.api.text_indexes.create_worker(index_id, request)
83
+ CONSOLE.print(f"Created worker {response.worker_id} for {index_id}.")
84
+
85
+
86
+ @app.command(name="workers", help="List search workers.")
87
+ def workers(
88
+ project: ProjectArg,
89
+ index: Annotated[str | None, Option(help="Index name.")] = None,
90
+ ):
91
+ """List text search workers."""
92
+ index_id = get_index_id(project, index)
93
+ worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
94
+
95
+ rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
96
+ for worker_id in worker_ids:
97
+ try:
98
+ worker = state.spiral.api.text_indexes.get_worker(worker_id)
99
+ rich_table.add_row(
100
+ worker_id,
101
+ worker.url,
102
+ )
103
+ except Exception:
104
+ rich_table.add_row(
105
+ worker_id,
106
+ "Unavailable",
107
+ )
108
+ CONSOLE.print(rich_table)
109
+
110
+
111
+ @app.command(name="shutdown", help="Shutdown a search worker.")
112
+ def shutdown(worker_id: str):
113
+ """Shutdown a worker."""
114
+ state.spiral.api.text_indexes.shutdown_worker(worker_id)
115
+ CONSOLE.print(f"Requested worker {worker_id} to shutdown.")
spiral/cli/types.py ADDED
@@ -0,0 +1,50 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import typer
5
+ from questionary import Choice
6
+ from typer import Argument
7
+
8
+ from spiral.api.types import OrgId, ProjectId
9
+ from spiral.cli import ERR_CONSOLE, state
10
+
11
+
12
+ def ask_project(title="Select a project"):
13
+ projects = list(state.spiral.api.project.list())
14
+
15
+ if not projects:
16
+ ERR_CONSOLE.print("No projects found")
17
+ raise typer.Exit(1)
18
+
19
+ return questionary.select(
20
+ title,
21
+ choices=[
22
+ Choice(title=f"{project.id} - {project.name}" if project.name else project.id, value=project.id)
23
+ for project in projects
24
+ ],
25
+ ).ask()
26
+
27
+
28
+ ProjectArg = Annotated[ProjectId, Argument(help="Project ID", show_default=False, default_factory=ask_project)]
29
+
30
+
31
+ def _org_default():
32
+ memberships = list(state.spiral.api.organization.list_memberships())
33
+
34
+ if not memberships:
35
+ ERR_CONSOLE.print("No organizations found")
36
+ raise typer.Exit(1)
37
+
38
+ return questionary.select(
39
+ "Select an organization",
40
+ choices=[
41
+ Choice(
42
+ title=f"{m.org.id} - {m.org.name}" if m.org.name else m.org.id,
43
+ value=m.org.id,
44
+ )
45
+ for m in memberships
46
+ ],
47
+ ).ask()
48
+
49
+
50
+ OrganizationArg = Annotated[OrgId, Argument(help="Organization ID", show_default=False, default_factory=_org_default)]