pyspiral 0.4.4__cp310-abi3-macosx_11_0_arm64.whl → 0.6.0__cp310-abi3-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {pyspiral-0.4.4.dist-info → pyspiral-0.6.0.dist-info}/METADATA +10 -5
  2. pyspiral-0.6.0.dist-info/RECORD +99 -0
  3. {pyspiral-0.4.4.dist-info → pyspiral-0.6.0.dist-info}/WHEEL +1 -1
  4. spiral/__init__.py +10 -3
  5. spiral/_lib.abi3.so +0 -0
  6. spiral/adbc.py +29 -11
  7. spiral/api/__init__.py +14 -0
  8. spiral/api/client.py +5 -1
  9. spiral/api/key_space_indexes.py +23 -0
  10. spiral/api/projects.py +17 -2
  11. spiral/api/text_indexes.py +56 -0
  12. spiral/api/types.py +2 -0
  13. spiral/api/workers.py +40 -0
  14. spiral/cli/__init__.py +15 -6
  15. spiral/cli/admin.py +2 -4
  16. spiral/cli/app.py +4 -2
  17. spiral/cli/fs.py +5 -6
  18. spiral/cli/iceberg.py +97 -0
  19. spiral/cli/key_spaces.py +68 -0
  20. spiral/cli/login.py +6 -7
  21. spiral/cli/orgs.py +7 -8
  22. spiral/cli/printer.py +3 -3
  23. spiral/cli/projects.py +5 -6
  24. spiral/cli/tables.py +131 -0
  25. spiral/cli/telemetry.py +3 -4
  26. spiral/cli/text.py +115 -0
  27. spiral/cli/types.py +3 -4
  28. spiral/cli/workloads.py +7 -8
  29. spiral/client.py +111 -8
  30. spiral/core/authn/__init__.pyi +27 -0
  31. spiral/core/client/__init__.pyi +135 -63
  32. spiral/core/table/__init__.pyi +36 -26
  33. spiral/core/table/metastore/__init__.pyi +0 -4
  34. spiral/core/table/spec/__init__.pyi +0 -2
  35. spiral/{tables/dataset.py → dataset.py} +13 -7
  36. spiral/{tables/debug → debug}/manifests.py +17 -6
  37. spiral/{tables/debug → debug}/scan.py +7 -7
  38. spiral/expressions/base.py +3 -3
  39. spiral/expressions/udf.py +1 -1
  40. spiral/{iceberg/client.py → iceberg.py} +1 -3
  41. spiral/key_space_index.py +44 -0
  42. spiral/project.py +171 -18
  43. spiral/protogen/_/arrow/flight/protocol/sql/__init__.py +1668 -1110
  44. spiral/protogen/_/google/protobuf/__init__.py +2190 -0
  45. spiral/protogen/_/message_pool.py +3 -0
  46. spiral/protogen/_/py.typed +0 -0
  47. spiral/protogen/_/scandal/__init__.py +138 -126
  48. spiral/protogen/_/spfs/__init__.py +72 -0
  49. spiral/protogen/_/spql/__init__.py +61 -0
  50. spiral/protogen/_/substrait/__init__.py +5256 -2459
  51. spiral/protogen/_/substrait/extensions/__init__.py +103 -49
  52. spiral/{tables/scan.py → scan.py} +37 -44
  53. spiral/settings.py +14 -3
  54. spiral/snapshot.py +55 -0
  55. spiral/streaming_/__init__.py +3 -0
  56. spiral/streaming_/reader.py +117 -0
  57. spiral/streaming_/stream.py +146 -0
  58. spiral/substrait_.py +9 -9
  59. spiral/table.py +257 -0
  60. spiral/text_index.py +17 -0
  61. spiral/{tables/transaction.py → transaction.py} +11 -15
  62. pyspiral-0.4.4.dist-info/RECORD +0 -98
  63. spiral/cli/iceberg/__init__.py +0 -7
  64. spiral/cli/iceberg/namespaces.py +0 -47
  65. spiral/cli/iceberg/tables.py +0 -60
  66. spiral/cli/indexes/__init__.py +0 -19
  67. spiral/cli/tables/__init__.py +0 -121
  68. spiral/core/index/__init__.pyi +0 -15
  69. spiral/iceberg/__init__.py +0 -3
  70. spiral/indexes/__init__.py +0 -5
  71. spiral/indexes/client.py +0 -137
  72. spiral/indexes/index.py +0 -34
  73. spiral/indexes/scan.py +0 -22
  74. spiral/protogen/_/spiral/table/__init__.py +0 -22
  75. spiral/protogen/substrait/__init__.py +0 -3399
  76. spiral/protogen/substrait/extensions/__init__.py +0 -115
  77. spiral/tables/__init__.py +0 -12
  78. spiral/tables/client.py +0 -130
  79. spiral/tables/maintenance.py +0 -12
  80. spiral/tables/snapshot.py +0 -78
  81. spiral/tables/table.py +0 -145
  82. {pyspiral-0.4.4.dist-info → pyspiral-0.6.0.dist-info}/entry_points.txt +0 -0
  83. /spiral/{protogen/_/spiral → debug}/__init__.py +0 -0
  84. /spiral/{tables/debug → debug}/metrics.py +0 -0
  85. /spiral/{tables/debug → protogen/_/google}/__init__.py +0 -0
spiral/cli/iceberg.py ADDED
@@ -0,0 +1,97 @@
1
+ import sys
2
+ from typing import Annotated
3
+
4
+ import rich
5
+ import typer
6
+ from typer import Argument
7
+
8
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
9
+ from spiral.cli.types import ProjectArg
10
+
11
+ app = AsyncTyper(short_help="Apache Iceberg Catalog")
12
+
13
+
14
+ @app.command(help="List namespaces.")
15
+ def namespaces(
16
+ project: ProjectArg,
17
+ namespace: Annotated[str | None, Argument(help="List only namespaces under this namespace.")] = None,
18
+ ):
19
+ """List Iceberg namespaces."""
20
+ import pyiceberg.exceptions
21
+
22
+ catalog = state.spiral.iceberg.catalog()
23
+
24
+ if namespace is None:
25
+ try:
26
+ namespaces = catalog.list_namespaces(project)
27
+ except pyiceberg.exceptions.ForbiddenError:
28
+ ERR_CONSOLE.print(
29
+ f"The project, {repr(project)}, does not exist or you lack the "
30
+ f"`iceberg:view` permission to list namespaces in it.",
31
+ )
32
+ raise typer.Exit(code=1)
33
+ else:
34
+ try:
35
+ namespaces = catalog.list_namespaces((project, namespace))
36
+ except pyiceberg.exceptions.ForbiddenError:
37
+ ERR_CONSOLE.print(
38
+ f"The namespace, {repr(project)}.{repr(namespace)}, does not exist or you lack the "
39
+ f"`iceberg:view` permission to list namespaces in it.",
40
+ )
41
+ raise typer.Exit(code=1)
42
+
43
+ table = CONSOLE.table.Table("Namespace ID", title="Iceberg namespaces")
44
+ for ns in namespaces:
45
+ table.add_row(".".join(ns))
46
+ CONSOLE.print(table)
47
+
48
+
49
+ @app.command(help="List tables.")
50
+ def tables(
51
+ project: ProjectArg,
52
+ namespace: Annotated[str | None, Argument(help="Show only tables in the given namespace.")] = None,
53
+ ):
54
+ import pyiceberg.exceptions
55
+
56
+ catalog = state.spiral.iceberg.catalog()
57
+
58
+ try:
59
+ if namespace is None:
60
+ tables = catalog.list_tables(project)
61
+ else:
62
+ tables = catalog.list_tables((project, namespace))
63
+ except pyiceberg.exceptions.ForbiddenError:
64
+ ERR_CONSOLE.print(
65
+ f"The namespace, {repr(project)}.{repr(namespace)}, does not exist or you lack the "
66
+ f"`iceberg:view` permission to list tables in it.",
67
+ )
68
+ raise typer.Exit(code=1)
69
+
70
+ rich_table = rich.table.Table("table id", title="Iceberg tables")
71
+ for table in tables:
72
+ rich_table.add_row(".".join(table))
73
+ CONSOLE.print(rich_table)
74
+
75
+
76
+ @app.command(help="Show the table schema.")
77
+ def schema(
78
+ project: ProjectArg,
79
+ namespace: Annotated[str, Argument(help="Table namespace.")],
80
+ table: Annotated[str, Argument(help="Table name.")],
81
+ ):
82
+ import pyiceberg.exceptions
83
+
84
+ catalog = state.spiral.iceberg.catalog()
85
+
86
+ try:
87
+ tbl = catalog.load_table((project, namespace, table))
88
+ except pyiceberg.exceptions.NoSuchTableError:
89
+ ERR_CONSOLE.print(f"No table {repr(table)} found in {repr(project)}.{repr(namespace)}", file=sys.stderr)
90
+ raise typer.Exit(code=1)
91
+
92
+ rich_table = rich.table.Table(
93
+ "Field ID", "Field name", "Type", "Required", "Doc", title=f"{project}.{namespace}.{table}"
94
+ )
95
+ for col in tbl.schema().columns:
96
+ rich_table.add_row(str(col.field_id), col.name, str(col.field_type), str(col.required), col.doc)
97
+ CONSOLE.print(rich_table)
@@ -0,0 +1,68 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import rich
5
+ import typer
6
+ from questionary import Choice
7
+ from typer import Option
8
+
9
+ from spiral.api.key_space_indexes import SyncIndexRequest
10
+ from spiral.api.projects import KeySpaceIndexResource
11
+ from spiral.api.types import IndexId
12
+ from spiral.api.workers import ResourceClass
13
+ from spiral.cli import CONSOLE, AsyncTyper, state
14
+ from spiral.cli.types import ProjectArg
15
+
16
+ app = AsyncTyper(short_help="Key Space Indexes.")
17
+
18
+
19
+ def ask_index(project_id, title="Select an index"):
20
+ indexes: list[KeySpaceIndexResource] = list(state.spiral.project(project_id).list_key_space_indexes())
21
+
22
+ if not indexes:
23
+ CONSOLE.print("[red]No indexes found[/red]")
24
+ raise typer.Exit(1)
25
+
26
+ return questionary.select(
27
+ title,
28
+ choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
29
+ ).ask()
30
+
31
+
32
+ def get_index_id(
33
+ project: ProjectArg,
34
+ name: Annotated[str | None, Option(help="Index name.")] = None,
35
+ ) -> IndexId:
36
+ if name is None:
37
+ return ask_index(project)
38
+
39
+ indexes: list[KeySpaceIndexResource] = list(state.spiral.project(project).list_key_space_indexes())
40
+ for index in indexes:
41
+ if index.name == name:
42
+ return index.id
43
+ raise ValueError(f"Index not found: {name}")
44
+
45
+
46
+ @app.command(help="List indexes.")
47
+ def ls(
48
+ project: ProjectArg,
49
+ ):
50
+ """List indexes."""
51
+ indexes = state.spiral.project(project).list_key_space_indexes()
52
+
53
+ rich_table = rich.table.Table("id", "name", title="Key Space Indexes")
54
+ for index in indexes:
55
+ rich_table.add_row(index.id, index.name)
56
+ CONSOLE.print(rich_table)
57
+
58
+
59
+ @app.command(help="Trigger a sync job for an index.")
60
+ def sync(
61
+ project: ProjectArg,
62
+ name: Annotated[str | None, Option(help="Index name.")] = None,
63
+ resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
64
+ ):
65
+ """Trigger a sync job."""
66
+ index_id = get_index_id(project, name)
67
+ response = state.spiral.api.key_space_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
68
+ CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
spiral/cli/login.py CHANGED
@@ -1,22 +1,21 @@
1
1
  import jwt
2
- from rich import print
3
2
 
4
- from spiral.cli import state
3
+ from spiral.cli import CONSOLE, state
5
4
 
6
5
 
7
6
  def command(org_id: str | None = None, force: bool = False):
8
7
  token = state.settings.device_code_auth.authenticate(force=force, org_id=org_id)
9
- print("Successfully logged in.")
10
- print(token.expose_secret())
8
+ CONSOLE.print("Successfully logged in.")
9
+ CONSOLE.print(token.expose_secret(), soft_wrap=True)
11
10
 
12
11
 
13
12
  def whoami():
14
13
  """Display the current user's information."""
15
14
  payload = jwt.decode(state.settings.authn.token().expose_secret(), options={"verify_signature": False})
16
- print(f"{payload['org_id']}")
17
- print(f"{payload['sub']}")
15
+ CONSOLE.print(f"{payload['org_id']}")
16
+ CONSOLE.print(f"{payload['sub']}")
18
17
 
19
18
 
20
19
  def logout():
21
20
  state.settings.device_code_auth.logout()
22
- print("Logged out.")
21
+ CONSOLE.print("Logged out.")
spiral/cli/orgs.py CHANGED
@@ -2,13 +2,12 @@ import webbrowser
2
2
  from typing import Annotated
3
3
 
4
4
  import jwt
5
- import rich
6
5
  import typer
7
6
  from rich.table import Table
8
7
  from typer import Option
9
8
 
10
9
  from spiral.api.organizations import CreateOrgRequest, InviteUserRequest, OrgRole, PortalLinkIntent, PortalLinkRequest
11
- from spiral.cli import AsyncTyper, state
10
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
12
11
  from spiral.cli.types import OrganizationArg
13
12
 
14
13
  app = AsyncTyper(short_help="Org admin.")
@@ -17,7 +16,7 @@ app = AsyncTyper(short_help="Org admin.")
17
16
  @app.command(help="Switch the active organization.")
18
17
  def switch(org_id: OrganizationArg):
19
18
  state.settings.device_code_auth.authenticate(org_id=org_id)
20
- rich.print(f"Switched to organization: {org_id}")
19
+ CONSOLE.print(f"Switched to organization: {org_id}")
21
20
 
22
21
 
23
22
  @app.command(help="Create a new organization.")
@@ -29,7 +28,7 @@ def create(
29
28
  # Authenticate to the new organization
30
29
  state.settings.device_code_auth.authenticate(org_id=res.org.id)
31
30
 
32
- rich.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
31
+ CONSOLE.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
33
32
 
34
33
 
35
34
  @app.command(help="List organizations.")
@@ -40,7 +39,7 @@ def ls():
40
39
  for m in state.settings.api.organization.list_memberships():
41
40
  table.add_row("👉" if m.org.id == org_id else "", m.org.id, m.org.name, m.role)
42
41
 
43
- rich.print(table)
42
+ CONSOLE.print(table)
44
43
 
45
44
 
46
45
  @app.command(help="Invite a user to the organization.")
@@ -48,7 +47,7 @@ def invite(email: str, role: OrgRole = OrgRole.MEMBER, expires_in_days: int = 7)
48
47
  state.settings.api.organization.invite_user(
49
48
  InviteUserRequest(email=email, role=role, expires_in_days=expires_in_days)
50
49
  )
51
- rich.print(f"Invited {email} as a {role.value}.")
50
+ CONSOLE.print(f"Invited {email} as a {role.value}.")
52
51
 
53
52
 
54
53
  @app.command(help="Configure single sign-on for your organization.")
@@ -78,7 +77,7 @@ def domains():
78
77
 
79
78
  def _do_action(intent: PortalLinkIntent):
80
79
  res = state.settings.api.organization.portal_link(PortalLinkRequest(intent=intent))
81
- rich.print(f"Opening the configuration portal:\n{res.url}")
80
+ CONSOLE.print(f"Opening the configuration portal:\n{res.url}")
82
81
  webbrowser.open(res.url)
83
82
 
84
83
 
@@ -86,5 +85,5 @@ def current_org_id():
86
85
  if token := state.settings.authn.token():
87
86
  if org_id := jwt.decode(token.expose_secret(), options={"verify_signature": False}).get("org_id"):
88
87
  return org_id
89
- rich.print("[red]You are not logged in to an organization.[/red]")
88
+ ERR_CONSOLE.print("You are not logged in to an organization.")
90
89
  raise typer.Exit(1)
spiral/cli/printer.py CHANGED
@@ -2,14 +2,14 @@ import json
2
2
  from collections.abc import Iterable
3
3
  from typing import TypeVar
4
4
 
5
- import betterproto
5
+ import betterproto2
6
6
  from pydantic import BaseModel
7
7
  from rich.console import ConsoleRenderable, Group
8
8
  from rich.padding import Padding
9
9
  from rich.pretty import Pretty
10
10
  from rich.table import Table
11
11
 
12
- T = TypeVar("T", bound=betterproto.Message)
12
+ T = TypeVar("T", bound=betterproto2.Message)
13
13
  M = TypeVar("M", bound=BaseModel)
14
14
 
15
15
 
@@ -34,7 +34,7 @@ def table_of_protos(cls: type[T], messages: Iterable[T], fields: list[str] = Non
34
34
 
35
35
  TODO(ngates): add a CLI switch to emit JSON results instead of tables.
36
36
  """
37
- cols = fields or cls()._betterproto.sorted_field_names
37
+ cols = fields or cls()._betterproto2.sorted_field_names
38
38
  table = Table(*cols, title=title or f"{cls.__name__}s")
39
39
  for msg in messages:
40
40
  table.add_row(*[getattr(msg, col, "") for col in cols])
spiral/cli/projects.py CHANGED
@@ -1,6 +1,5 @@
1
1
  from typing import Annotated
2
2
 
3
- import rich
4
3
  import typer
5
4
  from typer import Option
6
5
 
@@ -19,7 +18,7 @@ from spiral.api.projects import (
19
18
  Project,
20
19
  WorkloadPrincipalConditions,
21
20
  )
22
- from spiral.cli import AsyncTyper, printer, state
21
+ from spiral.cli import CONSOLE, AsyncTyper, printer, state
23
22
  from spiral.cli.types import ProjectArg
24
23
 
25
24
  app = AsyncTyper(short_help="Projects and grants.")
@@ -28,7 +27,7 @@ app = AsyncTyper(short_help="Projects and grants.")
28
27
  @app.command(help="List projects.")
29
28
  def ls():
30
29
  projects = list(state.settings.api.project.list())
31
- rich.print(printer.table_of_models(Project, projects))
30
+ CONSOLE.print(printer.table_of_models(Project, projects))
32
31
 
33
32
 
34
33
  @app.command(help="Create a new project.")
@@ -39,7 +38,7 @@ def create(
39
38
  name: Annotated[str | None, Option(help="Friendly name for the project.")] = None,
40
39
  ):
41
40
  res = state.settings.api.project.create(CreateProjectRequest(id_prefix=id_prefix, name=name))
42
- rich.print(f"Created project {res.project.id}")
41
+ CONSOLE.print(f"Created project {res.project.id}")
43
42
 
44
43
 
45
44
  @app.command(help="Grant a role on a project.")
@@ -127,10 +126,10 @@ def grant(
127
126
  ),
128
127
  )
129
128
 
130
- rich.print(f"Granted role {role} on project {project}")
129
+ CONSOLE.print(f"Granted role {role} on project {project}")
131
130
 
132
131
 
133
132
  @app.command(help="List project grants.")
134
133
  def grants(project: ProjectArg):
135
134
  project_grants = list(state.settings.api.project.list_grants(project))
136
- rich.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
135
+ CONSOLE.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
spiral/cli/tables.py ADDED
@@ -0,0 +1,131 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import rich
5
+ import rich.table
6
+ import typer
7
+ from questionary import Choice
8
+ from typer import Argument, Option
9
+
10
+ from spiral import Spiral
11
+ from spiral.api.projects import TableResource
12
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
13
+ from spiral.cli.types import ProjectArg
14
+ from spiral.debug.manifests import display_manifests
15
+ from spiral.table import Table
16
+
17
+ app = AsyncTyper(short_help="Spiral Tables.")
18
+
19
+
20
+ def ask_table(project_id: str, title: str = "Select a table") -> str:
21
+ tables: list[TableResource] = list(state.spiral.project(project_id).list_tables())
22
+
23
+ if not tables:
24
+ ERR_CONSOLE.print("No tables found")
25
+ raise typer.Exit(1)
26
+
27
+ return questionary.select( # pyright: ignore[reportAny]
28
+ title,
29
+ choices=[
30
+ Choice(title=f"{table.dataset}.{table.table}", value=f"{table.dataset}.{table.table}")
31
+ for table in sorted(tables, key=lambda t: (t.dataset, t.table))
32
+ ],
33
+ ).ask()
34
+
35
+
36
+ def get_table(
37
+ project: ProjectArg,
38
+ table: Annotated[str | None, Option(help="Table name.")] = None,
39
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
40
+ ) -> tuple[str, Table]:
41
+ if table is None:
42
+ identifier = ask_table(project)
43
+ else:
44
+ identifier = table
45
+ if dataset is not None:
46
+ identifier = f"{dataset}.{table}"
47
+ return identifier, state.spiral.project(project).table(identifier)
48
+
49
+
50
+ @app.command(help="List tables.")
51
+ def ls(
52
+ project: ProjectArg,
53
+ ):
54
+ tables = Spiral().project(project).list_tables()
55
+
56
+ rich_table = rich.table.Table("id", "dataset", "name", title="Spiral tables")
57
+ for table in tables:
58
+ rich_table.add_row(table.id, table.dataset, table.table)
59
+ CONSOLE.print(rich_table)
60
+
61
+
62
+ @app.command(help="Show the table key schema.")
63
+ def key_schema(
64
+ project: ProjectArg,
65
+ table: Annotated[str | None, Option(help="Table name.")] = None,
66
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
67
+ ):
68
+ _, t = get_table(project, table, dataset)
69
+ CONSOLE.print(t.key_schema)
70
+
71
+
72
+ @app.command(help="Compute the full table schema.")
73
+ def schema(
74
+ project: ProjectArg,
75
+ table: Annotated[str | None, Option(help="Table name.")] = None,
76
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
77
+ ):
78
+ _, t = get_table(project, table, dataset)
79
+ CONSOLE.print(t.schema())
80
+
81
+
82
+ @app.command(help="Flush Write-Ahead-Log.")
83
+ def flush(
84
+ project: ProjectArg,
85
+ table: Annotated[str | None, Option(help="Table name.")] = None,
86
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
87
+ ):
88
+ identifier, t = get_table(project, table, dataset)
89
+ state.spiral._ops().flush_wal(t.core) # pyright: ignore[reportPrivateUsage]
90
+ CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
91
+
92
+
93
+ @app.command(help="Display scan.")
94
+ def debug(
95
+ project: ProjectArg,
96
+ table: Annotated[str | None, Option(help="Table name.")] = None,
97
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
98
+ column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
99
+ ):
100
+ _, t = get_table(project, table, dataset)
101
+ scan = state.spiral.scan(t[column_group] if column_group != "." else t)
102
+ scan._debug() # pyright: ignore[reportPrivateUsage]
103
+
104
+
105
+ @app.command(help="Display all manifests.")
106
+ def manifests(
107
+ project: ProjectArg,
108
+ table: Annotated[str | None, Option(help="Table name.")] = None,
109
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
110
+ ):
111
+ _, t = get_table(project, table, dataset)
112
+ s = t.snapshot()
113
+
114
+ key_space_state = state.spiral._ops().key_space_state(t.core, asof=s.asof) # pyright: ignore[reportPrivateUsage]
115
+ key_space_manifest = key_space_state.manifest
116
+
117
+ column_groups_states = state.spiral._ops().column_groups_states(t.core, key_space_state, asof=s.asof) # pyright: ignore[reportPrivateUsage]
118
+
119
+ display_manifests(key_space_manifest, ((x.column_group, x.manifest) for x in column_groups_states))
120
+
121
+
122
+ @app.command(help="Display the manifests which would be read by a scan of the given column group.")
123
+ def scan_manifests(
124
+ project: ProjectArg,
125
+ table: Annotated[str | None, Option(help="Table name.")] = None,
126
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
127
+ column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
128
+ ):
129
+ _, t = get_table(project, table, dataset)
130
+ scan = state.spiral.scan(t[column_group] if column_group != "." else t)
131
+ scan._dump_manifests() # pyright: ignore[reportPrivateUsage]
spiral/cli/telemetry.py CHANGED
@@ -1,8 +1,7 @@
1
1
  import pyperclip
2
- import rich
3
2
 
4
3
  from spiral.api.telemetry import IssueExportTokenResponse
5
- from spiral.cli import AsyncTyper, state
4
+ from spiral.cli import CONSOLE, AsyncTyper, state
6
5
 
7
6
  app = AsyncTyper(short_help="Client-side telemetry.")
8
7
 
@@ -14,5 +13,5 @@ def export():
14
13
  command = f"export SPIRAL_OTEL_TOKEN={res.token}"
15
14
  pyperclip.copy(command)
16
15
 
17
- rich.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
18
- rich.print("[dim]Token is valid for 1h.[/dim]")
16
+ CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
17
+ CONSOLE.print("[dim]Token is valid for 1h.[/dim]")
spiral/cli/text.py ADDED
@@ -0,0 +1,115 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import rich
5
+ import typer
6
+ from questionary import Choice
7
+ from typer import Option
8
+
9
+ from spiral.api.projects import TextIndexResource
10
+ from spiral.api.text_indexes import CreateWorkerRequest, SyncIndexRequest
11
+ from spiral.api.types import IndexId
12
+ from spiral.api.workers import CPU, GcpRegion, Memory, ResourceClass
13
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
14
+ from spiral.cli.types import ProjectArg
15
+
16
+ app = AsyncTyper(short_help="Text Indexes.")
17
+
18
+
19
+ def ask_index(project_id, title="Select an index"):
20
+ indexes: list[TextIndexResource] = list(state.spiral.project(project_id).list_text_indexes())
21
+
22
+ if not indexes:
23
+ ERR_CONSOLE.print("No indexes found")
24
+ raise typer.Exit(1)
25
+
26
+ return questionary.select(
27
+ title,
28
+ choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
29
+ ).ask()
30
+
31
+
32
+ def get_index_id(
33
+ project: ProjectArg,
34
+ name: Annotated[str | None, Option(help="Index name.")] = None,
35
+ ) -> IndexId:
36
+ if name is None:
37
+ return ask_index(project)
38
+
39
+ indexes: list[TextIndexResource] = list(state.spiral.project(project).list_text_indexes())
40
+ for index in indexes:
41
+ if index.name == name:
42
+ return index.id
43
+ raise ValueError(f"Index not found: {name}")
44
+
45
+
46
+ @app.command(help="List indexes.")
47
+ def ls(
48
+ project: ProjectArg,
49
+ ):
50
+ """List indexes."""
51
+ indexes = state.spiral.project(project).list_text_indexes()
52
+
53
+ rich_table = rich.table.Table("id", "name", title="Text Indexes")
54
+ for index in indexes:
55
+ rich_table.add_row(index.id, index.name)
56
+ CONSOLE.print(rich_table)
57
+
58
+
59
+ @app.command(help="Trigger a sync job for an index.")
60
+ def sync(
61
+ project: ProjectArg,
62
+ name: Annotated[str | None, Option(help="Index name.")] = None,
63
+ resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
64
+ ):
65
+ """Trigger a sync job."""
66
+ index_id = get_index_id(project, name)
67
+ response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
68
+ CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
69
+
70
+
71
+ @app.command(name="serve", help="Spin up a worker to serve an index.")
72
+ def serve(
73
+ project: ProjectArg,
74
+ index: Annotated[str | None, Option(help="Index name.")] = None,
75
+ region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
76
+ cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
77
+ memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
78
+ ):
79
+ """Spin up a worker."""
80
+ index_id = get_index_id(project, index)
81
+ request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
82
+ response = state.spiral.api.text_indexes.create_worker(index_id, request)
83
+ CONSOLE.print(f"Created worker {response.worker_id} for {index_id}.")
84
+
85
+
86
+ @app.command(name="workers", help="List search workers.")
87
+ def workers(
88
+ project: ProjectArg,
89
+ index: Annotated[str | None, Option(help="Index name.")] = None,
90
+ ):
91
+ """List text search workers."""
92
+ index_id = get_index_id(project, index)
93
+ worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
94
+
95
+ rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
96
+ for worker_id in worker_ids:
97
+ try:
98
+ worker = state.spiral.api.text_indexes.get_worker(worker_id)
99
+ rich_table.add_row(
100
+ worker_id,
101
+ worker.url,
102
+ )
103
+ except Exception:
104
+ rich_table.add_row(
105
+ worker_id,
106
+ "Unavailable",
107
+ )
108
+ CONSOLE.print(rich_table)
109
+
110
+
111
+ @app.command(name="shutdown", help="Shutdown a search worker.")
112
+ def shutdown(worker_id: str):
113
+ """Shutdown a worker."""
114
+ state.spiral.api.text_indexes.shutdown_worker(worker_id)
115
+ CONSOLE.print(f"Requested worker {worker_id} to shutdown.")
spiral/cli/types.py CHANGED
@@ -1,20 +1,19 @@
1
1
  from typing import Annotated
2
2
 
3
3
  import questionary
4
- import rich
5
4
  import typer
6
5
  from questionary import Choice
7
6
  from typer import Argument
8
7
 
9
8
  from spiral.api.types import OrgId, ProjectId
10
- from spiral.cli import state
9
+ from spiral.cli import ERR_CONSOLE, state
11
10
 
12
11
 
13
12
  def ask_project(title="Select a project"):
14
13
  projects = list(state.settings.api.project.list())
15
14
 
16
15
  if not projects:
17
- rich.print("[red]No projects found[/red]")
16
+ ERR_CONSOLE.print("No projects found")
18
17
  raise typer.Exit(1)
19
18
 
20
19
  return questionary.select(
@@ -33,7 +32,7 @@ def _org_default():
33
32
  memberships = list(state.settings.api.organization.list_memberships())
34
33
 
35
34
  if not memberships:
36
- rich.print("[red]No organizations found[/red]")
35
+ ERR_CONSOLE.print("No organizations found")
37
36
  raise typer.Exit(1)
38
37
 
39
38
  return questionary.select(
spiral/cli/workloads.py CHANGED
@@ -2,12 +2,11 @@ from typing import Annotated
2
2
 
3
3
  import pyperclip
4
4
  import questionary
5
- import rich
6
5
  from questionary import Choice
7
6
  from typer import Argument, Option
8
7
 
9
8
  from spiral.api.workloads import CreateWorkloadRequest, IssueWorkloadCredentialsResponse, Workload
10
- from spiral.cli import AsyncTyper, printer, state
9
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, printer, state
11
10
  from spiral.cli.types import ProjectArg
12
11
 
13
12
  app = AsyncTyper()
@@ -19,7 +18,7 @@ def create(
19
18
  name: Annotated[str | None, Option(help="Friendly name for the workload.")] = None,
20
19
  ):
21
20
  res = state.settings.api.workload.create(project, CreateWorkloadRequest(name=name))
22
- rich.print(f"Created workload {res.workload.id}")
21
+ CONSOLE.print(f"Created workload {res.workload.id}")
23
22
 
24
23
 
25
24
  @app.command(help="List workloads.")
@@ -27,7 +26,7 @@ def ls(
27
26
  project: ProjectArg,
28
27
  ):
29
28
  workloads = list(state.settings.api.workload.list(project))
30
- rich.print(printer.table_of_models(Workload, workloads, fields=["id", "project_id", "name"]))
29
+ CONSOLE.print(printer.table_of_models(Workload, workloads, fields=["id", "project_id", "name"]))
31
30
 
32
31
 
33
32
  @app.command(help="Issue new workflow credentials.")
@@ -46,14 +45,14 @@ def issue_credentials(workload_id: Annotated[str, Argument(help="Workload ID.")]
46
45
 
47
46
  if choice == 1:
48
47
  pyperclip.copy(res.client_secret)
49
- rich.print("[green]Secret copied to clipboard![/green]")
48
+ CONSOLE.print("[green]Secret copied to clipboard![/green]")
50
49
  break
51
50
  elif choice == 2:
52
- rich.print(f"[green]Token Secret:[/green] {res.client_secret}")
51
+ CONSOLE.print(f"[green]Token Secret:[/green] {res.client_secret}")
53
52
  break
54
53
  elif choice == 3:
55
54
  break
56
55
  else:
57
- rich.print("[red]Invalid choice. Please try again.[/red]")
56
+ ERR_CONSOLE.print("Invalid choice. Please try again.")
58
57
 
59
- rich.print(f"[green]Token ID:[/green] {res.client_id}")
58
+ CONSOLE.print(f"[green]Token ID:[/green] {res.client_id}")