pyspiral 0.5.0__cp310-abi3-macosx_11_0_arm64.whl → 0.6.1__cp310-abi3-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {pyspiral-0.5.0.dist-info → pyspiral-0.6.1.dist-info}/METADATA +7 -3
  2. pyspiral-0.6.1.dist-info/RECORD +99 -0
  3. {pyspiral-0.5.0.dist-info → pyspiral-0.6.1.dist-info}/WHEEL +1 -1
  4. spiral/__init__.py +11 -3
  5. spiral/_lib.abi3.so +0 -0
  6. spiral/adbc.py +6 -6
  7. spiral/api/__init__.py +8 -2
  8. spiral/api/client.py +1 -1
  9. spiral/api/key_space_indexes.py +23 -0
  10. spiral/api/projects.py +15 -0
  11. spiral/api/text_indexes.py +1 -1
  12. spiral/cli/__init__.py +15 -6
  13. spiral/cli/admin.py +2 -4
  14. spiral/cli/app.py +4 -2
  15. spiral/cli/fs.py +5 -6
  16. spiral/cli/iceberg.py +97 -0
  17. spiral/cli/key_spaces.py +89 -0
  18. spiral/cli/login.py +6 -7
  19. spiral/cli/orgs.py +7 -8
  20. spiral/cli/printer.py +3 -3
  21. spiral/cli/projects.py +5 -6
  22. spiral/cli/tables.py +131 -0
  23. spiral/cli/telemetry.py +3 -4
  24. spiral/cli/text.py +115 -0
  25. spiral/cli/types.py +3 -4
  26. spiral/cli/workloads.py +7 -8
  27. spiral/client.py +111 -8
  28. spiral/core/authn/__init__.pyi +27 -0
  29. spiral/core/client/__init__.pyi +152 -63
  30. spiral/core/table/__init__.pyi +17 -27
  31. spiral/core/table/metastore/__init__.pyi +0 -4
  32. spiral/core/table/spec/__init__.pyi +0 -2
  33. spiral/{tables/dataset.py → dataset.py} +13 -7
  34. spiral/{tables/debug → debug}/manifests.py +15 -6
  35. spiral/{tables/debug → debug}/scan.py +3 -3
  36. spiral/expressions/base.py +3 -3
  37. spiral/expressions/udf.py +1 -1
  38. spiral/{iceberg/client.py → iceberg.py} +1 -3
  39. spiral/key_space_index.py +44 -0
  40. spiral/project.py +171 -18
  41. spiral/protogen/_/arrow/flight/protocol/sql/__init__.py +1668 -1110
  42. spiral/protogen/_/google/protobuf/__init__.py +2190 -0
  43. spiral/protogen/_/message_pool.py +3 -0
  44. spiral/protogen/_/py.typed +0 -0
  45. spiral/protogen/_/scandal/__init__.py +138 -126
  46. spiral/protogen/_/spfs/__init__.py +72 -0
  47. spiral/protogen/_/spql/__init__.py +61 -0
  48. spiral/protogen/_/substrait/__init__.py +5256 -2459
  49. spiral/protogen/_/substrait/extensions/__init__.py +103 -49
  50. spiral/{tables/scan.py → scan.py} +38 -44
  51. spiral/settings.py +14 -3
  52. spiral/snapshot.py +55 -0
  53. spiral/streaming_/__init__.py +3 -0
  54. spiral/streaming_/reader.py +131 -0
  55. spiral/streaming_/stream.py +146 -0
  56. spiral/substrait_.py +9 -9
  57. spiral/table.py +259 -0
  58. spiral/text_index.py +17 -0
  59. spiral/{tables/transaction.py → transaction.py} +11 -15
  60. pyspiral-0.5.0.dist-info/RECORD +0 -103
  61. spiral/cli/iceberg/__init__.py +0 -7
  62. spiral/cli/iceberg/namespaces.py +0 -47
  63. spiral/cli/iceberg/tables.py +0 -60
  64. spiral/cli/indexes/__init__.py +0 -40
  65. spiral/cli/indexes/args.py +0 -39
  66. spiral/cli/indexes/workers.py +0 -59
  67. spiral/cli/tables/__init__.py +0 -88
  68. spiral/cli/tables/args.py +0 -42
  69. spiral/core/index/__init__.pyi +0 -7
  70. spiral/iceberg/__init__.py +0 -3
  71. spiral/indexes/__init__.py +0 -5
  72. spiral/indexes/client.py +0 -137
  73. spiral/indexes/index.py +0 -28
  74. spiral/indexes/scan.py +0 -22
  75. spiral/protogen/_/spiral/table/__init__.py +0 -22
  76. spiral/protogen/substrait/__init__.py +0 -3399
  77. spiral/protogen/substrait/extensions/__init__.py +0 -115
  78. spiral/tables/__init__.py +0 -12
  79. spiral/tables/client.py +0 -133
  80. spiral/tables/maintenance.py +0 -12
  81. spiral/tables/snapshot.py +0 -78
  82. spiral/tables/table.py +0 -145
  83. {pyspiral-0.5.0.dist-info → pyspiral-0.6.1.dist-info}/entry_points.txt +0 -0
  84. /spiral/{protogen/_/spiral → debug}/__init__.py +0 -0
  85. /spiral/{tables/debug → debug}/metrics.py +0 -0
  86. /spiral/{tables/debug → protogen/_/google}/__init__.py +0 -0
spiral/cli/login.py CHANGED
@@ -1,22 +1,21 @@
1
1
  import jwt
2
- from rich import print
3
2
 
4
- from spiral.cli import state
3
+ from spiral.cli import CONSOLE, state
5
4
 
6
5
 
7
6
  def command(org_id: str | None = None, force: bool = False):
8
7
  token = state.settings.device_code_auth.authenticate(force=force, org_id=org_id)
9
- print("Successfully logged in.")
10
- print(token.expose_secret())
8
+ CONSOLE.print("Successfully logged in.")
9
+ CONSOLE.print(token.expose_secret(), soft_wrap=True)
11
10
 
12
11
 
13
12
  def whoami():
14
13
  """Display the current user's information."""
15
14
  payload = jwt.decode(state.settings.authn.token().expose_secret(), options={"verify_signature": False})
16
- print(f"{payload['org_id']}")
17
- print(f"{payload['sub']}")
15
+ CONSOLE.print(f"{payload['org_id']}")
16
+ CONSOLE.print(f"{payload['sub']}")
18
17
 
19
18
 
20
19
  def logout():
21
20
  state.settings.device_code_auth.logout()
22
- print("Logged out.")
21
+ CONSOLE.print("Logged out.")
spiral/cli/orgs.py CHANGED
@@ -2,13 +2,12 @@ import webbrowser
2
2
  from typing import Annotated
3
3
 
4
4
  import jwt
5
- import rich
6
5
  import typer
7
6
  from rich.table import Table
8
7
  from typer import Option
9
8
 
10
9
  from spiral.api.organizations import CreateOrgRequest, InviteUserRequest, OrgRole, PortalLinkIntent, PortalLinkRequest
11
- from spiral.cli import AsyncTyper, state
10
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
12
11
  from spiral.cli.types import OrganizationArg
13
12
 
14
13
  app = AsyncTyper(short_help="Org admin.")
@@ -17,7 +16,7 @@ app = AsyncTyper(short_help="Org admin.")
17
16
  @app.command(help="Switch the active organization.")
18
17
  def switch(org_id: OrganizationArg):
19
18
  state.settings.device_code_auth.authenticate(org_id=org_id)
20
- rich.print(f"Switched to organization: {org_id}")
19
+ CONSOLE.print(f"Switched to organization: {org_id}")
21
20
 
22
21
 
23
22
  @app.command(help="Create a new organization.")
@@ -29,7 +28,7 @@ def create(
29
28
  # Authenticate to the new organization
30
29
  state.settings.device_code_auth.authenticate(org_id=res.org.id)
31
30
 
32
- rich.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
31
+ CONSOLE.print(f"{res.org.name} [dim]{res.org.id}[/dim]")
33
32
 
34
33
 
35
34
  @app.command(help="List organizations.")
@@ -40,7 +39,7 @@ def ls():
40
39
  for m in state.settings.api.organization.list_memberships():
41
40
  table.add_row("👉" if m.org.id == org_id else "", m.org.id, m.org.name, m.role)
42
41
 
43
- rich.print(table)
42
+ CONSOLE.print(table)
44
43
 
45
44
 
46
45
  @app.command(help="Invite a user to the organization.")
@@ -48,7 +47,7 @@ def invite(email: str, role: OrgRole = OrgRole.MEMBER, expires_in_days: int = 7)
48
47
  state.settings.api.organization.invite_user(
49
48
  InviteUserRequest(email=email, role=role, expires_in_days=expires_in_days)
50
49
  )
51
- rich.print(f"Invited {email} as a {role.value}.")
50
+ CONSOLE.print(f"Invited {email} as a {role.value}.")
52
51
 
53
52
 
54
53
  @app.command(help="Configure single sign-on for your organization.")
@@ -78,7 +77,7 @@ def domains():
78
77
 
79
78
  def _do_action(intent: PortalLinkIntent):
80
79
  res = state.settings.api.organization.portal_link(PortalLinkRequest(intent=intent))
81
- rich.print(f"Opening the configuration portal:\n{res.url}")
80
+ CONSOLE.print(f"Opening the configuration portal:\n{res.url}")
82
81
  webbrowser.open(res.url)
83
82
 
84
83
 
@@ -86,5 +85,5 @@ def current_org_id():
86
85
  if token := state.settings.authn.token():
87
86
  if org_id := jwt.decode(token.expose_secret(), options={"verify_signature": False}).get("org_id"):
88
87
  return org_id
89
- rich.print("[red]You are not logged in to an organization.[/red]")
88
+ ERR_CONSOLE.print("You are not logged in to an organization.")
90
89
  raise typer.Exit(1)
spiral/cli/printer.py CHANGED
@@ -2,14 +2,14 @@ import json
2
2
  from collections.abc import Iterable
3
3
  from typing import TypeVar
4
4
 
5
- import betterproto
5
+ import betterproto2
6
6
  from pydantic import BaseModel
7
7
  from rich.console import ConsoleRenderable, Group
8
8
  from rich.padding import Padding
9
9
  from rich.pretty import Pretty
10
10
  from rich.table import Table
11
11
 
12
- T = TypeVar("T", bound=betterproto.Message)
12
+ T = TypeVar("T", bound=betterproto2.Message)
13
13
  M = TypeVar("M", bound=BaseModel)
14
14
 
15
15
 
@@ -34,7 +34,7 @@ def table_of_protos(cls: type[T], messages: Iterable[T], fields: list[str] = Non
34
34
 
35
35
  TODO(ngates): add a CLI switch to emit JSON results instead of tables.
36
36
  """
37
- cols = fields or cls()._betterproto.sorted_field_names
37
+ cols = fields or cls()._betterproto2.sorted_field_names
38
38
  table = Table(*cols, title=title or f"{cls.__name__}s")
39
39
  for msg in messages:
40
40
  table.add_row(*[getattr(msg, col, "") for col in cols])
spiral/cli/projects.py CHANGED
@@ -1,6 +1,5 @@
1
1
  from typing import Annotated
2
2
 
3
- import rich
4
3
  import typer
5
4
  from typer import Option
6
5
 
@@ -19,7 +18,7 @@ from spiral.api.projects import (
19
18
  Project,
20
19
  WorkloadPrincipalConditions,
21
20
  )
22
- from spiral.cli import AsyncTyper, printer, state
21
+ from spiral.cli import CONSOLE, AsyncTyper, printer, state
23
22
  from spiral.cli.types import ProjectArg
24
23
 
25
24
  app = AsyncTyper(short_help="Projects and grants.")
@@ -28,7 +27,7 @@ app = AsyncTyper(short_help="Projects and grants.")
28
27
  @app.command(help="List projects.")
29
28
  def ls():
30
29
  projects = list(state.settings.api.project.list())
31
- rich.print(printer.table_of_models(Project, projects))
30
+ CONSOLE.print(printer.table_of_models(Project, projects))
32
31
 
33
32
 
34
33
  @app.command(help="Create a new project.")
@@ -39,7 +38,7 @@ def create(
39
38
  name: Annotated[str | None, Option(help="Friendly name for the project.")] = None,
40
39
  ):
41
40
  res = state.settings.api.project.create(CreateProjectRequest(id_prefix=id_prefix, name=name))
42
- rich.print(f"Created project {res.project.id}")
41
+ CONSOLE.print(f"Created project {res.project.id}")
43
42
 
44
43
 
45
44
  @app.command(help="Grant a role on a project.")
@@ -127,10 +126,10 @@ def grant(
127
126
  ),
128
127
  )
129
128
 
130
- rich.print(f"Granted role {role} on project {project}")
129
+ CONSOLE.print(f"Granted role {role} on project {project}")
131
130
 
132
131
 
133
132
  @app.command(help="List project grants.")
134
133
  def grants(project: ProjectArg):
135
134
  project_grants = list(state.settings.api.project.list_grants(project))
136
- rich.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
135
+ CONSOLE.print(printer.table_of_models(Grant, project_grants, title="Project Grants"))
spiral/cli/tables.py ADDED
@@ -0,0 +1,131 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import rich
5
+ import rich.table
6
+ import typer
7
+ from questionary import Choice
8
+ from typer import Argument, Option
9
+
10
+ from spiral import Spiral
11
+ from spiral.api.projects import TableResource
12
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
13
+ from spiral.cli.types import ProjectArg
14
+ from spiral.debug.manifests import display_manifests
15
+ from spiral.table import Table
16
+
17
+ app = AsyncTyper(short_help="Spiral Tables.")
18
+
19
+
20
+ def ask_table(project_id: str, title: str = "Select a table") -> str:
21
+ tables: list[TableResource] = list(state.spiral.project(project_id).list_tables())
22
+
23
+ if not tables:
24
+ ERR_CONSOLE.print("No tables found")
25
+ raise typer.Exit(1)
26
+
27
+ return questionary.select( # pyright: ignore[reportAny]
28
+ title,
29
+ choices=[
30
+ Choice(title=f"{table.dataset}.{table.table}", value=f"{table.dataset}.{table.table}")
31
+ for table in sorted(tables, key=lambda t: (t.dataset, t.table))
32
+ ],
33
+ ).ask()
34
+
35
+
36
+ def get_table(
37
+ project: ProjectArg,
38
+ table: Annotated[str | None, Option(help="Table name.")] = None,
39
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
40
+ ) -> tuple[str, Table]:
41
+ if table is None:
42
+ identifier = ask_table(project)
43
+ else:
44
+ identifier = table
45
+ if dataset is not None:
46
+ identifier = f"{dataset}.{table}"
47
+ return identifier, state.spiral.project(project).table(identifier)
48
+
49
+
50
+ @app.command(help="List tables.")
51
+ def ls(
52
+ project: ProjectArg,
53
+ ):
54
+ tables = Spiral().project(project).list_tables()
55
+
56
+ rich_table = rich.table.Table("id", "dataset", "name", title="Spiral tables")
57
+ for table in tables:
58
+ rich_table.add_row(table.id, table.dataset, table.table)
59
+ CONSOLE.print(rich_table)
60
+
61
+
62
+ @app.command(help="Show the table key schema.")
63
+ def key_schema(
64
+ project: ProjectArg,
65
+ table: Annotated[str | None, Option(help="Table name.")] = None,
66
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
67
+ ):
68
+ _, t = get_table(project, table, dataset)
69
+ CONSOLE.print(t.key_schema)
70
+
71
+
72
+ @app.command(help="Compute the full table schema.")
73
+ def schema(
74
+ project: ProjectArg,
75
+ table: Annotated[str | None, Option(help="Table name.")] = None,
76
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
77
+ ):
78
+ _, t = get_table(project, table, dataset)
79
+ CONSOLE.print(t.schema())
80
+
81
+
82
+ @app.command(help="Flush Write-Ahead-Log.")
83
+ def flush(
84
+ project: ProjectArg,
85
+ table: Annotated[str | None, Option(help="Table name.")] = None,
86
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
87
+ ):
88
+ identifier, t = get_table(project, table, dataset)
89
+ state.spiral._ops().flush_wal(t.core) # pyright: ignore[reportPrivateUsage]
90
+ CONSOLE.print(f"Flushed WAL for table {identifier} in project {project}.")
91
+
92
+
93
+ @app.command(help="Display scan.")
94
+ def debug(
95
+ project: ProjectArg,
96
+ table: Annotated[str | None, Option(help="Table name.")] = None,
97
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
98
+ column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
99
+ ):
100
+ _, t = get_table(project, table, dataset)
101
+ scan = state.spiral.scan(t[column_group] if column_group != "." else t)
102
+ scan._debug() # pyright: ignore[reportPrivateUsage]
103
+
104
+
105
+ @app.command(help="Display all manifests.")
106
+ def manifests(
107
+ project: ProjectArg,
108
+ table: Annotated[str | None, Option(help="Table name.")] = None,
109
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
110
+ ):
111
+ _, t = get_table(project, table, dataset)
112
+ s = t.snapshot()
113
+
114
+ key_space_state = state.spiral._ops().key_space_state(s.core) # pyright: ignore[reportPrivateUsage]
115
+ key_space_manifest = key_space_state.manifest
116
+
117
+ column_groups_states = state.spiral._ops().column_groups_states(s.core, key_space_state) # pyright: ignore[reportPrivateUsage]
118
+
119
+ display_manifests(key_space_manifest, [(x.column_group, x.manifest) for x in column_groups_states])
120
+
121
+
122
+ @app.command(help="Display the manifests which would be read by a scan of the given column group.")
123
+ def scan_manifests(
124
+ project: ProjectArg,
125
+ table: Annotated[str | None, Option(help="Table name.")] = None,
126
+ dataset: Annotated[str | None, Option(help="Dataset name.")] = None,
127
+ column_group: Annotated[str, Argument(help="Dot-separated column group path.")] = ".",
128
+ ):
129
+ _, t = get_table(project, table, dataset)
130
+ scan = state.spiral.scan(t[column_group] if column_group != "." else t)
131
+ scan._dump_manifests() # pyright: ignore[reportPrivateUsage]
spiral/cli/telemetry.py CHANGED
@@ -1,8 +1,7 @@
1
1
  import pyperclip
2
- import rich
3
2
 
4
3
  from spiral.api.telemetry import IssueExportTokenResponse
5
- from spiral.cli import AsyncTyper, state
4
+ from spiral.cli import CONSOLE, AsyncTyper, state
6
5
 
7
6
  app = AsyncTyper(short_help="Client-side telemetry.")
8
7
 
@@ -14,5 +13,5 @@ def export():
14
13
  command = f"export SPIRAL_OTEL_TOKEN={res.token}"
15
14
  pyperclip.copy(command)
16
15
 
17
- rich.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
18
- rich.print("[dim]Token is valid for 1h.[/dim]")
16
+ CONSOLE.print("Export command copied to clipboard! Paste and run to set [green]SPIRAL_OTEL_TOKEN[/green].")
17
+ CONSOLE.print("[dim]Token is valid for 1h.[/dim]")
spiral/cli/text.py ADDED
@@ -0,0 +1,115 @@
1
+ from typing import Annotated
2
+
3
+ import questionary
4
+ import rich
5
+ import typer
6
+ from questionary import Choice
7
+ from typer import Option
8
+
9
+ from spiral.api.projects import TextIndexResource
10
+ from spiral.api.text_indexes import CreateWorkerRequest, SyncIndexRequest
11
+ from spiral.api.types import IndexId
12
+ from spiral.api.workers import CPU, GcpRegion, Memory, ResourceClass
13
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, state
14
+ from spiral.cli.types import ProjectArg
15
+
16
+ app = AsyncTyper(short_help="Text Indexes.")
17
+
18
+
19
+ def ask_index(project_id, title="Select an index"):
20
+ indexes: list[TextIndexResource] = list(state.spiral.project(project_id).list_text_indexes())
21
+
22
+ if not indexes:
23
+ ERR_CONSOLE.print("No indexes found")
24
+ raise typer.Exit(1)
25
+
26
+ return questionary.select(
27
+ title,
28
+ choices=[Choice(title=index.name, value=index.id) for index in sorted(indexes, key=lambda t: (t.name, t.id))],
29
+ ).ask()
30
+
31
+
32
+ def get_index_id(
33
+ project: ProjectArg,
34
+ name: Annotated[str | None, Option(help="Index name.")] = None,
35
+ ) -> IndexId:
36
+ if name is None:
37
+ return ask_index(project)
38
+
39
+ indexes: list[TextIndexResource] = list(state.spiral.project(project).list_text_indexes())
40
+ for index in indexes:
41
+ if index.name == name:
42
+ return index.id
43
+ raise ValueError(f"Index not found: {name}")
44
+
45
+
46
+ @app.command(help="List indexes.")
47
+ def ls(
48
+ project: ProjectArg,
49
+ ):
50
+ """List indexes."""
51
+ indexes = state.spiral.project(project).list_text_indexes()
52
+
53
+ rich_table = rich.table.Table("id", "name", title="Text Indexes")
54
+ for index in indexes:
55
+ rich_table.add_row(index.id, index.name)
56
+ CONSOLE.print(rich_table)
57
+
58
+
59
+ @app.command(help="Trigger a sync job for an index.")
60
+ def sync(
61
+ project: ProjectArg,
62
+ name: Annotated[str | None, Option(help="Index name.")] = None,
63
+ resources: Annotated[ResourceClass, Option(help="Resources to use for the sync job.")] = ResourceClass.SMALL,
64
+ ):
65
+ """Trigger a sync job."""
66
+ index_id = get_index_id(project, name)
67
+ response = state.spiral.api.text_indexes.sync_index(index_id, SyncIndexRequest(resources=resources))
68
+ CONSOLE.print(f"Triggered sync job {response.worker_id} for index {index_id}.")
69
+
70
+
71
+ @app.command(name="serve", help="Spin up a worker to serve an index.")
72
+ def serve(
73
+ project: ProjectArg,
74
+ index: Annotated[str | None, Option(help="Index name.")] = None,
75
+ region: Annotated[GcpRegion, Option(help="GCP region for the worker.")] = GcpRegion.US_EAST4,
76
+ cpu: Annotated[CPU, Option(help="CPU resources for the worker.")] = CPU.ONE,
77
+ memory: Annotated[Memory, Option(help="Memory resources for the worker in MB.")] = Memory.MB_512,
78
+ ):
79
+ """Spin up a worker."""
80
+ index_id = get_index_id(project, index)
81
+ request = CreateWorkerRequest(cpu=cpu, memory=memory, region=region)
82
+ response = state.spiral.api.text_indexes.create_worker(index_id, request)
83
+ CONSOLE.print(f"Created worker {response.worker_id} for {index_id}.")
84
+
85
+
86
+ @app.command(name="workers", help="List search workers.")
87
+ def workers(
88
+ project: ProjectArg,
89
+ index: Annotated[str | None, Option(help="Index name.")] = None,
90
+ ):
91
+ """List text search workers."""
92
+ index_id = get_index_id(project, index)
93
+ worker_ids = state.spiral.api.text_indexes.list_workers(index_id)
94
+
95
+ rich_table = rich.table.Table("Worker ID", "URL", title=f"Text Search Workers for {index_id}")
96
+ for worker_id in worker_ids:
97
+ try:
98
+ worker = state.spiral.api.text_indexes.get_worker(worker_id)
99
+ rich_table.add_row(
100
+ worker_id,
101
+ worker.url,
102
+ )
103
+ except Exception:
104
+ rich_table.add_row(
105
+ worker_id,
106
+ "Unavailable",
107
+ )
108
+ CONSOLE.print(rich_table)
109
+
110
+
111
+ @app.command(name="shutdown", help="Shutdown a search worker.")
112
+ def shutdown(worker_id: str):
113
+ """Shutdown a worker."""
114
+ state.spiral.api.text_indexes.shutdown_worker(worker_id)
115
+ CONSOLE.print(f"Requested worker {worker_id} to shutdown.")
spiral/cli/types.py CHANGED
@@ -1,20 +1,19 @@
1
1
  from typing import Annotated
2
2
 
3
3
  import questionary
4
- import rich
5
4
  import typer
6
5
  from questionary import Choice
7
6
  from typer import Argument
8
7
 
9
8
  from spiral.api.types import OrgId, ProjectId
10
- from spiral.cli import state
9
+ from spiral.cli import ERR_CONSOLE, state
11
10
 
12
11
 
13
12
  def ask_project(title="Select a project"):
14
13
  projects = list(state.settings.api.project.list())
15
14
 
16
15
  if not projects:
17
- rich.print("[red]No projects found[/red]")
16
+ ERR_CONSOLE.print("No projects found")
18
17
  raise typer.Exit(1)
19
18
 
20
19
  return questionary.select(
@@ -33,7 +32,7 @@ def _org_default():
33
32
  memberships = list(state.settings.api.organization.list_memberships())
34
33
 
35
34
  if not memberships:
36
- rich.print("[red]No organizations found[/red]")
35
+ ERR_CONSOLE.print("No organizations found")
37
36
  raise typer.Exit(1)
38
37
 
39
38
  return questionary.select(
spiral/cli/workloads.py CHANGED
@@ -2,12 +2,11 @@ from typing import Annotated
2
2
 
3
3
  import pyperclip
4
4
  import questionary
5
- import rich
6
5
  from questionary import Choice
7
6
  from typer import Argument, Option
8
7
 
9
8
  from spiral.api.workloads import CreateWorkloadRequest, IssueWorkloadCredentialsResponse, Workload
10
- from spiral.cli import AsyncTyper, printer, state
9
+ from spiral.cli import CONSOLE, ERR_CONSOLE, AsyncTyper, printer, state
11
10
  from spiral.cli.types import ProjectArg
12
11
 
13
12
  app = AsyncTyper()
@@ -19,7 +18,7 @@ def create(
19
18
  name: Annotated[str | None, Option(help="Friendly name for the workload.")] = None,
20
19
  ):
21
20
  res = state.settings.api.workload.create(project, CreateWorkloadRequest(name=name))
22
- rich.print(f"Created workload {res.workload.id}")
21
+ CONSOLE.print(f"Created workload {res.workload.id}")
23
22
 
24
23
 
25
24
  @app.command(help="List workloads.")
@@ -27,7 +26,7 @@ def ls(
27
26
  project: ProjectArg,
28
27
  ):
29
28
  workloads = list(state.settings.api.workload.list(project))
30
- rich.print(printer.table_of_models(Workload, workloads, fields=["id", "project_id", "name"]))
29
+ CONSOLE.print(printer.table_of_models(Workload, workloads, fields=["id", "project_id", "name"]))
31
30
 
32
31
 
33
32
  @app.command(help="Issue new workflow credentials.")
@@ -46,14 +45,14 @@ def issue_credentials(workload_id: Annotated[str, Argument(help="Workload ID.")]
46
45
 
47
46
  if choice == 1:
48
47
  pyperclip.copy(res.client_secret)
49
- rich.print("[green]Secret copied to clipboard![/green]")
48
+ CONSOLE.print("[green]Secret copied to clipboard![/green]")
50
49
  break
51
50
  elif choice == 2:
52
- rich.print(f"[green]Token Secret:[/green] {res.client_secret}")
51
+ CONSOLE.print(f"[green]Token Secret:[/green] {res.client_secret}")
53
52
  break
54
53
  elif choice == 3:
55
54
  break
56
55
  else:
57
- rich.print("[red]Invalid choice. Please try again.[/red]")
56
+ ERR_CONSOLE.print("Invalid choice. Please try again.")
58
57
 
59
- rich.print(f"[green]Token ID:[/green] {res.client_id}")
58
+ CONSOLE.print(f"[green]Token ID:[/green] {res.client_id}")
spiral/client.py CHANGED
@@ -1,24 +1,29 @@
1
+ from datetime import datetime, timedelta
1
2
  from typing import TYPE_CHECKING
2
3
 
3
4
  import jwt
5
+ import pyarrow as pa
4
6
 
5
7
  from spiral.api import SpiralAPI
6
8
  from spiral.api.projects import CreateProjectRequest, CreateProjectResponse
9
+ from spiral.core.client import Operations
7
10
  from spiral.core.client import Spiral as CoreSpiral
11
+ from spiral.datetime_ import timestamp_micros
12
+ from spiral.expressions import ExprLike
13
+ from spiral.scan import Scan
8
14
  from spiral.settings import Settings, settings
9
15
 
10
16
  if TYPE_CHECKING:
11
17
  from spiral.iceberg import Iceberg
18
+ from spiral.key_space_index import KeySpaceIndex
12
19
  from spiral.project import Project
20
+ from spiral.table import Table
21
+ from spiral.text_index import TextIndex
13
22
 
14
23
 
15
24
  class Spiral:
16
25
  def __init__(self, config: Settings | None = None):
17
26
  self._config = config or settings()
18
- self._api = self._config.api
19
- self._core = CoreSpiral(
20
- api_url=self._config.spiraldb.uri, spfs_url=self._config.spfs.uri, authn=self._config.authn
21
- )
22
27
  self._org = None
23
28
 
24
29
  @property
@@ -27,7 +32,11 @@ class Spiral:
27
32
 
28
33
  @property
29
34
  def api(self) -> SpiralAPI:
30
- return self._api
35
+ return self._config.api
36
+
37
+ @property
38
+ def _core(self) -> CoreSpiral:
39
+ return self._config.core
31
40
 
32
41
  @property
33
42
  def organization(self) -> str:
@@ -45,7 +54,7 @@ class Spiral:
45
54
  """List project IDs."""
46
55
  from .project import Project
47
56
 
48
- return [Project(self, id=p.id, name=p.name) for p in self.api.project.list()]
57
+ return [Project(self, project_id=p.id, name=p.name) for p in self.api.project.list()]
49
58
 
50
59
  def create_project(
51
60
  self,
@@ -64,7 +73,102 @@ class Spiral:
64
73
  from spiral.project import Project
65
74
 
66
75
  # We avoid an API call since we'd just be fetching a human-readable name. Seems a waste in most cases.
67
- return Project(self, id=project_id, name=project_id)
76
+ return Project(self, project_id=project_id, name=project_id)
77
+
78
+ def table(self, table_id: str) -> "Table":
79
+ """Open a table using an ID."""
80
+ from spiral.table import Table
81
+
82
+ return Table(self, self._core.table(table_id))
83
+
84
+ def text_index(self, index_id: str) -> "TextIndex":
85
+ """Open a text index using an ID."""
86
+ from spiral.text_index import TextIndex
87
+
88
+ return TextIndex(self._core.text_index(index_id))
89
+
90
+ def key_space_index(self, index_id: str) -> "KeySpaceIndex":
91
+ """Open a key space index using an ID."""
92
+ from spiral.key_space_index import KeySpaceIndex
93
+
94
+ return KeySpaceIndex(self._core.key_space_index(index_id))
95
+
96
+ def scan(
97
+ self,
98
+ *projections: ExprLike,
99
+ where: ExprLike | None = None,
100
+ asof: datetime | int | None = None,
101
+ exclude_keys: bool = False,
102
+ ) -> Scan:
103
+ """Starts a read transaction on the Spiral.
104
+
105
+ Args:
106
+ projections: a set of expressions that return struct arrays.
107
+ where: a query expression to apply to the data.
108
+ asof: only data written before the given timestamp will be returned, caveats around compaction.
109
+ exclude_keys: whether to exclude the key columns in the scan result, defaults to False.
110
+ Note that if a projection includes a key column, it will be included in the result.
111
+ """
112
+ from spiral import expressions as se
113
+
114
+ if isinstance(asof, datetime):
115
+ asof = timestamp_micros(asof)
116
+
117
+ # Combine all projections into a single struct.
118
+ projection = se.merge(*projections)
119
+ if where is not None:
120
+ where = se.lift(where)
121
+
122
+ return Scan(
123
+ self._core.scan(
124
+ projection.__expr__,
125
+ filter=where.__expr__ if where else None,
126
+ asof=asof,
127
+ exclude_keys=exclude_keys,
128
+ ),
129
+ )
130
+
131
+ # TODO(marko): This should be query, and search should be query + scan.
132
+ def search(
133
+ self,
134
+ top_k: int,
135
+ *rank_by: ExprLike,
136
+ filters: ExprLike | None = None,
137
+ freshness_window: timedelta | None = None,
138
+ ) -> pa.RecordBatchReader:
139
+ """Queries the index with the given rank by and filters clauses. Returns a steam of scored keys.
140
+
141
+ Args:
142
+ top_k: The number of top results to return.
143
+ rank_by: Rank by expressions are combined for scoring.
144
+ See `se.text.find` and `se.text.boost` for scoring expressions.
145
+ filters: The `filters` expression is used to filter the results.
146
+ It must return a boolean value and use only conjunctions (ANDs). Expressions in filters
147
+ statement are considered either a `must` or `must_not` clause in search terminology.
148
+ freshness_window: If provided, the index will not be refreshed if its freshness does not exceed this window.
149
+ """
150
+ from spiral import expressions as se
151
+
152
+ if not rank_by:
153
+ raise ValueError("At least one rank by expression is required.")
154
+ rank_by = se.or_(*rank_by)
155
+ if filters is not None:
156
+ filters = se.lift(filters)
157
+
158
+ if freshness_window is None:
159
+ freshness_window = timedelta(seconds=0)
160
+ freshness_window_s = int(freshness_window.total_seconds())
161
+
162
+ return self._core.search(
163
+ top_k=top_k,
164
+ rank_by=rank_by.__expr__,
165
+ filters=filters.__expr__ if filters else None,
166
+ freshness_window_s=freshness_window_s,
167
+ )
168
+
169
+ def _ops(self) -> Operations:
170
+ """Access maintenance operations."""
171
+ return self._core._ops(format=settings().file_format)
68
172
 
69
173
  @property
70
174
  def iceberg(self) -> "Iceberg":
@@ -72,7 +176,6 @@ class Spiral:
72
176
  Apache Iceberg is a powerful open-source table format designed for high-performance data lakes.
73
177
  Iceberg brings reliability, scalability, and advanced features like time travel, schema evolution,
74
178
  and ACID transactions to your warehouse.
75
-
76
179
  """
77
180
  from spiral.iceberg import Iceberg
78
181