arrowbricks 6.1.0__tar.gz → 6.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/PKG-INFO +10 -7
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/README.md +9 -6
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/pyproject.toml +3 -1
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/Cargo.toml +1 -1
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +4 -3
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/agent/mod.rs +7 -3
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/agent/render.rs +304 -31
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/lib.rs +5 -2
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/__init__.py +2 -0
- arrowbricks-6.2.0/src/arrowbricks/_agent.py +11 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/_core.pyi +1 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/cli.py +26 -4
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/client.py +12 -1
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/LICENSE +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/download.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/__main__.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/_sql.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/_streaming.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.2.0}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arrowbricks
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.2.0
|
|
4
4
|
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
5
|
Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
|
|
6
6
|
Provides-Extra: arro3
|
|
@@ -21,7 +21,7 @@ Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
|
21
21
|
|
|
22
22
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
23
23
|
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
24
|
-
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
24
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
25
25
|
|
|
26
26
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
27
27
|
Bring a token or your own token-refresh callable.
|
|
@@ -48,14 +48,17 @@ export DATABRICKS_TOKEN='your-access-token'
|
|
|
48
48
|
arrowbricks -c 'SELECT 1 AS answer'
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
-
```
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
| 1 |
|
|
51
|
+
```text
|
|
52
|
+
answer:Int32
|
|
53
|
+
1
|
|
55
54
|
|
|
56
55
|
rows_shown=1; complete=true; total_rows=1
|
|
57
56
|
```
|
|
58
57
|
|
|
58
|
+
Output is tab-separated by default (cheapest to read; tabs, newlines and
|
|
59
|
+
backslashes in values are escaped, NULL is `\N`). `--format markdown` /
|
|
60
|
+
`output_format="markdown"` gives a pipe table instead.
|
|
61
|
+
|
|
59
62
|
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
60
63
|
|
|
61
64
|
```bash
|
|
@@ -75,7 +78,7 @@ from arrowbricks import DatabricksClient
|
|
|
75
78
|
|
|
76
79
|
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
77
80
|
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
78
|
-
print(text) #
|
|
81
|
+
print(text) # TSV on success; JSON on query failure
|
|
79
82
|
```
|
|
80
83
|
|
|
81
84
|
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
9
9
|
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
10
|
-
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
10
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
11
11
|
|
|
12
12
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
13
13
|
Bring a token or your own token-refresh callable.
|
|
@@ -34,14 +34,17 @@ export DATABRICKS_TOKEN='your-access-token'
|
|
|
34
34
|
arrowbricks -c 'SELECT 1 AS answer'
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
| 1 |
|
|
37
|
+
```text
|
|
38
|
+
answer:Int32
|
|
39
|
+
1
|
|
41
40
|
|
|
42
41
|
rows_shown=1; complete=true; total_rows=1
|
|
43
42
|
```
|
|
44
43
|
|
|
44
|
+
Output is tab-separated by default (cheapest to read; tabs, newlines and
|
|
45
|
+
backslashes in values are escaped, NULL is `\N`). `--format markdown` /
|
|
46
|
+
`output_format="markdown"` gives a pipe table instead.
|
|
47
|
+
|
|
45
48
|
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
46
49
|
|
|
47
50
|
```bash
|
|
@@ -61,7 +64,7 @@ from arrowbricks import DatabricksClient
|
|
|
61
64
|
|
|
62
65
|
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
63
66
|
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
64
|
-
print(text) #
|
|
67
|
+
print(text) # TSV on success; JSON on query failure
|
|
65
68
|
```
|
|
66
69
|
|
|
67
70
|
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.2.0"
|
|
4
4
|
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -103,3 +103,5 @@ select = ["E", "F", "I", "UP", "S", "B"]
|
|
|
103
103
|
"tests/*" = ["S101", "S106"]
|
|
104
104
|
"rust/arrowbricks_core/tests_py/*" = ["S101", "S106"]
|
|
105
105
|
"tests/arro3_free/*" = ["S101", "S106"]
|
|
106
|
+
# One-off benchmark harness: terse scripts, seeded (non-crypto) randomness, asserts as checks.
|
|
107
|
+
"benchmarks/agent-output-format/*" = ["B905", "E501", "E731", "E741", "S101", "S311"]
|
{arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/examples/agent_render_bench.rs
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
use arrow_array::types::Int32Type;
|
|
2
2
|
use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
|
|
3
3
|
use arrow_schema::{Field, Schema};
|
|
4
|
-
use arrowbricks_core::agent::render::
|
|
4
|
+
use arrowbricks_core::agent::render::{AgentRenderer, OutputFormat};
|
|
5
5
|
use std::{hint::black_box, sync::Arc, time::Instant};
|
|
6
6
|
|
|
7
7
|
fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
|
|
@@ -14,16 +14,17 @@ fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
|
|
|
14
14
|
}
|
|
15
15
|
fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
|
|
16
16
|
let mut renderer = if interrupts {
|
|
17
|
-
|
|
17
|
+
AgentRenderer::new_with_interrupt(
|
|
18
18
|
b.schema(),
|
|
19
19
|
1000,
|
|
20
20
|
bytes,
|
|
21
21
|
500,
|
|
22
|
+
OutputFormat::Markdown,
|
|
22
23
|
Instant::now() + std::time::Duration::from_secs(60),
|
|
23
24
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
|
24
25
|
)
|
|
25
26
|
} else {
|
|
26
|
-
|
|
27
|
+
AgentRenderer::new(b.schema(), 1000, bytes, 500, OutputFormat::Markdown)
|
|
27
28
|
}
|
|
28
29
|
.unwrap();
|
|
29
30
|
renderer.push_batch(b).unwrap();
|
|
@@ -10,7 +10,8 @@ use serde_json::Value;
|
|
|
10
10
|
|
|
11
11
|
use crate::client::{ApiError, DbClient, Protocol};
|
|
12
12
|
use crate::pipeline;
|
|
13
|
-
use render::
|
|
13
|
+
use render::AgentRenderer;
|
|
14
|
+
pub use render::OutputFormat;
|
|
14
15
|
|
|
15
16
|
#[derive(Clone, Copy)]
|
|
16
17
|
pub struct AgentOptions {
|
|
@@ -18,6 +19,7 @@ pub struct AgentOptions {
|
|
|
18
19
|
pub max_bytes: usize,
|
|
19
20
|
pub max_cell_chars: usize,
|
|
20
21
|
pub total_timeout_s: f64,
|
|
22
|
+
pub format: OutputFormat,
|
|
21
23
|
}
|
|
22
24
|
|
|
23
25
|
impl Default for AgentOptions {
|
|
@@ -27,6 +29,7 @@ impl Default for AgentOptions {
|
|
|
27
29
|
max_bytes: 10_000,
|
|
28
30
|
max_cell_chars: 500,
|
|
29
31
|
total_timeout_s: 60.0,
|
|
32
|
+
format: OutputFormat::default(),
|
|
30
33
|
}
|
|
31
34
|
}
|
|
32
35
|
}
|
|
@@ -158,7 +161,7 @@ pub async fn query(
|
|
|
158
161
|
while !finished {
|
|
159
162
|
let want = renderer
|
|
160
163
|
.as_ref()
|
|
161
|
-
.map_or(128, |r: &
|
|
164
|
+
.map_or(128, |r: &AgentRenderer| {
|
|
162
165
|
options.max_rows.saturating_sub(r.rows()).min(128)
|
|
163
166
|
})
|
|
164
167
|
.min(options.max_rows);
|
|
@@ -167,11 +170,12 @@ pub async fn query(
|
|
|
167
170
|
let mut render_task = tokio::task::spawn_blocking(move || {
|
|
168
171
|
let mut renderer = match renderer {
|
|
169
172
|
Some(renderer) => renderer,
|
|
170
|
-
None =>
|
|
173
|
+
None => AgentRenderer::new_with_interrupt(
|
|
171
174
|
schema.unwrap_or_else(|| Arc::new(Schema::empty())),
|
|
172
175
|
options.max_rows,
|
|
173
176
|
options.max_bytes,
|
|
174
177
|
options.max_cell_chars,
|
|
178
|
+
options.format,
|
|
175
179
|
deadline.into_std(),
|
|
176
180
|
interrupted,
|
|
177
181
|
)?,
|
|
@@ -12,7 +12,31 @@ use std::time::Instant;
|
|
|
12
12
|
const FOOTER_RESERVE: usize = 160;
|
|
13
13
|
const CLIPPED: &str = "…[truncated]";
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
/// Text layout of an agent preview.
|
|
16
|
+
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
17
|
+
pub enum OutputFormat {
|
|
18
|
+
/// Tab-separated values, `name:Type` header. `\\`, tab, newline and carriage
|
|
19
|
+
/// return are backslash-escaped, SQL NULL is `\N` and the empty string is an
|
|
20
|
+
/// empty field (PostgreSQL `COPY` text conventions), so a cell can never
|
|
21
|
+
/// contain a raw delimiter.
|
|
22
|
+
#[default]
|
|
23
|
+
Tsv,
|
|
24
|
+
/// Pipe table without padding; strings are quoted and Markdown-escaped.
|
|
25
|
+
Markdown,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
impl OutputFormat {
|
|
29
|
+
pub fn parse(name: &str) -> Result<Self, &'static str> {
|
|
30
|
+
match name {
|
|
31
|
+
"tsv" => Ok(Self::Tsv),
|
|
32
|
+
"markdown" => Ok(Self::Markdown),
|
|
33
|
+
_ => Err("output_format must be 'tsv' or 'markdown'"),
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
pub struct AgentRenderer {
|
|
39
|
+
format: OutputFormat,
|
|
16
40
|
schema: SchemaRef,
|
|
17
41
|
output: String,
|
|
18
42
|
scratch: String,
|
|
@@ -25,15 +49,22 @@ pub struct MarkdownRenderer {
|
|
|
25
49
|
cells_truncated: bool,
|
|
26
50
|
interrupt: Option<Interrupt>,
|
|
27
51
|
}
|
|
28
|
-
impl
|
|
29
|
-
pub fn new(
|
|
30
|
-
|
|
52
|
+
impl AgentRenderer {
|
|
53
|
+
pub fn new(
|
|
54
|
+
schema: SchemaRef,
|
|
55
|
+
max_rows: usize,
|
|
56
|
+
max_bytes: usize,
|
|
57
|
+
max_cell_chars: usize,
|
|
58
|
+
format: OutputFormat,
|
|
59
|
+
) -> Result<Self, ApiError> {
|
|
60
|
+
Self::new_inner(schema, max_rows, max_bytes, max_cell_chars, format, None)
|
|
31
61
|
}
|
|
32
62
|
pub fn new_with_interrupt(
|
|
33
63
|
schema: SchemaRef,
|
|
34
64
|
max_rows: usize,
|
|
35
65
|
max_bytes: usize,
|
|
36
66
|
max_cell_chars: usize,
|
|
67
|
+
format: OutputFormat,
|
|
37
68
|
deadline: Instant,
|
|
38
69
|
cancelled: Arc<AtomicBool>,
|
|
39
70
|
) -> Result<Self, ApiError> {
|
|
@@ -42,6 +73,7 @@ impl MarkdownRenderer {
|
|
|
42
73
|
max_rows,
|
|
43
74
|
max_bytes,
|
|
44
75
|
max_cell_chars,
|
|
76
|
+
format,
|
|
45
77
|
Some(Interrupt { deadline, cancelled }),
|
|
46
78
|
)
|
|
47
79
|
}
|
|
@@ -53,6 +85,7 @@ impl MarkdownRenderer {
|
|
|
53
85
|
max_rows: usize,
|
|
54
86
|
max_bytes: usize,
|
|
55
87
|
max_cell_chars: usize,
|
|
88
|
+
format: OutputFormat,
|
|
56
89
|
interrupt: Option<Interrupt>,
|
|
57
90
|
) -> Result<Self, ApiError> {
|
|
58
91
|
if max_bytes < 256 || max_rows == 0 || max_cell_chars == 0 {
|
|
@@ -65,6 +98,7 @@ impl MarkdownRenderer {
|
|
|
65
98
|
}
|
|
66
99
|
let mut output = String::with_capacity(max_bytes.min(10_000));
|
|
67
100
|
let mut header = EscapedWriter::new(&mut output, usize::MAX, max_bytes - FOOTER_RESERVE);
|
|
101
|
+
header.tsv = format == OutputFormat::Tsv;
|
|
68
102
|
header.interrupt = interrupt.as_ref();
|
|
69
103
|
let mut full = false;
|
|
70
104
|
if schema.fields().is_empty() {
|
|
@@ -73,18 +107,30 @@ impl MarkdownRenderer {
|
|
|
73
107
|
.map_err(|_| ApiError::permanent(header.interrupted.unwrap_or("agent schema formatting failed")))?;
|
|
74
108
|
} else {
|
|
75
109
|
let result = (|| -> fmt::Result {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
110
|
+
match format {
|
|
111
|
+
OutputFormat::Markdown => {
|
|
112
|
+
header.raw("|")?;
|
|
113
|
+
for field in schema.fields() {
|
|
114
|
+
header.write_str(field.name())?;
|
|
115
|
+
header.raw(" (")?;
|
|
116
|
+
write!(header, "{}", field.data_type())?;
|
|
117
|
+
header.raw(")|")?;
|
|
118
|
+
}
|
|
119
|
+
header.raw("\n|")?;
|
|
120
|
+
for _ in schema.fields() {
|
|
121
|
+
header.raw("-|")?;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
OutputFormat::Tsv => {
|
|
125
|
+
for (index, field) in schema.fields().iter().enumerate() {
|
|
126
|
+
if index > 0 {
|
|
127
|
+
header.raw("\t")?;
|
|
128
|
+
}
|
|
129
|
+
header.write_str(field.name())?;
|
|
130
|
+
header.raw(":")?;
|
|
131
|
+
write!(header, "{}", field.data_type())?;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
88
134
|
}
|
|
89
135
|
header.raw("\n")
|
|
90
136
|
})();
|
|
@@ -98,6 +144,7 @@ impl MarkdownRenderer {
|
|
|
98
144
|
}
|
|
99
145
|
}
|
|
100
146
|
Ok(Self {
|
|
147
|
+
format,
|
|
101
148
|
schema,
|
|
102
149
|
output,
|
|
103
150
|
scratch: String::new(),
|
|
@@ -149,9 +196,12 @@ impl MarkdownRenderer {
|
|
|
149
196
|
}
|
|
150
197
|
self.scratch.clear();
|
|
151
198
|
let available = self.max_bytes - FOOTER_RESERVE - self.output.len();
|
|
152
|
-
self.
|
|
199
|
+
let tsv = self.format == OutputFormat::Tsv;
|
|
200
|
+
if !tsv {
|
|
201
|
+
self.scratch.push('|');
|
|
202
|
+
}
|
|
153
203
|
let mut row_clipped = false;
|
|
154
|
-
for (column, formatter) in batch.columns().iter().zip(&formatters) {
|
|
204
|
+
for (index, (column, formatter)) in batch.columns().iter().zip(&formatters).enumerate() {
|
|
155
205
|
// Reserve closing quote, clipping marker, and delimiter before formatting.
|
|
156
206
|
let reserve = CLIPPED.len() + 5;
|
|
157
207
|
if self.scratch.len().saturating_add(reserve + 2) > available {
|
|
@@ -159,8 +209,11 @@ impl MarkdownRenderer {
|
|
|
159
209
|
self.omitted_rows = true;
|
|
160
210
|
return Ok(false);
|
|
161
211
|
}
|
|
162
|
-
|
|
163
|
-
|
|
212
|
+
if tsv && index > 0 {
|
|
213
|
+
self.scratch.push('\t');
|
|
214
|
+
}
|
|
215
|
+
let quote = !tsv
|
|
216
|
+
&& !column.is_null(row)
|
|
164
217
|
&& matches!(
|
|
165
218
|
column.data_type(),
|
|
166
219
|
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
|
|
@@ -170,11 +223,13 @@ impl MarkdownRenderer {
|
|
|
170
223
|
}
|
|
171
224
|
let cell_bytes = available - self.scratch.len() - reserve;
|
|
172
225
|
let mut sink = EscapedWriter::new(&mut self.scratch, self.max_cell_chars, cell_bytes);
|
|
226
|
+
sink.tsv = tsv;
|
|
173
227
|
sink.interrupt = self.interrupt.as_ref();
|
|
174
228
|
let result = if column.is_null(row) {
|
|
175
229
|
sink.chars = usize::MAX;
|
|
176
230
|
sink.bytes = sink.bytes.max(4);
|
|
177
|
-
|
|
231
|
+
// `\N` is the TSV NULL marker; a literal backslash-N string renders as `\\N`.
|
|
232
|
+
sink.raw(if tsv { "\\N" } else { "NULL" })
|
|
178
233
|
.map_err(|_| arrow_schema::ArrowError::CastError("bounded output".into()))
|
|
179
234
|
} else if let Some(a) = column.as_any().downcast_ref::<StringArray>() {
|
|
180
235
|
sink.write_str(a.value(row))
|
|
@@ -204,7 +259,9 @@ impl MarkdownRenderer {
|
|
|
204
259
|
if quote {
|
|
205
260
|
self.scratch.push('"');
|
|
206
261
|
}
|
|
207
|
-
|
|
262
|
+
if !tsv {
|
|
263
|
+
self.scratch.push('|');
|
|
264
|
+
}
|
|
208
265
|
}
|
|
209
266
|
self.scratch.push('\n');
|
|
210
267
|
if self.scratch.len() > available {
|
|
@@ -321,6 +378,7 @@ struct EscapedWriter<'a> {
|
|
|
321
378
|
interrupt: Option<&'a Interrupt>,
|
|
322
379
|
interrupted: Option<&'static str>,
|
|
323
380
|
until_check: usize,
|
|
381
|
+
tsv: bool,
|
|
324
382
|
}
|
|
325
383
|
impl<'a> EscapedWriter<'a> {
|
|
326
384
|
fn new(output: &'a mut String, chars: usize, bytes: usize) -> Self {
|
|
@@ -332,6 +390,7 @@ impl<'a> EscapedWriter<'a> {
|
|
|
332
390
|
interrupt: None,
|
|
333
391
|
interrupted: None,
|
|
334
392
|
until_check: 0,
|
|
393
|
+
tsv: false,
|
|
335
394
|
}
|
|
336
395
|
}
|
|
337
396
|
fn raw(&mut self, text: &str) -> fmt::Result {
|
|
@@ -362,6 +421,10 @@ impl Write for EscapedWriter<'_> {
|
|
|
362
421
|
return Err(fmt::Error);
|
|
363
422
|
}
|
|
364
423
|
match ch {
|
|
424
|
+
// TSV needs no Markdown/HTML escaping, only delimiter and control-character safety.
|
|
425
|
+
'&' | '<' | '>' | '|' | '`' | '*' | '_' | '[' | ']' | '~' | '"' if self.tsv => {
|
|
426
|
+
self.raw(ch.encode_utf8(&mut [0; 4]))?
|
|
427
|
+
}
|
|
365
428
|
'&' => self.raw("&")?,
|
|
366
429
|
'<' => self.raw("<")?,
|
|
367
430
|
'>' => self.raw(">")?,
|
|
@@ -412,7 +475,7 @@ mod tests {
|
|
|
412
475
|
RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap()
|
|
413
476
|
}
|
|
414
477
|
fn render(batch: &RecordBatch, rows: usize, bytes: usize, chars: usize, exhausted: bool) -> String {
|
|
415
|
-
let mut r =
|
|
478
|
+
let mut r = AgentRenderer::new(batch.schema(), rows, bytes, chars, OutputFormat::Markdown).unwrap();
|
|
416
479
|
r.push_batch(batch).unwrap();
|
|
417
480
|
r.finish(exhausted)
|
|
418
481
|
}
|
|
@@ -454,8 +517,19 @@ mod tests {
|
|
|
454
517
|
fn rendering_interrupts_are_errors_not_clipped_success() {
|
|
455
518
|
let b = batch(&["a"], vec![Arc::new(StringArray::from(vec!["long value"]))]);
|
|
456
519
|
let flag = Arc::new(AtomicBool::new(false));
|
|
457
|
-
assert!(
|
|
458
|
-
|
|
520
|
+
assert!(
|
|
521
|
+
AgentRenderer::new_with_interrupt(
|
|
522
|
+
b.schema(),
|
|
523
|
+
10,
|
|
524
|
+
1000,
|
|
525
|
+
500,
|
|
526
|
+
OutputFormat::Markdown,
|
|
527
|
+
Instant::now(),
|
|
528
|
+
flag.clone()
|
|
529
|
+
)
|
|
530
|
+
.is_err()
|
|
531
|
+
);
|
|
532
|
+
let mut r = AgentRenderer::new(b.schema(), 10, 1000, 500, OutputFormat::Markdown).unwrap();
|
|
459
533
|
r.set_interrupt(Instant::now() + std::time::Duration::from_secs(10), flag.clone());
|
|
460
534
|
flag.store(true, Ordering::Relaxed);
|
|
461
535
|
assert!(r.push_batch(&b).unwrap_err().message.contains("cancelled"));
|
|
@@ -491,7 +565,7 @@ mod tests {
|
|
|
491
565
|
out.contains("total_rows=2") && out.contains("complete=false") && out.contains("cells_truncated=true"),
|
|
492
566
|
"{out}"
|
|
493
567
|
);
|
|
494
|
-
assert!(out.contains("|
|
|
568
|
+
assert!(out.contains("|NULL|"), "{out}");
|
|
495
569
|
assert!(out.contains("next=filter/aggregate SQL or raise budgets"));
|
|
496
570
|
}
|
|
497
571
|
#[test]
|
|
@@ -511,7 +585,7 @@ mod tests {
|
|
|
511
585
|
let out = render(&b, 10, 2000, 500, true);
|
|
512
586
|
assert!(out.contains("complete=true") && out.contains("\"NULL\""), "{out}");
|
|
513
587
|
for (rows, bytes, chars) in [(0, 256, 1), (1, 255, 1), (1, 256, 0)] {
|
|
514
|
-
assert!(
|
|
588
|
+
assert!(AgentRenderer::new(b.schema(), rows, bytes, chars, OutputFormat::Markdown).is_err());
|
|
515
589
|
}
|
|
516
590
|
}
|
|
517
591
|
#[test]
|
|
@@ -538,8 +612,8 @@ mod tests {
|
|
|
538
612
|
);
|
|
539
613
|
let out = render(&b, 10, 1000, 500, true);
|
|
540
614
|
assert!(out.contains("x (Int64)") && out.contains("x (Utf8)"), "{out}");
|
|
541
|
-
assert!(out.contains("|
|
|
542
|
-
assert!(out.contains("|
|
|
615
|
+
assert!(out.contains("|1|\"NULL\"|"), "{out}");
|
|
616
|
+
assert!(out.contains("|NULL|NULL|"), "{out}");
|
|
543
617
|
assert!(out.contains("complete=true") && out.contains("total_rows=2"), "{out}");
|
|
544
618
|
}
|
|
545
619
|
#[test]
|
|
@@ -629,10 +703,209 @@ mod tests {
|
|
|
629
703
|
assert!(out.contains(value), "missing {value}: {out}");
|
|
630
704
|
}
|
|
631
705
|
}
|
|
706
|
+
fn render_tsv(batch: &RecordBatch, rows: usize, bytes: usize, chars: usize) -> String {
|
|
707
|
+
let mut r = AgentRenderer::new(batch.schema(), rows, bytes, chars, OutputFormat::Tsv).unwrap();
|
|
708
|
+
r.push_batch(batch).unwrap();
|
|
709
|
+
r.finish(true)
|
|
710
|
+
}
|
|
711
|
+
/// Reference decoder for the documented TSV cell grammar.
|
|
712
|
+
fn tsv_cell(raw: &str) -> Option<String> {
|
|
713
|
+
if raw == "\\N" {
|
|
714
|
+
return None;
|
|
715
|
+
}
|
|
716
|
+
let mut out = String::new();
|
|
717
|
+
let mut chars = raw.chars();
|
|
718
|
+
while let Some(c) = chars.next() {
|
|
719
|
+
if c != '\\' {
|
|
720
|
+
out.push(c);
|
|
721
|
+
continue;
|
|
722
|
+
}
|
|
723
|
+
match chars.next().unwrap() {
|
|
724
|
+
'\\' => out.push('\\'),
|
|
725
|
+
't' => out.push('\t'),
|
|
726
|
+
'n' => out.push('\n'),
|
|
727
|
+
'r' => out.push('\r'),
|
|
728
|
+
'u' => {
|
|
729
|
+
let hex: String = chars.by_ref().take(4).collect();
|
|
730
|
+
out.push(char::from_u32(u32::from_str_radix(&hex, 16).unwrap()).unwrap());
|
|
731
|
+
}
|
|
732
|
+
other => panic!("unexpected escape \\{other}"),
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
Some(out)
|
|
736
|
+
}
|
|
737
|
+
#[test]
|
|
738
|
+
fn tsv_is_the_default_layout_with_typed_header() {
|
|
739
|
+
assert_eq!(OutputFormat::default(), OutputFormat::Tsv);
|
|
740
|
+
let b = batch(
|
|
741
|
+
&["id", "label"],
|
|
742
|
+
vec![
|
|
743
|
+
Arc::new(Int64Array::from(vec![1, 2])),
|
|
744
|
+
Arc::new(StringArray::from(vec![Some("a_b"), None])),
|
|
745
|
+
],
|
|
746
|
+
);
|
|
747
|
+
let out = render_tsv(&b, 10, 1000, 500);
|
|
748
|
+
assert_eq!(
|
|
749
|
+
out,
|
|
750
|
+
"id:Int64\tlabel:Utf8\n1\ta_b\n2\t\\N\n\nrows_shown=2; complete=true; total_rows=2"
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
#[test]
|
|
754
|
+
fn tsv_tabs_newlines_backslashes_and_markers_cannot_break_the_table() {
|
|
755
|
+
let nasty = [
|
|
756
|
+
"plain",
|
|
757
|
+
"a\tb",
|
|
758
|
+
"tab at end\t",
|
|
759
|
+
"\tleading tab",
|
|
760
|
+
"line1\nline2",
|
|
761
|
+
"crlf\r\n",
|
|
762
|
+
"back\\slash",
|
|
763
|
+
"ends with backslash\\",
|
|
764
|
+
"\\t is not a tab",
|
|
765
|
+
"\\N",
|
|
766
|
+
"NULL",
|
|
767
|
+
"",
|
|
768
|
+
" padded ",
|
|
769
|
+
"pipe|and,comma\"quote\"",
|
|
770
|
+
"bell\u{7} nul\u{0} nel\u{85}",
|
|
771
|
+
"Zürich 東京 🚀",
|
|
772
|
+
"\t\t\t",
|
|
773
|
+
];
|
|
774
|
+
let values: Vec<Option<&str>> = nasty.iter().map(|v| Some(*v)).chain([None]).collect();
|
|
775
|
+
let b = batch(
|
|
776
|
+
&["k", "v", "w"],
|
|
777
|
+
vec![
|
|
778
|
+
Arc::new(Int64Array::from_iter_values(0..values.len() as i64)),
|
|
779
|
+
Arc::new(StringArray::from(values.clone())),
|
|
780
|
+
Arc::new(StringArray::from(values.clone())),
|
|
781
|
+
],
|
|
782
|
+
);
|
|
783
|
+
let out = render_tsv(&b, 100, 10_000, 500);
|
|
784
|
+
let (table, footer) = out.split_once("\n\nrows_shown=").unwrap();
|
|
785
|
+
assert!(footer.contains("complete=true"), "{footer}");
|
|
786
|
+
let lines: Vec<&str> = table.lines().collect();
|
|
787
|
+
assert_eq!(lines.len(), values.len() + 1, "a cell leaked a line break: {out:?}");
|
|
788
|
+
for (line, expected) in lines[1..].iter().zip(&values) {
|
|
789
|
+
let cells: Vec<&str> = line.split('\t').collect();
|
|
790
|
+
assert_eq!(cells.len(), 3, "a cell leaked a tab: {line:?}");
|
|
791
|
+
for cell in &cells[1..] {
|
|
792
|
+
assert_eq!(tsv_cell(cell).as_deref(), *expected, "{line:?}");
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
#[test]
|
|
797
|
+
fn tsv_string_null_marker_and_empty_string_are_distinct() {
|
|
798
|
+
let b = batch(
|
|
799
|
+
&["s"],
|
|
800
|
+
vec![Arc::new(StringArray::from(vec![
|
|
801
|
+
Some("\\N"),
|
|
802
|
+
None,
|
|
803
|
+
Some(""),
|
|
804
|
+
Some("NULL"),
|
|
805
|
+
]))],
|
|
806
|
+
);
|
|
807
|
+
let out = render_tsv(&b, 10, 1000, 500);
|
|
808
|
+
assert!(out.contains("\n\\\\N\n\\N\n\nNULL\n"), "{out:?}");
|
|
809
|
+
}
|
|
810
|
+
#[test]
|
|
811
|
+
fn tsv_clips_cells_and_never_exceeds_the_byte_budget() {
|
|
812
|
+
let b = batch(
|
|
813
|
+
&["a", "b"],
|
|
814
|
+
vec![
|
|
815
|
+
Arc::new(StringArray::from(vec!["\t🦀\n".repeat(300); 10])),
|
|
816
|
+
Arc::new(Int64Array::from(vec![42; 10])),
|
|
817
|
+
],
|
|
818
|
+
);
|
|
819
|
+
for bytes in 256..700 {
|
|
820
|
+
let out = render_tsv(&b, 10, bytes, 500);
|
|
821
|
+
assert!(out.len() <= bytes, "{} > {bytes}", out.len());
|
|
822
|
+
assert!(out.contains("complete=false"));
|
|
823
|
+
for line in out.lines().take_while(|l| !l.is_empty()) {
|
|
824
|
+
assert_eq!(line.matches('\t').count(), 1, "{line:?}");
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
let out = render_tsv(&b, 10, 4000, 3);
|
|
828
|
+
assert!(
|
|
829
|
+
out.contains("\\t🦀\\n…[truncated]\t42") || out.contains("…[truncated]\t42"),
|
|
830
|
+
"{out:?}"
|
|
831
|
+
);
|
|
832
|
+
assert!(out.contains("cells_truncated=true"));
|
|
833
|
+
}
|
|
834
|
+
#[test]
|
|
835
|
+
fn tsv_nested_values_and_header_names_are_escaped_too() {
|
|
836
|
+
use arrow_array::builder::{ListBuilder, StringBuilder};
|
|
837
|
+
let mut builder = ListBuilder::new(StringBuilder::new());
|
|
838
|
+
builder.values().append_value("x\ty\nz");
|
|
839
|
+
builder.append(true);
|
|
840
|
+
let b = batch(
|
|
841
|
+
&["we\tird\nname", "list"],
|
|
842
|
+
vec![Arc::new(Int64Array::from(vec![1])), Arc::new(builder.finish())],
|
|
843
|
+
);
|
|
844
|
+
let out = render_tsv(&b, 10, 1000, 500);
|
|
845
|
+
let lines: Vec<&str> = out.lines().take_while(|l| !l.is_empty()).collect();
|
|
846
|
+
assert_eq!(lines.len(), 2, "{out:?}");
|
|
847
|
+
assert!(lines[0].starts_with("we\\tird\\nname:Int64\tlist:"), "{out:?}");
|
|
848
|
+
assert_eq!(lines[1].matches('\t').count(), 1, "{out:?}");
|
|
849
|
+
assert!(lines[1].contains("x\\ty\\nz"), "{out:?}");
|
|
850
|
+
}
|
|
851
|
+
proptest::proptest! {
|
|
852
|
+
#[test]
|
|
853
|
+
fn tsv_roundtrips_arbitrary_strings(values in proptest::collection::vec(proptest::option::of("\\PC{0,12}|[\t\n\r\\\\N|]{0,6}"), 1..8)) {
|
|
854
|
+
let b = batch(&["a", "b"], vec![
|
|
855
|
+
Arc::new(StringArray::from(values.clone())),
|
|
856
|
+
Arc::new(StringArray::from(values.clone())),
|
|
857
|
+
]);
|
|
858
|
+
let out = render_tsv(&b, 100, 100_000, 500);
|
|
859
|
+
let table = out.split_once("\n\nrows_shown=").unwrap().0;
|
|
860
|
+
let lines: Vec<&str> = table.split('\n').collect();
|
|
861
|
+
proptest::prop_assert_eq!(lines.len(), values.len() + 1);
|
|
862
|
+
for (line, expected) in lines[1..].iter().zip(&values) {
|
|
863
|
+
let cells: Vec<&str> = line.split('\t').collect();
|
|
864
|
+
proptest::prop_assert_eq!(cells.len(), 2);
|
|
865
|
+
for cell in cells {
|
|
866
|
+
proptest::prop_assert_eq!(tsv_cell(cell), expected.clone());
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
#[test]
|
|
872
|
+
fn tsv_round_trips_every_unicode_scalar_value() {
|
|
873
|
+
let all: Vec<char> = (0..=0x10ffff_u32).filter_map(char::from_u32).collect();
|
|
874
|
+
let chunks: Vec<String> = all.chunks(512).map(|c| c.iter().collect()).collect();
|
|
875
|
+
let b = batch(&["v"], vec![Arc::new(StringArray::from(chunks.clone()))]);
|
|
876
|
+
let mut r = AgentRenderer::new(b.schema(), chunks.len(), 64 << 20, 1 << 20, OutputFormat::Tsv).unwrap();
|
|
877
|
+
r.push_batch(&b).unwrap();
|
|
878
|
+
let out = r.finish(true);
|
|
879
|
+
let table = out.split_once("\n\nrows_shown=").unwrap().0;
|
|
880
|
+
let lines: Vec<&str> = table.split('\n').collect();
|
|
881
|
+
assert_eq!(lines.len(), chunks.len() + 1, "a character leaked a line break");
|
|
882
|
+
for (line, expected) in lines[1..].iter().zip(&chunks) {
|
|
883
|
+
assert!(!line.contains('\t'), "a character leaked a tab");
|
|
884
|
+
assert_eq!(tsv_cell(line).as_deref(), Some(expected.as_str()));
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
#[test]
|
|
888
|
+
fn markdown_has_no_padding_and_a_minimal_separator() {
|
|
889
|
+
let b = batch(
|
|
890
|
+
&["a", "b"],
|
|
891
|
+
vec![
|
|
892
|
+
Arc::new(Int64Array::from(vec![1])),
|
|
893
|
+
Arc::new(StringArray::from(vec!["x"])),
|
|
894
|
+
],
|
|
895
|
+
);
|
|
896
|
+
let out = render(&b, 10, 1000, 500, true);
|
|
897
|
+
assert!(out.starts_with("|a (Int64)|b (Utf8)|\n|-|-|\n|1|\"x\"|\n"), "{out}");
|
|
898
|
+
}
|
|
899
|
+
#[test]
|
|
900
|
+
fn output_format_names() {
|
|
901
|
+
assert_eq!(OutputFormat::parse("tsv"), Ok(OutputFormat::Tsv));
|
|
902
|
+
assert_eq!(OutputFormat::parse("markdown"), Ok(OutputFormat::Markdown));
|
|
903
|
+
assert!(OutputFormat::parse("csv").is_err());
|
|
904
|
+
}
|
|
632
905
|
#[test]
|
|
633
906
|
fn batch_schema_mismatch_errors_and_multiple_batches_append() {
|
|
634
907
|
let b = batch(&["a"], vec![Arc::new(Int64Array::from(vec![1, 2]))]);
|
|
635
|
-
let mut r =
|
|
908
|
+
let mut r = AgentRenderer::new(b.schema(), 3, 1000, 500, OutputFormat::Markdown).unwrap();
|
|
636
909
|
assert!(r.push_batch(&b).unwrap());
|
|
637
910
|
assert!(!r.push_batch(&b).unwrap());
|
|
638
911
|
assert_eq!(r.rows(), 3);
|
|
@@ -640,7 +913,7 @@ mod tests {
|
|
|
640
913
|
let out = r.finish(false);
|
|
641
914
|
assert!(out.contains("rows_shown=3") && !out.contains("total_rows="));
|
|
642
915
|
let other = batch(&["b"], vec![Arc::new(Int64Array::from(vec![1]))]);
|
|
643
|
-
let mut r =
|
|
916
|
+
let mut r = AgentRenderer::new(b.schema(), 3, 1000, 500, OutputFormat::Markdown).unwrap();
|
|
644
917
|
assert!(r.push_batch(&other).is_err());
|
|
645
918
|
}
|
|
646
919
|
}
|
|
@@ -828,9 +828,9 @@ impl PyDbClient {
|
|
|
828
828
|
})
|
|
829
829
|
}
|
|
830
830
|
|
|
831
|
-
/// Returns bounded typed Markdown, or actionable JSON for a query failure.
|
|
831
|
+
/// Returns a bounded typed preview (TSV or Markdown), or actionable JSON for a query failure.
|
|
832
832
|
#[allow(clippy::too_many_arguments)]
|
|
833
|
-
#[pyo3(signature = (statement, *, max_rows=1000, max_bytes=10000, max_cell_chars=500, total_timeout_s=60.0, catalog=None, schema=None, parameters=None))]
|
|
833
|
+
#[pyo3(signature = (statement, *, max_rows=1000, max_bytes=10000, max_cell_chars=500, total_timeout_s=60.0, output_format="tsv", catalog=None, schema=None, parameters=None))]
|
|
834
834
|
fn query_agent<'py>(
|
|
835
835
|
&self,
|
|
836
836
|
py: Python<'py>,
|
|
@@ -839,15 +839,18 @@ impl PyDbClient {
|
|
|
839
839
|
max_bytes: usize,
|
|
840
840
|
max_cell_chars: usize,
|
|
841
841
|
total_timeout_s: f64,
|
|
842
|
+
output_format: &str,
|
|
842
843
|
catalog: Option<String>,
|
|
843
844
|
schema: Option<String>,
|
|
844
845
|
parameters: Option<Py<PyAny>>,
|
|
845
846
|
) -> PyResult<Bound<'py, PyAny>> {
|
|
847
|
+
let format = agent::OutputFormat::parse(output_format).map_err(PyValueError::new_err)?;
|
|
846
848
|
let options = agent::AgentOptions {
|
|
847
849
|
max_rows,
|
|
848
850
|
max_bytes,
|
|
849
851
|
max_cell_chars,
|
|
850
852
|
total_timeout_s,
|
|
853
|
+
format,
|
|
851
854
|
};
|
|
852
855
|
options.validate().map_err(PyValueError::new_err)?;
|
|
853
856
|
let client = self.inner.clone();
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# them is first used. Older Pythons ignore this list and import eagerly.
|
|
3
3
|
__lazy_modules__ = ["arrowbricks._streaming", "arrowbricks.client", "arrowbricks.cursor"]
|
|
4
4
|
|
|
5
|
+
from ._agent import AgentFormat
|
|
5
6
|
from ._core import ArrowbricksError, AuthError, QueryStats, StatementError, TransientError
|
|
6
7
|
from ._sql import Sql, unsafe_sql
|
|
7
8
|
from ._streaming import (
|
|
@@ -16,6 +17,7 @@ from .client import DatabricksClient
|
|
|
16
17
|
from .cursor import Connection, Cursor, connect
|
|
17
18
|
|
|
18
19
|
__all__ = [
|
|
20
|
+
"AgentFormat",
|
|
19
21
|
"HEARTBEAT",
|
|
20
22
|
"ArrowbricksError",
|
|
21
23
|
"AuthError",
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Option values for `DatabricksClient.query_agent` and the CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Literal, TypeAlias, get_args
|
|
6
|
+
|
|
7
|
+
AgentFormat: TypeAlias = Literal["tsv", "markdown"]
|
|
8
|
+
"""`"tsv"`: tab-separated, `name:Type` header. `"markdown"`: unpadded pipe table."""
|
|
9
|
+
|
|
10
|
+
AGENT_FORMATS: tuple[AgentFormat, ...] = get_args(AgentFormat)
|
|
11
|
+
DEFAULT_AGENT_FORMAT: AgentFormat = "tsv"
|
|
@@ -109,6 +109,7 @@ class Client:
|
|
|
109
109
|
max_bytes: int = 10000,
|
|
110
110
|
max_cell_chars: int = 500,
|
|
111
111
|
total_timeout_s: float = 60.0,
|
|
112
|
+
output_format: str = "tsv",
|
|
112
113
|
catalog: str | None = None,
|
|
113
114
|
schema: str | None = None,
|
|
114
115
|
parameters: list[dict[str, Any]] | None = None,
|
|
@@ -9,6 +9,8 @@ import sys
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
from typing import NoReturn
|
|
11
11
|
|
|
12
|
+
from ._agent import AGENT_FORMATS, DEFAULT_AGENT_FORMAT
|
|
13
|
+
|
|
12
14
|
|
|
13
15
|
def _configuration_error(message: str) -> None:
|
|
14
16
|
print(
|
|
@@ -30,11 +32,13 @@ class _Parser(argparse.ArgumentParser):
|
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
def _parser() -> argparse.ArgumentParser:
|
|
33
|
-
parser = _Parser(
|
|
35
|
+
parser = _Parser(
|
|
36
|
+
description="Databricks SQL to a bounded typed preview (TSV by default); errors are JSON on stderr."
|
|
37
|
+
)
|
|
34
38
|
source = parser.add_mutually_exclusive_group()
|
|
35
39
|
source.add_argument("-c", "--sql", "--command", dest="sql", help="SQL statement, sent unchanged")
|
|
36
40
|
source.add_argument("-f", "--file", help="UTF-8 SQL file, or - for stdin; defaults to piped stdin")
|
|
37
|
-
parser.add_argument("--agent", "-agent", action="store_true", help="agent
|
|
41
|
+
parser.add_argument("--agent", "-agent", action="store_true", help="agent output (the default; see --format)")
|
|
38
42
|
parser.add_argument("--host", default=os.environ.get("DATABRICKS_HOST"), help="workspace host (DATABRICKS_HOST)")
|
|
39
43
|
parser.add_argument(
|
|
40
44
|
"--warehouse-id",
|
|
@@ -48,10 +52,27 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
48
52
|
parser.add_argument("--max-rows", type=int, default=1000)
|
|
49
53
|
parser.add_argument("--max-bytes", type=int, default=10000)
|
|
50
54
|
parser.add_argument("--max-cell-chars", type=int, default=500)
|
|
55
|
+
parser.add_argument(
|
|
56
|
+
"--format",
|
|
57
|
+
dest="output_format",
|
|
58
|
+
choices=AGENT_FORMATS,
|
|
59
|
+
default=DEFAULT_AGENT_FORMAT,
|
|
60
|
+
help="output layout: tab-separated values (default) or a Markdown pipe table",
|
|
61
|
+
)
|
|
51
62
|
parser.add_argument("--total-timeout-s", "--timeout", dest="total_timeout_s", type=float, default=60.0)
|
|
52
63
|
return parser
|
|
53
64
|
|
|
54
65
|
|
|
66
|
+
def _is_error_json(result: str) -> bool:
|
|
67
|
+
"""Error diagnostics are one JSON object; a TSV header or table never parses as JSON."""
|
|
68
|
+
if not result.startswith("{"):
|
|
69
|
+
return False
|
|
70
|
+
try:
|
|
71
|
+
return isinstance(json.loads(result), dict)
|
|
72
|
+
except ValueError:
|
|
73
|
+
return False
|
|
74
|
+
|
|
75
|
+
|
|
55
76
|
async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
56
77
|
from ._sql import unsafe_sql
|
|
57
78
|
from .client import DatabricksClient
|
|
@@ -69,6 +90,7 @@ async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
|
69
90
|
max_bytes=args.max_bytes,
|
|
70
91
|
max_cell_chars=args.max_cell_chars,
|
|
71
92
|
total_timeout_s=args.total_timeout_s,
|
|
93
|
+
output_format=args.output_format,
|
|
72
94
|
catalog=args.catalog,
|
|
73
95
|
schema=args.schema,
|
|
74
96
|
)
|
|
@@ -76,7 +98,7 @@ async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
|
76
98
|
_configuration_error(str(exc))
|
|
77
99
|
return 2
|
|
78
100
|
# Locale encoding and newline translation must not change the byte budget.
|
|
79
|
-
if result
|
|
101
|
+
if _is_error_json(result):
|
|
80
102
|
sys.stderr.buffer.write(result.encode("utf-8"))
|
|
81
103
|
return 1
|
|
82
104
|
sys.stdout.buffer.write(result.encode("utf-8"))
|
|
@@ -84,7 +106,7 @@ async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
|
84
106
|
|
|
85
107
|
|
|
86
108
|
def main(argv: list[str] | None = None) -> int:
|
|
87
|
-
"""Run one SQL statement; return 0 for
|
|
109
|
+
"""Run one SQL statement; return 0 for a preview, 1 for query errors, 2 for configuration."""
|
|
88
110
|
parser = _parser()
|
|
89
111
|
args = parser.parse_args(argv)
|
|
90
112
|
if not all((args.host, args.warehouse_id, args.token)):
|
|
@@ -11,6 +11,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
from . import _core
|
|
14
|
+
from ._agent import DEFAULT_AGENT_FORMAT, AgentFormat
|
|
14
15
|
from ._sql import Sql, prepare_sql
|
|
15
16
|
|
|
16
17
|
TokenProvider = Callable[[], "str | Awaitable[str]"]
|
|
@@ -155,11 +156,20 @@ class DatabricksClient:
|
|
|
155
156
|
max_bytes: int = 10000,
|
|
156
157
|
max_cell_chars: int = 500,
|
|
157
158
|
total_timeout_s: float = 60.0,
|
|
159
|
+
output_format: AgentFormat = DEFAULT_AGENT_FORMAT,
|
|
158
160
|
catalog: str | None = None,
|
|
159
161
|
schema: str | None = None,
|
|
160
162
|
params: list[dict[str, Any]] | None = None,
|
|
161
163
|
) -> str:
|
|
162
|
-
"""Return bounded typed
|
|
164
|
+
r"""Return a bounded typed preview, or actionable JSON on query failure.
|
|
165
|
+
|
|
166
|
+
`output_format="tsv"` (the default) is the cheapest to read: a
|
|
167
|
+
`name:Type` header, then one tab-separated line per row. Backslash, tab,
|
|
168
|
+
newline and carriage return in a value are written as `\\`, `\t`, `\n`
|
|
169
|
+
and `\r`, SQL NULL is `\N` and the empty string is an empty field, so a
|
|
170
|
+
cell can never contain a raw delimiter. `"markdown"` is a pipe table
|
|
171
|
+
without padding (`|a (Int64)|b (Utf8)|`) whose strings are quoted and
|
|
172
|
+
Markdown-escaped. Both end with the same completeness footer.
|
|
163
173
|
|
|
164
174
|
Budgets cover rows, the entire UTF-8 output, cell characters and one
|
|
165
175
|
deadline across submission and fetching. The completeness footer marks
|
|
@@ -176,6 +186,7 @@ class DatabricksClient:
|
|
|
176
186
|
max_bytes=max_bytes,
|
|
177
187
|
max_cell_chars=max_cell_chars,
|
|
178
188
|
total_timeout_s=total_timeout_s,
|
|
189
|
+
output_format=output_format,
|
|
179
190
|
catalog=catalog,
|
|
180
191
|
schema=schema,
|
|
181
192
|
parameters=params,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests/wiremock_volume_files.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.1.0 → arrowbricks-6.2.0}/rust/arrowbricks_core/tests_py/test_token_provider.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|