fivetran-cli 1.1.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/PKG-INFO +3 -3
  2. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/README.md +1 -1
  3. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/pyproject.toml +1 -1
  4. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/__init__.py +41 -15
  5. fivetran_cli-1.2.0/src/fivetran_cli/cli_utils/__init__.py +0 -0
  6. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/dispatch/base.py +3 -0
  7. fivetran_cli-1.2.0/src/fivetran_cli/display/__init__.py +0 -0
  8. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/display/_renderers.py +17 -7
  9. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/display/output.py +40 -5
  10. fivetran_cli-1.2.0/src/fivetran_cli/display/page_stream.py +98 -0
  11. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/http/client.py +46 -24
  12. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli.egg-info/PKG-INFO +3 -3
  13. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli.egg-info/SOURCES.txt +3 -0
  14. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/setup.cfg +0 -0
  15. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli.py +0 -0
  16. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_parser.py +0 -0
  17. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/args.py +0 -0
  18. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/dispatch/__init__.py +0 -0
  19. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/dispatch/reserved_flags.py +0 -0
  20. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/dispatch/timeout.py +0 -0
  21. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/parser/__init__.py +0 -0
  22. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/parser/builder.py +0 -0
  23. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/__init__.py +0 -0
  24. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/account.py +0 -0
  25. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/certificate.py +0 -0
  26. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/connection.py +0 -0
  27. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/connection_schema.py +0 -0
  28. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/connector_metadata.py +0 -0
  29. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/connector_sdk_package.py +0 -0
  30. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/destination.py +0 -0
  31. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/fingerprint.py +0 -0
  32. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/group.py +0 -0
  33. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/hybrid_deployment_agent.py +0 -0
  34. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/log_service.py +0 -0
  35. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/private_link.py +0 -0
  36. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/proxy_agent.py +0 -0
  37. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/public_connector.py +0 -0
  38. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/role.py +0 -0
  39. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/system_key.py +0 -0
  40. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/team.py +0 -0
  41. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/transformation.py +0 -0
  42. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/transformation_project.py +0 -0
  43. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/user.py +0 -0
  44. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/cli_utils/resources/webhook.py +0 -0
  45. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/constants.py +0 -0
  46. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/display/_formatters.py +0 -0
  47. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/display/error_output.py +0 -0
  48. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/display/views.py +0 -0
  49. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/http/__init__.py +0 -0
  50. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/http/config.py +0 -0
  51. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/http/exceptions.py +0 -0
  52. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/plugin_api.py +0 -0
  53. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli/telemetry_helper.py +0 -0
  54. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli.egg-info/dependency_links.txt +0 -0
  55. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli.egg-info/entry_points.txt +0 -0
  56. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli.egg-info/requires.txt +0 -0
  57. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli.egg-info/top_level.txt +0 -0
  58. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli_telemetry/__init__.py +0 -0
  59. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli_telemetry/_sender.py +0 -0
  60. {fivetran_cli-1.1.0 → fivetran_cli-1.2.0}/src/fivetran_cli_telemetry/telemetry.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fivetran_cli
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: Command line wrapper for the Fivetran REST API and Connector SDK
5
5
  Author-email: Fivetran <developers@fivetran.com>
6
6
  Project-URL: Homepage, https://fivetran.com
@@ -8,7 +8,7 @@ Project-URL: Github, https://github.com/fivetran/engineering
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.10
11
+ Requires-Python: >=3.11
12
12
  Description-Content-Type: text/markdown
13
13
  Requires-Dist: rich>=13.0
14
14
  Requires-Dist: requests==2.34.2
@@ -30,7 +30,7 @@ Verify the installation:
30
30
 
31
31
  ## Requirements
32
32
 
33
- - Python 3.10 or later
33
+ - Python 3.11 or later
34
34
  - A Fivetran account and API key for commands that communicate with Fivetran
35
35
 
36
36
  ## Authentication
@@ -14,7 +14,7 @@ Verify the installation:
14
14
 
15
15
  ## Requirements
16
16
 
17
- - Python 3.10 or later
17
+ - Python 3.11 or later
18
18
  - A Fivetran account and API key for commands that communicate with Fivetran
19
19
 
20
20
  ## Authentication
@@ -6,7 +6,7 @@ authors = [
6
6
  ]
7
7
  description = "Command line wrapper for the Fivetran REST API and Connector SDK"
8
8
  readme = "README.md"
9
- requires-python = ">=3.10"
9
+ requires-python = ">=3.11"
10
10
  classifiers = [
11
11
  "Programming Language :: Python :: 3",
12
12
  "License :: OSI Approved :: MIT License",
@@ -9,8 +9,7 @@ from pathlib import Path
9
9
  from typing import Sequence, TextIO
10
10
 
11
11
  # Version format: <major_version>.<minor_version>.<patch_version>
12
- # (where Major Version = 1, Minor Version is incremental MM from Sep 26 onwards, Patch Version is incremental within a month)
13
- __version__ = "1.1.0"
12
+ __version__ = "1.2.0"
14
13
 
15
14
  from fivetran_cli import telemetry_helper
16
15
  from fivetran_cli.cli_utils.dispatch.timeout import cancel_command_timeout, command_timed_out, install_command_timeout
@@ -18,7 +17,8 @@ from fivetran_cli.cli_utils.parser import build_parser, resolve_output_format, v
18
17
  from fivetran_cli.cli_parser import discover_plugins, first_command_token
19
18
  from fivetran_cli.constants import COMMAND_TIMEOUT, FIVETRAN_CLI_COMMANDS, UNKNOWN_FIELDS_ERROR_PREFIX
20
19
  from fivetran_cli.display.error_output import emit_error
21
- from fivetran_cli.display.output import emit_output, emit_raw_json, ensure_writable_parent
20
+ from fivetran_cli.display.output import create_page_stream, emit_output, emit_raw_json, ensure_writable_parent
21
+ from fivetran_cli.display.page_stream import PageStream
22
22
  from fivetran_cli.http.client import FivetranClient
23
23
  from fivetran_cli.http.config import resolve_config, require_api_key
24
24
  from fivetran_cli.http.exceptions import FivetranCliError, FivetranPartialResultError, extract_api_error
@@ -172,6 +172,32 @@ def _validate_flags_before_execution(args: argparse.Namespace) -> None:
172
172
  validate_filters(args)
173
173
 
174
174
 
175
+ def _handle_partial_result_error(
176
+ exc: FivetranPartialResultError,
177
+ args: argparse.Namespace,
178
+ stdout: TextIO,
179
+ stderr: TextIO,
180
+ page_stream: PageStream | None,
181
+ ) -> None:
182
+ """Reports a pagination that stopped early. Data collected so far is shown
183
+ on stdout, unless it was already streamed page by page. An output file is
184
+ only ever written by a successful run."""
185
+ emit_error(exc, args=args, stderr=stderr)
186
+ output_file = getattr(args, "output_file", None)
187
+ if output_file:
188
+ stderr.write(f"Output file '{output_file}' was not written because the results are incomplete.\n")
189
+ return
190
+ if exc.partial_data is None or page_stream is not None:
191
+ return
192
+ try:
193
+ emit_output(exc.partial_data, args=args, stdout=stdout)
194
+ except Exception as render_exc:
195
+ stderr.write(
196
+ f"fivetran: failed to render partial results: "
197
+ f"{type(render_exc).__name__}: {render_exc}\n"
198
+ )
199
+
200
+
175
201
  def _execute_command(
176
202
  args: argparse.Namespace,
177
203
  stdout: TextIO,
@@ -184,22 +210,29 @@ def _execute_command(
184
210
  API's HTTP status_code) instead of a generic 1 — see _telemetry_exit_code."""
185
211
  operation_completed = False
186
212
  payload = None
213
+ page_stream: PageStream | None = None
187
214
  client: FivetranClient | None = None
188
215
  try:
189
216
  _validate_flags_before_execution(args)
190
217
  config = resolve_config(args)
191
218
  require_api_key(config)
192
219
  client = FivetranClient(config, stderr=stderr)
220
+ page_stream = create_page_stream(args, stdout)
193
221
 
194
222
  install_command_timeout(COMMAND_TIMEOUT, stderr)
195
223
 
196
224
  args._stdout = stdout
197
225
  args._stderr = stderr
226
+ args._page_stream = page_stream
198
227
 
199
228
  try:
200
229
  payload = args.handler(args, client)
201
230
  operation_completed = True
202
- emit_output(payload, args=args, stdout=stdout)
231
+ # finish the page stream and don't emit output again as page by page rendering happened already
232
+ if page_stream is not None:
233
+ page_stream.finish()
234
+ else:
235
+ emit_output(payload, args=args, stdout=stdout)
203
236
  return 0, 0
204
237
  finally:
205
238
  cancel_command_timeout()
@@ -210,17 +243,10 @@ def _execute_command(
210
243
  return 1, _telemetry_exit_code(exc)
211
244
 
212
245
  except FivetranPartialResultError as exc:
213
- # Pagination stopped early on a malformed page — still a failure
214
- # (exit non-zero), but any data collected so far is still shown.
215
- emit_error(exc, args=args, stderr=stderr)
216
- if exc.partial_data is not None:
217
- try:
218
- emit_output(exc.partial_data, args=args, stdout=stdout)
219
- except Exception as render_exc:
220
- stderr.write(
221
- f"fivetran: failed to render partial results: "
222
- f"{type(render_exc).__name__}: {render_exc}\n"
223
- )
246
+ # Pagination stopped early — still a failure (exit non-zero). Data
247
+ # collected so far is shown on stdout, unless it was already streamed
248
+ # page by page. An output file is only ever written by a successful run.
249
+ _handle_partial_result_error(exc, args, stdout, stderr, page_stream)
224
250
  return 1, _telemetry_exit_code(exc)
225
251
 
226
252
  except FivetranCliError as exc:
@@ -283,6 +283,7 @@ def _make_handler(endpoint: Endpoint) -> Handler:
283
283
  payload = _load_payload(args) if endpoint.payload is not PayloadMode.NONE else None
284
284
  path = _build_path(endpoint.path_template, args, endpoint.id_args)
285
285
  query = {arg: getattr(args, arg, None) for arg in endpoint.query_args}
286
+ page_stream = getattr(args, "_page_stream", None)
286
287
 
287
288
  if endpoint.paginated:
288
289
  limit_val: int | None = getattr(args, "limit", None)
@@ -295,6 +296,8 @@ def _make_handler(endpoint: Endpoint) -> Handler:
295
296
  payload=payload,
296
297
  limit=limit_val,
297
298
  filters=resolve_filters(args),
299
+ on_page=page_stream.write_page if page_stream else None,
300
+ output_to_file=bool(getattr(args, "output_file", None)),
298
301
  )
299
302
 
300
303
  return client.request(
@@ -34,15 +34,19 @@ def render_to_console(
34
34
  *,
35
35
  console: Console,
36
36
  show_all_columns: bool = False,
37
+ output_columns: tuple[str, ...] = (),
37
38
  ) -> None:
38
39
  """Entry point for the rendering pipeline: unwraps the API envelope,
39
- prints any status lines, then dispatches to the right view."""
40
+ prints any status lines, then dispatches to the right view.
41
+
42
+ output_columns, when given, are the table's columns (instead of the keys
43
+ found in the rows), the same way a streamed table gets them."""
40
44
  data, header_lines = _unwrap(payload)
41
45
 
42
46
  for line in header_lines:
43
47
  console.print(Text(line, style="dim"))
44
48
 
45
- _render_value(data, console=console, show_all_columns=show_all_columns)
49
+ _render_value(data, console=console, show_all_columns=show_all_columns, output_columns=output_columns)
46
50
 
47
51
 
48
52
  def _unwrap(payload: Any) -> tuple[Any, list[str]]:
@@ -68,6 +72,7 @@ def _render_value(
68
72
  *,
69
73
  console: Console,
70
74
  show_all_columns: bool,
75
+ output_columns: tuple[str, ...],
71
76
  ) -> None:
72
77
  """Looks at the shape of `value` and renders it accordingly: a dict with
73
78
  an 'items' list or a raw list becomes a table, a plain dict becomes a
@@ -75,13 +80,13 @@ def _render_value(
75
80
  if isinstance(value, dict):
76
81
  items = value.get("items")
77
82
  if isinstance(items, list):
78
- render_table(items, console=console, show_all_columns=show_all_columns)
83
+ render_table(items, console=console, show_all_columns=show_all_columns, output_columns=output_columns)
79
84
  return
80
85
  render_detail(value, console=console)
81
86
  return
82
87
 
83
88
  if isinstance(value, list):
84
- render_table(value, console=console, show_all_columns=show_all_columns)
89
+ render_table(value, console=console, show_all_columns=show_all_columns, output_columns=output_columns)
85
90
  return
86
91
 
87
92
  console.print(Text(format_cell_value(value)))
@@ -96,9 +101,10 @@ def render_table(
96
101
  *,
97
102
  console: Console,
98
103
  show_all_columns: bool = False,
104
+ output_columns: tuple[str, ...] = (),
99
105
  ) -> None:
100
106
  """Renders a collection of items as a multi-column Rich table using the
101
- API's own field names as columns. Values are never truncated — they
107
+ API's own field names as columns (output_columns if given, else the keys found in the rows). Values are never truncated — they
102
108
  wrap within their cell instead — and only the first 8 columns show
103
109
  by default (pass --all-columns, or --fields, to see the rest).
104
110
 
@@ -116,7 +122,7 @@ def render_table(
116
122
  return
117
123
 
118
124
  rows = non_null_items
119
- columns, widths, hidden_column_count = _resolve_columns_and_widths(rows, show_all_columns)
125
+ columns, widths, hidden_column_count = _resolve_columns_and_widths(rows, show_all_columns, output_columns)
120
126
  if columns is None:
121
127
  console.print("No items found.")
122
128
  return
@@ -193,9 +199,13 @@ def _validate_requested_fields(
193
199
  def _resolve_columns_and_widths(
194
200
  rows: list[dict[str, Any]],
195
201
  show_all_columns: bool,
202
+ output_columns: tuple[str, ...],
196
203
  ) -> tuple[list[Column] | None, list[int] | None, int]:
197
204
  """Determine columns and their fixed widths for this render."""
198
- available_keys = collect_ordered_keys(rows)
205
+ if not rows:
206
+ return None, None, 0
207
+
208
+ available_keys = list(output_columns) or collect_ordered_keys(rows)
199
209
  if not available_keys:
200
210
  return None, None, 0
201
211
 
@@ -22,9 +22,32 @@ from fivetran_cli.display._renderers import (
22
22
  filter_payload_fields,
23
23
  render_to_console,
24
24
  )
25
+ from fivetran_cli.display.page_stream import PageStream
25
26
  from fivetran_cli.http.exceptions import FivetranCliError
26
27
 
27
28
 
29
+ def _output_columns(args: argparse.Namespace) -> tuple[str, ...]:
30
+ """The endpoint's full list of response columns; empty when it has none."""
31
+ metadata = getattr(args, "command_metadata", None)
32
+ return metadata.output_columns if metadata else ()
33
+
34
+
35
+ def create_page_stream(args: argparse.Namespace, stdout: TextIO) -> PageStream | None:
36
+ """Return a PageStream when this command's output is a paginated table
37
+ printed to the console, otherwise None (the payload is rendered once,
38
+ after all pages are fetched)."""
39
+ output_columns = _output_columns(args)
40
+ if not output_columns or getattr(args, "output_file", None) or resolve_output_format(args) != "text":
41
+ return None
42
+
43
+ return PageStream(
44
+ _make_stdout_console(stdout),
45
+ output_columns,
46
+ fields=resolve_fields(args),
47
+ show_all_columns=getattr(args, "show_all_columns", False),
48
+ )
49
+
50
+
28
51
  def emit_output(
29
52
  payload: Any,
30
53
  *,
@@ -58,6 +81,8 @@ def emit_output(
58
81
  output_path=output_path,
59
82
  stdout=stdout,
60
83
  show_all_columns=show_all_columns,
84
+ # With --fields the payload is already reduced to the requested columns.
85
+ output_columns=() if fields else _output_columns(args),
61
86
  )
62
87
  except FivetranCliError:
63
88
  raise # Re-raise explicit CLI errors as-is (e.g., file write issues)
@@ -87,18 +112,21 @@ def _emit_text(
87
112
  output_path: str | None,
88
113
  stdout: TextIO,
89
114
  show_all_columns: bool,
115
+ output_columns: tuple[str, ...],
90
116
  ) -> None:
91
117
  """Render formatted text to stdout or a file."""
92
118
  if output_path:
93
119
  with _atomic_write_file(output_path) as file_handle:
94
120
  console = _make_console(file=file_handle, color=False, width=DEFAULT_PRINTER_WIDTH)
95
- render_to_console(payload, console=console, show_all_columns=show_all_columns)
121
+ render_to_console(
122
+ payload, console=console, show_all_columns=show_all_columns, output_columns=output_columns,
123
+ )
96
124
  return
97
125
 
98
- use_color = stdout.isatty()
99
- effective_width = shutil.get_terminal_size(fallback=(DEFAULT_PRINTER_WIDTH, 24)).columns
100
- console = _make_console(file=stdout, color=use_color, width=effective_width)
101
- render_to_console(payload, console=console, show_all_columns=show_all_columns)
126
+ render_to_console(
127
+ payload, console=_make_stdout_console(stdout),
128
+ show_all_columns=show_all_columns, output_columns=output_columns,
129
+ )
102
130
 
103
131
 
104
132
  def ensure_writable_parent(output_path: str) -> Path:
@@ -141,6 +169,13 @@ def _atomic_write_file(output_path: str) -> Iterator[TextIO]:
141
169
  os.unlink(temporary_path)
142
170
  raise
143
171
 
172
+
173
+ def _make_stdout_console(stdout: TextIO) -> Console:
174
+ """Create a console sized to the terminal, with color only on a TTY."""
175
+ terminal_width = shutil.get_terminal_size(fallback=(DEFAULT_PRINTER_WIDTH, 24)).columns
176
+ return _make_console(file=stdout, color=stdout.isatty(), width=terminal_width)
177
+
178
+
144
179
  def _make_console(*, file: TextIO, color: bool, width: int | None) -> Console:
145
180
  """Create a Rich console configured for CLI output."""
146
181
  return Console(
@@ -0,0 +1,98 @@
1
+ """Render a paginated table one page at a time, as pages are fetched."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from rich.console import Console
8
+ from rich.table import Table
9
+
10
+ from fivetran_cli.constants import UNKNOWN_FIELDS_ERROR_PREFIX
11
+ from fivetran_cli.display._formatters import format_cell_rich, select_columns
12
+ from fivetran_cli.display._renderers import (
13
+ _WIDTH_SAMPLE_SIZE,
14
+ _add_table_columns,
15
+ _compute_natural_widths,
16
+ _unwrap,
17
+ print_columns_footer,
18
+ )
19
+ from fivetran_cli.http.exceptions import FivetranCliError
20
+
21
+
22
+ class PageStream:
23
+ """Renders a paginated table one page at a time as pages arrive.
24
+
25
+ The columns are fixed up front, from the endpoint's known output columns
26
+ (or the --fields selection, which must be one of them), so a field that
27
+ first shows up on a later page is never missed. Column widths (capped at
28
+ the console width) are taken from the first page that has rows and reused
29
+ afterward, so every page lines up under the single header.
30
+ """
31
+
32
+ def __init__(
33
+ self,
34
+ console: Console,
35
+ output_columns: tuple[str, ...],
36
+ *,
37
+ fields: list[str] | None,
38
+ show_all_columns: bool,
39
+ ) -> None:
40
+ if fields:
41
+ unknown_fields = [field for field in fields if field not in output_columns]
42
+ if unknown_fields:
43
+ raise FivetranCliError(f"{UNKNOWN_FIELDS_ERROR_PREFIX}{', '.join(unknown_fields)}")
44
+
45
+ self._console = console
46
+ self._columns, self._hidden_column_count = select_columns(
47
+ fields or list(output_columns), show_all_columns or bool(fields),
48
+ )
49
+ self._widths: list[int] = []
50
+
51
+ @property
52
+ def _has_printed_rows(self) -> bool:
53
+ """Column widths stay empty until the first page with rows is printed,
54
+ so having widths means a table has already been printed."""
55
+ return bool(self._widths)
56
+
57
+ def write_page(self, page: Any, kept_item_count: int) -> None:
58
+ """Print the first `kept_item_count` items of one fetched page."""
59
+ # The envelope's status lines (code, message) describe one page, so they aren't printed here.
60
+ data, _ = _unwrap(page)
61
+ rows = data["items"][:kept_item_count]
62
+ if not all(isinstance(row, dict) for row in rows):
63
+ raise FivetranCliError("malformed page — every item must be an object, found null or a non-object value")
64
+
65
+ if not rows:
66
+ return
67
+
68
+ is_first_table = not self._has_printed_rows
69
+ if is_first_table:
70
+ natural_widths = _compute_natural_widths(self._columns, rows[:_WIDTH_SAMPLE_SIZE])
71
+ # Cap now: _add_table_columns widens the console below, so capping again
72
+ # on a later page would give a wider column than this first one.
73
+ self._widths = [min(width, self._console.width) for width in natural_widths]
74
+
75
+ table = Table(
76
+ box=None,
77
+ show_header=is_first_table,
78
+ header_style="bold cyan",
79
+ show_edge=False,
80
+ pad_edge=False,
81
+ highlight=False,
82
+ )
83
+ _add_table_columns(table, self._columns, self._widths, self._console)
84
+ for row in rows:
85
+ table.add_row(*[format_cell_rich(row.get(column.field_key)) for column in self._columns])
86
+
87
+ self._console.print(table)
88
+ self._console.file.flush()
89
+
90
+ def finish(self) -> None:
91
+ """Print what follows the last page: the hidden-columns hint, or a
92
+ notice when no page had any rows."""
93
+ if not self._has_printed_rows:
94
+ self._console.print("No items found.")
95
+ elif self._hidden_column_count:
96
+ print_columns_footer(
97
+ self._console, len(self._columns), len(self._columns) + self._hidden_column_count,
98
+ )
@@ -6,7 +6,7 @@ import platform
6
6
  import sys
7
7
  from dataclasses import dataclass
8
8
  from decimal import Decimal, InvalidOperation
9
- from typing import Any, TextIO
9
+ from typing import Any, Callable, TextIO
10
10
 
11
11
  import requests
12
12
  import requests.exceptions
@@ -24,7 +24,6 @@ from fivetran_cli.http.exceptions import (
24
24
  raise_for_status,
25
25
  )
26
26
 
27
-
28
27
  # A single --filter predicate: (dotted_path, operator, raw_value). raw_value
29
28
  # is the unparsed CLI string, typed later against the actual field's JSON type.
30
29
  FilterCondition = tuple[str, str, str]
@@ -47,7 +46,7 @@ class _ProgressReporter:
47
46
  self.enabled: bool = self._stream.isatty()
48
47
  self.width: int = width
49
48
  self._line_active: bool = False
50
- #If filter flag is provided, send progress label as matched else fetched
49
+ # If filter flag is provided, send progress label as matched else fetched
51
50
  self._item_label: str = "matched" if filtered else "fetched"
52
51
 
53
52
  def _write(self, message: str, end: str = "") -> None:
@@ -142,6 +141,8 @@ class FivetranClient:
142
141
  payload: Any = None,
143
142
  limit: int | None = None,
144
143
  filters: list[FilterCondition] | None = None,
144
+ on_page: Callable[[Any, int], None] | None = None,
145
+ output_to_file: bool = False,
145
146
  ) -> Any:
146
147
  """Follow cursor pagination and merge all pages into one response.
147
148
 
@@ -152,6 +153,13 @@ class FivetranClient:
152
153
  filters: --filter predicates; every predicate must match for a
153
154
  row to be kept. Applied per page, before limit
154
155
  counting/slicing. If None or empty, no filtering occurs.
156
+ on_page: Called as on_page(page, kept_item_count) once per page, after
157
+ filtering and merging, so output can be rendered while
158
+ later pages are still being fetched. kept_item_count is
159
+ how many of the page's items count toward --limit.
160
+ output_to_file: True when the result is going to an --output-file. A
161
+ failed run never writes the file, so failure messages
162
+ then leave out the count of items fetched so far.
155
163
 
156
164
  If a page request itself fails (network error, or a 4xx/5xx from the
157
165
  API) or a fetched page cannot be merged (unexpected response shape),
@@ -173,14 +181,20 @@ class FivetranClient:
173
181
 
174
182
  page_query = _build_page_query(query, cursor)
175
183
  page = self._fetch_page_or_raise(
176
- method, path, page_query, payload, page_number, total_items, merged,
184
+ method, path, page_query, payload, page_number, total_items, merged, output_to_file,
177
185
  )
178
186
 
179
187
  remaining = None if limit is None else limit - total_items
180
- page = _filter_page_or_raise(page, filters, merged, page_number, total_items)
181
- merged = _merge_page_or_raise(merged, page, remaining, page_number, total_items, filtered=bool(filters))
188
+ page = _filter_page_or_raise(page, filters, merged, page_number, total_items, output_to_file)
189
+ merged = _merge_page_or_raise(
190
+ merged, page, remaining, page_number, total_items, bool(filters), output_to_file,
191
+ )
182
192
 
183
- total_items += _kept_item_count(page, remaining)
193
+ kept_item_count = _kept_item_count(page, remaining)
194
+ total_items += kept_item_count
195
+ if on_page is not None:
196
+ progress.clear()
197
+ on_page(page, kept_item_count)
184
198
  progress.page_done(page_number, total_items)
185
199
  next_cursor = _next_pagination_cursor(_next_cursor(page), limit, total_items)
186
200
  if next_cursor is None:
@@ -189,7 +203,9 @@ class FivetranClient:
189
203
 
190
204
  if next_cursor in seen_cursors:
191
205
  raise FivetranPartialResultError(
192
- _repeated_cursor_message(page_number, total_items, filtered=bool(filters)),
206
+ _repeated_cursor_message(
207
+ page_number, total_items, filtered=bool(filters), output_to_file=output_to_file,
208
+ ),
193
209
  partial_data=_redact_stale_success_message(merged),
194
210
  )
195
211
  seen_cursors.add(next_cursor)
@@ -200,7 +216,7 @@ class FivetranClient:
200
216
  except Exception as exc:
201
217
  progress.clear()
202
218
  raise FivetranPartialResultError(
203
- _unexpected_error_message(page_number, total_items, exc),
219
+ _unexpected_error_message(page_number, total_items, exc, output_to_file),
204
220
  partial_data=_redact_stale_success_message(merged),
205
221
  ) from exc
206
222
 
@@ -215,6 +231,7 @@ class FivetranClient:
215
231
  page_number: int,
216
232
  total_items: int,
217
233
  merged: Any,
234
+ output_to_file: bool,
218
235
  ) -> Any:
219
236
  """Fetch one page. If `self.request` raises any exception other than a
220
237
  FivetranPartialResultError (which is re-raised unchanged), it is wrapped into a
@@ -226,7 +243,7 @@ class FivetranClient:
226
243
  raise
227
244
  except Exception as exc:
228
245
  raise FivetranPartialResultError(
229
- _fetch_failure_message(page_number, total_items, exc),
246
+ _fetch_failure_message(page_number, total_items, exc, output_to_file),
230
247
  partial_data=_redact_stale_success_message(merged),
231
248
  ) from exc
232
249
 
@@ -237,6 +254,7 @@ def _filter_page_or_raise(
237
254
  merged: Any,
238
255
  page_number: int,
239
256
  total_items: int,
257
+ output_to_file: bool,
240
258
  ) -> Any:
241
259
  """Filter a fetched page, converting a malformed-page error into a
242
260
  FivetranPartialResultError that preserves whatever was already merged."""
@@ -244,7 +262,7 @@ def _filter_page_or_raise(
244
262
  return _filter_page_items(page, filters)
245
263
  except _MergeError as exc:
246
264
  raise FivetranPartialResultError(
247
- _merge_failure_message(page_number, total_items, exc, filtered=bool(filters)),
265
+ _merge_failure_message(page_number, total_items, exc, bool(filters), output_to_file),
248
266
  partial_data=_redact_stale_success_message(merged),
249
267
  ) from exc
250
268
 
@@ -256,6 +274,7 @@ def _merge_page_or_raise(
256
274
  page_number: int,
257
275
  total_items: int,
258
276
  filtered: bool,
277
+ output_to_file: bool,
259
278
  ) -> Any:
260
279
  """Merge a fetched page, converting a malformed-page error into a
261
280
  FivetranPartialResultError that preserves whatever was already merged."""
@@ -263,7 +282,7 @@ def _merge_page_or_raise(
263
282
  return _merge_page(merged, page, remaining=remaining)
264
283
  except _MergeError as exc:
265
284
  raise FivetranPartialResultError(
266
- _merge_failure_message(page_number, total_items, exc, filtered=filtered),
285
+ _merge_failure_message(page_number, total_items, exc, filtered, output_to_file),
267
286
  partial_data=_redact_stale_success_message(merged),
268
287
  ) from exc
269
288
 
@@ -323,8 +342,8 @@ def _merge_failure_message(
323
342
  page_number: int,
324
343
  total_items: int,
325
344
  exc: "_MergeError",
326
- *,
327
- filtered: bool = False,
345
+ filtered: bool,
346
+ output_to_file: bool,
328
347
  ) -> str:
329
348
  """Describe where pagination stopped and how much data was preserved."""
330
349
  if page_number == 1:
@@ -332,38 +351,41 @@ def _merge_failure_message(
332
351
  label = "matched" if filtered else "fetched"
333
352
  return (
334
353
  f"Stopped fetching after page {page_number - 1} — unexpected response "
335
- f"on page {page_number}: {exc}. Showing {total_items} item(s) {label} so far."
354
+ f"on page {page_number}: {exc}.{_item_count_note(total_items, label, output_to_file)}"
336
355
  )
337
356
 
338
357
 
339
- def _fetch_failure_message(page_number: int, total_items: int, exc: Exception) -> str:
358
+ def _fetch_failure_message(page_number: int, total_items: int, exc: Exception, output_to_file: bool) -> str:
340
359
  """Describe where pagination stopped and how much data was preserved, when
341
360
  fetching a page itself failed."""
342
361
  if page_number == 1:
343
362
  return f"Error while fetching page 1: {exc}"
344
363
  return (
345
364
  f"Stopped fetching after page {page_number - 1} — error while fetching "
346
- f"page {page_number}: {exc}. Showing {total_items} item(s) fetched so far."
365
+ f"page {page_number}: {exc}.{_item_count_note(total_items, 'fetched', output_to_file)}"
347
366
  )
348
367
 
349
368
 
350
- def _unexpected_error_message(page_number: int, total_items: int, exc: Exception) -> str:
369
+ def _unexpected_error_message(page_number: int, total_items: int, exc: Exception, output_to_file: bool) -> str:
351
370
  """Describe an unexpected failure during pagination processing."""
352
- return (
353
- f"Unexpected error during pagination processing on page {page_number}: "
354
- f"{exc}. Preserved {total_items} item(s) fetched so far."
355
- )
371
+ preserved_note = "" if output_to_file else f" Preserved {total_items} item(s) fetched so far."
372
+ return f"Unexpected error during pagination processing on page {page_number}: {exc}.{preserved_note}"
356
373
 
357
374
 
358
- def _repeated_cursor_message(page_number: int, total_items: int, *, filtered: bool = False) -> str:
375
+ def _repeated_cursor_message(page_number: int, total_items: int, *, filtered: bool, output_to_file: bool) -> str:
359
376
  """Describe a repeated cursor that would cause a pagination loop."""
360
377
  label = "matched" if filtered else "fetched"
361
378
  return (
362
379
  f"API returned a cursor it had already used earlier in this run (page {page_number}) — "
363
- f"aborting to avoid an infinite loop. Showing {total_items} item(s) {label} so far."
380
+ f"aborting to avoid an infinite loop.{_item_count_note(total_items, label, output_to_file)}"
364
381
  )
365
382
 
366
383
 
384
+ def _item_count_note(total_items: int, label: str, output_to_file: bool) -> str:
385
+ """The trailing sentence of a pagination failure message, or nothing."""
386
+ return "" if output_to_file else f" Showing {total_items} item(s) {label} so far."
387
+
388
+
367
389
  class _MergeError(Exception):
368
390
  """Raised when a paginated page cannot be merged due to an unexpected response shape."""
369
391
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fivetran_cli
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: Command line wrapper for the Fivetran REST API and Connector SDK
5
5
  Author-email: Fivetran <developers@fivetran.com>
6
6
  Project-URL: Homepage, https://fivetran.com
@@ -8,7 +8,7 @@ Project-URL: Github, https://github.com/fivetran/engineering
8
8
  Classifier: Programming Language :: Python :: 3
9
9
  Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.10
11
+ Requires-Python: >=3.11
12
12
  Description-Content-Type: text/markdown
13
13
  Requires-Dist: rich>=13.0
14
14
  Requires-Dist: requests==2.34.2
@@ -30,7 +30,7 @@ Verify the installation:
30
30
 
31
31
  ## Requirements
32
32
 
33
- - Python 3.10 or later
33
+ - Python 3.11 or later
34
34
  - A Fivetran account and API key for commands that communicate with Fivetran
35
35
 
36
36
  ## Authentication
@@ -12,6 +12,7 @@ src/fivetran_cli.egg-info/dependency_links.txt
12
12
  src/fivetran_cli.egg-info/entry_points.txt
13
13
  src/fivetran_cli.egg-info/requires.txt
14
14
  src/fivetran_cli.egg-info/top_level.txt
15
+ src/fivetran_cli/cli_utils/__init__.py
15
16
  src/fivetran_cli/cli_utils/args.py
16
17
  src/fivetran_cli/cli_utils/dispatch/__init__.py
17
18
  src/fivetran_cli/cli_utils/dispatch/base.py
@@ -41,10 +42,12 @@ src/fivetran_cli/cli_utils/resources/transformation.py
41
42
  src/fivetran_cli/cli_utils/resources/transformation_project.py
42
43
  src/fivetran_cli/cli_utils/resources/user.py
43
44
  src/fivetran_cli/cli_utils/resources/webhook.py
45
+ src/fivetran_cli/display/__init__.py
44
46
  src/fivetran_cli/display/_formatters.py
45
47
  src/fivetran_cli/display/_renderers.py
46
48
  src/fivetran_cli/display/error_output.py
47
49
  src/fivetran_cli/display/output.py
50
+ src/fivetran_cli/display/page_stream.py
48
51
  src/fivetran_cli/display/views.py
49
52
  src/fivetran_cli/http/__init__.py
50
53
  src/fivetran_cli/http/client.py
File without changes