spice-mcp 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of spice-mcp might be problematic. Click here for more details.
- spice_mcp/adapters/dune/__init__.py +4 -2
- spice_mcp/adapters/dune/cache.py +2 -34
- spice_mcp/adapters/dune/client.py +9 -4
- spice_mcp/adapters/dune/extract.py +79 -633
- spice_mcp/adapters/dune/query_wrapper.py +86 -0
- spice_mcp/adapters/dune/user_agent.py +9 -0
- spice_mcp/adapters/spellbook/__init__.py +6 -0
- spice_mcp/adapters/spellbook/explorer.py +313 -0
- spice_mcp/config.py +1 -1
- spice_mcp/core/models.py +0 -8
- spice_mcp/core/ports.py +0 -15
- spice_mcp/mcp/server.py +407 -137
- spice_mcp/mcp/tools/base.py +1 -1
- spice_mcp/mcp/tools/execute_query.py +32 -63
- spice_mcp-0.1.4.dist-info/METADATA +121 -0
- {spice_mcp-0.1.2.dist-info → spice_mcp-0.1.4.dist-info}/RECORD +19 -17
- spice_mcp/mcp/tools/sui_package_overview.py +0 -56
- spice_mcp/service_layer/sui_service.py +0 -131
- spice_mcp-0.1.2.dist-info/METADATA +0 -193
- {spice_mcp-0.1.2.dist-info → spice_mcp-0.1.4.dist-info}/WHEEL +0 -0
- {spice_mcp-0.1.2.dist-info → spice_mcp-0.1.4.dist-info}/entry_points.txt +0 -0
- {spice_mcp-0.1.2.dist-info → spice_mcp-0.1.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
|
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
# ---------------------------------------------------------------------------
|
|
48
|
-
#
|
|
48
|
+
# Internal helpers used by adapter code and tests
|
|
49
49
|
|
|
50
50
|
def _is_sql(query: int | str) -> bool:
|
|
51
51
|
if isinstance(query, int):
|
|
@@ -289,6 +289,14 @@ def query(
|
|
|
289
289
|
|
|
290
290
|
# execute or retrieve query
|
|
291
291
|
if query_id:
|
|
292
|
+
# Check if this is a parameterized query (raw SQL via template or parameterized query)
|
|
293
|
+
# For parameterized queries, results don't exist until execution, so skip GET attempt
|
|
294
|
+
is_parameterized = (
|
|
295
|
+
parameters is not None
|
|
296
|
+
and len(parameters) > 0
|
|
297
|
+
and ('query' in parameters or any(k != 'query' for k in parameters))
|
|
298
|
+
)
|
|
299
|
+
|
|
292
300
|
if cache and load_from_cache and not refresh:
|
|
293
301
|
cache_result, cache_execution = _cache.load_from_cache(
|
|
294
302
|
execute_kwargs, result_kwargs, output_kwargs
|
|
@@ -301,15 +309,29 @@ def query(
|
|
|
301
309
|
age = get_query_latest_age(**execute_kwargs, verbose=verbose) # type: ignore
|
|
302
310
|
if age is None or age > max_age:
|
|
303
311
|
refresh = True
|
|
304
|
-
|
|
312
|
+
# Skip GET results attempt for parameterized queries - they need execution first
|
|
313
|
+
if not refresh and not is_parameterized:
|
|
305
314
|
df = get_results(**execute_kwargs, **result_kwargs)
|
|
306
315
|
if df is not None:
|
|
307
316
|
return process_result(df, execution, **output_kwargs)
|
|
308
|
-
|
|
317
|
+
try:
|
|
318
|
+
execution = execute_query(**execute_kwargs, verbose=verbose)
|
|
319
|
+
except Exception as e:
|
|
320
|
+
# Re-raise with more context about the failure
|
|
321
|
+
if verbose:
|
|
322
|
+
print(f'execute_query failed for query_id={query_id}, parameters={parameters}')
|
|
323
|
+
raise Exception(f'failed to execute query {query_id}: {e}') from e
|
|
324
|
+
else:
|
|
325
|
+
# query_id is None or falsy - this shouldn't happen for valid inputs
|
|
326
|
+
if verbose:
|
|
327
|
+
print(f'query_id is falsy: {query_id}, query_or_execution={query_or_execution}')
|
|
309
328
|
|
|
310
|
-
#
|
|
329
|
+
# check execution status
|
|
311
330
|
if execution is None:
|
|
312
|
-
|
|
331
|
+
error_detail = f'query_id={query_id}, query_type={type(query_or_execution).__name__}'
|
|
332
|
+
if isinstance(query_or_execution, str):
|
|
333
|
+
error_detail += f', query_preview={query_or_execution[:100]}'
|
|
334
|
+
raise Exception(f'could not determine execution ({error_detail})')
|
|
313
335
|
if poll:
|
|
314
336
|
poll_execution(execution, **poll_kwargs)
|
|
315
337
|
df = get_results(execution, api_key, **result_kwargs)
|
|
@@ -321,205 +343,42 @@ def query(
|
|
|
321
343
|
return execution
|
|
322
344
|
|
|
323
345
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
*,
|
|
361
|
-
verbose: bool = True,
|
|
362
|
-
refresh: bool = False,
|
|
363
|
-
max_age: float | None = None,
|
|
364
|
-
parameters: Mapping[str, Any] | None = None,
|
|
365
|
-
api_key: str | None = None,
|
|
366
|
-
performance: Performance = 'medium',
|
|
367
|
-
poll: Literal[True] = True,
|
|
368
|
-
poll_interval: float = 1.0,
|
|
369
|
-
limit: int | None = None,
|
|
370
|
-
offset: int | None = None,
|
|
371
|
-
sample_count: int | None = None,
|
|
372
|
-
sort_by: str | None = None,
|
|
373
|
-
columns: Sequence[str] | None = None,
|
|
374
|
-
extras: Mapping[str, Any] | None = None,
|
|
375
|
-
types: Sequence[type[pl.DataType]]
|
|
376
|
-
| Mapping[str, type[pl.DataType]]
|
|
377
|
-
| None = None,
|
|
378
|
-
all_types: Sequence[type[pl.DataType]]
|
|
379
|
-
| Mapping[str, type[pl.DataType]]
|
|
380
|
-
| None = None,
|
|
381
|
-
cache: bool = True,
|
|
382
|
-
cache_dir: str | None = None,
|
|
383
|
-
save_to_cache: bool = True,
|
|
384
|
-
load_from_cache: bool = True,
|
|
385
|
-
include_execution: Literal[False] = False,
|
|
386
|
-
) -> pl.DataFrame: ...
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
@overload
|
|
390
|
-
async def async_query(
|
|
391
|
-
query_or_execution: Query | Execution,
|
|
392
|
-
*,
|
|
393
|
-
verbose: bool = True,
|
|
394
|
-
refresh: bool = False,
|
|
395
|
-
max_age: float | None = None,
|
|
396
|
-
parameters: Mapping[str, Any] | None = None,
|
|
397
|
-
api_key: str | None = None,
|
|
398
|
-
performance: Performance = 'medium',
|
|
399
|
-
poll: Literal[True] = True,
|
|
400
|
-
poll_interval: float = 1.0,
|
|
401
|
-
limit: int | None = None,
|
|
402
|
-
offset: int | None = None,
|
|
403
|
-
sample_count: int | None = None,
|
|
404
|
-
sort_by: str | None = None,
|
|
405
|
-
columns: Sequence[str] | None = None,
|
|
406
|
-
extras: Mapping[str, Any] | None = None,
|
|
407
|
-
types: Sequence[type[pl.DataType]]
|
|
408
|
-
| Mapping[str, type[pl.DataType]]
|
|
409
|
-
| None = None,
|
|
410
|
-
all_types: Sequence[type[pl.DataType]]
|
|
411
|
-
| Mapping[str, type[pl.DataType]]
|
|
412
|
-
| None = None,
|
|
413
|
-
cache: bool = True,
|
|
414
|
-
cache_dir: str | None = None,
|
|
415
|
-
save_to_cache: bool = True,
|
|
416
|
-
load_from_cache: bool = True,
|
|
417
|
-
include_execution: Literal[True],
|
|
418
|
-
) -> tuple[pl.DataFrame, Execution]: ...
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
async def async_query(
|
|
422
|
-
query_or_execution: Query | Execution,
|
|
423
|
-
*,
|
|
424
|
-
verbose: bool = True,
|
|
425
|
-
refresh: bool = False,
|
|
426
|
-
max_age: float | None = None,
|
|
427
|
-
parameters: Mapping[str, Any] | None = None,
|
|
428
|
-
api_key: str | None = None,
|
|
429
|
-
performance: Performance = 'medium',
|
|
430
|
-
poll: bool = True,
|
|
431
|
-
poll_interval: float = 1.0,
|
|
432
|
-
timeout_seconds: float | None = None,
|
|
433
|
-
limit: int | None = None,
|
|
434
|
-
offset: int | None = None,
|
|
435
|
-
sample_count: int | None = None,
|
|
436
|
-
sort_by: str | None = None,
|
|
437
|
-
columns: Sequence[str] | None = None,
|
|
438
|
-
extras: Mapping[str, Any] | None = None,
|
|
439
|
-
types: Sequence[type[pl.DataType]]
|
|
440
|
-
| Mapping[str, type[pl.DataType]]
|
|
441
|
-
| None = None,
|
|
442
|
-
all_types: Sequence[type[pl.DataType]]
|
|
443
|
-
| Mapping[str, type[pl.DataType]]
|
|
444
|
-
| None = None,
|
|
445
|
-
cache: bool = True,
|
|
446
|
-
cache_dir: str | None = None,
|
|
447
|
-
save_to_cache: bool = True,
|
|
448
|
-
load_from_cache: bool = True,
|
|
449
|
-
include_execution: bool = False,
|
|
450
|
-
):
|
|
451
|
-
|
|
452
|
-
# determine whether target is a query or an execution
|
|
453
|
-
query_id, execution, parameters = _determine_input_type(
|
|
454
|
-
query_or_execution,
|
|
455
|
-
parameters,
|
|
456
|
-
)
|
|
457
|
-
|
|
458
|
-
# gather arguments
|
|
459
|
-
execute_kwargs: ExecuteKwargs = {
|
|
460
|
-
'query_id': query_id,
|
|
461
|
-
'api_key': api_key,
|
|
462
|
-
'parameters': parameters,
|
|
463
|
-
'performance': performance,
|
|
464
|
-
}
|
|
465
|
-
poll_kwargs: PollKwargs = {
|
|
466
|
-
'poll_interval': poll_interval,
|
|
467
|
-
'api_key': api_key,
|
|
468
|
-
'verbose': verbose,
|
|
469
|
-
'timeout_seconds': timeout_seconds,
|
|
470
|
-
}
|
|
471
|
-
result_kwargs: RetrievalKwargs = {
|
|
472
|
-
'limit': limit,
|
|
473
|
-
'offset': offset,
|
|
474
|
-
'sample_count': sample_count,
|
|
475
|
-
'sort_by': sort_by,
|
|
476
|
-
'columns': columns,
|
|
477
|
-
'extras': extras,
|
|
478
|
-
'types': types,
|
|
479
|
-
'all_types': all_types,
|
|
480
|
-
'verbose': verbose,
|
|
481
|
-
}
|
|
482
|
-
output_kwargs: OutputKwargs = {
|
|
483
|
-
'execute_kwargs': execute_kwargs,
|
|
484
|
-
'result_kwargs': result_kwargs,
|
|
485
|
-
'cache': cache,
|
|
486
|
-
'save_to_cache': save_to_cache,
|
|
487
|
-
'cache_dir': cache_dir,
|
|
488
|
-
'include_execution': include_execution,
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
# execute or retrieve query
|
|
492
|
-
if query_id:
|
|
493
|
-
if cache and cache_dir is not None and not refresh:
|
|
494
|
-
cache_result, cache_execution = await _cache.async_load_from_cache(
|
|
495
|
-
execute_kwargs, result_kwargs, output_kwargs
|
|
496
|
-
)
|
|
497
|
-
if cache_result is not None:
|
|
498
|
-
return cache_result
|
|
499
|
-
if execution is None and cache_execution is not None:
|
|
500
|
-
execution = cache_execution
|
|
501
|
-
if max_age is not None and not refresh:
|
|
502
|
-
age = await async_get_query_latest_age(**execute_kwargs, verbose=verbose) # type: ignore
|
|
503
|
-
if age is None or age > max_age:
|
|
504
|
-
refresh = True
|
|
505
|
-
if not refresh:
|
|
506
|
-
df = await async_get_results(**execute_kwargs, **result_kwargs)
|
|
507
|
-
if df is not None:
|
|
508
|
-
return await async_process_result(df, execution, **output_kwargs)
|
|
509
|
-
execution = await async_execute_query(**execute_kwargs, verbose=verbose)
|
|
510
|
-
|
|
511
|
-
# await execution completion
|
|
512
|
-
if execution is None:
|
|
513
|
-
raise Exception('could not determine execution')
|
|
514
|
-
if poll:
|
|
515
|
-
await async_poll_execution(execution, **poll_kwargs)
|
|
516
|
-
df = await async_get_results(execution, api_key, **result_kwargs)
|
|
517
|
-
if df is not None:
|
|
518
|
-
return await async_process_result(df, execution, **output_kwargs)
|
|
519
|
-
else:
|
|
520
|
-
raise Exception('no successful execution for query')
|
|
521
|
-
else:
|
|
522
|
-
return execution
|
|
346
|
+
if TYPE_CHECKING:
|
|
347
|
+
@overload
|
|
348
|
+
def _process_result(
|
|
349
|
+
df: pl.DataFrame,
|
|
350
|
+
execution: Execution | None,
|
|
351
|
+
execute_kwargs: ExecuteKwargs,
|
|
352
|
+
result_kwargs: RetrievalKwargs,
|
|
353
|
+
cache: bool,
|
|
354
|
+
save_to_cache: bool,
|
|
355
|
+
cache_dir: str | None,
|
|
356
|
+
include_execution: Literal[False],
|
|
357
|
+
) -> pl.DataFrame: ...
|
|
358
|
+
|
|
359
|
+
@overload
|
|
360
|
+
def _process_result(
|
|
361
|
+
df: pl.DataFrame,
|
|
362
|
+
execution: Execution | None,
|
|
363
|
+
execute_kwargs: ExecuteKwargs,
|
|
364
|
+
result_kwargs: RetrievalKwargs,
|
|
365
|
+
cache: bool,
|
|
366
|
+
save_to_cache: bool,
|
|
367
|
+
cache_dir: str | None,
|
|
368
|
+
include_execution: Literal[True],
|
|
369
|
+
) -> tuple[pl.DataFrame, Execution]: ...
|
|
370
|
+
|
|
371
|
+
@overload
|
|
372
|
+
def _process_result(
|
|
373
|
+
df: pl.DataFrame,
|
|
374
|
+
execution: Execution | None,
|
|
375
|
+
execute_kwargs: ExecuteKwargs,
|
|
376
|
+
result_kwargs: RetrievalKwargs,
|
|
377
|
+
cache: bool,
|
|
378
|
+
save_to_cache: bool,
|
|
379
|
+
cache_dir: str | None,
|
|
380
|
+
include_execution: bool,
|
|
381
|
+
) -> pl.DataFrame | tuple[pl.DataFrame, Execution]: ...
|
|
523
382
|
|
|
524
383
|
|
|
525
384
|
def _process_result(
|
|
@@ -551,35 +410,6 @@ def _process_result(
|
|
|
551
410
|
return df
|
|
552
411
|
|
|
553
412
|
|
|
554
|
-
async def _async_process_result(
|
|
555
|
-
df: pl.DataFrame,
|
|
556
|
-
execution: Execution | None,
|
|
557
|
-
execute_kwargs: ExecuteKwargs,
|
|
558
|
-
result_kwargs: RetrievalKwargs,
|
|
559
|
-
cache: bool,
|
|
560
|
-
save_to_cache: bool,
|
|
561
|
-
cache_dir: str | None,
|
|
562
|
-
include_execution: bool,
|
|
563
|
-
) -> pl.DataFrame | tuple[pl.DataFrame, Execution]:
|
|
564
|
-
if cache and save_to_cache and execute_kwargs['query_id'] is not None:
|
|
565
|
-
if execution is None:
|
|
566
|
-
execution = await async_get_latest_execution(execute_kwargs)
|
|
567
|
-
if execution is None:
|
|
568
|
-
raise Exception('could not get execution')
|
|
569
|
-
_cache.save_to_cache(
|
|
570
|
-
df, execution, execute_kwargs, result_kwargs, cache_dir
|
|
571
|
-
)
|
|
572
|
-
|
|
573
|
-
if include_execution:
|
|
574
|
-
if execution is None:
|
|
575
|
-
execution = await async_get_latest_execution(execute_kwargs)
|
|
576
|
-
if execution is None:
|
|
577
|
-
raise Exception('could not get execution')
|
|
578
|
-
return df, execution
|
|
579
|
-
else:
|
|
580
|
-
return df
|
|
581
|
-
|
|
582
|
-
|
|
583
413
|
def _get_query_latest_age(
|
|
584
414
|
query_id: int,
|
|
585
415
|
*,
|
|
@@ -654,83 +484,6 @@ def _parse_timestamp(timestamp: str) -> int:
|
|
|
654
484
|
return int(timestamp_float)
|
|
655
485
|
|
|
656
486
|
|
|
657
|
-
async def _async_get_query_latest_age(
|
|
658
|
-
query_id: int,
|
|
659
|
-
*,
|
|
660
|
-
verbose: bool = True,
|
|
661
|
-
parameters: Mapping[str, Any] | None = None,
|
|
662
|
-
performance: Performance = 'medium',
|
|
663
|
-
api_key: str | None = None,
|
|
664
|
-
) -> float | None:
|
|
665
|
-
import datetime
|
|
666
|
-
import json
|
|
667
|
-
|
|
668
|
-
import aiohttp
|
|
669
|
-
|
|
670
|
-
# process inputs
|
|
671
|
-
if api_key is None:
|
|
672
|
-
api_key = _urls.get_api_key()
|
|
673
|
-
headers = {'X-Dune-API-Key': api_key, 'User-Agent': get_user_agent()}
|
|
674
|
-
data = {}
|
|
675
|
-
if parameters is not None:
|
|
676
|
-
data['query_parameters'] = parameters
|
|
677
|
-
url = _urls.get_query_results_url(query_id, parameters=data, csv=False)
|
|
678
|
-
|
|
679
|
-
# print summary
|
|
680
|
-
if verbose:
|
|
681
|
-
print('checking age of last execution, query_id = ' + str(query_id))
|
|
682
|
-
|
|
683
|
-
# perform request with retry/backoff for 429/502
|
|
684
|
-
timeout = aiohttp.ClientTimeout(total=30)
|
|
685
|
-
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
686
|
-
attempts = 0
|
|
687
|
-
backoff = 0.5
|
|
688
|
-
while True:
|
|
689
|
-
async with session.get(url, headers=headers) as response:
|
|
690
|
-
if response.status in (429, 502):
|
|
691
|
-
attempts += 1
|
|
692
|
-
if attempts >= 3:
|
|
693
|
-
break
|
|
694
|
-
import asyncio
|
|
695
|
-
import random
|
|
696
|
-
await asyncio.sleep(backoff * random.uniform(1.5, 2.5))
|
|
697
|
-
backoff = min(5.0, backoff * 2)
|
|
698
|
-
continue
|
|
699
|
-
result: Mapping[str, Any] = await response.json()
|
|
700
|
-
break
|
|
701
|
-
|
|
702
|
-
# check if result is error
|
|
703
|
-
try:
|
|
704
|
-
if 'error' in result:
|
|
705
|
-
if (
|
|
706
|
-
result['error']
|
|
707
|
-
== 'not found: No execution found for the latest version of the given query'
|
|
708
|
-
):
|
|
709
|
-
if verbose:
|
|
710
|
-
print(
|
|
711
|
-
'no age for query, because no previous executions exist'
|
|
712
|
-
)
|
|
713
|
-
return None
|
|
714
|
-
raise Exception(result['error'])
|
|
715
|
-
except json.JSONDecodeError:
|
|
716
|
-
pass
|
|
717
|
-
|
|
718
|
-
# process result
|
|
719
|
-
if 'execution_started_at' in result:
|
|
720
|
-
now = datetime.datetime.now(datetime.UTC).timestamp()
|
|
721
|
-
started = _parse_timestamp(result['execution_started_at'])
|
|
722
|
-
age = now - started
|
|
723
|
-
|
|
724
|
-
if verbose:
|
|
725
|
-
print('latest result age:', age)
|
|
726
|
-
|
|
727
|
-
return age
|
|
728
|
-
else:
|
|
729
|
-
if verbose:
|
|
730
|
-
print('no age for query, because no previous executions exist')
|
|
731
|
-
return None
|
|
732
|
-
|
|
733
|
-
|
|
734
487
|
def _execute(
|
|
735
488
|
query_id: int | str,
|
|
736
489
|
*,
|
|
@@ -755,50 +508,25 @@ def _execute(
|
|
|
755
508
|
|
|
756
509
|
# perform request
|
|
757
510
|
response = _transport_post(url, headers=headers, json=data, timeout=_POST_TIMEOUT)
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
async def _async_execute(
|
|
770
|
-
query_id: int | str,
|
|
771
|
-
*,
|
|
772
|
-
parameters: Mapping[str, Any] | None = None,
|
|
773
|
-
performance: Performance = 'medium',
|
|
774
|
-
api_key: str | None = None,
|
|
775
|
-
verbose: bool = True,
|
|
776
|
-
) -> Execution:
|
|
777
|
-
import aiohttp
|
|
778
|
-
|
|
779
|
-
# process inputs
|
|
780
|
-
url = _urls.get_query_execute_url(query_id)
|
|
781
|
-
if api_key is None:
|
|
782
|
-
api_key = _urls.get_api_key()
|
|
783
|
-
headers = {'X-Dune-API-Key': api_key, 'User-Agent': get_user_agent()}
|
|
784
|
-
data = {}
|
|
785
|
-
if parameters is not None:
|
|
786
|
-
data['query_parameters'] = parameters
|
|
787
|
-
data['performance'] = performance
|
|
788
|
-
|
|
789
|
-
# print summary
|
|
790
|
-
if verbose:
|
|
791
|
-
print('executing query, query_id = ' + str(query_id))
|
|
792
|
-
|
|
793
|
-
# perform request
|
|
794
|
-
timeout = aiohttp.ClientTimeout(total=_POST_TIMEOUT)
|
|
795
|
-
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
796
|
-
async with session.post(url, headers=headers, json=data) as response:
|
|
797
|
-
result: Mapping[str, Any] = await response.json()
|
|
511
|
+
|
|
512
|
+
# Parse response with better error handling
|
|
513
|
+
try:
|
|
514
|
+
result: Mapping[str, Any] = response.json()
|
|
515
|
+
except Exception as e:
|
|
516
|
+
if verbose:
|
|
517
|
+
print(f'failed to parse response JSON: {e}')
|
|
518
|
+
print(f'response status: {response.status_code}')
|
|
519
|
+
print(f'response text: {response.text[:500]}')
|
|
520
|
+
raise Exception(f'failed to parse response: {e}') from e
|
|
798
521
|
|
|
799
522
|
# check for errors
|
|
800
523
|
if 'execution_id' not in result:
|
|
801
|
-
|
|
524
|
+
error_msg = result.get('error', f'response missing execution_id: {result}')
|
|
525
|
+
if verbose:
|
|
526
|
+
print(f'execution failed: {error_msg}')
|
|
527
|
+
print(f'response status: {response.status_code}')
|
|
528
|
+
print(f'full response: {result}')
|
|
529
|
+
raise Exception(error_msg)
|
|
802
530
|
|
|
803
531
|
# process result
|
|
804
532
|
execution_id = result['execution_id']
|
|
@@ -912,124 +640,6 @@ def _get_results(
|
|
|
912
640
|
return df
|
|
913
641
|
|
|
914
642
|
|
|
915
|
-
async def _async_get_results(
|
|
916
|
-
execution: Execution | None = None,
|
|
917
|
-
api_key: str | None = None,
|
|
918
|
-
*,
|
|
919
|
-
query_id: int | None = None,
|
|
920
|
-
parameters: Mapping[str, Any] | None = None,
|
|
921
|
-
performance: Performance = 'medium',
|
|
922
|
-
limit: int | None = None,
|
|
923
|
-
offset: int | None = None,
|
|
924
|
-
sample_count: int | None = None,
|
|
925
|
-
sort_by: str | None = None,
|
|
926
|
-
columns: Sequence[str] | None = None,
|
|
927
|
-
extras: Mapping[str, Any] | None = None,
|
|
928
|
-
types: Sequence[type[pl.DataType]]
|
|
929
|
-
| Mapping[str, type[pl.DataType]]
|
|
930
|
-
| None = None,
|
|
931
|
-
all_types: Sequence[type[pl.DataType]]
|
|
932
|
-
| Mapping[str, type[pl.DataType]]
|
|
933
|
-
| None = None,
|
|
934
|
-
verbose: bool = True,
|
|
935
|
-
) -> pl.DataFrame | None:
|
|
936
|
-
import asyncio
|
|
937
|
-
import random
|
|
938
|
-
|
|
939
|
-
import aiohttp
|
|
940
|
-
import polars as pl
|
|
941
|
-
|
|
942
|
-
if api_key is None:
|
|
943
|
-
api_key = _urls.get_api_key()
|
|
944
|
-
headers = {'X-Dune-API-Key': api_key, 'User-Agent': get_user_agent()}
|
|
945
|
-
params: dict[str, Any] = {
|
|
946
|
-
'limit': limit,
|
|
947
|
-
'offset': offset,
|
|
948
|
-
'sample_count': sample_count,
|
|
949
|
-
'sort_by': sort_by,
|
|
950
|
-
'columns': columns,
|
|
951
|
-
}
|
|
952
|
-
if extras is not None:
|
|
953
|
-
params.update(extras)
|
|
954
|
-
if parameters is not None:
|
|
955
|
-
params['query_parameters'] = parameters
|
|
956
|
-
if query_id is not None:
|
|
957
|
-
url = _urls.get_query_results_url(query_id, parameters=params)
|
|
958
|
-
elif execution is not None:
|
|
959
|
-
url = _urls.get_execution_results_url(execution['execution_id'], params)
|
|
960
|
-
else:
|
|
961
|
-
raise Exception('must specify query_id or execution')
|
|
962
|
-
|
|
963
|
-
# print summary
|
|
964
|
-
if verbose:
|
|
965
|
-
if query_id is not None:
|
|
966
|
-
print('getting results, query_id = ' + str(query_id))
|
|
967
|
-
elif execution is not None:
|
|
968
|
-
print('getting results, execution_id = ' + str(execution['execution_id']))
|
|
969
|
-
|
|
970
|
-
# perform request
|
|
971
|
-
timeout = aiohttp.ClientTimeout(total=_GET_TIMEOUT)
|
|
972
|
-
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
973
|
-
# GET with simple retry/backoff for 429/502
|
|
974
|
-
attempts = 0
|
|
975
|
-
backoff = 0.5
|
|
976
|
-
while True:
|
|
977
|
-
async with session.get(url, headers=headers) as response:
|
|
978
|
-
if response.status in (429, 502):
|
|
979
|
-
attempts += 1
|
|
980
|
-
if attempts >= 3:
|
|
981
|
-
break
|
|
982
|
-
await asyncio.sleep(backoff * random.uniform(1.5, 2.5))
|
|
983
|
-
backoff = min(5.0, backoff * 2)
|
|
984
|
-
continue
|
|
985
|
-
if response.status == 404:
|
|
986
|
-
return None
|
|
987
|
-
result = await response.text()
|
|
988
|
-
response_headers = response.headers
|
|
989
|
-
break
|
|
990
|
-
|
|
991
|
-
# process result
|
|
992
|
-
df = _process_raw_table(result, types=types, all_types=all_types)
|
|
993
|
-
|
|
994
|
-
# support pagination when using limit
|
|
995
|
-
if limit is not None:
|
|
996
|
-
import polars as pl
|
|
997
|
-
|
|
998
|
-
n_rows = len(df)
|
|
999
|
-
pages = []
|
|
1000
|
-
timeout = aiohttp.ClientTimeout(total=30)
|
|
1001
|
-
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
1002
|
-
while 'x-dune-next-uri' in response_headers and n_rows < limit:
|
|
1003
|
-
if verbose:
|
|
1004
|
-
off = response.headers.get('x-dune-next-offset', 'unknown')
|
|
1005
|
-
print('gathering additional page, offset = ' + str(off))
|
|
1006
|
-
next_url = response_headers['x-dune-next-uri']
|
|
1007
|
-
# Pager GET with retry/backoff
|
|
1008
|
-
attempts = 0
|
|
1009
|
-
backoff = 0.5
|
|
1010
|
-
while True:
|
|
1011
|
-
async with session.get(next_url, headers=headers) as response:
|
|
1012
|
-
if response.status in (429, 502):
|
|
1013
|
-
attempts += 1
|
|
1014
|
-
if attempts >= 3:
|
|
1015
|
-
break
|
|
1016
|
-
await asyncio.sleep(backoff * random.uniform(1.5, 2.5))
|
|
1017
|
-
backoff = min(5.0, backoff * 2)
|
|
1018
|
-
continue
|
|
1019
|
-
result = await response.text()
|
|
1020
|
-
response_headers = response.headers
|
|
1021
|
-
break
|
|
1022
|
-
page = _process_raw_table(
|
|
1023
|
-
result, types=types, all_types=all_types
|
|
1024
|
-
)
|
|
1025
|
-
n_rows += len(page)
|
|
1026
|
-
pages.append(page)
|
|
1027
|
-
|
|
1028
|
-
df = pl.concat([df, *pages]).limit(limit)
|
|
1029
|
-
|
|
1030
|
-
return df
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
643
|
def _process_raw_table(
|
|
1034
644
|
raw_csv: str,
|
|
1035
645
|
types: Sequence[type[pl.DataType] | None]
|
|
@@ -1224,97 +834,6 @@ def _poll_execution(
|
|
|
1224
834
|
)
|
|
1225
835
|
|
|
1226
836
|
|
|
1227
|
-
async def _async_poll_execution(
|
|
1228
|
-
execution: Execution,
|
|
1229
|
-
*,
|
|
1230
|
-
api_key: str | None,
|
|
1231
|
-
poll_interval: float,
|
|
1232
|
-
verbose: bool,
|
|
1233
|
-
timeout_seconds: float | None,
|
|
1234
|
-
) -> None:
|
|
1235
|
-
import asyncio
|
|
1236
|
-
import random
|
|
1237
|
-
|
|
1238
|
-
import aiohttp
|
|
1239
|
-
|
|
1240
|
-
# process inputs
|
|
1241
|
-
url = _urls.get_execution_status_url(execution['execution_id'])
|
|
1242
|
-
execution_id = execution['execution_id']
|
|
1243
|
-
if api_key is None:
|
|
1244
|
-
api_key = _urls.get_api_key()
|
|
1245
|
-
headers = {'X-Dune-API-Key': api_key, 'User-Agent': get_user_agent()}
|
|
1246
|
-
|
|
1247
|
-
# print summary
|
|
1248
|
-
t_start = time.time()
|
|
1249
|
-
|
|
1250
|
-
# poll until completion
|
|
1251
|
-
timeout = aiohttp.ClientTimeout(total=_GET_TIMEOUT)
|
|
1252
|
-
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
1253
|
-
sleep_amount = poll_interval
|
|
1254
|
-
while True:
|
|
1255
|
-
t_poll = time.time()
|
|
1256
|
-
|
|
1257
|
-
# print summary
|
|
1258
|
-
if verbose:
|
|
1259
|
-
print(
|
|
1260
|
-
'waiting for results, execution_id = '
|
|
1261
|
-
+ str(execution['execution_id'])
|
|
1262
|
-
+ ', t = '
|
|
1263
|
-
+ str(t_poll - t_start)
|
|
1264
|
-
)
|
|
1265
|
-
|
|
1266
|
-
# poll
|
|
1267
|
-
async with session.get(url, headers=headers) as response:
|
|
1268
|
-
result = await response.json()
|
|
1269
|
-
if (
|
|
1270
|
-
'is_execution_finished' not in result
|
|
1271
|
-
and response.status == 429
|
|
1272
|
-
):
|
|
1273
|
-
sleep_amount = sleep_amount * random.uniform(1, 2)
|
|
1274
|
-
await asyncio.sleep(sleep_amount)
|
|
1275
|
-
continue
|
|
1276
|
-
if result['is_execution_finished']:
|
|
1277
|
-
if result['state'] == 'QUERY_STATE_FAILED':
|
|
1278
|
-
err_detail = ''
|
|
1279
|
-
try:
|
|
1280
|
-
if 'error' in result and result['error']:
|
|
1281
|
-
err_detail = f", error={result['error']}"
|
|
1282
|
-
except Exception:
|
|
1283
|
-
pass
|
|
1284
|
-
raise Exception(
|
|
1285
|
-
f"QUERY FAILED execution_id={execution_id} state={result.get('state')}{err_detail}"
|
|
1286
|
-
)
|
|
1287
|
-
execution['timestamp'] = _parse_timestamp(
|
|
1288
|
-
result['execution_started_at']
|
|
1289
|
-
)
|
|
1290
|
-
break
|
|
1291
|
-
|
|
1292
|
-
# timeout check
|
|
1293
|
-
if timeout_seconds is not None and (t_poll - t_start) > timeout_seconds:
|
|
1294
|
-
raise TimeoutError(
|
|
1295
|
-
f'query polling timed out after {timeout_seconds} seconds'
|
|
1296
|
-
)
|
|
1297
|
-
|
|
1298
|
-
# wait until polling interval
|
|
1299
|
-
t_wait = time.time() - t_poll
|
|
1300
|
-
if t_wait < poll_interval:
|
|
1301
|
-
import asyncio
|
|
1302
|
-
|
|
1303
|
-
await asyncio.sleep(poll_interval - t_wait)
|
|
1304
|
-
|
|
1305
|
-
# check for errors
|
|
1306
|
-
if result['state'] == 'QUERY_STATE_FAILED':
|
|
1307
|
-
err_detail = ''
|
|
1308
|
-
try:
|
|
1309
|
-
if 'error' in result and result['error']:
|
|
1310
|
-
err_detail = f", error={result['error']}"
|
|
1311
|
-
except Exception:
|
|
1312
|
-
pass
|
|
1313
|
-
raise Exception(
|
|
1314
|
-
f"QUERY FAILED execution_id={execution_id} state={result.get('state')}{err_detail}"
|
|
1315
|
-
)
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
837
|
def get_latest_execution(
|
|
1319
838
|
execute_kwargs: ExecuteKwargs,
|
|
1320
839
|
*,
|
|
@@ -1378,74 +897,6 @@ def get_latest_execution(
|
|
|
1378
897
|
return execution
|
|
1379
898
|
|
|
1380
899
|
|
|
1381
|
-
async def async_get_latest_execution(
|
|
1382
|
-
execute_kwargs: ExecuteKwargs,
|
|
1383
|
-
*,
|
|
1384
|
-
allow_unfinished: bool = False,
|
|
1385
|
-
) -> Execution | None:
|
|
1386
|
-
import json
|
|
1387
|
-
import random
|
|
1388
|
-
|
|
1389
|
-
import aiohttp
|
|
1390
|
-
|
|
1391
|
-
query_id = execute_kwargs['query_id']
|
|
1392
|
-
api_key = execute_kwargs['api_key']
|
|
1393
|
-
parameters = execute_kwargs['parameters']
|
|
1394
|
-
if query_id is None:
|
|
1395
|
-
raise Exception('query_id required for async_get_latest_execution')
|
|
1396
|
-
|
|
1397
|
-
# process inputs
|
|
1398
|
-
if api_key is None:
|
|
1399
|
-
api_key = _urls.get_api_key()
|
|
1400
|
-
headers = {'X-Dune-API-Key': api_key, 'User-Agent': get_user_agent()}
|
|
1401
|
-
data: dict[str, Any] = {}
|
|
1402
|
-
if parameters is not None:
|
|
1403
|
-
data['query_parameters'] = parameters
|
|
1404
|
-
data['limit'] = 0
|
|
1405
|
-
url = _urls.get_query_results_url(query_id, parameters=data, csv=False)
|
|
1406
|
-
|
|
1407
|
-
# perform request
|
|
1408
|
-
timeout = aiohttp.ClientTimeout(total=30)
|
|
1409
|
-
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
1410
|
-
sleep_amount = 1.0
|
|
1411
|
-
while True:
|
|
1412
|
-
async with session.get(url, headers=headers) as response:
|
|
1413
|
-
if response.status in (429, 502):
|
|
1414
|
-
sleep_amount = sleep_amount * random.uniform(1, 2)
|
|
1415
|
-
import asyncio
|
|
1416
|
-
await asyncio.sleep(sleep_amount)
|
|
1417
|
-
continue
|
|
1418
|
-
result: Mapping[str, Any] = await response.json()
|
|
1419
|
-
|
|
1420
|
-
# check if result is error
|
|
1421
|
-
try:
|
|
1422
|
-
if 'error' in result:
|
|
1423
|
-
if (
|
|
1424
|
-
result['error']
|
|
1425
|
-
== 'not found: No execution found for the latest version of the given query'
|
|
1426
|
-
):
|
|
1427
|
-
return None
|
|
1428
|
-
if response.status == 429:
|
|
1429
|
-
import asyncio
|
|
1430
|
-
|
|
1431
|
-
sleep_amount = sleep_amount * random.uniform(1, 2)
|
|
1432
|
-
await asyncio.sleep(sleep_amount)
|
|
1433
|
-
raise Exception(result['error'])
|
|
1434
|
-
except json.JSONDecodeError:
|
|
1435
|
-
pass
|
|
1436
|
-
break
|
|
1437
|
-
|
|
1438
|
-
# process result
|
|
1439
|
-
if not result['is_execution_finished'] and not allow_unfinished:
|
|
1440
|
-
return None
|
|
1441
|
-
execution: Execution = {'execution_id': result['execution_id']}
|
|
1442
|
-
if 'execution_started_at' in result:
|
|
1443
|
-
execution['timestamp'] = int(
|
|
1444
|
-
_parse_timestamp(result['execution_started_at'])
|
|
1445
|
-
)
|
|
1446
|
-
return execution
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
900
|
def get_user_agent() -> str:
|
|
1450
901
|
# Identify as spice-mcp vendored spice client
|
|
1451
902
|
return 'spice-mcp/' + ADAPTER_VERSION
|
|
@@ -1455,12 +906,7 @@ def get_user_agent() -> str:
|
|
|
1455
906
|
|
|
1456
907
|
determine_input_type = _determine_input_type
|
|
1457
908
|
get_query_latest_age = _get_query_latest_age
|
|
1458
|
-
async_get_query_latest_age = _async_get_query_latest_age
|
|
1459
909
|
execute_query = _execute
|
|
1460
|
-
async_execute_query = _async_execute
|
|
1461
910
|
get_results = _get_results
|
|
1462
|
-
async_get_results = _async_get_results
|
|
1463
911
|
process_result = _process_result
|
|
1464
|
-
async_process_result = _async_process_result
|
|
1465
912
|
poll_execution = _poll_execution
|
|
1466
|
-
async_poll_execution = _async_poll_execution
|