gooddata-eval 1.73.1.dev3__tar.gz → 1.74.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. gooddata_eval-1.74.0/.gitignore +8 -0
  2. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/PKG-INFO +2 -2
  3. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/pyproject.toml +2 -2
  4. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/chat/sse_client.py +5 -7
  5. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_runner.py +1 -5
  6. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tox.ini +0 -1
  7. gooddata_eval-1.73.1.dev3/.gitignore +0 -44
  8. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/LICENSE.txt +0 -0
  9. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/Makefile +0 -0
  10. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/README.md +0 -0
  11. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/__init__.py +0 -0
  12. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/_version.py +0 -0
  13. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/cli/__init__.py +0 -0
  14. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/cli/agentic_runner.py +0 -0
  15. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/cli/main.py +0 -0
  16. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/__init__.py +0 -0
  17. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/__init__.py +0 -0
  18. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/_catalog.py +0 -0
  19. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/_langfuse.py +0 -0
  20. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/alert_skill.py +0 -0
  21. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/conversation.py +0 -0
  22. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/general_question.py +0 -0
  23. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/guardrail.py +0 -0
  24. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/kda_skill.py +0 -0
  25. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/metric_skill.py +0 -0
  26. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/search_tool.py +0 -0
  27. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/agentic/visualization.py +0 -0
  28. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/chat/__init__.py +0 -0
  29. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/config.py +0 -0
  30. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/connection.py +0 -0
  31. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/dataset/__init__.py +0 -0
  32. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/dataset/langfuse_source.py +0 -0
  33. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/dataset/local.py +0 -0
  34. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/__init__.py +0 -0
  35. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/_deep_subset.py +0 -0
  36. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/_llm_judge.py +0 -0
  37. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/_text_utils.py +0 -0
  38. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/alert_skill.py +0 -0
  39. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/base.py +0 -0
  40. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/general_question.py +0 -0
  41. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/guardrail.py +0 -0
  42. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/metric_skill.py +0 -0
  43. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/search_tool.py +0 -0
  44. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/summary.py +0 -0
  45. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/evaluators/visualization.py +0 -0
  46. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/langfuse/__init__.py +0 -0
  47. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/langfuse/sink.py +0 -0
  48. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/models.py +0 -0
  49. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/reporting/__init__.py +0 -0
  50. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/reporting/console.py +0 -0
  51. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/reporting/json_report.py +0 -0
  52. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/runner.py +0 -0
  53. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/scoring.py +0 -0
  54. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/summary/__init__.py +0 -0
  55. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/summary/http_client.py +0 -0
  56. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/src/gooddata_eval/core/workspace.py +0 -0
  57. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/__init__.py +0 -0
  58. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/conftest.py +0 -0
  59. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/fixtures/sample_dataset/metric_skill_create.json +0 -0
  60. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/fixtures/sample_dataset/visualization_revenue.json +0 -0
  61. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/fixtures/sse_visualization_stream.txt +0 -0
  62. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_alert_skill.py +0 -0
  63. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_conversation.py +0 -0
  64. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_general_question.py +0 -0
  65. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_guardrail.py +0 -0
  66. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_kda_skill.py +0 -0
  67. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_langfuse_trace.py +0 -0
  68. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_metric_skill.py +0 -0
  69. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_run_context.py +0 -0
  70. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_search_tool.py +0 -0
  71. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_agentic_visualization.py +0 -0
  72. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_alert_skill_evaluator.py +0 -0
  73. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_cli.py +0 -0
  74. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_connection.py +0 -0
  75. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_deep_subset.py +0 -0
  76. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_langfuse_sink.py +0 -0
  77. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_langfuse_source.py +0 -0
  78. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_llm_judge.py +0 -0
  79. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_local_loader.py +0 -0
  80. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_metric_skill_evaluator.py +0 -0
  81. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_models.py +0 -0
  82. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_reporting.py +0 -0
  83. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_runner.py +0 -0
  84. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_scoring.py +0 -0
  85. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_search_tool_evaluator.py +0 -0
  86. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_sse_client.py +0 -0
  87. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_summary_client.py +0 -0
  88. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_summary_evaluator.py +0 -0
  89. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_text_evaluators.py +0 -0
  90. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_visualization_evaluator.py +0 -0
  91. {gooddata_eval-1.73.1.dev3 → gooddata_eval-1.74.0}/tests/test_workspace.py +0 -0
@@ -0,0 +1,8 @@
1
+ # (C) 2026 GoodData Corporation
2
+
3
+ # Unit test / coverage reports
4
+ .tox/
5
+ .coverage
6
+ .coverage.*
7
+ coverage.xml
8
+ .json-report-*.json
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: gooddata-eval
3
- Version: 1.73.1.dev3
3
+ Version: 1.74.0
4
4
  Summary: Evaluate the GoodData AI agent against your own questions and models.
5
5
  Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
6
6
  Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
17
17
  Classifier: Topic :: Software Development
18
18
  Classifier: Typing :: Typed
19
19
  Requires-Python: >=3.10
20
- Requires-Dist: gooddata-sdk~=1.73.1.dev3
20
+ Requires-Dist: gooddata-sdk~=1.74.0
21
21
  Requires-Dist: httpx<1.0,>=0.27
22
22
  Requires-Dist: orjson<4.0.0,>=3.9.15
23
23
  Requires-Dist: pydantic<3.0,>=2.6
@@ -1,7 +1,7 @@
1
1
  # (C) 2026 GoodData Corporation
2
2
  [project]
3
3
  name = "gooddata-eval"
4
- version = "1.73.1.dev3"
4
+ version = "1.74.0"
5
5
  description = "Evaluate the GoodData AI agent against your own questions and models."
6
6
  readme = "README.md"
7
7
  license = "MIT"
@@ -11,7 +11,7 @@ authors = [
11
11
  keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
12
12
  requires-python = ">=3.10"
13
13
  dependencies = [
14
- "gooddata-sdk~=1.73.1.dev3",
14
+ "gooddata-sdk~=1.74.0",
15
15
  "httpx>=0.27,<1.0",
16
16
  "orjson>=3.9.15,<4.0.0",
17
17
  "pydantic>=2.6,<3.0",
@@ -89,13 +89,11 @@ def _is_retryable_exc(exc: Exception) -> bool:
89
89
  return True
90
90
  if isinstance(exc, httpx.HTTPStatusError):
91
91
  return exc.response.status_code in _RETRYABLE_STATUS_CODES
92
- if isinstance(exc, httpx.RemoteProtocolError):
93
- # Mid-stream disconnect ("peer closed connection without sending complete
94
- # message body") -- pure network flake, not a real agent/content failure.
95
- # Confirmed live: contaminated ~1-4% of visualization runs with a hard
96
- # fail and zero retry attempts.
97
- return True
98
- return False
92
+ # Mid-stream disconnect ("peer closed connection without sending complete
93
+ # message body") -- pure network flake, not a real agent/content failure.
94
+ # Confirmed live: contaminated ~1-4% of visualization runs with a hard
95
+ # fail and zero retry attempts.
96
+ return isinstance(exc, httpx.RemoteProtocolError)
99
97
 
100
98
 
101
99
  def _retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T:
@@ -3,7 +3,7 @@
3
3
  from unittest.mock import patch
4
4
 
5
5
  import pytest
6
- from gooddata_eval.cli.agentic_runner import _dispatch_agentic, run_agentic_items
6
+ from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, _dispatch_agentic, run_agentic_items
7
7
  from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError
8
8
  from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem
9
9
 
@@ -81,8 +81,6 @@ def test_all_agentic_kind_cases_covers_every_registered_kind():
81
81
  """Guards the two parametrized tests below against silently going stale: a kind added
82
82
  to AGENTIC_TEST_KINDS without a matching case here would otherwise just not get tested,
83
83
  not fail loudly."""
84
- from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS
85
-
86
84
  covered = {kind for kind, _, _ in _ALL_AGENTIC_KIND_CASES}
87
85
  assert covered == set(AGENTIC_TEST_KINDS)
88
86
 
@@ -194,8 +192,6 @@ def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_o
194
192
  evaluator produced -- not None, not the outcome's reasoning_steps list alone, not any
195
193
  other bare value the old `isinstance(outcome, tuple)`/`isinstance(outcome, AgenticEvalOutcome)`
196
194
  fallback could silently swallow."""
197
- from gooddata_eval.core.models import AgenticEvalOutcome
198
-
199
195
  item = DatasetItem(
200
196
  id="q1",
201
197
  dataset_name="ds",
@@ -5,7 +5,6 @@ envlist = py3{10,11,12,13,14}
5
5
  [testenv]
6
6
  runner = uv-venv-lock-runner
7
7
  package = wheel
8
- wheel_build_env = .pkg
9
8
  extras =
10
9
  llm-judge
11
10
  dependency_groups =
@@ -1,44 +0,0 @@
1
- .idea
2
- .vscode
3
- .cursor
4
- *.iml
5
- .env
6
- .env.test
7
- .venv
8
- test_clients.py
9
- packages/gooddata-sdk/tests/catalog/store
10
- packages/gooddata-sdk/tests/catalog/translate
11
- .DS_Store
12
- .vscode
13
- .ruff_cache
14
-
15
- # Pytest JSON reports
16
- **/.json-report-*.json
17
-
18
- # Python build artifacts
19
- .tox
20
- *.egg-info
21
- dist/
22
- build/
23
- __pycache__/
24
- *.pyc
25
- *.pyo
26
- *.pyd
27
-
28
- docs/node_modules
29
- docs/public
30
- docs/resources/_gen
31
- docs/tmp/
32
- docs/versioned_docs
33
- docs/.hugo_build.lock
34
-
35
- # Export artifacts from Docker export-controller service
36
- packages/gooddata-sdk/tests/export/exports/default/
37
-
38
- # Staging test fixture backups (created by conftest.py, self-heal on next run)
39
- *.staging-backup
40
- .cursor/rules/aida.mdc
41
- .claude/CLAUDE.md
42
- AGENTS.md
43
- .aiassistant/rules/aida.md
44
- .junie/guidelines.md