tiltmeter 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. tiltmeter-0.1.0/.gitignore +26 -0
  2. tiltmeter-0.1.0/CHANGELOG.md +33 -0
  3. tiltmeter-0.1.0/LICENSE +21 -0
  4. tiltmeter-0.1.0/PKG-INFO +240 -0
  5. tiltmeter-0.1.0/README.md +214 -0
  6. tiltmeter-0.1.0/examples/README.md +9 -0
  7. tiltmeter-0.1.0/examples/github-action.yml +46 -0
  8. tiltmeter-0.1.0/examples/tiltmeter.yaml +62 -0
  9. tiltmeter-0.1.0/pyproject.toml +158 -0
  10. tiltmeter-0.1.0/scripts/demo.py +96 -0
  11. tiltmeter-0.1.0/scripts/release.py +122 -0
  12. tiltmeter-0.1.0/src/tiltmeter/__init__.py +5 -0
  13. tiltmeter-0.1.0/src/tiltmeter/__main__.py +6 -0
  14. tiltmeter-0.1.0/src/tiltmeter/cli.py +331 -0
  15. tiltmeter-0.1.0/src/tiltmeter/config.py +246 -0
  16. tiltmeter-0.1.0/src/tiltmeter/console.py +151 -0
  17. tiltmeter-0.1.0/src/tiltmeter/core/__init__.py +5 -0
  18. tiltmeter-0.1.0/src/tiltmeter/core/analyze.py +117 -0
  19. tiltmeter-0.1.0/src/tiltmeter/core/classify.py +75 -0
  20. tiltmeter-0.1.0/src/tiltmeter/core/diff.py +220 -0
  21. tiltmeter-0.1.0/src/tiltmeter/core/fetch.py +253 -0
  22. tiltmeter-0.1.0/src/tiltmeter/core/models.py +271 -0
  23. tiltmeter-0.1.0/src/tiltmeter/core/paths.py +204 -0
  24. tiltmeter-0.1.0/src/tiltmeter/core/shape.py +343 -0
  25. tiltmeter-0.1.0/src/tiltmeter/core/store.py +395 -0
  26. tiltmeter-0.1.0/src/tiltmeter/core/timeutil.py +39 -0
  27. tiltmeter-0.1.0/src/tiltmeter/errors.py +27 -0
  28. tiltmeter-0.1.0/src/tiltmeter/graph/__init__.py +10 -0
  29. tiltmeter-0.1.0/src/tiltmeter/graph/builder.py +38 -0
  30. tiltmeter-0.1.0/src/tiltmeter/graph/nodes.py +166 -0
  31. tiltmeter-0.1.0/src/tiltmeter/graph/state.py +64 -0
  32. tiltmeter-0.1.0/src/tiltmeter/llm/__init__.py +1 -0
  33. tiltmeter-0.1.0/src/tiltmeter/llm/explainer.py +179 -0
  34. tiltmeter-0.1.0/src/tiltmeter/llm/prompts.py +77 -0
  35. tiltmeter-0.1.0/src/tiltmeter/mcp_server/__init__.py +1 -0
  36. tiltmeter-0.1.0/src/tiltmeter/mcp_server/server.py +170 -0
  37. tiltmeter-0.1.0/src/tiltmeter/notify/__init__.py +37 -0
  38. tiltmeter-0.1.0/src/tiltmeter/notify/base.py +66 -0
  39. tiltmeter-0.1.0/src/tiltmeter/notify/slack.py +72 -0
  40. tiltmeter-0.1.0/src/tiltmeter/notify/webhook.py +48 -0
  41. tiltmeter-0.1.0/src/tiltmeter/py.typed +0 -0
  42. tiltmeter-0.1.0/src/tiltmeter/service.py +276 -0
  43. tiltmeter-0.1.0/tests/__init__.py +0 -0
  44. tiltmeter-0.1.0/tests/conftest.py +29 -0
  45. tiltmeter-0.1.0/tests/core/__init__.py +0 -0
  46. tiltmeter-0.1.0/tests/core/test_analyze.py +112 -0
  47. tiltmeter-0.1.0/tests/core/test_classify.py +68 -0
  48. tiltmeter-0.1.0/tests/core/test_diff.py +173 -0
  49. tiltmeter-0.1.0/tests/core/test_fetch.py +177 -0
  50. tiltmeter-0.1.0/tests/core/test_models.py +119 -0
  51. tiltmeter-0.1.0/tests/core/test_paths.py +85 -0
  52. tiltmeter-0.1.0/tests/core/test_shape.py +187 -0
  53. tiltmeter-0.1.0/tests/core/test_store.py +176 -0
  54. tiltmeter-0.1.0/tests/core/test_timeutil.py +42 -0
  55. tiltmeter-0.1.0/tests/fixtures/pairs/array_item_shape_changed/after.json +20 -0
  56. tiltmeter-0.1.0/tests/fixtures/pairs/array_item_shape_changed/before.json +17 -0
  57. tiltmeter-0.1.0/tests/fixtures/pairs/array_item_shape_changed/expected.json +7 -0
  58. tiltmeter-0.1.0/tests/fixtures/pairs/enum_value_added/after.json +12 -0
  59. tiltmeter-0.1.0/tests/fixtures/pairs/enum_value_added/before.json +52 -0
  60. tiltmeter-0.1.0/tests/fixtures/pairs/enum_value_added/expected.json +7 -0
  61. tiltmeter-0.1.0/tests/fixtures/pairs/field_added_optional/after.json +5 -0
  62. tiltmeter-0.1.0/tests/fixtures/pairs/field_added_optional/before.json +4 -0
  63. tiltmeter-0.1.0/tests/fixtures/pairs/field_added_optional/expected.json +7 -0
  64. tiltmeter-0.1.0/tests/fixtures/pairs/field_now_optional/after.json +11 -0
  65. tiltmeter-0.1.0/tests/fixtures/pairs/field_now_optional/before.json +12 -0
  66. tiltmeter-0.1.0/tests/fixtures/pairs/field_now_optional/expected.json +7 -0
  67. tiltmeter-0.1.0/tests/fixtures/pairs/field_removed/after.json +4 -0
  68. tiltmeter-0.1.0/tests/fixtures/pairs/field_removed/before.json +5 -0
  69. tiltmeter-0.1.0/tests/fixtures/pairs/field_removed/expected.json +7 -0
  70. tiltmeter-0.1.0/tests/fixtures/pairs/float_accepts_integer/after.json +3 -0
  71. tiltmeter-0.1.0/tests/fixtures/pairs/float_accepts_integer/before.json +3 -0
  72. tiltmeter-0.1.0/tests/fixtures/pairs/float_accepts_integer/expected.json +1 -0
  73. tiltmeter-0.1.0/tests/fixtures/pairs/integer_becomes_float/after.json +3 -0
  74. tiltmeter-0.1.0/tests/fixtures/pairs/integer_becomes_float/before.json +3 -0
  75. tiltmeter-0.1.0/tests/fixtures/pairs/integer_becomes_float/expected.json +7 -0
  76. tiltmeter-0.1.0/tests/fixtures/pairs/nested_rename/after.json +8 -0
  77. tiltmeter-0.1.0/tests/fixtures/pairs/nested_rename/before.json +8 -0
  78. tiltmeter-0.1.0/tests/fixtures/pairs/nested_rename/expected.json +14 -0
  79. tiltmeter-0.1.0/tests/fixtures/pairs/no_change_reordered/after.json +9 -0
  80. tiltmeter-0.1.0/tests/fixtures/pairs/no_change_reordered/before.json +10 -0
  81. tiltmeter-0.1.0/tests/fixtures/pairs/no_change_reordered/expected.json +1 -0
  82. tiltmeter-0.1.0/tests/fixtures/pairs/nullable_introduced/after.json +6 -0
  83. tiltmeter-0.1.0/tests/fixtures/pairs/nullable_introduced/before.json +6 -0
  84. tiltmeter-0.1.0/tests/fixtures/pairs/nullable_introduced/expected.json +7 -0
  85. tiltmeter-0.1.0/tests/fixtures/pairs/odd_key_removed/after.json +3 -0
  86. tiltmeter-0.1.0/tests/fixtures/pairs/odd_key_removed/before.json +4 -0
  87. tiltmeter-0.1.0/tests/fixtures/pairs/odd_key_removed/expected.json +7 -0
  88. tiltmeter-0.1.0/tests/fixtures/pairs/root_array_field_removed/after.json +5 -0
  89. tiltmeter-0.1.0/tests/fixtures/pairs/root_array_field_removed/before.json +10 -0
  90. tiltmeter-0.1.0/tests/fixtures/pairs/root_array_field_removed/expected.json +7 -0
  91. tiltmeter-0.1.0/tests/fixtures/pairs/root_type_changed/after.json +1 -0
  92. tiltmeter-0.1.0/tests/fixtures/pairs/root_type_changed/before.json +3 -0
  93. tiltmeter-0.1.0/tests/fixtures/pairs/root_type_changed/expected.json +7 -0
  94. tiltmeter-0.1.0/tests/fixtures/pairs/type_changed/after.json +5 -0
  95. tiltmeter-0.1.0/tests/fixtures/pairs/type_changed/before.json +5 -0
  96. tiltmeter-0.1.0/tests/fixtures/pairs/type_changed/expected.json +12 -0
  97. tiltmeter-0.1.0/tests/graph/__init__.py +0 -0
  98. tiltmeter-0.1.0/tests/graph/test_graph.py +49 -0
  99. tiltmeter-0.1.0/tests/helpers.py +108 -0
  100. tiltmeter-0.1.0/tests/service_kit.py +46 -0
  101. tiltmeter-0.1.0/tests/test_cli.py +227 -0
  102. tiltmeter-0.1.0/tests/test_config.py +124 -0
  103. tiltmeter-0.1.0/tests/test_explainer.py +187 -0
  104. tiltmeter-0.1.0/tests/test_mcp_server.py +124 -0
  105. tiltmeter-0.1.0/tests/test_notify.py +132 -0
  106. tiltmeter-0.1.0/tests/test_release_script.py +129 -0
  107. tiltmeter-0.1.0/tests/test_service.py +264 -0
@@ -0,0 +1,26 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .venv/
6
+ build/
7
+ dist/
8
+
9
+ # Tooling caches
10
+ .pytest_cache/
11
+ .mypy_cache/
12
+ .ruff_cache/
13
+ .coverage
14
+ coverage.xml
15
+ htmlcov/
16
+
17
+ # tiltmeter runtime state (baselines, history)
18
+ .tiltmeter/
19
+
20
+ # Editors / OS
21
+ .idea/
22
+ .vscode/
23
+ .DS_Store
24
+
25
+ # Local secrets
26
+ .env
@@ -0,0 +1,33 @@
1
+ # Changelog
2
+
3
+ All notable changes to Tiltmeter are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
6
+ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). Until 1.0,
7
+ a minor release may include breaking changes.
8
+
9
+ ## [Unreleased]
10
+
11
+ ## [0.1.0] - 2026-10-21
12
+
13
+ First public release, with the same features as 0.1.0rc1.
14
+
15
+ ## [0.1.0rc1] - 2026-10-21
16
+
17
+ First release candidate, published to TestPyPI only.
18
+
19
+ ### Added
20
+
21
+ - Learn the shape of JSON responses from real traffic and compare every check with a
22
+ rolling baseline of recent responses, with no spec required.
23
+ - Classify changes as breaking, warning or info: removed, newly optional, retyped and newly
24
+ nullable fields, new enum values, status and content-type changes, latency regressions
25
+ and new fields, with hints for probable renames.
26
+ - The `tiltmeter` command with `init`, `check`, `watch`, `history`, `show`, `schema`,
27
+ `explain` and `serve`, and exit codes for CI.
28
+ - An MCP server with `has_drifted`, `get_schema`, `get_changes`, `list_endpoints`,
29
+ `check_api` and `check_all`, plus endpoint and schema resources.
30
+ - Optional drift explanations written by Claude through LangChain, using structured
31
+ outputs.
32
+ - Slack and webhook notifications.
33
+ - A local SQLite history with content-addressed shapes and retention.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Agustin Torres
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,240 @@
1
+ Metadata-Version: 2.5
2
+ Name: tiltmeter
3
+ Version: 0.1.0
4
+ Summary: An early-warning system for the third-party APIs your AI agents depend on: CLI, CI gate and MCP server.
5
+ Project-URL: Homepage, https://github.com/tiltmeter-dev/tiltmeter
6
+ Project-URL: Documentation, https://github.com/tiltmeter-dev/tiltmeter/tree/main/docs
7
+ Project-URL: Changelog, https://github.com/tiltmeter-dev/tiltmeter/blob/main/CHANGELOG.md
8
+ Project-URL: Issues, https://github.com/tiltmeter-dev/tiltmeter/issues
9
+ Project-URL: Source, https://github.com/tiltmeter-dev/tiltmeter
10
+ Author: Agustin Torres
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ Keywords: ai-agents,api,contract-testing,json-schema,langgraph,mcp,monitoring,schema-drift
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Environment :: Console
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Programming Language :: Python :: 3.14
23
+ Classifier: Topic :: Software Development :: Quality Assurance
24
+ Classifier: Topic :: Software Development :: Testing
25
+ Classifier: Topic :: System :: Monitoring
26
+ Classifier: Typing :: Typed
27
+ Requires-Python: >=3.12
28
+ Requires-Dist: httpx<1,>=0.27
29
+ Requires-Dist: langchain-anthropic<2,>=1.7
30
+ Requires-Dist: langchain-core<2,>=1.0
31
+ Requires-Dist: langgraph<2,>=1.0
32
+ Requires-Dist: langsmith>=0.3.0
33
+ Requires-Dist: mcp<3,>=2.3
34
+ Requires-Dist: pydantic<3,>=2.7
35
+ Requires-Dist: pyyaml>=6.0
36
+ Requires-Dist: rich>=13
37
+ Requires-Dist: typer>=0.12
38
+ Description-Content-Type: text/markdown
39
+
40
+ # Tiltmeter
41
+
42
+ [![PyPI](https://img.shields.io/pypi/v/tiltmeter.svg)](https://pypi.org/project/tiltmeter/)
43
+ [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13%20%7C%203.14-blue.svg)](https://github.com/tiltmeter-dev/tiltmeter/blob/main/pyproject.toml)
44
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/tiltmeter-dev/tiltmeter/blob/main/LICENSE)
45
+
46
+ **An early-warning system for the third-party APIs your AI agents depend on.**
47
+
48
+
49
+
50
+ Volcanologists bolt tiltmeters to the ground to sense tiny deformations long before an
51
+ eruption. Tiltmeter does the same for the APIs under your agents. It learns the shape of
52
+ their JSON responses from real traffic, notices when a vendor renames a field, starts
53
+ sending `null` or turns a number into a string, and tells you, your CI and the agents
54
+ themselves before anything goes on tilt.
55
+
56
+ Typed code crashes loudly when an API changes. Agents don't. They read the new response,
57
+ fill the gaps with guesses, or retry until the budget is gone.
58
+
59
+ ## Highlights
60
+
61
+ - **Agents can check before they act.** An MCP server lets Claude Code, Cursor, LangChain
62
+ or any MCP client call `has_drifted` before trusting an API, and `get_schema` to see
63
+ which fields to expect.
64
+ - **No spec required.** Tiltmeter learns from real responses, so it works with any JSON
65
+ API, documented or not.
66
+ - **Breaking or harmless, at a glance.** Every change is classified as breaking, warning
67
+ or info, and a removed field next to a similar new one is flagged as a probable rename.
68
+ - **Explanations on demand.** With an Anthropic API key, Claude explains what changed,
69
+ the likely cause and what to do about it.
70
+ - **A CI gate with alerts.** Exit codes fail the build on breaking drift. Slack and
71
+ webhook alerts take one line of config.
72
+ - **Local and private.** One SQLite file and no account. Credentials are expanded at
73
+ request time and never stored. Response bodies are not stored or sent to the LLM, only
74
+ their shape.
75
+
76
+ ## Quickstart
77
+
78
+ ```bash
79
+ pip install tiltmeter # or: uv tool install tiltmeter
80
+ tiltmeter init # writes tiltmeter.yaml with a public example API
81
+ tiltmeter check # the first check records the baseline
82
+ ```
83
+
84
+ Run `tiltmeter check` again from cron or CI, or leave `tiltmeter watch` running. This is
85
+ what you get when a vendor renames `phone` and turns `balance` into a string:
86
+
87
+ ```console
88
+ $ tiltmeter check
89
+ ✗ billing_customers 200 · 13 ms BREAKING · 3 changes
90
+ breaking type_changed $.data[*].balance integer → string
91
+ breaking field_removed $.data[*].phone string → (absent) possibly renamed to $.data[*].phone_number
92
+ info field_added $.data[*].phone_number (absent) → string possibly replaces $.data[*].phone
93
+ 1 endpoint checked · 1 breaking
94
+ $ echo $?
95
+ 2
96
+ ```
97
+
98
+ The [getting started guide](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/getting-started.md) runs that exact scenario against a
99
+ fake API on your machine in about five minutes.
100
+
101
+ ## Give your agents a tiltmeter
102
+
103
+ ```bash
104
+ claude mcp add tiltmeter -- tiltmeter serve --config /absolute/path/to/tiltmeter.yaml
105
+ ```
106
+
107
+ Then add a line like this to your agent's instructions:
108
+
109
+ > Before you rely on a third-party API, call `has_drifted` for it. If it drifted, read the
110
+ > summary and adapt before trusting the response.
111
+
112
+ | Tool | What it answers |
113
+ |---|---|
114
+ | `has_drifted(endpoint_id, since?)` | Did the responses change at warning level or worse recently? Returns a summary and the changes. |
115
+ | `get_schema(endpoint_id)` | Which fields, types and enum values to expect, as JSON Schema. |
116
+ | `get_changes(endpoint_id?, since?, min_severity?, limit?)` | The change history, newest first. |
117
+ | `list_endpoints()` | Which APIs are watched and how their last check went. |
118
+ | `check_api(endpoint_id, explain?)` | Calls one API now and records the result. |
119
+ | `check_all(explain?)` | Calls every API now. |
120
+
121
+ Setups for Cursor, Claude Desktop and LangChain agents are in the
122
+ [agents guide](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/agents.md).
123
+
124
+ ## What counts as drift
125
+
126
+ | Change | Example | Default severity |
127
+ |---|---|---|
128
+ | `field_removed` | `phone` was in every response and is gone | breaking |
129
+ | `field_now_optional` | `email` was always there and is now missing from some objects | breaking |
130
+ | `type_changed` | `balance` went from integer to string | breaking |
131
+ | `nullable_introduced` | `phone` can now be `null` | breaking |
132
+ | `content_type_changed` | the JSON body became an HTML error page | breaking |
133
+ | `status_changed` | 200 to 503 is breaking, 404 to 200 is info, 200 to 202 is a warning | depends |
134
+ | `enum_value_added` | `status` returned `suspended` besides `active` and `past_due` | warning |
135
+ | `latency_regression` | median latency over 3x the baseline and at least 250 ms slower | warning |
136
+ | `field_added` | a new `phone_number` field | info, or warning with `strict: true` |
137
+
138
+ Override any of them globally or per endpoint with `severity_overrides`.
139
+
140
+ ## Use it in CI
141
+
142
+ | Exit code | Meaning |
143
+ |---|---|
144
+ | 0 | No drift. Info-only changes and new baselines count as no drift. |
145
+ | 1 | Warning-level drift. |
146
+ | 2 | Breaking drift. |
147
+ | 3 | A check could not run (network error, unexpected status, invalid JSON) and nothing broke. |
148
+ | 4 | Usage or configuration error. |
149
+
150
+ The baseline lives in `.tiltmeter/`, so keep that folder between runs. The
151
+ [CI guide](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/ci.md) has ready-to-copy jobs for GitHub Actions, GitLab CI and cron.
152
+
153
+ ## How it works
154
+
155
+ 1. **Observe.** A check calls the endpoint and turns the JSON body into a *shape*: for
156
+ every path, which JSON types appeared, how many objects contained each field and, for
157
+ fields that look categorical (`active`, `past_due`, `USD`), which values were seen.
158
+ 2. **Compare.** The baseline is the merge of the shapes from the last 20 successful
159
+ checks. Tiltmeter reports only what *contradicts* it: a missing optional field is
160
+ fine, a field that was in every object and is now gone is not.
161
+ 3. **Report once.** The new shape joins the baseline, so the check that sees a change
162
+ reports it, and from then on it is the new normal.
163
+
164
+
165
+
166
+ The pipeline is a small [LangGraph](https://github.com/langchain-ai/langgraph) graph that
167
+ only sequences the steps. The diff itself is plain, deterministic Python with no LLM in the
168
+ loop. [How it works](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/how-it-works.md) covers the details.
169
+
170
+ ## Configuration
171
+
172
+ ```yaml
173
+ endpoints:
174
+ - id: stripe_customers
175
+ url: https://api.stripe.com/v1/customers
176
+ params:
177
+ limit: 10
178
+ headers:
179
+ Authorization: Bearer ${STRIPE_API_KEY} # expanded from the environment at request time
180
+ ignore_paths:
181
+ - $.data[*].metadata # free-form, changes all the time
182
+
183
+ notify:
184
+ - type: slack
185
+ webhook_url: ${SLACK_WEBHOOK_URL}
186
+ ```
187
+
188
+ Every option is described in the [configuration reference](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/configuration.md), and
189
+ [`examples/`](https://github.com/tiltmeter-dev/tiltmeter/blob/main/examples) has a config you can run as is.
190
+
191
+ ## How it compares
192
+
193
+ - **OpenAPI diff tools** such as oasdiff compare two versions of a spec. They are the right
194
+ tool when the vendor publishes and maintains one, and they cannot see changes the vendor
195
+ did not document.
196
+ - **Contract testing** such as Pact needs the provider to run your contracts, which
197
+ third-party vendors don't do.
198
+ - **Uptime and synthetic monitors** tell you an API is up. You write and maintain the
199
+ response assertions yourself.
200
+ - **Tiltmeter** learns the contract from real responses, so it needs nothing from the
201
+ vendor and catches undocumented changes. It also hands what it learns to your agents.
202
+
203
+ ## Privacy and security
204
+
205
+ - `${VAR}` placeholders are expanded right before a request or notification is sent. The
206
+ values are never written to disk, sent to the LLM or kept in the LangGraph state, and
207
+ they are scrubbed from error messages.
208
+ - The history stores shapes, status codes, latencies and changes, never response bodies
209
+ unless you set `store_samples: true`.
210
+ - Agents can only check the endpoints in your config. They cannot make Tiltmeter call an
211
+ arbitrary URL.
212
+
213
+ [SECURITY.md](https://github.com/tiltmeter-dev/tiltmeter/blob/main/SECURITY.md) has the full threat model and how to report a vulnerability.
214
+
215
+ ## Roadmap
216
+
217
+ - **Auto-fix pull requests.** When a vendor ships a breaking change, an agent built on
218
+ LangChain Deep Agents finds the affected code in your repository and opens a PR.
219
+ - **Spec and changelog awareness.** Diff vendor OpenAPI specs and changelogs next to
220
+ observed responses.
221
+ - **Hosted checks** and a team dashboard.
222
+
223
+ ## Documentation
224
+
225
+ - [Getting started](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/getting-started.md): a five-minute tour on your machine.
226
+ - [Configuration reference](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/configuration.md): every option, placeholders and ignore patterns.
227
+ - [Agents and MCP](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/agents.md): Claude Code, Cursor, Claude Desktop and LangChain.
228
+ - [CI and scheduling](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/ci.md): GitHub Actions, GitLab CI and cron.
229
+ - [How it works](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/how-it-works.md): shapes, baselines, the diff rules and the pipeline.
230
+ - [FAQ and troubleshooting](https://github.com/tiltmeter-dev/tiltmeter/blob/main/docs/faq.md).
231
+
232
+ ## Contributing
233
+
234
+ Bug reports, ideas and pull requests are welcome. [CONTRIBUTING.md](https://github.com/tiltmeter-dev/tiltmeter/blob/main/CONTRIBUTING.md)
235
+ explains how to set up the project and what a good change looks like. Everyone taking
236
+ part is expected to follow the [code of conduct](https://github.com/tiltmeter-dev/tiltmeter/blob/main/CODE_OF_CONDUCT.md).
237
+
238
+ ## License
239
+
240
+ [MIT](https://github.com/tiltmeter-dev/tiltmeter/blob/main/LICENSE)
@@ -0,0 +1,214 @@
1
+ # Tiltmeter
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/tiltmeter.svg)](https://pypi.org/project/tiltmeter/)
4
+ [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13%20%7C%203.14-blue.svg)](pyproject.toml)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
6
+
7
+ **An early-warning system for the third-party APIs your AI agents depend on.**
8
+
9
+ <!-- pypi-skip-image -->
10
+ ![Tiltmeter catching a renamed field and a type change](docs/demo.svg)
11
+ <!-- /pypi-skip-image -->
12
+
13
+ Volcanologists bolt tiltmeters to the ground to sense tiny deformations long before an
14
+ eruption. Tiltmeter does the same for the APIs under your agents. It learns the shape of
15
+ their JSON responses from real traffic, notices when a vendor renames a field, starts
16
+ sending `null` or turns a number into a string, and tells you, your CI and the agents
17
+ themselves before anything goes on tilt.
18
+
19
+ Typed code crashes loudly when an API changes. Agents don't. They read the new response,
20
+ fill the gaps with guesses, or retry until the budget is gone.
21
+
22
+ ## Highlights
23
+
24
+ - **Agents can check before they act.** An MCP server lets Claude Code, Cursor, LangChain
25
+ or any MCP client call `has_drifted` before trusting an API, and `get_schema` to see
26
+ which fields to expect.
27
+ - **No spec required.** Tiltmeter learns from real responses, so it works with any JSON
28
+ API, documented or not.
29
+ - **Breaking or harmless, at a glance.** Every change is classified as breaking, warning
30
+ or info, and a removed field next to a similar new one is flagged as a probable rename.
31
+ - **Explanations on demand.** With an Anthropic API key, Claude explains what changed,
32
+ the likely cause and what to do about it.
33
+ - **A CI gate with alerts.** Exit codes fail the build on breaking drift. Slack and
34
+ webhook alerts take one line of config.
35
+ - **Local and private.** One SQLite file and no account. Credentials are expanded at
36
+ request time and never stored. Response bodies are not stored or sent to the LLM, only
37
+ their shape.
38
+
39
+ ## Quickstart
40
+
41
+ ```bash
42
+ pip install tiltmeter # or: uv tool install tiltmeter
43
+ tiltmeter init # writes tiltmeter.yaml with a public example API
44
+ tiltmeter check # the first check records the baseline
45
+ ```
46
+
47
+ Run `tiltmeter check` again from cron or CI, or leave `tiltmeter watch` running. This is
48
+ what you get when a vendor renames `phone` and turns `balance` into a string:
49
+
50
+ ```console
51
+ $ tiltmeter check
52
+ ✗ billing_customers 200 · 13 ms BREAKING · 3 changes
53
+ breaking type_changed $.data[*].balance integer → string
54
+ breaking field_removed $.data[*].phone string → (absent) possibly renamed to $.data[*].phone_number
55
+ info field_added $.data[*].phone_number (absent) → string possibly replaces $.data[*].phone
56
+ 1 endpoint checked · 1 breaking
57
+ $ echo $?
58
+ 2
59
+ ```
60
+
61
+ The [getting started guide](docs/getting-started.md) runs that exact scenario against a
62
+ fake API on your machine in about five minutes.
63
+
64
+ ## Give your agents a tiltmeter
65
+
66
+ ```bash
67
+ claude mcp add tiltmeter -- tiltmeter serve --config /absolute/path/to/tiltmeter.yaml
68
+ ```
69
+
70
+ Then add a line like this to your agent's instructions:
71
+
72
+ > Before you rely on a third-party API, call `has_drifted` for it. If it drifted, read the
73
+ > summary and adapt before trusting the response.
74
+
75
+ | Tool | What it answers |
76
+ |---|---|
77
+ | `has_drifted(endpoint_id, since?)` | Did the responses change at warning level or worse recently? Returns a summary and the changes. |
78
+ | `get_schema(endpoint_id)` | Which fields, types and enum values to expect, as JSON Schema. |
79
+ | `get_changes(endpoint_id?, since?, min_severity?, limit?)` | The change history, newest first. |
80
+ | `list_endpoints()` | Which APIs are watched and how their last check went. |
81
+ | `check_api(endpoint_id, explain?)` | Calls one API now and records the result. |
82
+ | `check_all(explain?)` | Calls every API now. |
83
+
84
+ Setups for Cursor, Claude Desktop and LangChain agents are in the
85
+ [agents guide](docs/agents.md).
86
+
87
+ ## What counts as drift
88
+
89
+ | Change | Example | Default severity |
90
+ |---|---|---|
91
+ | `field_removed` | `phone` was in every response and is gone | breaking |
92
+ | `field_now_optional` | `email` was always there and is now missing from some objects | breaking |
93
+ | `type_changed` | `balance` went from integer to string | breaking |
94
+ | `nullable_introduced` | `phone` can now be `null` | breaking |
95
+ | `content_type_changed` | the JSON body became an HTML error page | breaking |
96
+ | `status_changed` | 200 to 503 is breaking, 404 to 200 is info, 200 to 202 is a warning | depends |
97
+ | `enum_value_added` | `status` returned `suspended` besides `active` and `past_due` | warning |
98
+ | `latency_regression` | median latency over 3x the baseline and at least 250 ms slower | warning |
99
+ | `field_added` | a new `phone_number` field | info, or warning with `strict: true` |
100
+
101
+ Override any of them globally or per endpoint with `severity_overrides`.
102
+
103
+ ## Use it in CI
104
+
105
+ | Exit code | Meaning |
106
+ |---|---|
107
+ | 0 | No drift. Info-only changes and new baselines count as no drift. |
108
+ | 1 | Warning-level drift. |
109
+ | 2 | Breaking drift. |
110
+ | 3 | A check could not run (network error, unexpected status, invalid JSON) and nothing broke. |
111
+ | 4 | Usage or configuration error. |
112
+
113
+ The baseline lives in `.tiltmeter/`, so keep that folder between runs. The
114
+ [CI guide](docs/ci.md) has ready-to-copy jobs for GitHub Actions, GitLab CI and cron.
115
+
116
+ ## How it works
117
+
118
+ 1. **Observe.** A check calls the endpoint and turns the JSON body into a *shape*: for
119
+ every path, which JSON types appeared, how many objects contained each field and, for
120
+ fields that look categorical (`active`, `past_due`, `USD`), which values were seen.
121
+ 2. **Compare.** The baseline is the merge of the shapes from the last 20 successful
122
+ checks. Tiltmeter reports only what *contradicts* it: a missing optional field is
123
+ fine, a field that was in every object and is now gone is not.
124
+ 3. **Report once.** The new shape joins the baseline, so the check that sees a change
125
+ reports it, and from then on it is the new normal.
126
+
127
+ <!-- pypi-skip-diagram -->
128
+ ```mermaid
129
+ graph TD;
130
+ start([start]) --> observe
131
+ observe -. fetch failed .-> persist
132
+ observe --> load_baseline --> compare
133
+ compare -. drift worth explaining .-> explain --> persist
134
+ compare --> persist
135
+ persist -. a notifier wants it .-> notify --> finish([end])
136
+ persist --> finish
137
+ ```
138
+ <!-- /pypi-skip-diagram -->
139
+
140
+ The pipeline is a small [LangGraph](https://github.com/langchain-ai/langgraph) graph that
141
+ only sequences the steps. The diff itself is plain, deterministic Python with no LLM in the
142
+ loop. [How it works](docs/how-it-works.md) covers the details.
143
+
144
+ ## Configuration
145
+
146
+ ```yaml
147
+ endpoints:
148
+ - id: stripe_customers
149
+ url: https://api.stripe.com/v1/customers
150
+ params:
151
+ limit: 10
152
+ headers:
153
+ Authorization: Bearer ${STRIPE_API_KEY} # expanded from the environment at request time
154
+ ignore_paths:
155
+ - $.data[*].metadata # free-form, changes all the time
156
+
157
+ notify:
158
+ - type: slack
159
+ webhook_url: ${SLACK_WEBHOOK_URL}
160
+ ```
161
+
162
+ Every option is described in the [configuration reference](docs/configuration.md), and
163
+ [`examples/`](examples) has a config you can run as is.
164
+
165
+ ## How it compares
166
+
167
+ - **OpenAPI diff tools** such as oasdiff compare two versions of a spec. They are the right
168
+ tool when the vendor publishes and maintains one, and they cannot see changes the vendor
169
+ did not document.
170
+ - **Contract testing** such as Pact needs the provider to run your contracts, which
171
+ third-party vendors don't do.
172
+ - **Uptime and synthetic monitors** tell you an API is up. You write and maintain the
173
+ response assertions yourself.
174
+ - **Tiltmeter** learns the contract from real responses, so it needs nothing from the
175
+ vendor and catches undocumented changes. It also hands what it learns to your agents.
176
+
177
+ ## Privacy and security
178
+
179
+ - `${VAR}` placeholders are expanded right before a request or notification is sent. The
180
+ values are never written to disk, sent to the LLM or kept in the LangGraph state, and
181
+ they are scrubbed from error messages.
182
+ - The history stores shapes, status codes, latencies and changes, never response bodies
183
+ unless you set `store_samples: true`.
184
+ - Agents can only check the endpoints in your config. They cannot make Tiltmeter call an
185
+ arbitrary URL.
186
+
187
+ [SECURITY.md](SECURITY.md) has the full threat model and how to report a vulnerability.
188
+
189
+ ## Roadmap
190
+
191
+ - **Auto-fix pull requests.** When a vendor ships a breaking change, an agent built on
192
+ LangChain Deep Agents finds the affected code in your repository and opens a PR.
193
+ - **Spec and changelog awareness.** Diff vendor OpenAPI specs and changelogs next to
194
+ observed responses.
195
+ - **Hosted checks** and a team dashboard.
196
+
197
+ ## Documentation
198
+
199
+ - [Getting started](docs/getting-started.md): a five-minute tour on your machine.
200
+ - [Configuration reference](docs/configuration.md): every option, placeholders and ignore patterns.
201
+ - [Agents and MCP](docs/agents.md): Claude Code, Cursor, Claude Desktop and LangChain.
202
+ - [CI and scheduling](docs/ci.md): GitHub Actions, GitLab CI and cron.
203
+ - [How it works](docs/how-it-works.md): shapes, baselines, the diff rules and the pipeline.
204
+ - [FAQ and troubleshooting](docs/faq.md).
205
+
206
+ ## Contributing
207
+
208
+ Bug reports, ideas and pull requests are welcome. [CONTRIBUTING.md](CONTRIBUTING.md)
209
+ explains how to set up the project and what a good change looks like. Everyone taking
210
+ part is expected to follow the [code of conduct](CODE_OF_CONDUCT.md).
211
+
212
+ ## License
213
+
214
+ [MIT](LICENSE)
@@ -0,0 +1,9 @@
1
+ # Examples
2
+
3
+ | File | What it is |
4
+ |---|---|
5
+ | [`tiltmeter.yaml`](tiltmeter.yaml) | A config with three public APIs that need no keys (GitHub, JSONPlaceholder and Open-Meteo), plus commented Stripe and GraphQL endpoints. Run it with `tiltmeter check -c examples/tiltmeter.yaml`. |
6
+ | [`github-action.yml`](github-action.yml) | A scheduled GitHub Actions job that keeps the baseline in the Actions cache and fails on drift. Copy it to `.github/workflows/` in the repository that holds your config. |
7
+
8
+ The [getting started guide](../docs/getting-started.md) walks through a full drift cycle
9
+ against a fake API on your machine.
@@ -0,0 +1,46 @@
1
+ # Check your APIs every hour and fail the job on breaking drift.
2
+ # Copy this file to .github/workflows/tiltmeter.yml in the repository that holds
3
+ # tiltmeter.yaml, and add the secrets your endpoints reference.
4
+ name: API drift
5
+
6
+ on:
7
+ schedule:
8
+ - cron: "17 * * * *"
9
+ workflow_dispatch:
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ tiltmeter:
16
+ runs-on: ubuntu-latest
17
+ timeout-minutes: 15
18
+ steps:
19
+ - uses: actions/checkout@v7
20
+ with:
21
+ persist-credentials: false
22
+
23
+ - uses: astral-sh/setup-uv@v10
24
+
25
+ # The baseline lives in .tiltmeter/. Restore the newest copy...
26
+ - uses: actions/cache/restore@v6
27
+ with:
28
+ path: .tiltmeter
29
+ key: tiltmeter-${{ github.run_id }}
30
+ restore-keys: tiltmeter-
31
+
32
+ # Exit codes: 0 clean, 1 warning, 2 breaking, 3 a check failed, 4 bad config.
33
+ # Pin the version so a new release never changes your CI without a review.
34
+ - name: Check APIs
35
+ run: uvx tiltmeter@0.1.0 check
36
+ env:
37
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
38
+ STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }}
39
+ SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
40
+
41
+ # ...and save the new one even when the job fails, so each change is reported once.
42
+ - uses: actions/cache/save@v6
43
+ if: always()
44
+ with:
45
+ path: .tiltmeter
46
+ key: tiltmeter-${{ github.run_id }}
@@ -0,0 +1,62 @@
1
+ # tiltmeter example: public APIs you can check right away, no keys needed.
2
+ # tiltmeter check -c examples/tiltmeter.yaml
3
+ version: 1
4
+
5
+ endpoints:
6
+ - id: github_repo
7
+ description: Repository metadata from the GitHub REST API (60 requests/hour without a token)
8
+ url: https://api.github.com/repos/langchain-ai/langchain
9
+ headers:
10
+ Accept: application/vnd.github+json
11
+
12
+ - id: jsonplaceholder_users
13
+ description: Fake user directory from JSONPlaceholder
14
+ url: https://jsonplaceholder.typicode.com/users
15
+
16
+ - id: open_meteo_buenos_aires
17
+ description: Current weather in Buenos Aires from Open-Meteo
18
+ url: https://api.open-meteo.com/v1/forecast
19
+ params:
20
+ latitude: "-34.61"
21
+ longitude: "-58.38"
22
+ current: temperature_2m,wind_speed_10m
23
+
24
+ # Credentials come from environment variables and are expanded at request time.
25
+ # They are never written to disk or sent to the LLM.
26
+ #
27
+ # - id: stripe_customers
28
+ # url: https://api.stripe.com/v1/customers
29
+ # params:
30
+ # limit: 10
31
+ # headers:
32
+ # Authorization: Bearer ${STRIPE_API_KEY}
33
+ # ignore_paths:
34
+ # - $.data[*].metadata # free-form, changes all the time
35
+ # severity_overrides:
36
+ # field_now_optional: warning
37
+ #
38
+ # GraphQL and other POST APIs work too:
39
+ #
40
+ # - id: github_viewer
41
+ # method: POST
42
+ # url: https://api.github.com/graphql
43
+ # headers:
44
+ # Authorization: Bearer ${GITHUB_TOKEN}
45
+ # body:
46
+ # query: "{ viewer { login createdAt } }"
47
+
48
+ llm:
49
+ enabled: true # used only when ANTHROPIC_API_KEY is set
50
+ model: claude-haiku-5-5
51
+ effort: low
52
+
53
+ notify: []
54
+ # - type: slack
55
+ # webhook_url: ${SLACK_WEBHOOK_URL}
56
+ # min_severity: warning
57
+ # - type: webhook
58
+ # url: https://example.com/hooks/tiltmeter
59
+ # headers:
60
+ # Authorization: Bearer ${HOOK_TOKEN}
61
+ # min_severity: breaking
62
+ # on_error: true